From 08247c2b1d4a5883b3df0a4544b1bce4bc0d28c9 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sun, 19 Apr 2026 23:12:03 -0400 Subject: [PATCH] Drop DbRetryMiddleware, make logger async, bump django-bolt MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wedges resumed despite the April 16 DB-timeout fix, and this time with zero ESTABLISHED connections to Postgres — so not a DB deadlock. Most suspicious piece is DbRetryMiddleware itself: its async path awaits sync_to_async(...) inside an exception handler, a known-tricky pattern on the asgiref thread pool. Django already handles stale-connection recovery via conn_health_checks=True plus the psycopg connect_timeout/ statement_timeout added earlier, so the retry layer isn't load-bearing. Also: - RequestLoggingMiddleware is now async-capable, so the full chain stays async for async views instead of hopping through the thread pool on every request. - Bump django-bolt 0.7.4 → 0.7.5 for any upstream fixes. Co-Authored-By: Claude Opus 4.7 (1M context) --- core/middleware.py | 89 +++++++++++++++----------------------------- exiftree/settings.py | 1 - pyproject.toml | 2 +- uv.lock | 12 +++--- 4 files changed, 38 insertions(+), 66 deletions(-) diff --git a/core/middleware.py b/core/middleware.py index fdf45ad..f22c97c 100644 --- a/core/middleware.py +++ b/core/middleware.py @@ -2,8 +2,7 @@ import logging import re import time -from asgiref.sync import iscoroutinefunction, markcoroutinefunction, sync_to_async -from django.db import InterfaceError, OperationalError, connections +from asgiref.sync import iscoroutinefunction, markcoroutinefunction logger = logging.getLogger('core.requests') @@ -14,6 +13,9 @@ BOT_PATTERNS = re.compile( re.IGNORECASE, ) +SKIP_PATHS = ('/static/',) +SKIP_EXACT = {'/health', '/favicon.ico'} + def _detect_bot(user_agent: str) -> str | None: """Return the bot name from User-Agent, or None if not a bot.""" @@ -21,17 +23,28 @@ def _detect_bot(user_agent: str) -> str | None: return match.group(0) if match else None -def _drop_connections(): - for conn in connections.all(initialized_only=True): - conn.close() +def _log(request, response, duration_ms): + path = request.path + if path in SKIP_EXACT or path.startswith(SKIP_PATHS): + return + ua = request.META.get('HTTP_USER_AGENT', '') + bot = _detect_bot(ua) + if bot: + logger.info( + '[BOT:%s] %s %s %s %.0fms ua="%s"', + bot, request.method, path, response.status_code, duration_ms, ua[:200], + ) + else: + logger.info( + '%s %s %s %.0fms', + request.method, path, response.status_code, duration_ms, + ) -class DbRetryMiddleware: - """Retry once on stale DB connections (e.g. after Postgres restart). - - Supports both sync and async request paths — django-bolt mounts Django - under ASGI, so middleware must handle both. - """ +class RequestLoggingMiddleware: + """Logs every request. Async-capable so django-bolt's ASGI chain stays + async end-to-end — avoids hopping through the asgiref thread pool on + every request, which piles up under load.""" sync_capable = True async_capable = True @@ -45,53 +58,13 @@ class DbRetryMiddleware: def __call__(self, request): if self.async_mode: return self._acall(request) - try: - return self.get_response(request) - except (OperationalError, InterfaceError): - _drop_connections() - return self.get_response(request) - - async def _acall(self, request): - try: - return await self.get_response(request) - except (OperationalError, InterfaceError): - await sync_to_async(_drop_connections)() - return await self.get_response(request) - - -class RequestLoggingMiddleware: - def __init__(self, get_response): - self.get_response = get_response - - def __call__(self, request): start = time.time() response = self.get_response(request) - duration = (time.time() - start) * 1000 - - # Skip static/health/favicon - path = request.path - if path.startswith('/static/') or path == '/health' or path == '/favicon.ico': - return response - - ua = request.META.get('HTTP_USER_AGENT', '') - bot = _detect_bot(ua) - - if bot: - logger.info( - '[BOT:%s] %s %s %s %.0fms ua="%s"', - bot, - request.method, - path, - response.status_code, - duration, - ua[:200], - ) - else: - logger.info( - '%s %s %s %.0fms', - request.method, - path, - response.status_code, - duration, - ) + _log(request, response, (time.time() - start) * 1000) + return response + + async def _acall(self, request): + start = time.time() + response = await self.get_response(request) + _log(request, response, (time.time() - start) * 1000) return response diff --git a/exiftree/settings.py b/exiftree/settings.py index a091d4a..bbd715d 100644 --- a/exiftree/settings.py +++ b/exiftree/settings.py @@ -53,7 +53,6 @@ INSTALLED_APPS = [ AUTH_USER_MODEL = "core.User" MIDDLEWARE = [ - "core.middleware.DbRetryMiddleware", "core.middleware.RequestLoggingMiddleware", "django.middleware.security.SecurityMiddleware", "django.contrib.sessions.middleware.SessionMiddleware", diff --git a/pyproject.toml b/pyproject.toml index 077fdd4..e62336e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,7 @@ dependencies = [ "celery>=5.6.3", "dj-database-url>=3.1.2", "django>=6.0.4", - "django-bolt>=0.7.4", + "django-bolt>=0.7.5", "django-storages>=1.14.6", "exifread>=3.5.1", "httpx>=0.28.1", diff --git a/uv.lock b/uv.lock index dc65a45..3027159 100644 --- a/uv.lock +++ b/uv.lock @@ -279,7 +279,7 @@ wheels = [ [[package]] name = "django-bolt" -version = "0.7.4" +version = "0.7.5" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "click" }, @@ -291,11 +291,11 @@ dependencies = [ { name = "typing-extensions" }, { name = "uvloop", marker = "sys_platform != 'win32'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/bf/66/933e3bfdfc6e06ff1e9c004a9a576da55f9a8ee31e05ff8b0082c8e3156a/django_bolt-0.7.4.tar.gz", hash = "sha256:7604d01220ca5dae6473b4aeccd2202afe42fd5de9834df80597f0f20a004825", size = 15134915, upload-time = "2026-03-25T11:09:10.774Z" } +sdist = { url = "https://files.pythonhosted.org/packages/55/9d/66fea5c67d4b5fe853fd2a96e0ad447569efcde9d8fa8bb331829d7b82e7/django_bolt-0.7.5.tar.gz", hash = "sha256:8e7e0d371b109d24dc747bf93b7f5d568c103cd2b210034ff6cfb8622f51d182", size = 15146489, upload-time = "2026-04-17T13:01:29.216Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/cb/61/c48fbdb6c553f77593596c54533c4558350b47b8533721c732da5458a864/django_bolt-0.7.4-cp312-abi3-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:f0367f7e16573318d22ccbaef1bc68f0d6e1edfbe676c9aaad83e2fc04b620fb", size = 6387694, upload-time = "2026-03-25T11:09:06.842Z" }, - { url = "https://files.pythonhosted.org/packages/88/72/1698df5ea7452dd0752a2d7e2949a10aa03c89b375c531d998c588d67bf4/django_bolt-0.7.4-cp312-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:872c36d893787478f442ce4fa0b12029e78a466003723787e55c5c4a36182280", size = 3693799, upload-time = "2026-03-25T11:09:08.517Z" }, - { url = "https://files.pythonhosted.org/packages/ec/ae/e944d0754ac700038e5a86f2aa367fb60c387289de26cac7b04c35161686/django_bolt-0.7.4-cp312-abi3-win_amd64.whl", hash = "sha256:16ac49d0c46ab33f6e3d70fe117c78269ec4aad2488a6cfb48812665e0ff2e7e", size = 3409429, upload-time = "2026-03-25T11:09:04.973Z" }, + { url = "https://files.pythonhosted.org/packages/fb/72/a847d6229a2d0d2bc33a34577e7136a65eaf151d1ad07012b13a1361be37/django_bolt-0.7.5-cp312-abi3-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:bcf058fbc03fab287e653e7b31e1f4ceb45274e86a581962e6d95fb82c00893d", size = 6388402, upload-time = "2026-04-17T13:01:31.218Z" }, + { url = "https://files.pythonhosted.org/packages/00/97/98997c2cd24a43444ab8a9eb85822bc536c77f6100b48e0295dd6f6e4a00/django_bolt-0.7.5-cp312-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:41863ac06412c1b6b773245b999ffc85af9d4723fc39ef197035a48770499d98", size = 3679337, upload-time = "2026-04-17T13:01:27.167Z" }, + { url = "https://files.pythonhosted.org/packages/b5/f2/c1982157db0263e69d07a369ca29130c4ff9f5fb31f7ea04235e375eee96/django_bolt-0.7.5-cp312-abi3-win_amd64.whl", hash = "sha256:13d92ea376ed23c9c7124128676746f6ac701a91d3380366a7584d7c2b69762c", size = 3417120, upload-time = "2026-04-17T13:01:25.449Z" }, ] [[package]] @@ -349,7 +349,7 @@ requires-dist = [ { name = "celery", specifier = ">=5.6.3" }, { name = "dj-database-url", specifier = ">=3.1.2" }, { name = "django", specifier = ">=6.0.4" }, - { name = "django-bolt", specifier = ">=0.7.4" }, + { name = "django-bolt", specifier = ">=0.7.5" }, { name = "django-storages", specifier = ">=1.14.6" }, { name = "exifread", specifier = ">=3.5.1" }, { name = "httpx", specifier = ">=0.28.1" },