Reduce Fly.io resource usage and harden robots.txt

- Downscale VMs from performance-2x/4GB to shared-cpu-2x/2GB
- Set min_machines_running to 0 for autosleep
- Disable interlinear preload to prevent OOM kills
- Reduce workers from 4 to 2
- Add crawl-delay of 5s and block PDF/old verse-of-the-day crawling

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

fly.toml

@@ -16,7 +16,7 @@ internal_port = 8000
 force_https = true
 auto_stop_machines = 'suspend'
 auto_start_machines = true
-min_machines_running = 1
+min_machines_running = 0
 processes = ['app']
 
 [http_service.concurrency]
@@ -33,8 +33,8 @@ method = "GET"
 path = "/health"
 
 [[vm]]
-memory = '4gb'
-cpu_kind = 'performance'
+memory = '2gb'
+cpu_kind = 'shared'
 cpus = 2
 
 [env]
@@ -42,8 +42,8 @@ cpus = 2
 PYTHONUNBUFFERED = "1"
 PYTHONDONTWRITEBYTECODE = "1"
 
-# Preload interlinear data on startup for fast first requests
-PRELOAD_INTERLINEAR = "true"
+# Lazy-load interlinear data to reduce memory usage
+PRELOAD_INTERLINEAR = "false"
 
-# Number of Uvicorn workers (recommended: 2x CPU cores)
-WORKERS = "4"
+# Number of Uvicorn workers
+WORKERS = "2"

kjvstudy_org/routes/utility.py

@@ -49,12 +49,12 @@ async def robots_txt():
     robots_content = """User-agent: *
 Allow: /
 Disallow: /api/
+Disallow: /*/pdf
+Disallow: /verse-of-the-day/2
+Crawl-delay: 5
 
 # Sitemap location
 Sitemap: https://kjvstudy.org/sitemap.xml
-
-# Crawl delay (be nice to our servers)
-Crawl-delay: 1
 """
     return Response(content=robots_content, media_type="text/plain")