From 961a0d45ea23ddeaeda89a32cba96d86c67c60aa Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sun, 12 Apr 2026 10:41:31 -0400 Subject: [PATCH] Bump VM memory to 2GB and add crawler rate limits MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit OOM crash took the site down — increase memory from 1GB to 2GB. Add Crawl-delay to robots.txt to throttle aggressive bots. Co-Authored-By: Claude Opus 4.6 (1M context) --- fly.toml | 2 +- templates/robots.txt | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/fly.toml b/fly.toml index a9b4291..145f1b3 100644 --- a/fly.toml +++ b/fly.toml @@ -41,6 +41,6 @@ primary_region = 'iad' DEBUG = '0' [[vm]] - memory = '1gb' + memory = '2gb' cpu_kind = 'shared' cpus = 1 diff --git a/templates/robots.txt b/templates/robots.txt index 0c154be..b3546df 100644 --- a/templates/robots.txt +++ b/templates/robots.txt @@ -1,4 +1,23 @@ User-agent: * Allow: / +Crawl-delay: 2 + +User-agent: GPTBot +Crawl-delay: 5 + +User-agent: Bytespider +Crawl-delay: 10 + +User-agent: AhrefsBot +Crawl-delay: 10 + +User-agent: SemrushBot +Crawl-delay: 10 + +User-agent: DotBot +Crawl-delay: 10 + +User-agent: MJ12bot +Crawl-delay: 10 Sitemap: https://photos.kennethreitz.org/sitemap.xml