From f39e71472fd985be2352f732b86f50dcadd3dadc Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Fri, 20 Mar 2026 01:20:33 -0400 Subject: [PATCH] Performance: add Bible indexes, eliminate O(n) iterations, remove dead code Major performance improvements: 1. Build book/chapter indexes at Bible init for O(1) lookups instead of O(n) iteration through 31,102 verses on every API request 2. Pre-compute total word count at init instead of splitting all verse text on every /stats or /about request 3. Add get_verses_by_book() and get_total_words() methods to Bible class 4. Replace all iter_verses()/iter_chapters() calls in API routes with indexed get_verses_by_book_chapter() and get_chapters_for_book() 5. Remove unused Scofield commentary load at startup (27KB saved) 6. Increase GZip minimum_size from 500 to 1000 bytes to reduce CPU waste on tiny responses Affected endpoints (now O(1) instead of O(n)): - GET /api/books, /api/books/{book}, /api/books/{book}/text - GET /api/books/{book}/pdf, /api/books/{book}/chapters/{chapter} - GET /api/bible, /api/stats, /about/stats Co-Authored-By: Claude Opus 4.6 (1M context) --- kjvstudy_org/kjv.py | 64 ++++++++++++++++++------- kjvstudy_org/routes/about.py | 4 +- kjvstudy_org/routes/api.py | 92 ++++++++++++------------------------ kjvstudy_org/server.py | 12 +---- 4 files changed, 81 insertions(+), 91 deletions(-) diff --git a/kjvstudy_org/kjv.py b/kjvstudy_org/kjv.py index f51335d..4b64fe2 100644 --- a/kjvstudy_org/kjv.py +++ b/kjvstudy_org/kjv.py @@ -80,6 +80,34 @@ class Bible: for key, text in self.verses.items() } + # Build indexes for O(1) lookups instead of O(n) iteration + self._book_index = {} # book -> [verse_keys] + self._chapter_index = {} # (book, chapter) -> [verse_keys] + self._book_chapters = {} # book -> sorted list of chapter numbers + self._total_words = 0 + + for key in self.verses: + ref = VerseReference.from_string(key) + # Book index + if ref.book not in self._book_index: + self._book_index[ref.book] = [] + self._book_index[ref.book].append(key) + # Chapter index + ch_key = (ref.book, ref.chapter) + if ch_key not in self._chapter_index: + self._chapter_index[ch_key] = [] + self._chapter_index[ch_key].append(key) + # Book chapters + if ref.book not in self._book_chapters: + self._book_chapters[ref.book] = set() + self._book_chapters[ref.book].add(ref.chapter) + # Word count + self._total_words += len(self._cleaned_verses[key].split()) + + # Sort chapter sets into lists + for book in self._book_chapters: + self._book_chapters[book] = sorted(self._book_chapters[book]) + @lru_cache(maxsize=1024) def __getitem__(self, verse): """Returns the text of the verse.""" @@ -167,29 +195,22 @@ class Bible: @lru_cache(maxsize=256) def get_verses_by_book_chapter(self, book, chapter): """Returns a list of verses for a specific book and chapter.""" + keys = self._chapter_index.get((book, chapter), []) verses = [] - for verse in self.verses: - verse_ref = VerseReference.from_string(verse) - if verse_ref.book == book and verse_ref.chapter == chapter: - # Use pre-cleaned text for performance - text = self._cleaned_verses[verse] - verses.append(Verse( - book=verse_ref.book, - chapter=verse_ref.chapter, - verse=verse_ref.verse, - text=text, - )) + for key in keys: + ref = VerseReference.from_string(key) + verses.append(Verse( + book=ref.book, + chapter=ref.chapter, + verse=ref.verse, + text=self._cleaned_verses[key], + )) return sorted(verses, key=lambda v: v.verse) @lru_cache(maxsize=128) def get_chapters_for_book(self, book): """Returns a list of chapter numbers for a specific book.""" - chapters = set() - for verse in self.verses: - verse_ref = VerseReference.from_string(verse) - if verse_ref.book == book: - chapters.add(verse_ref.chapter) - return sorted(list(chapters)) + return self._book_chapters.get(book, []) @lru_cache(maxsize=2048) def get_verse_text(self, book, chapter, verse_num): @@ -205,6 +226,15 @@ class Bible: """Returns the total number of verses in the Bible.""" return len(self.verses) + def get_total_words(self): + """Returns the total word count across all verses (pre-computed at init).""" + return self._total_words + + def get_verses_by_book(self, book): + """Returns all verses for a specific book, grouped by chapter.""" + chapters = self._book_chapters.get(book, []) + return {ch: self.get_verses_by_book_chapter(book, ch) for ch in chapters} + # Create an instance of the Bible class. bible = Bible() diff --git a/kjvstudy_org/routes/about.py b/kjvstudy_org/routes/about.py index 854e2e0..1961fe9 100644 --- a/kjvstudy_org/routes/about.py +++ b/kjvstudy_org/routes/about.py @@ -35,8 +35,8 @@ def _compute_stats() -> dict: total_books = len(bible.get_books()) total_chapters = len(bible.get_chapters()) - # Calculate words in Bible - total_words = sum(len(verse.text.split()) for verse in bible.iter_verses()) + # Use pre-computed word count from Bible init + total_words = bible.get_total_words() # Count unique book types ot_books = len(OT_BOOKS) diff --git a/kjvstudy_org/routes/api.py b/kjvstudy_org/routes/api.py index a212b4b..62892c9 100644 --- a/kjvstudy_org/routes/api.py +++ b/kjvstudy_org/routes/api.py @@ -889,7 +889,7 @@ async def api_get_books(): new_testament = [] for book in books: - chapters = [ch for bk, ch in bible.iter_chapters() if bk == book] + chapters = bible.get_chapters_for_book(book) book_data = get_book_data(book) if has_book_data(book) else None book_info = { @@ -924,13 +924,13 @@ async def api_get_book(book: str = Path(..., description="Book name", example="G if canonical_name: book = canonical_name - chapters = [ch for bk, ch in bible.iter_chapters() if bk == book] + chapters = bible.get_chapters_for_book(book) if not chapters: raise HTTPException(status_code=404, detail="Book not found") chapter_details = [] for chapter in chapters: - verses = [v for v in bible.iter_verses() if v.book == book and v.chapter == chapter] + verses = bible.get_verses_by_book_chapter(book, chapter) chapter_details.append({ "chapter": chapter, "verses": len(verses) @@ -979,7 +979,7 @@ async def api_book_pdf(book: str = Path(..., description="Book name", example="G if canonical_name: book = canonical_name - chapters = [ch for bk, ch in bible.iter_chapters() if bk == book] + chapters = bible.get_chapters_for_book(book) if not chapters: raise HTTPException(status_code=404, detail="Book not found") @@ -990,14 +990,9 @@ async def api_book_pdf(book: str = Path(..., description="Book name", example="G chapters_data = [] total_verses = 0 for chapter in chapters: - verses = [v for v in bible.iter_verses() if v.book == book and v.chapter == chapter] + verses = bible.get_verses_by_book_chapter(book, chapter) if verses: - chapter_verses = [] - for v in verses: - chapter_verses.append({ - "verse": v.verse, - "text": v.text - }) + chapter_verses = [{"verse": v.verse, "text": v.text} for v in verses] chapters_data.append({ "chapter": chapter, "verses": chapter_verses @@ -1037,16 +1032,11 @@ async def api_get_chapter( if canonical_name: book = canonical_name - verses = [v for v in bible.iter_verses() if v.book == book and v.chapter == chapter] + verses = bible.get_verses_by_book_chapter(book, chapter) if not verses: raise HTTPException(status_code=404, detail="Chapter not found") - verse_list = [] - for v in verses: - verse_list.append({ - "verse": v.verse, - "text": v.text - }) + verse_list = [{"verse": v.verse, "text": v.text} for v in verses] return { "book": book, @@ -1075,7 +1065,7 @@ async def api_chapter_pdf( if canonical_name: book = canonical_name - verses = [v for v in bible.iter_verses() if v.book == book and v.chapter == chapter] + verses = bible.get_verses_by_book_chapter(book, chapter) if not verses: raise HTTPException(status_code=404, detail="Chapter not found") @@ -1083,12 +1073,7 @@ async def api_chapter_pdf( raise HTTPException(status_code=500, detail="Templates not initialized") # Prepare data for template - verse_list = [] - for v in verses: - verse_list.append({ - "verse": v.verse, - "text": v.text - }) + verse_list = [{"verse": v.verse, "text": v.text} for v in verses] # Render the PDF template html_content = templates.get_template("chapter_pdf.html").render( @@ -1117,30 +1102,24 @@ async def api_get_book_text(book: str = Path(..., description="Book name", examp if canonical_name: book = canonical_name - verses = [v for v in bible.iter_verses() if v.book == book] - if not verses: + book_chapters = bible.get_chapters_for_book(book) + if not book_chapters: raise HTTPException(status_code=404, detail="Book not found") - chapters = {} - for v in verses: - if v.chapter not in chapters: - chapters[v.chapter] = [] - chapters[v.chapter].append({ - "verse": v.verse, - "text": v.text - }) - chapter_list = [] - for chapter_num in sorted(chapters.keys()): + total_verses = 0 + for ch in book_chapters: + verses = bible.get_verses_by_book_chapter(book, ch) chapter_list.append({ - "chapter": chapter_num, - "verses": chapters[chapter_num] + "chapter": ch, + "verses": [{"verse": v.verse, "text": v.text} for v in verses] }) + total_verses += len(verses) return { "book": book, - "total_chapters": len(chapters), - "total_verses": len(verses), + "total_chapters": len(book_chapters), + "total_verses": total_verses, "chapters": chapter_list } @@ -1148,35 +1127,24 @@ async def api_get_book_text(book: str = Path(..., description="Book name", examp @router.get("/bible") async def api_get_bible(): """Get the entire Bible text.""" - books_data = {} - for v in bible.iter_verses(): - if v.book not in books_data: - books_data[v.book] = {} - if v.chapter not in books_data[v.book]: - books_data[v.book][v.chapter] = [] - books_data[v.book][v.chapter].append({ - "verse": v.verse, - "text": v.text - }) - books_list = [] - for book_name in books_data: + total_verses = 0 + for book_name in bible.get_books(): chapter_list = [] - for chapter_num in sorted(books_data[book_name].keys()): + for ch in bible.get_chapters_for_book(book_name): + verses = bible.get_verses_by_book_chapter(book_name, ch) chapter_list.append({ - "chapter": chapter_num, - "verses": books_data[book_name][chapter_num] + "chapter": ch, + "verses": [{"verse": v.verse, "text": v.text} for v in verses] }) - + total_verses += len(verses) books_list.append({ "book": book_name, "chapters": chapter_list }) - total_verses = sum(len(books_data[book][ch]) for book in books_data for ch in books_data[book]) - return { - "total_books": len(books_data), + "total_books": len(books_list), "total_verses": total_verses, "books": books_list } @@ -2394,8 +2362,8 @@ def _get_site_stats(): total_books = len(bible.get_books()) total_chapters = len(bible.get_chapters()) - # Calculate words in Bible - total_words = sum(len(verse.text.split()) for verse in bible.iter_verses()) + # Use pre-computed word count from Bible init + total_words = bible.get_total_words() # Count unique book types ot_books = len(OT_BOOKS) diff --git a/kjvstudy_org/server.py b/kjvstudy_org/server.py index 4dbeb1d..bc95d25 100644 --- a/kjvstudy_org/server.py +++ b/kjvstudy_org/server.py @@ -303,8 +303,8 @@ class TimeoutMiddleware(BaseHTTPMiddleware): ) -# Add GZip compression middleware (compress responses > 500 bytes) -app.add_middleware(GZipMiddleware, minimum_size=500) +# Add GZip compression middleware (compress responses > 1000 bytes) +app.add_middleware(GZipMiddleware, minimum_size=1000) # Add caching middleware app.add_middleware(CacheControlMiddleware) @@ -368,14 +368,6 @@ init_about_templates(templates) init_main_templates(templates) init_misc_templates(templates) -# Load Scofield commentary for cross-references -scofield_commentary = {} -try: - scofield_path = static_dir / "scofield_commentary.json" - with open(scofield_path, 'r') as f: - scofield_commentary = json.load(f) -except Exception as e: - print(f"Warning: Could not load Scofield commentary: {e}") @app.exception_handler(StarletteHTTPException)