updates to data storage

scripts/add_10_new_commentaries.py

@@ -1,9 +1,13 @@
 #!/usr/bin/env python3
-"""Add comprehensive commentary for 10 specific verses to verse_commentary.json"""
+"""Add comprehensive commentary for 10 specific verses to per-book commentary files."""
 
-import json
+import sys
 from pathlib import Path
 
+sys.path.insert(0, str(Path(__file__).parent.parent))
+
+from kjvstudy_org.utils.commentary_loader import merge_commentary_entries
+
 # Define the commentary data for the 10 requested verses
 NEW_COMMENTARY = {
     "Ecclesiastes": {
@@ -154,37 +158,17 @@ NEW_COMMENTARY = {
     }
 }
 
-def update_commentary_json():
-    """Update the verse_commentary.json file with new commentary"""
-    json_path = Path(__file__).parent.parent / "kjvstudy_org" / "data" / "verse_commentary.json"
-
-    print(f"Reading commentary file: {json_path}")
-
-    # Read existing commentary
-    with open(json_path, 'r', encoding='utf-8') as f:
-        commentary = json.load(f)
-
-    # Track additions
+def update_commentary_files():
+    """Update the per-book commentary files with new commentary."""
     additions = []
 
-    # Add new commentary
     for book, chapters in NEW_COMMENTARY.items():
-        if book not in commentary:
-            commentary[book] = {}
-
         for chapter, verses in chapters.items():
-            if chapter not in commentary[book]:
-                commentary[book][chapter] = {}
-
-            for verse, content in verses.items():
-                commentary[book][chapter][verse] = content
+            for verse in verses:
                 additions.append(f"{book} {chapter}:{verse}")
                 print(f"Added commentary for {book} {chapter}:{verse}")
 
-    # Write updated commentary
-    print(f"\nWriting updated commentary to: {json_path}")
-    with open(json_path, 'w', encoding='utf-8') as f:
-        json.dump(commentary, f, indent=2, ensure_ascii=False)
+    merge_commentary_entries(NEW_COMMENTARY)
 
     print(f"\n✓ Successfully added commentary for {len(additions)} verses:")
     for ref in additions:
@@ -198,5 +182,5 @@ def update_commentary_json():
     return len(additions)
 
 if __name__ == "__main__":
-    total = update_commentary_json()
+    total = update_commentary_files()
     print(f"\nTotal verses updated: {total}")

scripts/add_10_verses.py

@@ -1,13 +1,17 @@
 #!/usr/bin/env python3
 """
-Script to add comprehensive commentary for 10 specific verses to verse_commentary.json
+Script to add comprehensive commentary for 10 specific verses to per-book commentary files
 Daniel 4:5, Psalms 6:9, Luke 23:29, Ezekiel 40:33, Job 18:14,
 Deuteronomy 33:25, John 20:7, John 13:18, Psalms 109:20, Numbers 33:1
 """
 
-import json
+import sys
 from pathlib import Path
 
+sys.path.insert(0, str(Path(__file__).parent.parent))
+
+from kjvstudy_org.utils.commentary_loader import merge_commentary_entries
+
 # Define the commentary data
 commentaries = {
     "Daniel": {
@@ -159,30 +163,7 @@ commentaries = {
 }
 
 def main():
-    # Load existing commentary file
-    commentary_file = Path(__file__).parent.parent / "kjvstudy_org" / "data" / "verse_commentary.json"
-
-    print(f"Reading commentary file: {commentary_file}")
-    with open(commentary_file, 'r', encoding='utf-8') as f:
-        existing_data = json.load(f)
-
-    # Merge new commentaries
-    for book, chapters in commentaries.items():
-        if book not in existing_data:
-            existing_data[book] = {}
-
-        for chapter, verses in chapters.items():
-            if chapter not in existing_data[book]:
-                existing_data[book][chapter] = {}
-
-            for verse, content in verses.items():
-                existing_data[book][chapter][verse] = content
-                print(f"Added commentary for {book} {chapter}:{verse}")
-
-    # Save updated file
-    print(f"\nWriting updated commentary to: {commentary_file}")
-    with open(commentary_file, 'w', encoding='utf-8') as f:
-        json.dump(existing_data, f, indent=2, ensure_ascii=False)
+    merge_commentary_entries(commentaries)
 
     print("\n✓ Successfully added commentary for 10 verses:")
     print("  - Daniel 4:5")

scripts/add_commentary.py

@@ -1,9 +1,13 @@
 #!/usr/bin/env python3
-"""Script to add verse commentary to the JSON file."""
+"""Script to add verse commentary to the per-book JSON files."""
 
-import json
+import sys
 from pathlib import Path
 
+sys.path.insert(0, str(Path(__file__).parent.parent))
+
+from kjvstudy_org.utils.commentary_loader import merge_commentary_entries
+
 # New commentaries to add
 new_commentaries = {
     "Proverbs": {
@@ -24,29 +28,8 @@ new_commentaries = {
 }
 
 def main():
-    commentary_file = Path(__file__).parent.parent / "kjvstudy_org" / "data" / "verse_commentary.json"
-
-    # Read existing commentary
-    with open(commentary_file, 'r', encoding='utf-8') as f:
-        existing_data = json.load(f)
-
-    # Merge new commentaries
-    for book, chapters in new_commentaries.items():
-        if book not in existing_data:
-            existing_data[book] = {}
-
-        for chapter, verses in chapters.items():
-            if chapter not in existing_data[book]:
-                existing_data[book][chapter] = {}
-
-            for verse, commentary in verses.items():
-                existing_data[book][chapter][verse] = commentary
-
-    # Write back to file
-    with open(commentary_file, 'w', encoding='utf-8') as f:
-        json.dump(existing_data, f, ensure_ascii=False, indent=2)
-
-    print("Successfully added commentary")
+    merge_commentary_entries(new_commentaries)
+    print("Successfully added commentary to per-book files")
 
 if __name__ == "__main__":
     main()

scripts/add_requested_10_verses.py

@@ -1,13 +1,17 @@
 #!/usr/bin/env python3
 """
-Add comprehensive commentary for these 10 verses to verse_commentary.json:
+Add comprehensive commentary for these 10 verses to the per-book commentary files:
 Psalms 89:30, Ezekiel 37:23, Hebrews 2:4, Jeremiah 29:32, Acts 19:35,
 Numbers 30:3, Proverbs 1:24, Isaiah 24:19, Psalms 105:1, Numbers 16:2
 """
 
-import json
+import sys
 from pathlib import Path
 
+sys.path.insert(0, str(Path(__file__).parent.parent))
+
+from kjvstudy_org.utils.commentary_loader import merge_commentary_entries
+
 # Define the commentary data for the requested verses
 NEW_COMMENTARY = {
     "Psalms": {
@@ -158,37 +162,15 @@ NEW_COMMENTARY = {
     }
 }
 
-def update_commentary_json():
-    """Update the verse_commentary.json file with new commentary"""
-    json_path = Path(__file__).parent.parent / "kjvstudy_org" / "data" / "verse_commentary.json"
-
-    print(f"Reading commentary file: {json_path}")
-
-    # Read existing commentary
-    with open(json_path, 'r', encoding='utf-8') as f:
-        commentary = json.load(f)
-
-    # Track additions
+def update_commentary_files():
+    """Update the per-book commentary files with new entries."""
     additions = []
-
-    # Add new commentary
     for book, chapters in NEW_COMMENTARY.items():
-        if book not in commentary:
-            commentary[book] = {}
-
         for chapter, verses in chapters.items():
-            if chapter not in commentary[book]:
-                commentary[book][chapter] = {}
-
-            for verse, content in verses.items():
-                commentary[book][chapter][verse] = content
+            for verse in verses:
                 additions.append(f"{book} {chapter}:{verse}")
-                print(f"Added commentary for {book} {chapter}:{verse}")
 
-    # Write updated commentary
-    print(f"\nWriting updated commentary to: {json_path}")
-    with open(json_path, 'w', encoding='utf-8') as f:
-        json.dump(commentary, f, indent=2, ensure_ascii=False)
+    merge_commentary_entries(NEW_COMMENTARY)
 
     print(f"\n✓ Successfully added commentary for {len(additions)} verses:")
     for ref in additions:
@@ -197,5 +179,5 @@ def update_commentary_json():
     return len(additions)
 
 if __name__ == "__main__":
-    total = update_commentary_json()
+    total = update_commentary_files()
     print(f"\nTotal verses updated: {total}")

scripts/analyze_data_coverage.py

@@ -12,6 +12,7 @@ from collections import defaultdict
 sys.path.insert(0, str(Path(__file__).parent.parent))
 
 from kjvstudy_org.kjv import bible
+from kjvstudy_org.utils.commentary_loader import load_commentary_flat
 
 
 def analyze_commentary_coverage():
@@ -21,9 +22,7 @@ def analyze_commentary_coverage():
     print("=" * 80)
     print()
 
-    commentary_file = Path("kjvstudy_org/data/verse_commentary.json")
-    with open(commentary_file, 'r') as f:
-        commentary = json.load(f)
+    commentary = load_commentary_flat()
 
     total_verses = 31102
     verses_with_commentary = len(commentary)
@@ -142,9 +141,7 @@ def analyze_books_without_data():
     books = bible.get_books()
 
     # Load commentary
-    commentary_file = Path("kjvstudy_org/data/verse_commentary.json")
-    with open(commentary_file, 'r') as f:
-        commentary = json.load(f)
+    commentary = load_commentary_flat()
 
     # Count verses per book
     book_stats = {}

scripts/list_unwritten_commentary.py

@@ -10,7 +10,6 @@ Usage:
     python scripts/list_unwritten_commentary.py --random 10        # Show 10 random verses without commentary
 """
 
-import json
 import sys
 import argparse
 import random
@@ -20,13 +19,7 @@ from pathlib import Path
 sys.path.insert(0, str(Path(__file__).parent.parent))
 
 from kjvstudy_org.kjv import bible
-
-
-def load_commentary():
-    """Load the verse commentary JSON"""
-    commentary_path = Path(__file__).parent.parent / 'kjvstudy_org' / 'data' / 'verse_commentary.json'
-    with open(commentary_path, 'r') as f:
-        return json.load(f)
+from kjvstudy_org.utils.commentary_loader import load_commentary
 
 
 def get_all_verses():
@@ -45,8 +38,8 @@ def get_all_verses():
 def has_commentary(commentary_data, book, chapter, verse):
     """Check if a verse has commentary"""
     return (book in commentary_data and
-            str(chapter) in commentary_data[book] and
-            str(verse) in commentary_data[book][str(chapter)])
+            chapter in commentary_data[book] and
+            verse in commentary_data[book][chapter])
 
 
 def main():

scripts/show_commentary_coverage.py

@@ -2,7 +2,7 @@
 """
 Show which verses have enhanced commentary and which don't.
 
-This script analyzes the verse_commentary.json file and displays statistics
+This script analyzes the verse_commentary directory and displays statistics
 about commentary coverage across the Bible. It helps track progress on adding
 enhanced verse-by-verse commentary.
 
@@ -14,27 +14,21 @@ Usage:
     python scripts/show_commentary_coverage.py --stats       # Detailed statistics
 """
 
-import json
 import sys
 from pathlib import Path
 from collections import defaultdict
 from typing import Dict, List, Tuple
 import argparse
 
+# Ensure package imports work when running as a script
+sys.path.insert(0, str(Path(__file__).parent.parent))
 
-# Path to data directory
-DATA_DIR = Path(__file__).parent.parent / "kjvstudy_org" / "data"
-VERSE_COMMENTARY_PATH = DATA_DIR / "verse_commentary.json"
+from kjvstudy_org.utils.commentary_loader import load_commentary_flat
 
 
 def load_commentary() -> Dict[str, dict]:
     """Load verse commentary from JSON file."""
-    if not VERSE_COMMENTARY_PATH.exists():
-        print(f"Error: {VERSE_COMMENTARY_PATH} not found")
-        sys.exit(1)
-
-    with open(VERSE_COMMENTARY_PATH, 'r', encoding='utf-8') as f:
-        return json.load(f)
+    return load_commentary_flat()
 
 
 def parse_verse_ref(verse_ref: str) -> Tuple[str, int, int]:

scripts/validate_data.py

@@ -110,10 +110,21 @@ class GuideContent(BaseModel):
     sections: List[StudySection] = Field(..., min_length=1)
 
 
-class StudyGuides(BaseModel):
-    """Schema for study_guides.json"""
-    catalog: Dict[str, List[CatalogEntry]]
-    content: Dict[str, GuideContent]
+class StudyGuideFile(BaseModel):
+    """Schema for a single study guide file"""
+    content: GuideContent
+    catalog_entry: Optional[CatalogEntry] = None
+    category: Optional[str] = None
+
+
+class TopicsFile(BaseModel):
+    """Schema for a single topics file"""
+    root: Dict[str, dict]
+
+
+class ReadingPlanFile(BaseModel):
+    """Schema for a single reading plan file"""
+    plan: Dict[str, List[Dict[str, object]]]
 
 
 class VerseCommentaryEntry(BaseModel):
@@ -123,19 +134,25 @@ class VerseCommentaryEntry(BaseModel):
     questions: List[str] = Field(..., min_length=1)
 
 
-class VersesDict(RootModel[Dict[str, VerseCommentaryEntry]]):
-    """Dictionary of verse numbers to commentary entries"""
-    root: Dict[str, VerseCommentaryEntry]
+class VerseCommentaryBook(BaseModel):
+    """Schema for a single verse commentary book file"""
+    book: str = Field(..., min_length=1)
+    commentary: Dict[str, Dict[str, VerseCommentaryEntry]] = Field(..., min_length=1)
 
-
-class ChaptersDict(RootModel[Dict[str, VersesDict]]):
-    """Dictionary of chapter numbers to verses"""
-    root: Dict[str, VersesDict]
-
-
-class VerseCommentary(RootModel[Dict[str, ChaptersDict]]):
-    """Schema for verse_commentary.json - nested structure: Book -> Chapter -> Verse -> Commentary"""
-    root: Dict[str, ChaptersDict]
+    @field_validator('commentary')
+    @classmethod
+    def check_numeric_keys(cls, v):
+        for chapter_key, verses in v.items():
+            if not str(chapter_key).isdigit():
+                raise ValueError(f"Invalid chapter key: {chapter_key}")
+            if not isinstance(verses, dict) or len(verses) == 0:
+                raise ValueError(f"Chapter {chapter_key} must contain verse entries")
+            for verse_key, entry in verses.items():
+                if not str(verse_key).isdigit():
+                    raise ValueError(f"Invalid verse key: {verse_key}")
+                if not isinstance(entry, (dict, BaseModel)):
+                    raise ValueError(f"Verse {chapter_key}:{verse_key} must be an object")
+        return v
 
 
 class FeaturedVerse(BaseModel):
@@ -243,8 +260,10 @@ class BookIntroduction(BaseModel):
 MODEL_MAPPING = {
     "bible_metadata.json": BibleMetadata,
     "word_studies.json": WordStudies,
-    "study_guides.json": StudyGuides,
-    "verse_commentary.json": VerseCommentary,
+    "study_guides": StudyGuideFile,
+    "verse_commentary": VerseCommentaryBook,
+    "topics": TopicsFile,
+    "reading_plans": ReadingPlanFile,
     "featured_verses.json": FeaturedVerses,
     "red_letter_verses.json": RedLetterVerses,
     "resource_slugs.json": ResourceSlugs,
@@ -264,6 +283,15 @@ def load_json(file_path: Path) -> Tuple[dict, Optional[str]]:
 
 def validate_file(data_file: str, verbose: bool = False) -> bool:
     """Validate a single data file using its Pydantic model."""
+    if data_file == "verse_commentary":
+        return validate_verse_commentary_directory(verbose)
+    if data_file == "study_guides":
+        return validate_study_guides_directory(verbose)
+    if data_file == "topics":
+        return validate_topics_directory(verbose)
+    if data_file == "reading_plans":
+        return validate_reading_plans_directory(verbose)
+
     if data_file not in MODEL_MAPPING:
         if verbose:
             print(f"⚠️  {data_file}: No validation model defined (skipped)")
@@ -313,6 +341,173 @@ def validate_file(data_file: str, verbose: bool = False) -> bool:
         return False
 
 
+def validate_verse_commentary_directory(verbose: bool = False) -> bool:
+    """Validate all per-book verse commentary files."""
+    dir_path = DATA_DIR / "verse_commentary"
+    if not dir_path.exists():
+        print(f"❌ verse_commentary: Directory not found at {dir_path}")
+        return False
+
+    passed = 0
+    failed = 0
+
+    for file_path in sorted(dir_path.glob("*.json")):
+        data, error = load_json(file_path)
+        if error:
+            print(f"❌ {file_path.name}: {error}")
+            failed += 1
+            continue
+
+        try:
+            VerseCommentaryBook(**data)
+            if verbose:
+                print(f"✅ {file_path.name}: Valid")
+            passed += 1
+        except ValidationError as e:
+            print(f"❌ {file_path.name}: Validation failed")
+            for error_detail in e.errors():
+                location = " -> ".join(str(loc) for loc in error_detail['loc'])
+                print(f"   {location}: {error_detail['msg']}")
+            failed += 1
+        except Exception as e:
+            print(f"❌ {file_path.name}: Unexpected error")
+            print(f"   Error: {str(e)}")
+            failed += 1
+
+    if failed == 0:
+        print(f"✅ verse_commentary: Valid ({passed} files)")
+        return True
+
+    print(f"❌ verse_commentary: {failed} files failed validation")
+    return False
+
+
+def validate_study_guides_directory(verbose: bool = False) -> bool:
+    """Validate per-guide study guide files."""
+    dir_path = DATA_DIR / "study_guides"
+    if not dir_path.exists():
+        print(f"❌ study_guides: Directory not found at {dir_path}")
+        return False
+
+    passed = 0
+    failed = 0
+
+    for file_path in sorted(dir_path.glob("*.json")):
+        data, error = load_json(file_path)
+        if error:
+            print(f"❌ {file_path.name}: {error}")
+            failed += 1
+            continue
+
+        try:
+            StudyGuideFile(**data)
+            if verbose:
+                print(f"✅ {file_path.name}: Valid")
+            passed += 1
+        except ValidationError as e:
+            print(f"❌ {file_path.name}: Validation failed")
+            for error_detail in e.errors():
+                location = " -> ".join(str(loc) for loc in error_detail['loc'])
+                print(f"   {location}: {error_detail['msg']}")
+            failed += 1
+        except Exception as e:
+            print(f"❌ {file_path.name}: Unexpected error")
+            print(f"   Error: {str(e)}")
+            failed += 1
+
+    if failed == 0:
+        print(f"✅ study_guides: Valid ({passed} files)")
+        return True
+
+    print(f"❌ study_guides: {failed} files failed validation")
+    return False
+
+
+def validate_topics_directory(verbose: bool = False) -> bool:
+    """Validate per-topic files."""
+    dir_path = DATA_DIR / "topics"
+    if not dir_path.exists():
+        print(f"❌ topics: Directory not found at {dir_path}")
+        return False
+
+    passed = 0
+    failed = 0
+
+    for file_path in sorted(dir_path.glob("*.json")):
+        data, error = load_json(file_path)
+        if error:
+            print(f"❌ {file_path.name}: {error}")
+            failed += 1
+            continue
+
+        try:
+            TopicsFile(root=data)
+            if verbose:
+                print(f"✅ {file_path.name}: Valid")
+            passed += 1
+        except ValidationError as e:
+            print(f"❌ {file_path.name}: Validation failed")
+            for error_detail in e.errors():
+                location = " -> ".join(str(loc) for loc in error_detail['loc'])
+                print(f"   {location}: {error_detail['msg']}")
+            failed += 1
+        except Exception as e:
+            print(f"❌ {file_path.name}: Unexpected error")
+            print(f"   Error: {str(e)}")
+            failed += 1
+
+    if failed == 0:
+        print(f"✅ topics: Valid ({passed} files)")
+        return True
+
+    print(f"❌ topics: {failed} files failed validation")
+    return False
+
+
+def validate_reading_plans_directory(verbose: bool = False) -> bool:
+    """Validate per-plan reading plan files."""
+    dir_path = DATA_DIR / "reading_plans"
+    if not dir_path.exists():
+        print(f"❌ reading_plans: Directory not found at {dir_path}")
+        return False
+
+    passed = 0
+    failed = 0
+
+    for file_path in sorted(dir_path.glob("*.json")):
+        data, error = load_json(file_path)
+        if error:
+            print(f"❌ {file_path.name}: {error}")
+            failed += 1
+            continue
+
+        try:
+            # Each file has one key with the plan id mapping to the plan object
+            if len(data) != 1:
+                raise ValueError("Reading plan file must contain exactly one plan")
+            ReadingPlanFile(plan=data)
+            if verbose:
+                print(f"✅ {file_path.name}: Valid")
+            passed += 1
+        except ValidationError as e:
+            print(f"❌ {file_path.name}: Validation failed")
+            for error_detail in e.errors():
+                location = " -> ".join(str(loc) for loc in error_detail['loc'])
+                print(f"   {location}: {error_detail['msg']}")
+            failed += 1
+        except Exception as e:
+            print(f"❌ {file_path.name}: Unexpected error")
+            print(f"   Error: {str(e)}")
+            failed += 1
+
+    if failed == 0:
+        print(f"✅ reading_plans: Valid ({passed} files)")
+        return True
+
+    print(f"❌ reading_plans: {failed} files failed validation")
+    return False
+
+
 def validate_all(verbose: bool = False) -> Tuple[int, int]:
     """Validate all data files with models. Returns (passed, failed) counts."""
     passed = 0
@@ -401,7 +596,7 @@ def generate_json_schemas():
 
     # Generate schemas for main data files
     for data_file, model_class in MODEL_MAPPING.items():
-        schema_file = data_file.replace('.json', '.schema.json')
+        schema_file = data_file.replace('.json', '.schema.json') if data_file.endswith('.json') else f"{data_file}.schema.json"
         schema_path = SCHEMAS_DIR / schema_file
 
         try: