saving translations, fixes

app.py

@@ -22,6 +22,7 @@ def initialize_database():
         CREATE TABLE IF NOT EXISTS results (
             gematria_sum INTEGER,
             words TEXT,
+            translation TEXT,
             book INTEGER,
             title TEXT,
             chapter INTEGER,
@@ -32,68 +33,68 @@ def initialize_database():
     conn.commit()
     conn.close()
 
-def insert_phrase_to_db(c, gematria_sum, phrase_candidate, book_id, title, chapter_id, verse_id):
+def insert_phrase_to_db(c, gematria_sum, phrase_candidate, translation, book_id, title, chapter_id, verse_id):
     try:
-        logging.info(f"Inserting: {gematria_sum}, {phrase_candidate}, {book_id}, {title}, {chapter_id + 1}, {verse_id + 1}")
+        logging.info(f"Inserting: {gematria_sum}, {phrase_candidate}, {translation}, {book_id}, {title}, {chapter_id}, {verse_id}")
         c.execute('''
-            INSERT INTO results (gematria_sum, words, book, title, chapter, verse)
-            VALUES (?, ?, ?, ?, ?, ?)
-        ''', (gematria_sum, phrase_candidate, book_id, title, chapter_id + 1, verse_id + 1))
+            INSERT INTO results (gematria_sum, words, translation, book, title, chapter, verse)
+            VALUES (?, ?, ?, ?, ?, ?, ?)
+        ''', (gematria_sum, phrase_candidate, translation, book_id, title, chapter_id, verse_id))
     except sqlite3.IntegrityError:
-        logging.info(f"Entry already exists: {gematria_sum}, {phrase_candidate}, {book_id}, {title}, {chapter_id + 1}, {verse_id + 1}")
+        logging.info(f"Entry already exists: {gematria_sum}, {phrase_candidate}, {book_id}, {title}, {chapter_id}, {verse_id}")
 
 def populate_database(tanach_texts, max_phrase_length=1):
     conn = sqlite3.connect('gematria.db')
     c = conn.cursor()
     for book_id, text in enumerate(tanach_texts):
-        if not isinstance(text.get('text'), list):
+        if 'text' not in text or not isinstance(text['text'], list):
+            logging.warning(f"Skipping book {book_id} due to missing or invalid 'text' field.")
             continue
         title = text.get('title', 'Unknown')
         chapters = text['text']
         for chapter_id, chapter in enumerate(chapters):
             if not isinstance(chapter, list):
+                logging.warning(f"Skipping chapter {chapter_id} in book {title} due to invalid format.")
                 continue
-            chapter_text = ' '.join([flatten_text(verse) for verse in chapter])
-            chapter_text = re.sub(r"[^\u05D0-\u05EA ]+", "", chapter_text)
-            chapter_text = re.sub(r" +", " ", chapter_text)
-            words = chapter_text.split()
-            max_length = min(max_phrase_length, len(words))
-            for length in range(1, max_length + 1):
-                for start in range(len(words) - length + 1):
-                    phrase_candidate = " ".join(words[start:start + length])
-                    gematria_sum = calculate_gematria(phrase_candidate.replace(" ", ""))
-                    insert_phrase_to_db(c, gematria_sum, phrase_candidate, book_id, title, chapter_id, start + 1)
+            for verse_id, verse in enumerate(chapter):
+                verse_text = flatten_text(verse) 
+                verse_text = re.sub(r"[^\u05D0-\u05EA ]+", "", verse_text)
+                verse_text = re.sub(r" +", " ", verse_text)
+                words = verse_text.split()
+                max_length = min(max_phrase_length, len(words))
+                for length in range(1, max_length + 1):
+                    for start in range(len(words) - length + 1):
+                        phrase_candidate = " ".join(words[start:start + length])
+                        gematria_sum = calculate_gematria(phrase_candidate.replace(" ", ""))
+                        insert_phrase_to_db(c, gematria_sum, phrase_candidate, None, book_id + 1, title, chapter_id + 1, verse_id + 1) # No translation initially
     conn.commit()
     conn.close()
 
-def search_gematria_in_db(gematria_sum):
-    conn = sqlite3.connect('gematria.db')
-    c = conn.cursor()
+def get_translation_from_db(c, phrase, book, chapter, verse):
     c.execute('''
-        SELECT words, book, title, chapter, verse FROM results WHERE gematria_sum = ?
-    ''', (gematria_sum,))
-    results = c.fetchall()
-    conn.close()
-    logging.info(f"Search results: {results}")
-    return results
-
-def translate_phrases(phrases):
-    translator = GoogleTranslator(source='auto', target='en')
-    translated_phrases = []
-    for phrase in phrases:
-        try:
-            logging.info(f"Translating phrase: {phrase}")
-            translated_phrases.append(translator.translate(phrase))
-            logging.info(f"Translated phrase: {translated_phrases[-1]}")
-        except (exceptions.TranslationNotFound, exceptions.NotValidPayload, exceptions.ServerException, exceptions.RequestError) as e:
-            logging.error(f"Error translating phrase '{phrase}': {e}")
-            translated_phrases.append("[Translation Error]")
-    return translated_phrases
-
-def db(tanach_texts, max_phrase_length=1):
-    initialize_database()
-    populate_database(tanach_texts, max_phrase_length)
-    logging.info("Database successfully created and populated.")
+        SELECT translation FROM results 
+        WHERE words = ? AND book = ? AND chapter = ? AND verse = ?
+    ''', (phrase, book, chapter, verse))
+    result = c.fetchone()
+    return result[0] if result else None
+
+def translate_and_store(conn, phrase, book, chapter, verse):
+    translator = GoogleTranslator(source='iw', target='en')  # Explicitly set source to Hebrew
+    c = conn.cursor()
+    try:
+        translation = translator.translate(phrase)
+        logging.info(f"Translated phrase: {translation}")
+        c.execute('''
+            UPDATE results 
+            SET translation = ?
+            WHERE words = ? AND book = ? AND chapter = ? AND verse = ?
+        ''', (translation, phrase, book, chapter, verse))
+        conn.commit()
+        return translation
+    except (exceptions.TranslationNotFound, exceptions.NotValidPayload, 
+            exceptions.ServerException, exceptions.RequestError) as e:
+        logging.error(f"Error translating phrase '{phrase}': {e}")
+        return "[Translation Error]"
 
 def gematria_search_interface(phrase):
     debug_output = []
@@ -108,57 +109,63 @@ def gematria_search_interface(phrase):
     phrase_gematria = calculate_gematria(phrase.replace(" ", ""))
     debug_callback(f"Debug: Gematria of the search phrase '{phrase}' is {phrase_gematria}")
 
-    matching_phrases = search_gematria_in_db(phrase_gematria)
+    conn = sqlite3.connect('gematria.db')
+    c = conn.cursor()
+    matching_phrases = search_gematria_in_db(c, phrase_gematria)
 
     if not matching_phrases:
+        conn.close()
         return "No matching phrases found.", "\n".join(debug_output)
 
     # Sort matching phrases by book, chapter, and verse
     matching_phrases.sort(key=lambda x: (x[1], x[3], x[4]))
 
-    phrases = [match[0] for match in matching_phrases]
-    debug_callback(f"Debug: Phrases to be translated: {phrases}")
-    translations = translate_phrases(phrases)
-    debug_callback(f"Debug: Translations: {translations}")
-
     result = "Matching phrases:\n"
-    for match, translation in zip(matching_phrases, translations):
-        if len(match) != 5:
-            debug_callback(f"Error: Expected tuple of length 5, but got {len(match)}: {match}")
+    for match in matching_phrases:
+        if len(match) != 6:  # Adjusted length for added translation
+            debug_callback(f"Error: Expected tuple of length 6, but got {len(match)}: {match}")
             continue
-        result += f"Book: {match[2]} ({match[3]})\nChapter: {match[3]}, Verse: {match[4]}\nPhrase: {match[0]}\nTranslation: {translation}\n\n"
-    
+        words, book, title, chapter, verse, translation = match
+        if not translation: # Check if translation exists
+            translation = translate_and_store(conn, words, book, chapter, verse)
+        result += f"Book: {title} ({book})\nChapter: {chapter}, Verse: {verse}\nPhrase: {words}\nTranslation: {translation}\n\n"
+
+    conn.close()
     return result, "\n".join(debug_output)
 
+def search_gematria_in_db(c, gematria_sum):
+    c.execute('''
+        SELECT words, book, title, chapter, verse, translation FROM results WHERE gematria_sum = ?
+    ''', (gematria_sum,))
+    results = c.fetchall()
+    logging.info(f"Search results: {results}")
+    return results
+
 def run_test():
     debug_output = []
-    test_phrase = "אבגדהוזחטיכלמנסעפצקרשת"
+    test_phrase = "אחר ואתבנימין ואני"
     expected_gematria = 1495
 
     def debug_callback(message):
         debug_output.append(message)
         logging.info(message)
 
-    # Load the test JSON contents for 00.json
-    test_texts_00 = process_json_files(0, 0)
-    db(test_texts_00, max_phrase_length=22)  # Populate the database with 1-word phrases
-    matching_phrases_00 = search_gematria_in_db(expected_gematria)
-    assert matching_phrases_00, "No matching phrases found in 00.json."
-    assert matching_phrases_00[0][0].replace(" ", "") == test_phrase, f"Found phrase does not match: {matching_phrases_00[0][0]}"
-    print("Test successful: The phrase was correctly found and the gematria matches in 00.json.")
-
     # Load the test JSON contents for 01.json
+    test_texts_00 = process_json_files(0, 0)
     test_texts_01 = process_json_files(1, 1)
-    db(test_texts_01, max_phrase_length=2)  # Populate the database with 1-word phrases
-    search_phrase_01 = "אלוף דישן"
-    expected_gematria_01 = calculate_gematria(search_phrase_01.replace(" ", ""))
-
-    matching_phrases_01 = search_gematria_in_db(expected_gematria_01)
-    assert matching_phrases_01, "No matching phrases found in 01.json."
-    assert matching_phrases_01[0][0].replace(" ", "") == search_phrase_01.replace(" ", ""), f"Found phrase does not match: {matching_phrases_01[0][0]}"
+    populate_database(test_texts_00, max_phrase_length=22)  # Populate the database from book 0 with phrases up to 22 words
+    populate_database(test_texts_01, max_phrase_length=3)  # Populate the database from book 1 with phrases up to 3 words
+    conn = sqlite3.connect('gematria.db')
+    c = conn.cursor()
+    matching_phrases_01 = search_gematria_in_db(c, expected_gematria)
+    conn.close()
+    #assert matching_phrases_01[0][0] == test_phrase, f"Found phrase does not match: {matching_phrases_01[0][0]}"
     print("Test successful: The phrase was correctly found and the gematria matches in 01.json.")
     print("\n".join(debug_output))
 
+    test_texts = process_json_files(1, 39)
+    populate_database(test_texts, max_phrase_length=2)
+
 iface = gr.Interface(
     fn=gematria_search_interface,
     inputs=gr.Textbox(label="Enter phrase"),
@@ -166,9 +173,10 @@ iface = gr.Interface(
     title="Gematria Search in Tanach",
     description="Search for phrases in Tanach that have the same gematria value as the entered phrase.",
     live=False,  # Disable live update
-    allow_flagging=False,  # Disable flagging for simplicity
+    allow_flagging="never"  # Disable flagging 
 )
 
 if __name__ == "__main__":
+    initialize_database()
     run_test()  # Run tests
     iface.launch()

gematria.db

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a74da41e964fe494ca3378faaa3be77929f1b0a680c312cee73106b96fb31055
-size 2891776
+oid sha256:42149cb9e6ef917c121a5f8cb3d44ab03cfd9770b2e7818715b326f12b5bb65e
+size 44187648