diff --git a/.gitattributes b/.gitattributes
index a6344aac8c09253b3b630fb776ae94478aa0275b..e869236541f13ecd9892b56e491e5fa8be01afd6 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -33,3 +33,8 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
*.zip filter=lfs diff=lfs merge=lfs -text
*.zst filter=lfs diff=lfs merge=lfs -text
*tfevents* filter=lfs diff=lfs merge=lfs -text
+els_cache.db filter=lfs diff=lfs merge=lfs -text
+gematria.db filter=lfs diff=lfs merge=lfs -text
+translation_database.db filter=lfs diff=lfs merge=lfs -text
+translation_database.db.initial filter=lfs diff=lfs merge=lfs -text
+texts/bible/OpenGNT_version3_3.csv filter=lfs diff=lfs merge=lfs -text
diff --git a/README.md b/README.md
index c6509cc9bc5777026008566cb47e6b4c23228229..bbee331fd8851a03d039a50b633d68398047af5f 100644
--- a/README.md
+++ b/README.md
@@ -1,12 +1,45 @@
---
-title: Els Journal
-emoji: ЁЯРа
-colorFrom: pink
+title: Book of Souls - Search your name+day (journal) oracle with ELS over Torah, Bible, Quran, Rigveda, Tripitaka
+emoji: ЁЯУК
+colorFrom: green
colorTo: pink
sdk: gradio
-sdk_version: 5.7.1
+sdk_version: 4.39.0
app_file: app.py
pinned: false
---
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+This application searches for equidistant letter sequences (ELS) in the Torah, Bible, Quran, and Rigveda. It also integrates a network search functionality to find related phrases based on gematria.
+
+**Inputs:**
+
+* **Target Language for Translation:** The language to translate the results into.
+* **Date to investigate (optional):** A date to include in the gematria calculation.
+* **Language of the person/topic (optional) (Date Word Language):** The language to use for converting the date to words.
+* **Name and/or Topic (required):** The text to calculate the gematria for.
+* **Jump Width (Steps) (optional) for ELS:** The step size for the ELS search.
+* **Round (1) / Round (2) (optional):** The number of rounds for the ELS search (positive or negative).
+* **Include Torah / Include Bible / Include Quran / Include Rigveda:** Checkboxes to select which texts to search.
+* **Strip Spaces from Books / Strip Text in Braces from Books / Strip Diacritics from Books:** Options for text preprocessing.
+
+**Outputs:**
+
+* **ELS Results:** A dataframe containing the ELS search results.
+* **Most Frequent Phrase in Network Search:** The most frequent phrase found in the network search.
+* **JSON Output:** A JSON representation of the search results.
+
+
+**How to Use:**
+
+1. Enter the name or topic you want to investigate.
+2. Optionally, select a date and the language for its representation.
+3. Set the jump width (steps) and rounds for the ELS search.
+4. Choose which texts to include in the search.
+5. Configure text preprocessing options as needed.
+6. Click "Search with ELS".
+7. The results will be displayed in the output sections. You can copy the JSON output using the provided button.
+
+
+**Network Search:**
+
+The network search functionality uses the calculated gematria of the ELS results to search a database for phrases with the same gematria. It displays the most frequent matching phrase. If no exact match is found, it attempts to find the closest match based on similarity and word count difference.
\ No newline at end of file
diff --git a/__pycache__/bible.cpython-310.pyc b/__pycache__/bible.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..0ae9e74df91afe4265f5a85440ba634a204b2a6e
Binary files /dev/null and b/__pycache__/bible.cpython-310.pyc differ
diff --git a/__pycache__/gematria.cpython-310.pyc b/__pycache__/gematria.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..8b32bfe5528673e39456ad5214c0a9e09fc9acb3
Binary files /dev/null and b/__pycache__/gematria.cpython-310.pyc differ
diff --git a/__pycache__/hindu.cpython-310.pyc b/__pycache__/hindu.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..c59722dad0f67668326fc2ee654d78b737d60827
Binary files /dev/null and b/__pycache__/hindu.cpython-310.pyc differ
diff --git a/__pycache__/quran.cpython-310.pyc b/__pycache__/quran.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..3714766311e6d50b8ef4555d3355fa49dff3fd58
Binary files /dev/null and b/__pycache__/quran.cpython-310.pyc differ
diff --git a/__pycache__/torah.cpython-310.pyc b/__pycache__/torah.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..1d856eb921596f60bff6e6b4b9183a9ab50b503f
Binary files /dev/null and b/__pycache__/torah.cpython-310.pyc differ
diff --git a/__pycache__/translation_utils.cpython-310.pyc b/__pycache__/translation_utils.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..d1afe409fe7aabb8ed6ba2e5578a2237ab379825
Binary files /dev/null and b/__pycache__/translation_utils.cpython-310.pyc differ
diff --git a/__pycache__/tripitaka.cpython-310.pyc b/__pycache__/tripitaka.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..362e6397c0d669ab57fd85577ca668536b9f6312
Binary files /dev/null and b/__pycache__/tripitaka.cpython-310.pyc differ
diff --git a/__pycache__/utils.cpython-310.pyc b/__pycache__/utils.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..3be4106ab6ff168a5c9f7e55133cab205ae768cf
Binary files /dev/null and b/__pycache__/utils.cpython-310.pyc differ
diff --git a/app.py b/app.py
new file mode 100644
index 0000000000000000000000000000000000000000..7f46ca88c56f7ccefe0c7443858d58d6d82f558f
--- /dev/null
+++ b/app.py
@@ -0,0 +1,596 @@
+#TODO: Quran results have numbers
+
+import logging
+logger = logging.getLogger(__name__)
+logging.basicConfig(level=logging.INFO)
+
+import gradio as gr
+import torah
+import bible
+import quran
+import hindu
+import tripitaka
+from utils import number_to_ordinal_word, custom_normalize, date_to_words, translate_date_to_words
+from gematria import calculate_gematria, strip_diacritics
+
+import pandas as pd
+from deep_translator import GoogleTranslator
+from gradio_calendar import Calendar
+from datetime import datetime, timedelta
+import math
+import json
+import re
+import sqlite3
+from collections import defaultdict
+from typing import List, Tuple
+import rich
+from fuzzywuzzy import fuzz
+import calendar
+import translation_utils
+import hashlib
+
+translation_utils.create_translation_table()
+
+# Create a translator instance *once* globally
+translator = GoogleTranslator(source='auto', target='auto')
+LANGUAGES_SUPPORTED = translator.get_supported_languages(as_dict=True) # Corrected dictionary name
+
+LANGUAGE_CODE_MAP = LANGUAGES_SUPPORTED # Use deep_translator's mapping directly
+
+# --- Constants ---
+DATABASE_FILE = 'gematria.db'
+MAX_PHRASE_LENGTH_LIMIT = 20
+
+ELS_CACHE_DB = "els_cache.db"
+DATABASE_TIMEOUT = 60
+
+# --- Database Initialization ---
+def initialize_database():
+ global conn
+ conn = sqlite3.connect(DATABASE_FILE)
+ cursor = conn.cursor()
+ cursor.execute('''
+ CREATE TABLE IF NOT EXISTS results (
+ gematria_sum INTEGER,
+ words TEXT,
+ translation TEXT,
+ book TEXT,
+ chapter INTEGER,
+ verse INTEGER,
+ phrase_length INTEGER,
+ word_position TEXT,
+ PRIMARY KEY (gematria_sum, words, book, chapter, verse, word_position)
+ )
+ ''')
+ cursor.execute('''
+ CREATE INDEX IF NOT EXISTS idx_results_gematria
+ ON results (gematria_sum)
+ ''')
+ cursor.execute('''
+ CREATE TABLE IF NOT EXISTS processed_books (
+ book TEXT PRIMARY KEY,
+ max_phrase_length INTEGER
+ )
+ ''')
+ conn.commit()
+
+# --- Initialize Database ---
+initialize_database()
+
+# --- ELS Cache Functions ---
+def create_els_cache_table():
+ with sqlite3.connect(ELS_CACHE_DB) as conn:
+ conn.execute('''
+ CREATE TABLE IF NOT EXISTS els_cache (
+ query_hash TEXT PRIMARY KEY,
+ results TEXT
+ )
+ ''')
+
+def get_query_hash(func, *args, **kwargs):
+ key = (func.__name__, args, tuple(sorted(kwargs.items())))
+ return hashlib.sha256(json.dumps(key).encode()).hexdigest()
+
+
+def cached_process_json_files(func, *args, **kwargs):
+ query_hash = get_query_hash(func, *args, **kwargs)
+
+ try:
+ with sqlite3.connect(ELS_CACHE_DB, timeout=DATABASE_TIMEOUT) as conn:
+ cursor = conn.cursor()
+ cursor.execute("SELECT results FROM els_cache WHERE query_hash = ?", (query_hash,))
+ result = cursor.fetchone()
+ if result:
+ logger.info(f"Cache hit for query: {query_hash}")
+ return json.loads(result[0])
+ except sqlite3.Error as e:
+ logger.error(f"Database error checking cache: {e}")
+
+ logger.info(f"Cache miss for query: {query_hash}")
+ results = func(*args, **kwargs)
+
+ try:
+ with sqlite3.connect(ELS_CACHE_DB, timeout=DATABASE_TIMEOUT) as conn:
+ cursor = conn.cursor()
+ cursor.execute("INSERT INTO els_cache (query_hash, results) VALUES (?, ?)", (query_hash, json.dumps(results)))
+ conn.commit()
+ except sqlite3.Error as e:
+ logger.error(f"Database error caching results: {e}")
+
+ return results
+
+# --- Helper Functions (from Network app.py) ---
+def flatten_text(text: List) -> str:
+ if isinstance(text, list):
+ return " ".join(flatten_text(item) if isinstance(item, list) else item for item in text)
+ return text
+
+def search_gematria_in_db(gematria_sum: int, max_words: int) -> List[Tuple[str, str, int, int, int, str]]:
+ global conn
+ with sqlite3.connect(DATABASE_FILE) as conn:
+ cursor = conn.cursor()
+ cursor.execute('''
+ SELECT words, book, chapter, verse, phrase_length, word_position
+ FROM results
+ WHERE gematria_sum = ? AND phrase_length <= ?
+ ''', (gematria_sum, max_words))
+ results = cursor.fetchall()
+ return results
+
+def get_most_frequent_phrase(results):
+ phrase_counts = defaultdict(int)
+ for words, book, chapter, verse, phrase_length, word_position in results:
+ phrase_counts[words] += 1
+ most_frequent_phrase = max(phrase_counts, key=phrase_counts.get) if phrase_counts else None # Handle empty results
+ return most_frequent_phrase
+
+# --- Functions from BOS app.py ---
+def create_language_dropdown(label, default_value='English', show_label=True): # Default value must be in LANGUAGE_CODE_MAP
+ return gr.Dropdown(
+ choices=list(LANGUAGE_CODE_MAP.keys()), # Correct choices
+ label=label,
+ value=default_value,
+ show_label=show_label
+ )
+
+def calculate_gematria_sum(text, date_words):
+ if text or date_words:
+ combined_input = f"{text} {date_words}"
+ logger.info(f"searching for input: {combined_input}")
+ numbers = re.findall(r'\d+', combined_input)
+ text_without_numbers = re.sub(r'\d+', '', combined_input)
+ number_sum = sum(int(number) for number in numbers)
+ text_gematria = calculate_gematria(strip_diacritics(text_without_numbers))
+ total_sum = text_gematria + number_sum
+ return total_sum
+ else:
+ return None
+
+def perform_els_search(step, rounds_combination, tlang, strip_spaces, strip_in_braces, strip_diacritics_chk, include_torah, include_bible, include_quran, include_hindu, include_tripitaka):
+ if step == 0 or rounds_combination == "0,0":
+ return None
+
+ results = {}
+ length = 0
+
+ selected_language_long = tlang # From the Gradio dropdown (long form)
+ tlang = LANGUAGES_SUPPORTED.get(selected_language_long) #Get the short code.
+ if tlang is None: # Handle unsupported languages
+ tlang = "en"
+ logger.warning(f"Unsupported language selected: {selected_language_long}. Defaulting to English (en).")
+
+ if include_torah:
+ logger.debug(f"Arguments for Torah: {(1, 39, step, rounds_combination, length, tlang, strip_spaces, strip_in_braces, strip_diacritics_chk)}")
+ results["Torah"] = cached_process_json_files(torah.process_json_files, 1, 39, step, rounds_combination, length, tlang, strip_spaces, strip_in_braces, strip_diacritics_chk)
+ else:
+ results["Torah"] = []
+
+ if include_bible:
+ results["Bible"] = cached_process_json_files(bible.process_json_files, 40, 66, step, rounds_combination, length, tlang, strip_spaces, strip_in_braces, strip_diacritics_chk)
+ else:
+ results["Bible"] = []
+
+ if include_quran:
+ results["Quran"] = cached_process_json_files(quran.process_json_files, 1, 114, step, rounds_combination, length, tlang, strip_spaces, strip_in_braces, strip_diacritics_chk)
+ else:
+ results["Quran"] = []
+
+ if include_hindu:
+ results["Rig Veda"] = cached_process_json_files(hindu.process_json_files, 1, 10, step, rounds_combination, length, tlang, False, strip_in_braces, strip_diacritics_chk)
+ else:
+ results["Rig Veda"] = []
+
+ if include_tripitaka:
+ results["Tripitaka"] = cached_process_json_files(tripitaka.process_json_files, 1, 52, step, rounds_combination, length, tlang, strip_spaces, strip_in_braces, strip_diacritics_chk)
+ else:
+ results["Tripitaka"] = []
+
+ return results
+
+
+def add_24h_projection(results_dict): #Now takes a dictionary of results
+ for book_name, results in results_dict.items(): # Iterate per book
+ num_results = len(results)
+ if num_results > 0:
+ time_interval = timedelta(minutes=24 * 60 / num_results)
+ current_time = datetime.min.time()
+ for i in range(num_results):
+ next_time = (datetime.combine(datetime.min, current_time) + time_interval).time()
+ time_range_str = f"{current_time.strftime('%H:%M')}-{next_time.strftime('%H:%M')}"
+ results[i]['24h Projection'] = time_range_str
+ current_time = next_time
+ return results_dict
+
+
+def add_monthly_projection(results_dict, selected_date):
+ if selected_date is None:
+ return results_dict # Return if no date is selected
+
+ for book_name, results in results_dict.items(): # Iterate per book
+ num_results = len(results)
+ if num_results > 0:
+ days_in_month = calendar.monthrange(selected_date.year, selected_date.month)[1]
+ total_seconds = (days_in_month - 1) * 24 * 3600
+ seconds_interval = total_seconds / num_results
+ start_datetime = datetime(selected_date.year, selected_date.month, 1)
+ current_datetime = start_datetime
+
+
+ for i in range(num_results):
+ next_datetime = current_datetime + timedelta(seconds=seconds_interval)
+ current_date = current_datetime.date() # Moved assignment inside loop
+ next_date = next_datetime.date()
+ date_range_str = f"{current_date.strftime('%h %d')} - {next_date.strftime('%h %d')}"
+ results[i]['Monthly Projection'] = date_range_str
+ current_datetime = next_datetime # Add this
+ current_date = next_datetime.date() # Add this too
+ return results_dict
+
+
+def add_yearly_projection(results_dict, selected_date): #Correct name, handle dictionary input
+ if selected_date is None:
+ return results_dict # Return if no date is selected
+
+ for book_name, results in results_dict.items(): # Iterate per book
+ num_results = len(results)
+ if num_results > 0:
+ days_in_year = 366 if calendar.isleap(selected_date.year) else 365
+ total_seconds = (days_in_year - 1) * 24 * 3600
+ seconds_interval = total_seconds / num_results
+ start_datetime = datetime(selected_date.year, 1, 1)
+ current_datetime = start_datetime
+
+
+ for i in range(num_results):
+ next_datetime = current_datetime + timedelta(seconds=seconds_interval)
+ current_date = current_datetime.date() # Move assignment inside loop
+ next_date = next_datetime.date()
+ date_range_str = f"{current_date.strftime('%b %d')} - {next_date.strftime('%b %d')}"
+ results[i]['Yearly Projection'] = date_range_str
+ current_datetime = next_datetime # Update current datetime for next iteration
+
+ return results_dict
+
+
+def sort_results(results):
+ def parse_time(time_str):
+ try:
+ hours, minutes = map(int, time_str.split(':'))
+ return hours * 60 + minutes # Convert to total minutes
+ except ValueError:
+ return 24 * 60 # Sort invalid times to the end
+
+ return sorted(results, key=lambda x: (
+ parse_time(x.get('24h Projection', '23:59').split('-')[0]), # Sort by start time first
+ parse_time(x.get('24h Projection', '23:59').split('-')[1]) # Then by end time
+ ))
+
+# --- Main Gradio App ---
+with gr.Blocks() as app:
+ with gr.Column():
+ with gr.Row():
+ tlang = create_language_dropdown("Target Language for Result Translation", default_value='english')
+ selected_date = Calendar(type="datetime", label="Date to investigate (optional)", info="Pick a date from the calendar")
+ use_day = gr.Checkbox(label="Use Day", info="Check to include day in search", value=True)
+ use_month = gr.Checkbox(label="Use Month", info="Check to include month in search", value=True)
+ use_year = gr.Checkbox(label="Use Year", info="Check to include year in search", value=True)
+ date_language_input = create_language_dropdown("Language of the person/topic (optional) (Date Word Language)", default_value='english')
+ with gr.Row():
+ gematria_text = gr.Textbox(label="Name and/or Topic (required)", value="Hans Albert Einstein Mileva Marity-Einstein")
+ date_words_output = gr.Textbox(label="Date in Words Translated (optional)")
+ gematria_result = gr.Number(label="Journal Sum")
+ #with gr.Row():
+
+
+ with gr.Row():
+ step = gr.Number(label="Jump Width (Steps) for ELS")
+ float_step = gr.Number(visible=False, value=1)
+ half_step_btn = gr.Button("Steps / 2")
+ double_step_btn = gr.Button("Steps * 2")
+
+ with gr.Column():
+ round_x = gr.Number(label="Round (1)", value=1)
+ round_y = gr.Number(label="Round (2)", value=-1)
+
+ rounds_combination = gr.Textbox(label="Combined Rounds", value="1,-1")
+
+ with gr.Row():
+ include_torah_chk = gr.Checkbox(label="Include Torah", value=True)
+ include_bible_chk = gr.Checkbox(label="Include Bible", value=True)
+ include_quran_chk = gr.Checkbox(label="Include Quran", value=True)
+ include_hindu_chk = gr.Checkbox(label="Include Rigveda", value=True)
+ include_tripitaka_chk = gr.Checkbox(label="Include Tripitaka", value=True)
+ merge_results_chk = gr.Checkbox(label="Merge Results (Torah-Bible-Quran)", value=True)
+
+ strip_spaces = gr.Checkbox(label="Strip Spaces from Books", value=True)
+ strip_in_braces = gr.Checkbox(label="Strip Text in Braces from Books", value=True)
+ strip_diacritics_chk = gr.Checkbox(label="Strip Diacritics from Books", value=True)
+
+ translate_btn = gr.Button("Search with ELS")
+
+ # --- Output Components ---
+ markdown_output = gr.Dataframe(label="ELS Results")
+ most_frequent_phrase_output = gr.Textbox(label="Most Frequent Phrase in Network Search")
+ json_output = gr.Textbox(label="JSON Output")
+ copy_json_button = gr.Button("Copy JSON to Clipboard")
+
+ # --- Hidden HTML component to hold the JavaScript function ---
+ html_js = gr.HTML(
+ """
+
+ """, visible=False # Hide the HTML component
+ )
+
+
+ # --- Event Handlers ---
+
+ def update_date_words(selected_date, date_language_input, use_day, use_month, use_year):
+ if selected_date is None:
+ return ""
+
+ if not use_year and not use_month and not use_day:
+ return translate_date_to_words(selected_date, date_language_input)
+
+ year = selected_date.year if use_year else None
+ month = selected_date.month if use_month else None
+ day = selected_date.day if use_day else None
+
+ if year is not None and month is not None and day is not None:
+ date_obj = selected_date
+ elif year is not None and month is not None:
+ date_obj = str(f"{year}-{month}")
+ elif year is not None:
+ date_obj = str(f"{year}")
+ else: # Return empty string if no date components are selected
+ return ""
+
+
+ date_in_words = date_to_words(date_obj)
+
+
+ translator = GoogleTranslator(source='auto', target=date_language_input)
+ translated_date_words = translator.translate(date_in_words)
+ return custom_normalize(translated_date_words)
+
+ def update_journal_sum(gematria_text, date_words_output):
+ sum_value = calculate_gematria_sum(gematria_text, date_words_output)
+ return sum_value, sum_value, sum_value
+
+ def update_rounds_combination(round_x, round_y):
+ return f"{int(round_x)},{int(round_y)}"
+
+ def update_step_half(float_step):
+ new_step = math.ceil(float_step / 2)
+ return new_step, float_step / 2
+
+ def update_step_double(float_step):
+ new_step = math.ceil(float_step * 2)
+ return new_step, float_step * 2
+
+
+ def find_closest_phrase(target_phrase, phrases):
+ best_match = None
+ best_score = 0
+
+ logging.debug(f"Target phrase for similarity search: {target_phrase}") # Log target phrase
+
+ for phrase, _, _, _, _, _ in phrases:
+ word_length_diff = abs(len(target_phrase.split()) - len(phrase.split()))
+ similarity_score = fuzz.ratio(target_phrase, phrase)
+ combined_score = similarity_score - word_length_diff
+
+ logging.debug(f"Comparing with phrase: {phrase}") # Log each phrase being compared
+ logging.debug(
+ f"Word Length Difference: {word_length_diff}, Similarity Score: {similarity_score}, Combined Score: {combined_score}") # Log scores
+
+ if combined_score > best_score:
+ best_score = combined_score
+ best_match = phrase
+
+ logging.debug(f"Closest phrase found: {best_match} with score: {best_score}") # Log the best match
+ return best_match
+
+ def perform_search(step, rounds_combination, tlang, strip_spaces, strip_in_braces, strip_diacritics_chk, include_torah, include_bible, include_quran, include_hindu, include_tripitaka, gematria_text, date_words_output, selected_date):
+ # Inside perform_search
+ els_results = perform_els_search(step, rounds_combination, tlang, strip_spaces, strip_in_braces,
+ strip_diacritics_chk, include_torah, include_bible, include_quran,
+ include_hindu,
+ include_tripitaka)
+
+ # --- Network Search Integration ---
+ most_frequent_phrases = {}
+ combined_and_sorted_results = [] # Combined list to hold all results
+
+ for book_name, book_results in els_results.items():
+ if book_results: # Add this check to ensure book_results is not empty
+ most_frequent_phrases[book_name] = "" # Default value
+
+ for result in book_results:
+ try:
+ gematria_sum = calculate_gematria(result['result_text']) # Calculate gematria
+ max_words = len(result['result_text'].split())
+ matching_phrases = search_gematria_in_db(gematria_sum, max_words)
+ max_words_limit = 20
+ while not matching_phrases and max_words < max_words_limit: # Increase max_words for more results
+ max_words += 1
+ matching_phrases = search_gematria_in_db(gematria_sum, max_words)
+
+ if matching_phrases:
+ most_frequent_phrase = get_most_frequent_phrase(matching_phrases)
+ most_frequent_phrases[book_name] = most_frequent_phrase
+ else:
+ closest_phrase = find_closest_phrase(result['result_text'],
+ search_gematria_in_db(gematria_sum, max_words_limit))
+ most_frequent_phrases[
+ book_name] = closest_phrase or "" # Update most frequent phrases even if no phrase found
+
+ result['Most Frequent Phrase'] = most_frequent_phrases[book_name]
+ if 'book' in result:
+ if isinstance(result['book'], int): # Torah, Bible, Quran case
+ result['book'] = f"{book_name} {result['book']}."
+ combined_and_sorted_results.append(result)
+
+ except KeyError as e:
+ print(f"DEBUG: KeyError - Key '{e.args[0]}' not found in result. Skipping this result.")
+ continue
+
+ # --- Batch Translation ---
+ selected_language_long = tlang
+ tlang_short = LANGUAGES_SUPPORTED.get(selected_language_long)
+ if tlang_short is None:
+ tlang_short = "en"
+ logger.warning(f"Unsupported language selected: {selected_language_long}. Defaulting to English (en).")
+
+ # Prepare lists for batch translation, including source language
+ phrases_to_translate = []
+ phrases_source_langs = [] # Source languages for phrases
+ results_to_translate = []
+ results_source_langs = [] # Source languages for results
+ for result in combined_and_sorted_results:
+ phrases_to_translate.append(result.get('Most Frequent Phrase', ''))
+ phrases_source_langs.append(result.get("source_language", "auto"))
+ results_to_translate.append(result.get('result_text', ''))
+ results_source_langs.append(result.get("source_language", "auto"))
+
+
+ translated_phrases = translation_utils.batch_translate(phrases_to_translate, tlang_short, phrases_source_langs)
+ translated_result_texts = translation_utils.batch_translate(results_to_translate, tlang_short, results_source_langs)
+
+
+ for i, result in enumerate(combined_and_sorted_results):
+ result['translated_text'] = translated_result_texts.get(results_to_translate[i], None)
+ result['Translated Most Frequent Phrase'] = translated_phrases.get(phrases_to_translate[i], None)
+
+ # Time Projections (using els_results dictionary)
+ updated_els_results = add_24h_projection(els_results) # Use original els_results dictionary
+ updated_els_results = add_monthly_projection(updated_els_results, selected_date) # Call correct functions with correct params
+ updated_els_results = add_yearly_projection(updated_els_results, selected_date)
+
+ combined_and_sorted_results = []
+ for book_results in updated_els_results.values(): # Combine results for dataframe and json
+ combined_and_sorted_results.extend(book_results)
+ combined_and_sorted_results = sort_results(combined_and_sorted_results) # sort combined results
+
+ df = pd.DataFrame(combined_and_sorted_results)
+ df.index = range(1, len(df) + 1)
+ df.reset_index(inplace=True)
+ df.rename(columns={'index': 'Result Number'}, inplace=True)
+
+ for i, result in enumerate(combined_and_sorted_results): # Iterate through the combined list
+ result['Result Number'] = i + 1
+
+ search_config = {
+ "step": step,
+ "rounds_combination": rounds_combination,
+ "target_language": tlang,
+ "strip_spaces": strip_spaces,
+ "strip_in_braces": strip_in_braces,
+ "strip_diacritics": strip_diacritics_chk,
+ "include_torah": include_torah,
+ "include_bible": include_bible,
+ "include_quran": include_quran,
+ "include_hindu": include_hindu,
+ "include_tripitaka": include_tripitaka,
+ "gematria_text": gematria_text,
+ "date_words": date_words_output
+ }
+
+ output_data = {
+ "search_configuration": search_config,
+ "results": combined_and_sorted_results # Use the combined list here
+ }
+
+ json_data = json.dumps(output_data, ensure_ascii=False, indent=4)
+
+ # --- Return results ---
+ combined_most_frequent = "\n".join(
+ f"{book}: {phrase}" for book, phrase in most_frequent_phrases.items()) # Combine phrases
+ return df, combined_most_frequent, json_data
+
+
+
+ # --- Event Triggers ---
+ round_x.change(update_rounds_combination, inputs=[round_x, round_y], outputs=rounds_combination)
+ round_y.change(update_rounds_combination, inputs=[round_x, round_y], outputs=rounds_combination)
+
+ selected_date.change(update_date_words, inputs=[selected_date, date_language_input, use_day, use_month, use_year], outputs=[date_words_output])
+ date_language_input.change(update_date_words, inputs=[selected_date, date_language_input, use_day, use_month, use_year], outputs=[date_words_output])
+
+ gematria_text.change(update_journal_sum, inputs=[gematria_text, date_words_output], outputs=[gematria_result, step, float_step])
+ date_words_output.change(update_journal_sum, inputs=[gematria_text, date_words_output], outputs=[gematria_result, step, float_step])
+
+ half_step_btn.click(update_step_half, inputs=[float_step], outputs=[step, float_step])
+ double_step_btn.click(update_step_double, inputs=[float_step], outputs=[step, float_step])
+
+ translate_btn.click(
+ perform_search,
+ inputs=[step, rounds_combination, tlang, strip_spaces, strip_in_braces, strip_diacritics_chk, include_torah_chk, include_bible_chk, include_quran_chk, include_hindu_chk, include_tripitaka_chk, gematria_text, date_words_output, selected_date],
+ outputs=[markdown_output, most_frequent_phrase_output, json_output]
+ )
+
+ app.load(
+ update_date_words,
+ inputs=[selected_date, date_language_input, use_day, use_month, use_year], # Include all 5 inputs
+ outputs=[date_words_output]
+ )
+
+ copy_json_button.click(
+ js="""
+ (json_string) => {
+ copyJSON(json_string); // Call the JavaScript function defined in the HTML component
+ }""",
+ inputs=json_output, # Make sure json_output is an input
+ )
+
+ use_day.change(
+ update_date_words,
+ inputs=[selected_date, date_language_input, use_day, use_month, use_year],
+ outputs=[date_words_output]
+ )
+ use_month.change(
+ update_date_words,
+ inputs=[selected_date, date_language_input, use_day, use_month, use_year],
+ outputs=[date_words_output]
+ )
+ use_year.change(
+ update_date_words,
+ inputs=[selected_date, date_language_input, use_day, use_month, use_year],
+ outputs=[date_words_output]
+ )
+
+ def checkbox_behavior(use_day_value, use_month_value):
+ if use_day_value: # Tick month and year automatically when day is ticked.
+ return True, True
+
+ return use_month_value, True # return month value unchanged and automatically tick year if month is checked
+
+ use_day.change(checkbox_behavior, inputs=[use_day, use_month], outputs=[use_month, use_year])
+ use_month.change(checkbox_behavior, inputs=[use_day, use_month], outputs=[use_month, use_year]) #No need for use_day here, day won't be changed by month
+
+
+if __name__ == "__main__":
+ app.launch(share=False)
diff --git a/app.py.bak b/app.py.bak
new file mode 100644
index 0000000000000000000000000000000000000000..33246a99e448f0af1d4dd345480536ad8a866259
--- /dev/null
+++ b/app.py.bak
@@ -0,0 +1,456 @@
+import gradio as gr
+import json
+import re
+import sqlite3
+import logging
+from collections import defaultdict
+from typing import Tuple, Dict, List
+
+# Assuming you have these files in your project
+from util import process_json_files
+from gematria import calculate_gematria
+from deep_translator import GoogleTranslator, exceptions
+from urllib.parse import quote_plus
+from tqdm import tqdm
+
+# Constants
+DATABASE_FILE = 'gematria.db'
+MAX_PHRASE_LENGTH_LIMIT = 20
+BATCH_SIZE = 10000
+
+# Set up logging
+logging.basicConfig(level=logging.DEBUG, format='%(asctime)s - %(levelname)s - %(message)s')
+
+# Global variables
+conn: sqlite3.Connection = None
+translator: GoogleTranslator = None
+book_names: Dict[int, str] = {}
+gematria_cache: Dict[Tuple[int, int], List[Tuple[str, str, int, int, int, str]]] = {}
+translation_cache: Dict[str, str] = {}
+total_word_count: int = 0 # Global counter for word position
+
+
+def initialize_database() -> None:
+ """Initializes the SQLite database."""
+ global conn
+ conn = sqlite3.connect(DATABASE_FILE)
+ cursor = conn.cursor()
+
+ cursor.execute('''
+ CREATE TABLE IF NOT EXISTS results (
+ gematria_sum INTEGER,
+ words TEXT,
+ translation TEXT,
+ book TEXT,
+ chapter INTEGER,
+ verse INTEGER,
+ phrase_length INTEGER,
+ word_position TEXT,
+ PRIMARY KEY (gematria_sum, words, book, chapter, verse, word_position)
+ )
+ ''')
+
+ cursor.execute('''
+ CREATE INDEX IF NOT EXISTS idx_results_gematria
+ ON results (gematria_sum)
+ ''')
+
+ cursor.execute('''
+ CREATE TABLE IF NOT EXISTS processed_books (
+ book TEXT PRIMARY KEY,
+ max_phrase_length INTEGER
+ )
+ ''')
+
+ conn.commit()
+
+
+def initialize_translator() -> None:
+ """Initializes the Google Translator."""
+ global translator
+ translator = GoogleTranslator(source='iw', target='en')
+ logging.info("Translator initialized.")
+
+
+def process_book(book_id: int, max_phrase_length: int, cursor):
+ """Processes a single book and returns phrases to insert."""
+ global book_names, total_word_count
+ book_data = process_json_files(book_id, book_id)
+ phrases_to_insert = []
+
+ if book_id in book_data:
+ book_data = book_data[book_id]
+ if 'title' not in book_data or not isinstance(book_data['title'], str):
+ logging.warning(f"Skipping book {book_id} due to missing 'title' field.")
+ return phrases_to_insert
+
+ title = book_data['title']
+ book_names[book_id] = title
+
+ # Check if this book has already been processed for this phrase length
+ cursor.execute('''SELECT max_phrase_length FROM processed_books WHERE book = ?''', (title,))
+ result = cursor.fetchone()
+ if result and result[0] >= max_phrase_length:
+ logging.info(f"Skipping book {title}: Already processed with max_phrase_length {result[0]}")
+ return phrases_to_insert
+
+ if 'text' not in book_data or not isinstance(book_data['text'], list):
+ logging.warning(f"Skipping book {book_id} due to missing 'text' field.")
+ return phrases_to_insert
+
+ chapters = book_data['text']
+ for chapter_id, chapter in enumerate(chapters):
+ for verse_id, verse in enumerate(chapter):
+ verse_text = flatten_text(verse)
+ verse_text = re.sub(r'\[.*?\]', '', verse_text)
+ verse_text = re.sub(r"[^\u05D0-\u05EA ]+", "", verse_text)
+ verse_text = re.sub(r" +", " ", verse_text)
+ words = verse_text.split()
+
+ for length in range(1, max_phrase_length + 1):
+ for start in range(len(words) - length + 1):
+ phrase_candidate = " ".join(words[start:start + length])
+ gematria_sum = calculate_gematria(phrase_candidate.replace(" ", ""))
+
+ word_position_range = f"{total_word_count + start + 1}-{total_word_count + start + length}"
+
+ phrases_to_insert.append(
+ (gematria_sum, phrase_candidate, None, title, chapter_id + 1, verse_id + 1, length,
+ word_position_range))
+
+ total_word_count += len(words)
+
+ return phrases_to_insert
+
+
+def populate_database(start_book: int, end_book: int, max_phrase_length: int = 1) -> None:
+ """Populates the database with phrases from the Tanach."""
+ global conn, book_names, total_word_count
+ logging.info(f"Populating database with books from {start_book} to {end_book}...")
+
+ with sqlite3.connect(DATABASE_FILE) as conn:
+ cursor = conn.cursor()
+
+ for book_id in tqdm(range(start_book, end_book + 1), desc="Processing Books"):
+ phrases_to_insert = process_book(book_id, max_phrase_length, cursor)
+
+ if phrases_to_insert:
+ cursor.executemany('''
+ INSERT OR IGNORE INTO results (gematria_sum, words, translation, book, chapter, verse, phrase_length, word_position)
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+ ''', phrases_to_insert)
+
+ # Update processed_books after processing each book
+ cursor.execute('''
+ INSERT OR REPLACE INTO processed_books (book, max_phrase_length)
+ VALUES (?, ?)
+ ''', (book_names[book_id], max_phrase_length))
+
+ conn.commit()
+
+ total_word_count = 0 # Reset for the next set of phrase lengths
+
+
+def get_translation(phrase: str) -> str:
+ """Retrieves or generates the English translation of a Hebrew phrase
+ and caches it in the database.
+ """
+ global conn, translator, translation_cache
+
+ # Check if the translation exists in the database
+ with sqlite3.connect(DATABASE_FILE) as conn:
+ cursor = conn.cursor()
+ cursor.execute("SELECT translation FROM results WHERE words = ? LIMIT 1", (phrase,))
+ result = cursor.fetchone()
+ if result and result[0]: # If a translation exists, use it
+ return result[0]
+
+ # If no translation in the database, translate and store it
+ translation = translate_and_store(phrase)
+ translation_cache[phrase] = translation
+
+ # Update the database with the new translation
+ with sqlite3.connect(DATABASE_FILE) as conn:
+ cursor = conn.cursor()
+ cursor.execute("UPDATE results SET translation = ? WHERE words = ?", (translation, phrase))
+ conn.commit()
+
+ return translation
+
+
+def translate_and_store(phrase: str) -> str:
+ """Translates a Hebrew phrase to English using Google Translate."""
+ global translator
+ max_retries = 3
+ retries = 0
+ while retries < max_retries:
+ try:
+ translation = translator.translate(phrase)
+ return translation
+ except (exceptions.TranslationNotFound, exceptions.NotValidPayload,
+ exceptions.ServerException, exceptions.RequestError) as e:
+ retries += 1
+ logging.warning(f"Error translating phrase '{phrase}': {e}. Retrying... ({retries}/{max_retries})")
+ logging.error(f"Failed to translate phrase '{phrase}' after {max_retries} retries.")
+ return "[Translation Error]"
+
+
+def search_gematria_in_db(gematria_sum: int, max_words: int) -> List[Tuple[str, str, int, int, int, str]]:
+ """Searches the database for phrases with a given Gematria value."""
+ global conn
+ with sqlite3.connect(DATABASE_FILE) as conn:
+ cursor = conn.cursor()
+ cursor.execute('''
+ SELECT words, book, chapter, verse, phrase_length, word_position
+ FROM results
+ WHERE gematria_sum = ? AND phrase_length <= ?
+ ''', (gematria_sum, max_words))
+ results = cursor.fetchall()
+ return results
+
+
+def gematria_search_interface(phrases: str, max_words: int, show_translation: bool) -> str:
+ """The main function for the Gradio interface, handling multiple phrases."""
+ global conn, book_names, gematria_cache
+
+ results = []
+ all_results = [] # Store results for each phrase
+ middle_words_results = [] # Store middle word results for all books
+ all_names_average_position = 0 # Initialize variable for average position across all names and books
+ total_name_count = 0 # Initialize counter for the total number of names processed
+
+ phrases = phrases.strip().splitlines()
+ if not phrases:
+ return "Please enter at least one phrase."
+
+ for phrase in phrases:
+ if not phrase.strip():
+ continue # Skip empty lines
+
+ numbers = re.findall(r'\d+', phrase)
+ text_without_numbers = re.sub(r'\d+', '', phrase)
+ phrase_gematria = calculate_gematria(text_without_numbers.replace(" ", ""))
+ phrase_gematria += sum(int(number) for number in numbers)
+
+ if (phrase_gematria, max_words) in gematria_cache:
+ matching_phrases = gematria_cache[(phrase_gematria, max_words)]
+ else:
+ matching_phrases = search_gematria_in_db(phrase_gematria, max_words)
+ gematria_cache[(phrase_gematria, max_words)] = matching_phrases
+
+ if not matching_phrases:
+ results.append(f"No matching phrases found for: {phrase}")
+ continue
+
+ sorted_phrases = sorted(matching_phrases,
+ key=lambda x: (int(list(book_names.keys())[list(book_names.values()).index(x[1])]), x[2],
+ x[3]))
+ results_by_book = defaultdict(list)
+ for words, book, chapter, verse, phrase_length, word_position in sorted_phrases:
+ results_by_book[book].append((words, chapter, verse, phrase_length, word_position))
+
+ results.append(f"Results for: {phrase} (Gematria: {phrase_gematria})
")
+ results.append("")
+ for book, phrases in results_by_book.items():
+ for words, chapter, verse, phrase_length, word_position in phrases:
+ translation = get_translation(words) if show_translation else ""
+ link = f"https://www.biblegateway.com/passage/?search={quote_plus(book)}+{chapter}%3A{verse}&version=CJB"
+ results.append(f"""
+
+
Book: {book}
+
Chapter: {chapter}, Verse: {verse}
+
Hebrew Phrase: {words}
+
Translation: {translation}
+
Phrase Length: {phrase_length} words
+
Phrase Gematria: {phrase_gematria}
+
Word Position in the Tanach: {word_position}
+
[See on Bible Gateway]
+
+ """)
+
+ # Calculate average position for the current name across all books
+ name_average_position = calculate_average_position_for_name(results_by_book)
+ if name_average_position is not None:
+ results.append(f"
Average Word Position for '{phrase}' across all books: {name_average_position:.2f}
")
+ all_names_average_position += name_average_position
+ total_name_count += 1
+
+ results.append("
Average Word Position Across All Names and Books: {all_names_average_position:.2f}
")
+
+ # Calculate middle words for all input lines (common books)
+ if len(all_results) >= 2:
+ results.append("Middle Words (Common Books):
")
+ results.append("")
+
+ common_books = set.intersection(*[set(results.keys()) for results in all_results])
+ logging.debug(f"Common books: {common_books}")
+
+ for book in common_books:
+ logging.debug(f"Processing book: {book}")
+
+ # Find nearest positions for all phrases in the current book
+ nearest_positions = find_nearest_positions([results[book] for results in all_results])
+ logging.debug(f"Nearest positions in {book}: {nearest_positions}")
+
+ if nearest_positions:
+ middle_word_position = sum(nearest_positions) / len(nearest_positions)
+ logging.debug(f"Calculated middle word position in {book}: {middle_word_position}")
+
+ start_position = int(middle_word_position)
+ end_position = start_position + 1 if middle_word_position % 1 != 0 else start_position
+ logging.debug(f"Middle word position range in {book}: {start_position}-{end_position}")
+
+ middle_words_data = get_words_from_db(book, start_position, end_position)
+ logging.debug(f"Middle words data fetched from database: {middle_words_data}")
+
+ if middle_words_data:
+ # Store middle word data along with book name for sorting
+ middle_words_results.extend([(book, data) for data in middle_words_data])
+ else:
+ # Handle edge case: fetch words independently for start and end positions
+ logging.debug(f"No middle words found for range {start_position}-{end_position}. "
+ f"Fetching words independently.")
+ middle_words_data_start = get_words_from_db(book, start_position, start_position)
+ middle_words_data_end = get_words_from_db(book, end_position, end_position)
+
+ if middle_words_data_start or middle_words_data_end:
+ middle_words_results.extend([(book, data) for data in middle_words_data_start + middle_words_data_end])
+
+ # Sort middle words results by book order before displaying
+ middle_words_results.sort(key=lambda x: int(list(book_names.keys())[list(book_names.values()).index(x[0])]))
+
+ for book, (words, chapter, verse, phrase_length, word_position) in middle_words_results:
+ translation = get_translation(words) if show_translation else ""
+ link = f"https://www.biblegateway.com/passage/?search={quote_plus(book)}+{chapter}%3A{verse}&version=CJB"
+ results.append(f"""
+
+
Book: {book}
+
Chapter: {chapter}, Verse: {verse}
+
Hebrew Phrase: {words}
+
Translation: {translation}
+
Phrase Length: {phrase_length} words
+
Word Position in the Tanach: {word_position}
+
[See on Bible Gateway]
+
+ """)
+ results.append("
Results for: {phrase} (Gematria: {phrase_gematria})
")
+ results.append("")
+ for book, phrases in results_by_book.items():
+ for words, chapter, verse, phrase_length, word_position in phrases:
+ translation = get_translation(words) if show_translation else ""
+ link = f"https://www.biblegateway.com/passage/?search={quote_plus(book)}+{chapter}%3A{verse}&version=CJB"
+ results.append(f"""
+
+
Book: {book}
+
Chapter: {chapter}, Verse: {verse}
+
Hebrew Phrase: {words}
+
Translation: {translation}
+
Phrase Length: {phrase_length} words
+
Phrase Gematria: {phrase_gematria}
+
Word Position in the Tanach: {word_position}
+
[See on Bible Gateway]
+
+ """)
+
+ # Calculate average position for the current name across all books
+ name_average_position = calculate_average_position_for_name(results_by_book)
+ if name_average_position is not None:
+ results.append(f"
Average Word Position for '{phrase}' across all books: {name_average_position:.2f}
")
+ all_names_average_position += name_average_position
+ total_name_count += 1
+
+ results.append("
Average Word Position Across All Names and Books: {all_names_average_position:.2f}
")
+
+ # Calculate middle words for all input lines (common books)
+ if len(all_results) >= 2:
+ results.append("Middle Words (Common Books):
")
+ results.append("")
+
+ common_books = set.intersection(*[set(results.keys()) for results in all_results])
+ logging.debug(f"Common books: {common_books}")
+
+ for book in common_books:
+ logging.debug(f"Processing book: {book}")
+
+ # Find nearest positions for all phrases in the current book
+ nearest_positions = find_nearest_positions([results[book] for results in all_results])
+ logging.debug(f"Nearest positions in {book}: {nearest_positions}")
+
+ if nearest_positions:
+ middle_word_position = sum(nearest_positions) / len(nearest_positions)
+ logging.debug(f"Calculated middle word position in {book}: {middle_word_position}")
+
+ start_position = int(middle_word_position)
+ end_position = start_position + 1 if middle_word_position % 1 != 0 else start_position
+ logging.debug(f"Middle word position range in {book}: {start_position}-{end_position}")
+
+ middle_words_data = get_words_from_db(book, start_position, end_position)
+ logging.debug(f"Middle words data fetched from database: {middle_words_data}")
+
+ if middle_words_data:
+ # Store middle word data along with book name for sorting
+ middle_words_results.extend([(book, data) for data in middle_words_data])
+ else:
+ # Handle edge case: fetch words independently for start and end positions
+ logging.debug(f"No middle words found for range {start_position}-{end_position}. "
+ f"Fetching words independently.")
+ middle_words_data_start = get_words_from_db(book, start_position, start_position)
+ middle_words_data_end = get_words_from_db(book, end_position, end_position)
+
+ if middle_words_data_start or middle_words_data_end:
+ middle_words_results.extend([(book, data) for data in middle_words_data_start + middle_words_data_end])
+
+ # Sort middle words results by book order before displaying
+ middle_words_results.sort(key=lambda x: int(list(book_names.keys())[list(book_names.values()).index(x[0])]))
+
+ for book, (words, chapter, verse, phrase_length, word_position) in middle_words_results:
+ translation = get_translation(words) if show_translation else ""
+ link = f"https://www.biblegateway.com/passage/?search={quote_plus(book)}+{chapter}%3A{verse}&version=CJB"
+ results.append(f"""
+
+
Book: {book}
+
Chapter: {chapter}, Verse: {verse}
+
Hebrew Phrase: {words}
+
Translation: {translation}
+
Phrase Length: {phrase_length} words
+
Word Position in the Tanach: {word_position}
+
[See on Bible Gateway]
+
+ """)
+ results.append("