Upload app.py

app.py

@@ -0,0 +1,209 @@
+import subprocess
+
+# Install hunspell and its dependencies, pip wheels are completely broken
+subprocess.call(['sudo', 'apt', 'install', 'hunspell', 'hunspell-uk', 'libhunspell-dev'])
+subprocess.call(['sudo', 'pip', 'install', 'hunspell'])
+
+# Import hunspell
+import hunspell
+# Main imports
+import gradio as gr
+import re
+import stanza
+import spacy
+import pandas as pd
+
+def create_settlement_and_country_lists():
+    settlement_list = []
+    country_list = []
+
+    # Read Ukrainian settlement names from CSV file
+    df_settlements = pd.read_csv("assets/locations/ukrainian_settlement_mames.csv", encoding="utf-8")
+    ukrainian_settlements = df_settlements["Назва об'єкта українською мовою"].values.tolist()
+    settlement_list.extend(ukrainian_settlements)
+
+    # Read European settlement names from CSV file
+    df_eu_settlements = pd.read_csv("assets/locations/european_cities.csv", encoding="utf-8")
+    european_settlements = df_eu_settlements["City"].values.tolist()
+    settlement_list.extend(european_settlements)
+
+    # Convert settlement list to lowercase
+    settlement_list = [word.lower() for word in settlement_list]
+
+    # Read country names from text file
+    with open("assets/locations/countries.txt", "r", encoding="utf-8") as country_file:
+        country_list = [line.strip().lower() for line in country_file]
+
+    return settlement_list, country_list
+
+# Call the function to create settlement and country lists
+settlement_list, country_list = create_settlement_and_country_lists()
+
+spellchecker = hunspell.HunSpell('assets/dictionaries/uk_UA.dic', 'assets/dictionaries/uk_UA.aff')
+
+settlement_list = [s.lower() for s in settlement_list]  # Convert settlement list to lowercase
+country_list = [c.lower() for c in country_list]  # Convert country list to lowercase
+
+# Initialize Stanza NLP
+stanza.download('uk')
+nlp_stanza = stanza.Pipeline('uk', processors='tokenize,pos,ner')
+
+# Load SpaCy NER model
+nlp_spacy = spacy.load("uk_ner_web_trf_base")
+
+def process_text_with_stanza(text):
+    doc = nlp_stanza(text)
+    return format_output(process_text(doc))
+
+def process_text_with_spacy(text):
+    doc = nlp_spacy(text)
+    return format_output(process_text_spacy(doc))
+
+def format_output(matches):
+    formatted_matches = []
+    for match in matches:
+        location_type = match[0]
+        entity = match[1]
+        formatted_matches.append(f"{location_type}: {entity}")
+    return "\n".join(formatted_matches) if formatted_matches else notify_no_result()
+
+def notify_no_result():
+    return "No locations found in the text."
+
+def process_text(doc):
+    starting_point_patterns = [r'(з|із|із-за|від|от|од){pos:IN} (\w+{ner:LOC})']
+    destination_patterns = [r'(до|в|у|ув|к){pos:IN} (\w+{ner:LOC})']
+
+    starting_point_matches = []
+    for pattern in starting_point_patterns:
+        matches = re.findall(pattern, doc.text)
+        starting_point_matches.extend(matches)
+
+    destination_matches = []
+    for pattern in destination_patterns:
+        matches = re.findall(pattern, doc.text)
+        destination_matches.extend(matches)
+
+    loc_entities = [ent.text for ent in doc.ents if ent.type == 'LOC']
+    if len(loc_entities) == 2 and not starting_point_matches and not destination_matches:
+        starting_point = loc_entities[0]
+        destination = loc_entities[1]
+        return [
+            (starting_point, 'Starting Point', get_base_form_regex(starting_point, settlement_list, country_list, doc)),
+            (destination, 'Destination', get_base_form_regex(destination, settlement_list, country_list, doc))
+        ]
+
+    if len(loc_entities) == 1 and not starting_point_matches and not destination_matches:
+        return [(loc_entities[0], 'Unknown', get_base_form_regex(loc_entities[0], settlement_list, country_list, doc))]
+
+    treated_matches = [
+        (match[1], 'Starting Point', get_base_form_regex(match[1], settlement_list, country_list, doc))
+        for match in starting_point_matches
+    ] + [
+        (match[1], 'Destination', get_base_form_regex(match[1], settlement_list, country_list, doc))
+        for match in destination_matches
+    ]
+
+    formatted_matches = []
+    for match in treated_matches:
+        location_type = match[1]
+        lemma_results = match[2][0]  # Access the first element of the nested list
+        formatted_lemma = lemma_results[1].capitalize().strip('\n')
+        formatted_matches.append((location_type, lemma_results[0], formatted_lemma))
+
+    return formatted_matches
+
+def process_text_spacy(doc):
+    starting_point_patterns = [
+        r'(з|із|із-за|від|от|од){pos:ADP} (\w+{ner:LOC})',
+        r'(\w+{ner:LOC})\s+(з|із|із-за|від|от|од){pos:ADP}'
+    ]
+    destination_patterns = [
+        r'(до|в|у|ув|к){pos:ADP} (\w+{ner:LOC})',
+        r'(\w+{ner:LOC})\s+(до|в|у|ув|к){pos:ADP}'
+    ]
+
+    starting_point_matches = []
+    for pattern in starting_point_patterns:
+        matches = re.findall(pattern, doc.text)
+        starting_point_matches.extend(matches)
+
+    destination_matches = []
+    for pattern in destination_patterns:
+        matches = re.findall(pattern, doc.text)
+        destination_matches.extend(matches)
+
+    loc_entities = [ent.text for ent in doc.ents if ent.label_ == 'LOC']
+    if len(loc_entities) == 2 and not starting_point_matches and not destination_matches:
+        starting_point = loc_entities[0]
+        destination = loc_entities[1]
+        return [
+            (starting_point, 'Starting Point', get_base_form_stanza(starting_point, settlement_list, country_list, doc)),
+            (destination, 'Destination', get_base_form_stanza(destination, settlement_list, country_list, doc))
+        ]
+
+    if len(loc_entities) == 1 and not starting_point_matches and not destination_matches:
+        return [(loc_entities[0], 'Unknown', get_base_form_stanza(loc_entities[0], settlement_list, country_list, doc))]
+
+    treated_matches = [
+        (match[1], 'Starting Point', get_base_form_stanza(match[1], settlement_list, country_list, doc))
+        for match in starting_point_matches
+    ] + [
+        (match[1], 'Destination', get_base_form_stanza(match[1], settlement_list, country_list, doc))
+        for match in destination_matches
+    ]
+
+    formatted_matches = []
+    for match in treated_matches:
+        location_type = match[1]
+        lemma_results = match[2]  # Use directly, as it's already the required format
+        formatted_lemma = lemma_results.capitalize().strip('\n')
+        formatted_matches.append((location_type, lemma_results, formatted_lemma))
+
+    return formatted_matches
+
+def get_base_form_stanza(word, settlement_list, country_list, doc):
+    token = None
+    base_form = ""
+    for sent in doc.sentences:
+        for wrd in sent.words:
+            if wrd.text.lower() == word.lower():
+                token = wrd
+                break
+
+    if token is not None:
+        if token.upos == 'PROPN' and token.text.lower() not in settlement_list and token.text.lower() not in country_list:
+            base_form = token.lemma
+        else:
+            base_form = token.text
+
+    return base_form
+
+def get_base_form_regex(word, settlement_list, country_list, doc):
+    base_form = ""
+    base_form_regex = ""
+
+    if word.lower() in settlement_list or word.lower() in country_list:
+        base_form = word.lower()
+    else:
+        base_form = get_base_form_stanza(word, settlement_list, country_list, doc)
+
+    if base_form:
+        base_form_regex = base_form
+
+    return base_form_regex, base_form
+
+iface = gr.Interface(
+    fn=[process_text_with_stanza, process_text_with_spacy],
+    inputs=gr.inputs.Textbox(lines=5, label="Input Text"),
+    outputs=["text", "text"],
+    title="Text Processing Demo",
+    description="A demo to process text and extract locations using Stanza and SpaCy.",
+    examples=[
+        ["Автобус з Києва до Житомира"],
+        ["Автобус з Києва в Бердичів"],
+        ["Поїздка з Варшави до Івано-Франківська"],
+    ]
+)
+
+iface.launch()