added ner

app.py

@@ -4,6 +4,8 @@ from interfaces.cap import demo as cap_demo
 from interfaces.manifesto import demo as manifesto_demo
 from interfaces.sentiment import demo as sentiment_demo
 from interfaces.emotion import demo as emotion_demo
+from interfaces.ner import demo as ner_demo
+from interfaces.ner import download_models as download_spacy_models
 
 with gr.Blocks() as demo:
     gr.Markdown(
@@ -16,11 +18,12 @@ with gr.Blocks() as demo:
         """)
 
     gr.TabbedInterface(
-        interface_list=[cap_demo, manifesto_demo, sentiment_demo, emotion_demo],
-        tab_names=["CAP", "Manifesto", "Sentiment (3)", "Emotions (8)"],
+        interface_list=[cap_demo, manifesto_demo, sentiment_demo, emotion_demo, ner_demo],
+        tab_names=["CAP", "Manifesto", "Sentiment (3)", "Emotions (8)", "Named Entity Recognition"],
     )
 
 if __name__ == "__main__":
+    download_spacy_models()
     demo.launch()
 
 # TODO: add all languages & domains

interfaces/cap.py

@@ -14,19 +14,19 @@ from label_dicts import CAP_NUM_DICT, CAP_LABEL_NAMES
 HF_TOKEN = os.environ["hf_read"]
 
 languages = [
-    "danish",
-    "dutch",
-    "english",
-    "french",
-    "german",
-    "hungarian",
-    "italian",
-    "polish",
-    "portuguese",
-    "spanish",
-    "czech",
-    "slovak",
-    "norwegian"
+    "Danish",
+    "Dutch",
+    "English",
+    "French",
+    "German",
+    "Hungarian",
+    "Italian",
+    "Polish",
+    "Portuguese",
+    "Spanish",
+    "Czech",
+    "Slovak",
+    "Norwegian"
 ]
 
 domains = {

interfaces/emotion.py

@@ -12,7 +12,7 @@ from label_dicts import MANIFESTO_LABEL_NAMES
 HF_TOKEN = os.environ["hf_read"]
 
 languages = [
-    "czech", "english", "french", "german", "hungarian", "italian"
+    "Czech", "English", "French", "German", "Hungarian", "Italian"
 ]
 
 def build_huggingface_path(language: str):

interfaces/manifesto.py

@@ -12,11 +12,11 @@ from label_dicts import MANIFESTO_LABEL_NAMES
 HF_TOKEN = os.environ["hf_read"]
 
 languages = [
-    "armenian", "bulgarian", "croatian", "czech", "danish", "dutch", "english",
-    "estonian", "finnish", "french", "georgian", "german", "greek", "hebrew",
-    "hungarian", "icelandic", "italian", "japanese", "korean", "latvian",
-    "lithuanian", "norwegian", "polish", "portuguese", "romanian", "russian",
-    "serbian", "slovak", "slovenian", "spanish", "swedish", "turkish"
+    "Armenian", "Bulgarian", "Croatian", "Czech", "Danish", "Dutch", "English",
+    "Estonian", "Finnish", "French", "Georgian", "German", "Greek", "Hebrew",
+    "Hungarian", "Icelandic", "Italian", "Japanese", "Korean", "Latvian",
+    "Lithuanian", "Norwegian", "Polish", "Portuguese", "Romanian", "Russian",
+    "Serbian", "Slovak", "Slovenian", "Spanish", "Swedish", "Turkish"
 ]
 
 def build_huggingface_path(language: str):

interfaces/ner.py

@@ -0,0 +1,42 @@
+import gradio as gr
+
+import os
+import torch
+import numpy as np
+from transformers import AutoModelForSequenceClassification
+from transformers import AutoTokenizer
+from huggingface_hub import HfApi
+
+languages = [
+    "English", "Hungarian", "Multilingual"
+]
+
+def download_models(models=["en_core_web_lg", "xx_ent_wiki_sm", "hu_core_news_lg"]):
+    for model in models:
+        if model.startswith("hu"):
+            huspacy.download(model)
+        spacy.cli.download(model_name)
+
+def build_spacy_path(language: str):
+    language = language.lower()
+    if language  == "english":
+        return "en_core_web_lg"
+    if language == "hungarian":
+        return "hu_core_news_lg"
+    else:
+        return "xx_ent_wiki_sm"
+
+def named_entity_recognition(text, language):
+    model_id = build_spacy_path(language)
+    pipeline = spacy.load(model_id)
+    doc = pipeline(text)
+    entities = [(ent.text, ent.label_) for ent in doc.ents]
+    model_id_hf = f"huspacy/{model_id}" if model_id.startswith("hu") else f"spacy/{model_id}"
+    output_info = f'<p style="text-align: center; display: block">Prediction was made using the <a href="https://huggingface.co/{model_id_hf}">{model_id_hf}</a> model.</p>'
+    return entities, output_info
+
+demo = gr.Interface(
+    fn=named_entity_recognition,
+    inputs=[gr.Textbox(lines=6, label="Input"),
+            gr.Dropdown(languages, label="Language")],
+    outputs=[gr.HighlightedText(label='Output'), gr.Markdown()])

interfaces/sentiment.py

@@ -12,7 +12,7 @@ from label_dicts import MANIFESTO_LABEL_NAMES
 HF_TOKEN = os.environ["hf_read"]
 
 languages = [
-    "czech", "english", "french", "german", "hungarian", "italian"
+    "Czech", "English", "French", "German", "Hungarian", "Italian"
 ]
 
 def build_huggingface_path(language: str):

requirements.txt

@@ -1,4 +1,6 @@
 pandas
 torch==2.2.1
 transformers==4.39.1
-sentencepiece==0.2.0
+sentencepiece==0.2.0
+spacy
+huspacy