creating two requirements folders

all-requirements.txt

@@ -0,0 +1,11 @@
+transformers
+gradio
+gradio-client
+httpx
+idna
+langchain_openai
+python-dotenv
+torch
+spacy
+umap-learn
+plotly

app.py

@@ -1,11 +1,12 @@
 import gradio as gr
 from transformers import pipeline
-from embedding_gen import load_skills_from_date, visualize3D
+# from embedding_gen import load_skills_from_date, visualize3D
 import numpy as np
 import pickle
 
-token_skill_classifier = pipeline(model="jjzha/jobbert_skill_extraction", aggregation_strategy="first")
-token_knowledge_classifier = pipeline(model="Robzy/jobbert_knowledge_extraction", aggregation_strategy="first")
+# token_skill_classifier = pipeline(model="jjzha/jobbert_skill_extraction", aggregation_strategy="first")
+# token_knowledge_classifier = pipeline(model="jjzha/jobbert_knowledge_extraction")
+# token_knowledge_classifier = pipeline(model="Robzy/jobbert_knowledge_extraction")
 
 
 examples = [
@@ -30,19 +31,55 @@ def aggregate_span(results):
 
     return new_results
 
-def ner(text):
+# def ner(text):
 
 
-    output_knowledge = token_knowledge_classifier(text)
-    for result in output_knowledge:
-        if result.get("entity_group"):
-            result["entity"] = "Knowledge"
-            del result["entity_group"]
+#     output_knowledge = token_knowledge_classifier(text)
+#     for result in output_knowledge:
+#         if result.get("entity_group"):
+#             result["entity"] = "Knowledge"
+#             del result["entity_group"]
 
-    if len(output_knowledge) > 0:
-        output_knowledge = aggregate_span(output_knowledge)
+#     if len(output_knowledge) > 0:
+#         output_knowledge = aggregate_span(output_knowledge)
+
+#     return {"text": text, "entities": output_knowledge}
+
+### Visualisation 3D
+
+import os
+
+def load_skills_from_date(base_folder, date):
+    date_folder = os.path.join(base_folder, date)
+    all_skills = set()  # To ensure unique skills
+    if os.path.exists(date_folder) and os.path.isdir(date_folder):
+        for file_name in os.listdir(date_folder):
+            file_path = os.path.join(date_folder, file_name)
+            if file_name.endswith(".txt"):
+                with open(file_path, 'r', encoding='utf-8') as f:
+                    all_skills.update(line.strip() for line in f if line.strip())
+    return list(all_skills)
+
+def visualize3D(reduced_embeddings, labels, skills, n_clusters, output_folder, date):
+    
+    fig = px.scatter_3d(
+        x=reduced_embeddings[:, 0],
+        y=reduced_embeddings[:, 1],
+        z=reduced_embeddings[:, 2],
+        color=labels,
+        text=skills,
+        title=f"KMeans Clustering with {n_clusters} Clusters ({date})"
+    )
+    
+    # Save the clustered plot
+    # os.makedirs(output_folder, exist_ok=True)
+    # plot_path = os.path.join(output_folder, f"{date}_3D_clustering.html")
+    # fig.write_html(plot_path)
+    # print(f"3D clustered plot saved at {plot_path}")
+    
+    # fig.show()
+    return fig
 
-    return {"text": text, "entities": output_knowledge}
 
 
 import plotly.express as px
@@ -61,12 +98,12 @@ fig.update_layout(
 
 with gr.Blocks() as demo:
     
-    gr.Interface(fn=ner,
-        inputs=gr.Textbox(placeholder="Enter sentence here..."),
-        outputs=["highlight"],
-        examples=examples,
-        title="In-demand skills in machine learning (ML) industry"
-    )
+    # gr.Interface(fn=ner,
+    #     inputs=gr.Textbox(placeholder="Enter sentence here..."),
+    #     outputs=["highlight"],
+    #     examples=examples,
+    #     title="In-demand skills in machine learning (ML) industry"
+    # )
 
     # gr.Markdown("Embedding visualisation of sought skills in ML job posting in Stockholm, Sweden on LinkedIn")
     gr.Plot(fig)

requirements.txt

@@ -7,5 +7,4 @@ langchain_openai
 python-dotenv
 torch
 spacy
-umap-learn
 plotly