ready for text endpoint

tasks/text.py

@@ -10,27 +10,27 @@ from huggingface_hub import hf_hub_download
 import joblib
 
 REPO_ID = "kantundpeterpan/frugal-ai-toy"
-FILENAME = "tfidf.skops"
+FILENAME = "tfidf_rf.skops"
 
-import nltk
-from nltk.tokenize import WordPunctTokenizer
-from nltk.stem import WordNetLemmatizer
-from nltk.corpus import stopwords
-import string
-nltk.download('stopwords')
+# import nltk
+# from nltk.tokenize import WordPunctTokenizer
+# from nltk.stem import WordNetLemmatizer
+# from nltk.corpus import stopwords
+# import string
+# nltk.download('stopwords')
 
-stop = set(stopwords.words('english') + list(string.punctuation))
+# stop = set(stopwords.words('english') + list(string.punctuation))
 
-def tokenize_quote(r):
-    tokens = nltk.word_tokenize(r.lower())
-    cleaned = [word for word in tokens if word not in stop]
-    return cleaned
+# def tokenize_quote(r):
+#     tokens = nltk.word_tokenize(r.lower())
+#     cleaned = [word for word in tokens if word not in stop]
+#     return cleaned
 
-def lemmatize_tokens(tokens: list):
-    return [lemmatizer.lemmatize(t) for t in tokens]
+# def lemmatize_tokens(tokens: list):
+#     return [lemmatizer.lemmatize(t) for t in tokens]
 
-def lemmatize_X(X):
-    return X.quote.apply(tokenize_quote).apply(lemmatize_tokens).apply(lambda x: " ".join(x))
+# def lemmatize_X(X):
+#     return X.quote.apply(tokenize_quote).apply(lemmatize_tokens).apply(lambda x: " ".join(x))
 
 import random
 
@@ -86,10 +86,13 @@ async def evaluate_text(request: TextEvaluationRequest):
     # Update the code below to replace the random baseline by your model inference within the inference pass where the energy consumption and emissions are tracked.
     #--------------------------------------------------------------------------------------------   
 
+    #get unknown types
+    unknown = skops.io.get_untrusted_types(file = hf_hub_download(repo_id=REPO_ID, filename=FILENAME))
+    
     #load model
     model = sio.load(
         hf_hub_download(repo_id=REPO_ID, filename=FILENAME),
-        trusted=['__main__.lemmatize_X', '__main__.tokenize_quote']
+        trusted=unknown
     )
     
     # Make predictions