Update app.py

app.py

@@ -8,11 +8,12 @@ from nltk.data import load
 import streamlit as st
 import sklearn
 
+trans_table = {ord(c): None for c in string.punctuation + string.digits}
+
 def custom_tokenizer_with_Turkish_stemmer(text):
     # tokenize text
     # tokens = text.split(" ")
-    tokenizer = load("turkish.pickle")
-    tokens = tokenizer.tokenize(text)
+    tokens = [word for word in nltk.word_tokenize(text.translate(trans_table))]
     print(tokens)
     stems = [stemmerTR.stem(item.lower()) for item in tokens]
     return stems