Update app.py

app.py

@@ -3,12 +3,15 @@ from TurkishStemmer import TurkishStemmer
 import string
 # import for loading python objects (scikit-learn models)
 import pickle
+import tokenize
 import streamlit as st
 import sklearn
 
 def custom_tokenizer_with_Turkish_stemmer(text):
     # tokenize text
-    tokens = text.split(" ")
+    # tokens = text.split(" ")
+    tokens = tokenize.tokenize(text)
+    print(tokens)
     stems = [stemmerTR.stem(item.lower()) for item in tokens]
     return stems