akuysal commited on
Commit
4ae52b2
·
1 Parent(s): 9ecdbb6

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +2 -1
app.py CHANGED
@@ -10,7 +10,8 @@ import sklearn
10
  def custom_tokenizer_with_Turkish_stemmer(text):
11
  # tokenize text
12
  # tokens = text.split(" ")
13
- tokens = nltk.word_tokenize(text, language="turkish")
 
14
  print(tokens)
15
  stems = [stemmerTR.stem(item.lower()) for item in tokens]
16
  return stems
 
10
  def custom_tokenizer_with_Turkish_stemmer(text):
11
  # tokenize text
12
  # tokens = text.split(" ")
13
+ tokenizer = load("turkish.pickle")
14
+ tokens = tokenizer.tokenize(text)
15
  print(tokens)
16
  stems = [stemmerTR.stem(item.lower()) for item in tokens]
17
  return stems