Spaces:

nishantb06
/

hindi-tokenizer-bpe-v2

Sleeping

nishantb06 commited on Jan 8

Commit

0f464dc

verified ·

1 Parent(s): 8bf0de6

updating app.py

Files changed (1) hide show

app.py CHANGED Viewed

@@ -1,9 +1,9 @@
 import gradio as gr
-from minbpe.basic import BasicTokenizer
 from typing import List, Dict
 # Initialize the tokenizer
-tokenizer = BasicTokenizer()
 tokenizer.load("models/basic.model")
 def process_text(text: str) -> str:

 import gradio as gr
+from minbpe.regex import RegexTokenizer
 from typing import List, Dict
 # Initialize the tokenizer
+tokenizer = RegexTokenizer()
 tokenizer.load("models/basic.model")
 def process_text(text: str) -> str: