Spaces:
Runtime error
Runtime error
Update app.py
Browse files
app.py
CHANGED
@@ -4,12 +4,12 @@ import aranizer
|
|
4 |
from transformers import AutoTokenizer
|
5 |
import codecs
|
6 |
|
7 |
-
# Loading tokenizer instances from Transformers
|
8 |
gpt_13b_tokenizer = AutoTokenizer.from_pretrained("FreedomIntelligence/AceGPT-13B")
|
9 |
gpt_7b_tokenizer = AutoTokenizer.from_pretrained("FreedomIntelligence/AceGPT-7B")
|
10 |
jais_13b_tokenizer = AutoTokenizer.from_pretrained("inception-mbzuai/jais-13b")
|
11 |
|
12 |
-
# Assuming the existence of get_tokenizer() method for aranizer models in your setup
|
13 |
tokenizers = {
|
14 |
"aranizer_bpe50k": lambda: aranizer.aranizer_bpe50k.get_tokenizer(),
|
15 |
"aranizer_bpe64k": lambda: aranizer.aranizer_bpe64k.get_tokenizer(),
|
@@ -23,7 +23,7 @@ tokenizers = {
|
|
23 |
"inception-mbzuai/jais-13b": lambda: jais_13b_tokenizer,
|
24 |
}
|
25 |
|
26 |
-
# Define tokenizer options for dropdown menu
|
27 |
tokenizer_options = list(tokenizers.keys())
|
28 |
|
29 |
def compare_tokenizers(tokenizer_name, text):
|
@@ -44,7 +44,7 @@ def compare_tokenizers(tokenizer_name, text):
|
|
44 |
|
45 |
inputs_component = [
|
46 |
gr.Dropdown(choices=tokenizer_options, label="Select Tokenizer"),
|
47 |
-
gr.Textbox(lines=2, placeholder="Enter Arabic text here...", label="Input Text"
|
48 |
]
|
49 |
|
50 |
outputs_component = gr.Dataframe(
|
|
|
4 |
from transformers import AutoTokenizer
|
5 |
import codecs
|
6 |
|
7 |
+
# Loading tokenizer instances from Transformers
|
8 |
gpt_13b_tokenizer = AutoTokenizer.from_pretrained("FreedomIntelligence/AceGPT-13B")
|
9 |
gpt_7b_tokenizer = AutoTokenizer.from_pretrained("FreedomIntelligence/AceGPT-7B")
|
10 |
jais_13b_tokenizer = AutoTokenizer.from_pretrained("inception-mbzuai/jais-13b")
|
11 |
|
12 |
+
# Assuming the existence of get_tokenizer() method for aranizer models in your setup
|
13 |
tokenizers = {
|
14 |
"aranizer_bpe50k": lambda: aranizer.aranizer_bpe50k.get_tokenizer(),
|
15 |
"aranizer_bpe64k": lambda: aranizer.aranizer_bpe64k.get_tokenizer(),
|
|
|
23 |
"inception-mbzuai/jais-13b": lambda: jais_13b_tokenizer,
|
24 |
}
|
25 |
|
26 |
+
# Define tokenizer options for dropdown menu
|
27 |
tokenizer_options = list(tokenizers.keys())
|
28 |
|
29 |
def compare_tokenizers(tokenizer_name, text):
|
|
|
44 |
|
45 |
inputs_component = [
|
46 |
gr.Dropdown(choices=tokenizer_options, label="Select Tokenizer"),
|
47 |
+
gr.Textbox(lines=2, placeholder="Enter Arabic text here...", label="Input Text") # Removed `default` parameter
|
48 |
]
|
49 |
|
50 |
outputs_component = gr.Dataframe(
|