Spaces:
Build error
Build error
Update app.py
Browse files
app.py
CHANGED
@@ -9,7 +9,7 @@ models = ["bigcode/tokenizer", "facebook/incoder-6B", "Salesforce/codegen-16B-mo
|
|
9 |
@st.cache()
|
10 |
def load_tokenizer():
|
11 |
if selected_model == "bigcode/tokenizer":
|
12 |
-
tokenizer = AutoTokenizer.from_pretrained("bigcode/tokenizer", subfolder="digit-
|
13 |
else:
|
14 |
tokenizer = AutoTokenizer.from_pretrained(selected_model)
|
15 |
return tokenizer
|
|
|
9 |
@st.cache()
|
10 |
def load_tokenizer():
|
11 |
if selected_model == "bigcode/tokenizer":
|
12 |
+
tokenizer = AutoTokenizer.from_pretrained("bigcode/tokenizer", subfolder="digit-custom_punctuation-bytelevel-bpe-py-js-java-50k")
|
13 |
else:
|
14 |
tokenizer = AutoTokenizer.from_pretrained(selected_model)
|
15 |
return tokenizer
|