cakiki commited on
Commit
84a2246
·
1 Parent(s): 7490628

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +1 -1
app.py CHANGED
@@ -9,7 +9,7 @@ models = ["bigcode/tokenizer", "facebook/incoder-6B", "Salesforce/codegen-16B-mo
9
  @st.cache()
10
  def load_tokenizer():
11
  if selected_model == "bigcode/tokenizer":
12
- tokenizer = AutoTokenizer.from_pretrained("bigcode/tokenizer", subfolder="digit-punctuation-bytelevel-bpe-py-js-java-50k")
13
  else:
14
  tokenizer = AutoTokenizer.from_pretrained(selected_model)
15
  return tokenizer
 
9
  @st.cache()
10
  def load_tokenizer():
11
  if selected_model == "bigcode/tokenizer":
12
+ tokenizer = AutoTokenizer.from_pretrained("bigcode/tokenizer", subfolder="digit-custom_punctuation-bytelevel-bpe-py-js-java-50k")
13
  else:
14
  tokenizer = AutoTokenizer.from_pretrained(selected_model)
15
  return tokenizer