add HF_TOKEN

app.py

@@ -1,9 +1,12 @@
+import os
 from typing import List
 import gradio as gr
 from transformers import AutoTokenizer
 import tiktoken
 
 
+HF_TOKEN = os.getenv("HF_TOKEN")
+
 hf_tokenizer_list = [
     ("tugstugi/bert-large-mongolian-cased", False),
     ("tugstugi/bert-large-mongolian-uncased", False),
@@ -19,7 +22,7 @@ openai_tokenizer_list = [
 
 # load tokenizers
 hf_tokenizers = [
-    AutoTokenizer.from_pretrained(model_name_or_id, use_fast=use_fast, trust_remote_code=True)
+    AutoTokenizer.from_pretrained(model_name_or_id, use_fast=use_fast, trust_remote_code=True, token=HF_TOKEN)
     for model_name_or_id, use_fast in hf_tokenizer_list
 ]