AriNubar commited on
Commit
f2eda42
·
1 Parent(s): 4a0f728

Update translation.py

Browse files

added HF_TOKEN through secret.

Files changed (1) hide show
  1. translation.py +5 -2
translation.py CHANGED
@@ -1,3 +1,4 @@
 
1
  import re
2
  import sys
3
  import typing as tp
@@ -16,6 +17,8 @@ LANGUAGES = {
16
  "Անգլերէն | English": "eng_Latn",
17
  }
18
 
 
 
19
  def get_non_printing_char_replacer(replace_by: str = " "):
20
  non_printable_map = {
21
  ord(c): replace_by
@@ -106,12 +109,12 @@ def init_tokenizer(tokenizer, new_lang='hyw_Armn'):
106
 
107
  class Translator:
108
  def __init__(self) -> None:
109
- self.model = AutoModelForSeq2SeqLM.from_pretrained(MODEL_NAME)
110
 
111
  if torch.cuda.is_available():
112
  self.model = self.model.cuda()
113
 
114
- self.tokenizer = NllbTokenizer.from_pretrained(MODEL_NAME)
115
  init_tokenizer(self.tokenizer)
116
 
117
  self.hyw_splitter = pysbd.Segmenter(language="hy", clean=False)
 
1
+ import os
2
  import re
3
  import sys
4
  import typing as tp
 
17
  "Անգլերէն | English": "eng_Latn",
18
  }
19
 
20
+ HF_TOKEN = os.environ.get("HF_TOKEN")
21
+
22
  def get_non_printing_char_replacer(replace_by: str = " "):
23
  non_printable_map = {
24
  ord(c): replace_by
 
109
 
110
  class Translator:
111
  def __init__(self) -> None:
112
+ self.model = AutoModelForSeq2SeqLM.from_pretrained(MODEL_NAME, token=HF_TOKEN)
113
 
114
  if torch.cuda.is_available():
115
  self.model = self.model.cuda()
116
 
117
+ self.tokenizer = NllbTokenizer.from_pretrained(MODEL_NAME, token=HF_TOKEN)
118
  init_tokenizer(self.tokenizer)
119
 
120
  self.hyw_splitter = pysbd.Segmenter(language="hy", clean=False)