Spaces:
Sleeping
Sleeping
Update translation.py
Browse filesadded HF_TOKEN through secret.
- translation.py +5 -2
translation.py
CHANGED
@@ -1,3 +1,4 @@
|
|
|
|
1 |
import re
|
2 |
import sys
|
3 |
import typing as tp
|
@@ -16,6 +17,8 @@ LANGUAGES = {
|
|
16 |
"Անգլերէն | English": "eng_Latn",
|
17 |
}
|
18 |
|
|
|
|
|
19 |
def get_non_printing_char_replacer(replace_by: str = " "):
|
20 |
non_printable_map = {
|
21 |
ord(c): replace_by
|
@@ -106,12 +109,12 @@ def init_tokenizer(tokenizer, new_lang='hyw_Armn'):
|
|
106 |
|
107 |
class Translator:
|
108 |
def __init__(self) -> None:
|
109 |
-
self.model = AutoModelForSeq2SeqLM.from_pretrained(MODEL_NAME)
|
110 |
|
111 |
if torch.cuda.is_available():
|
112 |
self.model = self.model.cuda()
|
113 |
|
114 |
-
self.tokenizer = NllbTokenizer.from_pretrained(MODEL_NAME)
|
115 |
init_tokenizer(self.tokenizer)
|
116 |
|
117 |
self.hyw_splitter = pysbd.Segmenter(language="hy", clean=False)
|
|
|
1 |
+
import os
|
2 |
import re
|
3 |
import sys
|
4 |
import typing as tp
|
|
|
17 |
"Անգլերէն | English": "eng_Latn",
|
18 |
}
|
19 |
|
20 |
+
HF_TOKEN = os.environ.get("HF_TOKEN")
|
21 |
+
|
22 |
def get_non_printing_char_replacer(replace_by: str = " "):
|
23 |
non_printable_map = {
|
24 |
ord(c): replace_by
|
|
|
109 |
|
110 |
class Translator:
|
111 |
def __init__(self) -> None:
|
112 |
+
self.model = AutoModelForSeq2SeqLM.from_pretrained(MODEL_NAME, token=HF_TOKEN)
|
113 |
|
114 |
if torch.cuda.is_available():
|
115 |
self.model = self.model.cuda()
|
116 |
|
117 |
+
self.tokenizer = NllbTokenizer.from_pretrained(MODEL_NAME, token=HF_TOKEN)
|
118 |
init_tokenizer(self.tokenizer)
|
119 |
|
120 |
self.hyw_splitter = pysbd.Segmenter(language="hy", clean=False)
|