Update app.py
Browse files
app.py
CHANGED
@@ -62,7 +62,7 @@ llm_model = None
|
|
62 |
|
63 |
def trans(text):
|
64 |
text = "こんにちは"
|
65 |
-
input_text = f"<2ja>{text}"
|
66 |
tokens = llm.tokenize(input_text)
|
67 |
print("Tokens:", tokens)
|
68 |
initial_tokens = [llm.decoder_start_token()]
|
|
|
62 |
|
63 |
def trans(text):
|
64 |
text = "こんにちは"
|
65 |
+
input_text = f"<2ja>{text}".encode('utf-8')
|
66 |
tokens = llm.tokenize(input_text)
|
67 |
print("Tokens:", tokens)
|
68 |
initial_tokens = [llm.decoder_start_token()]
|