Update app.py
Browse files
app.py
CHANGED
@@ -24,7 +24,7 @@ model = AutoModelForCausalLM.from_pretrained(BASE_MODEL)
|
|
24 |
|
25 |
# === トークナイズ関数修正版 ===
|
26 |
def preprocess(examples):
|
27 |
-
texts = [
|
28 |
tokenized = tokenizer(texts, max_length=256, truncation=True)
|
29 |
tokenized["labels"] = tokenized["input_ids"].copy()
|
30 |
return tokenized
|
|
|
24 |
|
25 |
# === トークナイズ関数修正版 ===
|
26 |
def preprocess(examples):
|
27 |
+
texts = [english + " " + japanese for english, japanese in zip(examples["english"], examples["japanese"])]
|
28 |
tokenized = tokenizer(texts, max_length=256, truncation=True)
|
29 |
tokenized["labels"] = tokenized["input_ids"].copy()
|
30 |
return tokenized
|