Sakalti commited on
Commit
1090fe1
·
verified ·
1 Parent(s): dae2dae

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +1 -1
app.py CHANGED
@@ -24,7 +24,7 @@ model = AutoModelForCausalLM.from_pretrained(BASE_MODEL)
24
 
25
  # === トークナイズ関数修正版 ===
26
  def preprocess(examples):
27
- texts = [en + " " + ja for en, ja in zip(examples["en"], examples["ja"])]
28
  tokenized = tokenizer(texts, max_length=256, truncation=True)
29
  tokenized["labels"] = tokenized["input_ids"].copy()
30
  return tokenized
 
24
 
25
  # === トークナイズ関数修正版 ===
26
  def preprocess(examples):
27
+ texts = [english + " " + japanese for english, japanese in zip(examples["english"], examples["japanese"])]
28
  tokenized = tokenizer(texts, max_length=256, truncation=True)
29
  tokenized["labels"] = tokenized["input_ids"].copy()
30
  return tokenized