nroggendorff commited on
Commit
53e45ad
·
verified ·
1 Parent(s): 44388ec

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +2 -2
app.py CHANGED
@@ -30,7 +30,7 @@ def create_tokenizer(training_corpus):
30
  )
31
  return tokenizer
32
 
33
- def get_training_corpus():
34
  for i in range(0, len(dataset), 1000):
35
  yield dataset[i : i + 1000]["text"]
36
 
@@ -118,7 +118,7 @@ def train_model(model, tokenizer, dataset):
118
 
119
  def main():
120
  dataset = load_data()
121
- training_corpus = get_training_corpus()
122
  tokenizer = create_tokenizer(training_corpus)
123
  configure_tokenizer(tokenizer)
124
  model = create_model(tokenizer, FACTOR)
 
30
  )
31
  return tokenizer
32
 
33
+ def get_training_corpus(dataset):
34
  for i in range(0, len(dataset), 1000):
35
  yield dataset[i : i + 1000]["text"]
36
 
 
118
 
119
  def main():
120
  dataset = load_data()
121
+ training_corpus = get_training_corpus(dataset)
122
  tokenizer = create_tokenizer(training_corpus)
123
  configure_tokenizer(tokenizer)
124
  model = create_model(tokenizer, FACTOR)