We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Test data in language model
Is somewhere used test data?
https://github.com/zalandoresearch/flair/blob/d7c0f1733b398d59b3bffb62f5a422343c7e420e/flair/trainers/language_model_trainer.py#L35

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `flair/trainers/language_model_trainer.py`
Content:
```
1 import time, datetime
2 import os
3 import random
4 import logging
5 import math
6 import torch
7 from torch.autograd import Variable
8 from torch.optim.lr_scheduler import ReduceLROnPlateau
9 
10 from flair.data import Dictionary
11 from flair.models import LanguageModel
12 
13 
14 log = logging.getLogger(__name__)
15 
16 
17 class TextCorpus(object):
18     def __init__(self, path, dictionary: Dictionary, forward: bool = True, character_level: bool = True):
19 
20         self.forward = forward
21         self.split_on_char = character_level
22         self.train_path = os.path.join(path, 'train')
23 
24         self.train_files = sorted(
25             [f for f in os.listdir(self.train_path) if os.path.isfile(os.path.join(self.train_path, f))])
26 
27         self.dictionary: Dictionary = dictionary
28 
29         self.current_train_file_index = len(self.train_files)
30 
31         self.valid = self.charsplit(os.path.join(path, 'valid.txt'),
32                                     forward=forward,
33                                     split_on_char=self.split_on_char)
34 
35         self.test = self.charsplit(os.path.join(path, 'test.txt'),
36                                    forward=forward,
37                                    split_on_char=self.split_on_char)
38 
39     @property
40     def is_last_slice(self) -> bool:
41         if self.current_train_file_index >= len(self.train_files) - 1:
42             return True
43         return False
44 
45     def get_next_train_slice(self):
46 
47         self.current_train_file_index += 1
48 
49         if self.current_train_file_index >= len(self.train_files):
50             self.current_train_file_index = 0
51             random.shuffle(self.train_files)
52 
53         current_train_file = self.train_files[self.current_train_file_index]
54 
55         train_slice = self.charsplit(os.path.join(self.train_path, current_train_file),
56                                     expand_vocab=False,
57                                     forward=self.forward,
58                                     split_on_char=self.split_on_char)
59 
60         return train_slice
61 
62     def charsplit(self, path: str, expand_vocab=False, forward=True, split_on_char=True) -> torch.LongTensor:
63 
64         """Tokenizes a text file on character basis."""
65         assert os.path.exists(path)
66 
67         #
68         with open(path, 'r', encoding="utf-8") as f:
69             tokens = 0
70             for line in f:
71 
72                 if split_on_char:
73                     chars = list(line)
74                 else:
75                     chars = line.split()
76 
77                 tokens += len(chars)
78 
79                 # Add chars to the dictionary
80                 if expand_vocab:
81                     for char in chars:
82                         self.dictionary.add_item(char)
83 
84         if forward:
85             # charsplit file content
86             with open(path, 'r', encoding="utf-8") as f:
87                 ids = torch.LongTensor(tokens)
88                 token = 0
89                 for line in f:
90                     line = self.random_casechange(line)
91 
92                     if split_on_char:
93                         chars = list(line)
94                     else:
95                         chars = line.split()
96 
97                     for char in chars:
98                         if token >= tokens: break
99                         ids[token] = self.dictionary.get_idx_for_item(char)
100                         token += 1
101         else:
102             # charsplit file content
103             with open(path, 'r', encoding="utf-8") as f:
104                 ids = torch.LongTensor(tokens)
105                 token = tokens - 1
106                 for line in f:
107                     line = self.random_casechange(line)
108 
109                     if split_on_char:
110                         chars = list(line)
111                     else:
112                         chars = line.split()
113 
114                     for char in chars:
115                         if token >= tokens: break
116                         ids[token] = self.dictionary.get_idx_for_item(char)
117                         token -= 1
118 
119         return ids
120 
121     @staticmethod
122     def random_casechange(line: str) -> str:
123         no = random.randint(0, 99)
124         if no is 0:
125             line = line.lower()
126         if no is 1:
127             line = line.upper()
128         return line
129 
130     def tokenize(self, path):
131         """Tokenizes a text file."""
132         assert os.path.exists(path)
133         # Add words to the dictionary
134         with open(path, 'r') as f:
135             tokens = 0
136             for line in f:
137                 words = line.split() + ['<eos>']
138                 tokens += len(words)
139                 for word in words:
140                     self.dictionary.add_word(word)
141 
142         # Tokenize file content
143         with open(path, 'r') as f:
144             ids = torch.LongTensor(tokens)
145             token = 0
146             for line in f:
147                 words = line.split() + ['<eos>']
148                 for word in words:
149                     ids[token] = self.dictionary.word2idx[word]
150                     token += 1
151 
152         return ids
153 
154 
155 class LanguageModelTrainer:
156     def __init__(self, model: LanguageModel, corpus: TextCorpus, test_mode: bool = False):
157         self.model: LanguageModel = model
158         self.corpus: TextCorpus = corpus
159         self.test_mode: bool = test_mode
160 
161         self.loss_function = torch.nn.CrossEntropyLoss()
162         self.log_interval = 100
163 
164     def train(self,
165               base_path: str,
166               sequence_length: int,
167               learning_rate: float = 20,
168               mini_batch_size: int = 100,
169               anneal_factor: float = 0.25,
170               patience: int = 10,
171               clip=0.25,
172               max_epochs: int = 1000):
173 
174         number_of_splits: int = len(self.corpus.train_files)
175 
176         # an epoch has a number, so calculate total max splits bby multiplying max_epochs with number_of_splits
177         max_splits: int = number_of_splits * max_epochs
178 
179         val_data = self._batchify(self.corpus.valid, mini_batch_size)
180 
181         os.makedirs(base_path, exist_ok=True)
182         loss_txt = os.path.join(base_path, 'loss.txt')
183         savefile = os.path.join(base_path, 'best-lm.pt')
184 
185         try:
186 
187             epoch = 0
188             best_val_loss = 100000000
189             optimizer = torch.optim.SGD(self.model.parameters(), lr=learning_rate)
190             scheduler: ReduceLROnPlateau = ReduceLROnPlateau(optimizer, verbose=True, factor=anneal_factor,
191                                                              patience=patience)
192 
193             for split in range(1, max_splits + 1):
194 
195                 # after pass over all splits, increment epoch count
196                 if (split - 1) % number_of_splits == 0:
197                     epoch += 1
198 
199                 log.info('Split %d' % split + '\t - ({:%H:%M:%S})'.format(datetime.datetime.now()))
200 
201                 for group in optimizer.param_groups:
202                     learning_rate = group['lr']
203 
204                 train_slice = self.corpus.get_next_train_slice()
205 
206                 train_data = self._batchify(train_slice, mini_batch_size)
207                 log.info('\t({:%H:%M:%S})'.format(datetime.datetime.now()))
208 
209                 # go into train mode
210                 self.model.train()
211 
212                 # reset variables
213                 epoch_start_time = time.time()
214                 total_loss = 0
215                 start_time = time.time()
216 
217                 hidden = self.model.init_hidden(mini_batch_size)
218 
219                 # not really sure what this does
220                 ntokens = len(self.corpus.dictionary)
221 
222                 # do batches
223                 for batch, i in enumerate(range(0, train_data.size(0) - 1, sequence_length)):
224 
225                     data, targets = self._get_batch(train_data, i, sequence_length)
226 
227                     # Starting each batch, we detach the hidden state from how it was previously produced.
228                     # If we didn't, the model would try backpropagating all the way to start of the dataset.
229                     hidden = self._repackage_hidden(hidden)
230 
231                     self.model.zero_grad()
232                     optimizer.zero_grad()
233 
234                     # do the forward pass in the model
235                     output, rnn_output, hidden = self.model.forward(data, hidden)
236 
237                     # try to predict the targets
238                     loss = self.loss_function(output.view(-1, ntokens), targets)
239                     loss.backward()
240 
241                     # `clip_grad_norm` helps prevent the exploding gradient problem in RNNs / LSTMs.
242                     torch.nn.utils.clip_grad_norm_(self.model.parameters(), clip)
243 
244                     optimizer.step()
245 
246                     total_loss += loss.data
247 
248                     if batch % self.log_interval == 0 and batch > 0:
249                         cur_loss = total_loss.item() / self.log_interval
250                         elapsed = time.time() - start_time
251                         log.info('| split {:3d} /{:3d} | {:5d}/{:5d} batches | ms/batch {:5.2f} | '
252                               'loss {:5.2f} | ppl {:8.2f}'.format(
253                             split, number_of_splits, batch, len(train_data) // sequence_length,
254                                                             elapsed * 1000 / self.log_interval, cur_loss,
255                             math.exp(cur_loss)))
256                         total_loss = 0
257                         start_time = time.time()
258 
259                 log.info('training done! \t({:%H:%M:%S})'.format(datetime.datetime.now()))
260 
261                 ###############################################################################
262                 # TEST
263                 ###############################################################################
264                 self.model.eval()
265                 val_loss = self.evaluate(val_data, mini_batch_size, sequence_length)
266                 scheduler.step(val_loss)
267 
268                 log.info('best loss so far {:5.2f}'.format(best_val_loss))
269 
270                 # Save the model if the validation loss is the best we've seen so far.
271                 if val_loss < best_val_loss:
272                     self.model.save(savefile)
273                     best_val_loss = val_loss
274 
275                 ###############################################################################
276                 # print info
277                 ###############################################################################
278                 log.info('-' * 89)
279 
280                 local_split_number = split % number_of_splits
281                 if local_split_number == 0: local_split_number = number_of_splits
282 
283                 summary = '| end of split {:3d} /{:3d} | epoch {:3d} | time: {:5.2f}s | valid loss {:5.2f} | ' \
284                           'valid ppl {:8.2f} | learning rate {:3.2f}'.format(local_split_number,
285                                                                              number_of_splits,
286                                                                              epoch,
287                                                                              (time.time() - epoch_start_time),
288                                                                              val_loss,
289                                                                              math.exp(val_loss),
290                                                                              learning_rate)
291 
292                 with open(loss_txt, "a") as myfile:
293                     myfile.write('%s\n' % summary)
294 
295                 log.info(summary)
296                 log.info('-' * 89)
297 
298         except KeyboardInterrupt:
299             log.info('-' * 89)
300             log.info('Exiting from training early')
301 
302     def evaluate(self, data_source, eval_batch_size, sequence_length):
303         # Turn on evaluation mode which disables dropout.
304         self.model.eval()
305 
306         with torch.no_grad():
307             total_loss = 0
308             ntokens = len(self.corpus.dictionary)
309 
310             hidden = self.model.init_hidden(eval_batch_size)
311 
312             for i in range(0, data_source.size(0) - 1, sequence_length):
313                 data, targets = self._get_batch(data_source, i, sequence_length)
314                 prediction, rnn_output, hidden = self.model.forward(data, hidden)
315                 output_flat = prediction.view(-1, ntokens)
316                 total_loss += len(data) * self.loss_function(output_flat, targets).data
317                 hidden = self._repackage_hidden(hidden)
318             return total_loss.item() / len(data_source)
319 
320     @staticmethod
321     def _batchify(data, batch_size):
322         # Work out how cleanly we can divide the dataset into bsz parts.
323         nbatch = data.size(0) // batch_size
324         # Trim off any extra elements that wouldn't cleanly fit (remainders).
325         data = data.narrow(0, 0, nbatch * batch_size)
326         # Evenly divide the data across the bsz batches.
327         data = data.view(batch_size, -1).t().contiguous()
328         if torch.cuda.is_available():
329             data = data.cuda()
330         return data
331 
332     @staticmethod
333     def _get_batch(source, i, sequence_length):
334         seq_len = min(sequence_length, len(source) - 1 - i)
335         data = Variable(source[i:i + seq_len])
336         target = Variable(source[i + 1:i + 1 + seq_len].view(-1))
337         return data, target
338 
339     @staticmethod
340     def _repackage_hidden(h):
341         """Wraps hidden states in new Variables, to detach them from their history."""
342         return tuple(Variable(v) for v in h)
343 
344 
345
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/flair/trainers/language_model_trainer.py b/flair/trainers/language_model_trainer.py
--- a/flair/trainers/language_model_trainer.py
+++ b/flair/trainers/language_model_trainer.py
@@ -299,6 +299,19 @@
             log.info('-' * 89)
             log.info('Exiting from training early')
 
+        ###############################################################################
+        # final testing
+        ###############################################################################
+        test_data = self._batchify(self.corpus.test, mini_batch_size)
+        test_loss = self.evaluate(test_data, mini_batch_size, sequence_length)
+
+        summary = 'TEST: valid loss {:5.2f} | valid ppl {:8.2f}'.format(test_loss, math.exp(test_loss))
+        with open(loss_txt, "a") as myfile:
+            myfile.write('%s\n' % summary)
+
+        log.info(summary)
+        log.info('-' * 89)
+
     def evaluate(self, data_source, eval_batch_size, sequence_length):
         # Turn on evaluation mode which disables dropout.
         self.model.eval()

{"golden_diff": "diff --git a/flair/trainers/language_model_trainer.py b/flair/trainers/language_model_trainer.py\n--- a/flair/trainers/language_model_trainer.py\n+++ b/flair/trainers/language_model_trainer.py\n@@ -299,6 +299,19 @@\n             log.info('-' * 89)\n             log.info('Exiting from training early')\n \n+        ###############################################################################\n+        # final testing\n+        ###############################################################################\n+        test_data = self._batchify(self.corpus.test, mini_batch_size)\n+        test_loss = self.evaluate(test_data, mini_batch_size, sequence_length)\n+\n+        summary = 'TEST: valid loss {:5.2f} | valid ppl {:8.2f}'.format(test_loss, math.exp(test_loss))\n+        with open(loss_txt, \"a\") as myfile:\n+            myfile.write('%s\\n' % summary)\n+\n+        log.info(summary)\n+        log.info('-' * 89)\n+\n     def evaluate(self, data_source, eval_batch_size, sequence_length):\n         # Turn on evaluation mode which disables dropout.\n         self.model.eval()\n", "issue": "Test data in language model\nIs somewhere used test data?\r\nhttps://github.com/zalandoresearch/flair/blob/d7c0f1733b398d59b3bffb62f5a422343c7e420e/flair/trainers/language_model_trainer.py#L35\n", "before_files": [{"content": "import time, datetime\nimport os\nimport random\nimport logging\nimport math\nimport torch\nfrom torch.autograd import Variable\nfrom torch.optim.lr_scheduler import ReduceLROnPlateau\n\nfrom flair.data import Dictionary\nfrom flair.models import LanguageModel\n\n\nlog = logging.getLogger(__name__)\n\n\nclass TextCorpus(object):\n    def __init__(self, path, dictionary: Dictionary, forward: bool = True, character_level: bool = True):\n\n        self.forward = forward\n        self.split_on_char = character_level\n        self.train_path = os.path.join(path, 'train')\n\n        self.train_files = sorted(\n            [f for f in os.listdir(self.train_path) if os.path.isfile(os.path.join(self.train_path, f))])\n\n        self.dictionary: Dictionary = dictionary\n\n        self.current_train_file_index = len(self.train_files)\n\n        self.valid = self.charsplit(os.path.join(path, 'valid.txt'),\n                                    forward=forward,\n                                    split_on_char=self.split_on_char)\n\n        self.test = self.charsplit(os.path.join(path, 'test.txt'),\n                                   forward=forward,\n                                   split_on_char=self.split_on_char)\n\n    @property\n    def is_last_slice(self) -> bool:\n        if self.current_train_file_index >= len(self.train_files) - 1:\n            return True\n        return False\n\n    def get_next_train_slice(self):\n\n        self.current_train_file_index += 1\n\n        if self.current_train_file_index >= len(self.train_files):\n            self.current_train_file_index = 0\n            random.shuffle(self.train_files)\n\n        current_train_file = self.train_files[self.current_train_file_index]\n\n        train_slice = self.charsplit(os.path.join(self.train_path, current_train_file),\n                                    expand_vocab=False,\n                                    forward=self.forward,\n                                    split_on_char=self.split_on_char)\n\n        return train_slice\n\n    def charsplit(self, path: str, expand_vocab=False, forward=True, split_on_char=True) -> torch.LongTensor:\n\n        \"\"\"Tokenizes a text file on character basis.\"\"\"\n        assert os.path.exists(path)\n\n        #\n        with open(path, 'r', encoding=\"utf-8\") as f:\n            tokens = 0\n            for line in f:\n\n                if split_on_char:\n                    chars = list(line)\n                else:\n                    chars = line.split()\n\n                tokens += len(chars)\n\n                # Add chars to the dictionary\n                if expand_vocab:\n                    for char in chars:\n                        self.dictionary.add_item(char)\n\n        if forward:\n            # charsplit file content\n            with open(path, 'r', encoding=\"utf-8\") as f:\n                ids = torch.LongTensor(tokens)\n                token = 0\n                for line in f:\n                    line = self.random_casechange(line)\n\n                    if split_on_char:\n                        chars = list(line)\n                    else:\n                        chars = line.split()\n\n                    for char in chars:\n                        if token >= tokens: break\n                        ids[token] = self.dictionary.get_idx_for_item(char)\n                        token += 1\n        else:\n            # charsplit file content\n            with open(path, 'r', encoding=\"utf-8\") as f:\n                ids = torch.LongTensor(tokens)\n                token = tokens - 1\n                for line in f:\n                    line = self.random_casechange(line)\n\n                    if split_on_char:\n                        chars = list(line)\n                    else:\n                        chars = line.split()\n\n                    for char in chars:\n                        if token >= tokens: break\n                        ids[token] = self.dictionary.get_idx_for_item(char)\n                        token -= 1\n\n        return ids\n\n    @staticmethod\n    def random_casechange(line: str) -> str:\n        no = random.randint(0, 99)\n        if no is 0:\n            line = line.lower()\n        if no is 1:\n            line = line.upper()\n        return line\n\n    def tokenize(self, path):\n        \"\"\"Tokenizes a text file.\"\"\"\n        assert os.path.exists(path)\n        # Add words to the dictionary\n        with open(path, 'r') as f:\n            tokens = 0\n            for line in f:\n                words = line.split() + ['<eos>']\n                tokens += len(words)\n                for word in words:\n                    self.dictionary.add_word(word)\n\n        # Tokenize file content\n        with open(path, 'r') as f:\n            ids = torch.LongTensor(tokens)\n            token = 0\n            for line in f:\n                words = line.split() + ['<eos>']\n                for word in words:\n                    ids[token] = self.dictionary.word2idx[word]\n                    token += 1\n\n        return ids\n\n\nclass LanguageModelTrainer:\n    def __init__(self, model: LanguageModel, corpus: TextCorpus, test_mode: bool = False):\n        self.model: LanguageModel = model\n        self.corpus: TextCorpus = corpus\n        self.test_mode: bool = test_mode\n\n        self.loss_function = torch.nn.CrossEntropyLoss()\n        self.log_interval = 100\n\n    def train(self,\n              base_path: str,\n              sequence_length: int,\n              learning_rate: float = 20,\n              mini_batch_size: int = 100,\n              anneal_factor: float = 0.25,\n              patience: int = 10,\n              clip=0.25,\n              max_epochs: int = 1000):\n\n        number_of_splits: int = len(self.corpus.train_files)\n\n        # an epoch has a number, so calculate total max splits bby multiplying max_epochs with number_of_splits\n        max_splits: int = number_of_splits * max_epochs\n\n        val_data = self._batchify(self.corpus.valid, mini_batch_size)\n\n        os.makedirs(base_path, exist_ok=True)\n        loss_txt = os.path.join(base_path, 'loss.txt')\n        savefile = os.path.join(base_path, 'best-lm.pt')\n\n        try:\n\n            epoch = 0\n            best_val_loss = 100000000\n            optimizer = torch.optim.SGD(self.model.parameters(), lr=learning_rate)\n            scheduler: ReduceLROnPlateau = ReduceLROnPlateau(optimizer, verbose=True, factor=anneal_factor,\n                                                             patience=patience)\n\n            for split in range(1, max_splits + 1):\n\n                # after pass over all splits, increment epoch count\n                if (split - 1) % number_of_splits == 0:\n                    epoch += 1\n\n                log.info('Split %d' % split + '\\t - ({:%H:%M:%S})'.format(datetime.datetime.now()))\n\n                for group in optimizer.param_groups:\n                    learning_rate = group['lr']\n\n                train_slice = self.corpus.get_next_train_slice()\n\n                train_data = self._batchify(train_slice, mini_batch_size)\n                log.info('\\t({:%H:%M:%S})'.format(datetime.datetime.now()))\n\n                # go into train mode\n                self.model.train()\n\n                # reset variables\n                epoch_start_time = time.time()\n                total_loss = 0\n                start_time = time.time()\n\n                hidden = self.model.init_hidden(mini_batch_size)\n\n                # not really sure what this does\n                ntokens = len(self.corpus.dictionary)\n\n                # do batches\n                for batch, i in enumerate(range(0, train_data.size(0) - 1, sequence_length)):\n\n                    data, targets = self._get_batch(train_data, i, sequence_length)\n\n                    # Starting each batch, we detach the hidden state from how it was previously produced.\n                    # If we didn't, the model would try backpropagating all the way to start of the dataset.\n                    hidden = self._repackage_hidden(hidden)\n\n                    self.model.zero_grad()\n                    optimizer.zero_grad()\n\n                    # do the forward pass in the model\n                    output, rnn_output, hidden = self.model.forward(data, hidden)\n\n                    # try to predict the targets\n                    loss = self.loss_function(output.view(-1, ntokens), targets)\n                    loss.backward()\n\n                    # `clip_grad_norm` helps prevent the exploding gradient problem in RNNs / LSTMs.\n                    torch.nn.utils.clip_grad_norm_(self.model.parameters(), clip)\n\n                    optimizer.step()\n\n                    total_loss += loss.data\n\n                    if batch % self.log_interval == 0 and batch > 0:\n                        cur_loss = total_loss.item() / self.log_interval\n                        elapsed = time.time() - start_time\n                        log.info('| split {:3d} /{:3d} | {:5d}/{:5d} batches | ms/batch {:5.2f} | '\n                              'loss {:5.2f} | ppl {:8.2f}'.format(\n                            split, number_of_splits, batch, len(train_data) // sequence_length,\n                                                            elapsed * 1000 / self.log_interval, cur_loss,\n                            math.exp(cur_loss)))\n                        total_loss = 0\n                        start_time = time.time()\n\n                log.info('training done! \\t({:%H:%M:%S})'.format(datetime.datetime.now()))\n\n                ###############################################################################\n                # TEST\n                ###############################################################################\n                self.model.eval()\n                val_loss = self.evaluate(val_data, mini_batch_size, sequence_length)\n                scheduler.step(val_loss)\n\n                log.info('best loss so far {:5.2f}'.format(best_val_loss))\n\n                # Save the model if the validation loss is the best we've seen so far.\n                if val_loss < best_val_loss:\n                    self.model.save(savefile)\n                    best_val_loss = val_loss\n\n                ###############################################################################\n                # print info\n                ###############################################################################\n                log.info('-' * 89)\n\n                local_split_number = split % number_of_splits\n                if local_split_number == 0: local_split_number = number_of_splits\n\n                summary = '| end of split {:3d} /{:3d} | epoch {:3d} | time: {:5.2f}s | valid loss {:5.2f} | ' \\\n                          'valid ppl {:8.2f} | learning rate {:3.2f}'.format(local_split_number,\n                                                                             number_of_splits,\n                                                                             epoch,\n                                                                             (time.time() - epoch_start_time),\n                                                                             val_loss,\n                                                                             math.exp(val_loss),\n                                                                             learning_rate)\n\n                with open(loss_txt, \"a\") as myfile:\n                    myfile.write('%s\\n' % summary)\n\n                log.info(summary)\n                log.info('-' * 89)\n\n        except KeyboardInterrupt:\n            log.info('-' * 89)\n            log.info('Exiting from training early')\n\n    def evaluate(self, data_source, eval_batch_size, sequence_length):\n        # Turn on evaluation mode which disables dropout.\n        self.model.eval()\n\n        with torch.no_grad():\n            total_loss = 0\n            ntokens = len(self.corpus.dictionary)\n\n            hidden = self.model.init_hidden(eval_batch_size)\n\n            for i in range(0, data_source.size(0) - 1, sequence_length):\n                data, targets = self._get_batch(data_source, i, sequence_length)\n                prediction, rnn_output, hidden = self.model.forward(data, hidden)\n                output_flat = prediction.view(-1, ntokens)\n                total_loss += len(data) * self.loss_function(output_flat, targets).data\n                hidden = self._repackage_hidden(hidden)\n            return total_loss.item() / len(data_source)\n\n    @staticmethod\n    def _batchify(data, batch_size):\n        # Work out how cleanly we can divide the dataset into bsz parts.\n        nbatch = data.size(0) // batch_size\n        # Trim off any extra elements that wouldn't cleanly fit (remainders).\n        data = data.narrow(0, 0, nbatch * batch_size)\n        # Evenly divide the data across the bsz batches.\n        data = data.view(batch_size, -1).t().contiguous()\n        if torch.cuda.is_available():\n            data = data.cuda()\n        return data\n\n    @staticmethod\n    def _get_batch(source, i, sequence_length):\n        seq_len = min(sequence_length, len(source) - 1 - i)\n        data = Variable(source[i:i + seq_len])\n        target = Variable(source[i + 1:i + 1 + seq_len].view(-1))\n        return data, target\n\n    @staticmethod\n    def _repackage_hidden(h):\n        \"\"\"Wraps hidden states in new Variables, to detach them from their history.\"\"\"\n        return tuple(Variable(v) for v in h)\n\n\n", "path": "flair/trainers/language_model_trainer.py"}], "after_files": [{"content": "import time, datetime\nimport os\nimport random\nimport logging\nimport math\nimport torch\nfrom torch.autograd import Variable\nfrom torch.optim.lr_scheduler import ReduceLROnPlateau\n\nfrom flair.data import Dictionary\nfrom flair.models import LanguageModel\n\n\nlog = logging.getLogger(__name__)\n\n\nclass TextCorpus(object):\n    def __init__(self, path, dictionary: Dictionary, forward: bool = True, character_level: bool = True):\n\n        self.forward = forward\n        self.split_on_char = character_level\n        self.train_path = os.path.join(path, 'train')\n\n        self.train_files = sorted(\n            [f for f in os.listdir(self.train_path) if os.path.isfile(os.path.join(self.train_path, f))])\n\n        self.dictionary: Dictionary = dictionary\n\n        self.current_train_file_index = len(self.train_files)\n\n        self.valid = self.charsplit(os.path.join(path, 'valid.txt'),\n                                    forward=forward,\n                                    split_on_char=self.split_on_char)\n\n        self.test = self.charsplit(os.path.join(path, 'test.txt'),\n                                   forward=forward,\n                                   split_on_char=self.split_on_char)\n\n    @property\n    def is_last_slice(self) -> bool:\n        if self.current_train_file_index >= len(self.train_files) - 1:\n            return True\n        return False\n\n    def get_next_train_slice(self):\n\n        self.current_train_file_index += 1\n\n        if self.current_train_file_index >= len(self.train_files):\n            self.current_train_file_index = 0\n            random.shuffle(self.train_files)\n\n        current_train_file = self.train_files[self.current_train_file_index]\n\n        train_slice = self.charsplit(os.path.join(self.train_path, current_train_file),\n                                    expand_vocab=False,\n                                    forward=self.forward,\n                                    split_on_char=self.split_on_char)\n\n        return train_slice\n\n    def charsplit(self, path: str, expand_vocab=False, forward=True, split_on_char=True) -> torch.LongTensor:\n\n        \"\"\"Tokenizes a text file on character basis.\"\"\"\n        assert os.path.exists(path)\n\n        #\n        with open(path, 'r', encoding=\"utf-8\") as f:\n            tokens = 0\n            for line in f:\n\n                if split_on_char:\n                    chars = list(line)\n                else:\n                    chars = line.split()\n\n                tokens += len(chars)\n\n                # Add chars to the dictionary\n                if expand_vocab:\n                    for char in chars:\n                        self.dictionary.add_item(char)\n\n        if forward:\n            # charsplit file content\n            with open(path, 'r', encoding=\"utf-8\") as f:\n                ids = torch.LongTensor(tokens)\n                token = 0\n                for line in f:\n                    line = self.random_casechange(line)\n\n                    if split_on_char:\n                        chars = list(line)\n                    else:\n                        chars = line.split()\n\n                    for char in chars:\n                        if token >= tokens: break\n                        ids[token] = self.dictionary.get_idx_for_item(char)\n                        token += 1\n        else:\n            # charsplit file content\n            with open(path, 'r', encoding=\"utf-8\") as f:\n                ids = torch.LongTensor(tokens)\n                token = tokens - 1\n                for line in f:\n                    line = self.random_casechange(line)\n\n                    if split_on_char:\n                        chars = list(line)\n                    else:\n                        chars = line.split()\n\n                    for char in chars:\n                        if token >= tokens: break\n                        ids[token] = self.dictionary.get_idx_for_item(char)\n                        token -= 1\n\n        return ids\n\n    @staticmethod\n    def random_casechange(line: str) -> str:\n        no = random.randint(0, 99)\n        if no is 0:\n            line = line.lower()\n        if no is 1:\n            line = line.upper()\n        return line\n\n    def tokenize(self, path):\n        \"\"\"Tokenizes a text file.\"\"\"\n        assert os.path.exists(path)\n        # Add words to the dictionary\n        with open(path, 'r') as f:\n            tokens = 0\n            for line in f:\n                words = line.split() + ['<eos>']\n                tokens += len(words)\n                for word in words:\n                    self.dictionary.add_word(word)\n\n        # Tokenize file content\n        with open(path, 'r') as f:\n            ids = torch.LongTensor(tokens)\n            token = 0\n            for line in f:\n                words = line.split() + ['<eos>']\n                for word in words:\n                    ids[token] = self.dictionary.word2idx[word]\n                    token += 1\n\n        return ids\n\n\nclass LanguageModelTrainer:\n    def __init__(self, model: LanguageModel, corpus: TextCorpus, test_mode: bool = False):\n        self.model: LanguageModel = model\n        self.corpus: TextCorpus = corpus\n        self.test_mode: bool = test_mode\n\n        self.loss_function = torch.nn.CrossEntropyLoss()\n        self.log_interval = 100\n\n    def train(self,\n              base_path: str,\n              sequence_length: int,\n              learning_rate: float = 20,\n              mini_batch_size: int = 100,\n              anneal_factor: float = 0.25,\n              patience: int = 10,\n              clip=0.25,\n              max_epochs: int = 1000):\n\n        number_of_splits: int = len(self.corpus.train_files)\n\n        # an epoch has a number, so calculate total max splits bby multiplying max_epochs with number_of_splits\n        max_splits: int = number_of_splits * max_epochs\n\n        val_data = self._batchify(self.corpus.valid, mini_batch_size)\n\n        os.makedirs(base_path, exist_ok=True)\n        loss_txt = os.path.join(base_path, 'loss.txt')\n        savefile = os.path.join(base_path, 'best-lm.pt')\n\n        try:\n\n            epoch = 0\n            best_val_loss = 100000000\n            optimizer = torch.optim.SGD(self.model.parameters(), lr=learning_rate)\n            scheduler: ReduceLROnPlateau = ReduceLROnPlateau(optimizer, verbose=True, factor=anneal_factor,\n                                                             patience=patience)\n\n            for split in range(1, max_splits + 1):\n\n                # after pass over all splits, increment epoch count\n                if (split - 1) % number_of_splits == 0:\n                    epoch += 1\n\n                log.info('Split %d' % split + '\\t - ({:%H:%M:%S})'.format(datetime.datetime.now()))\n\n                for group in optimizer.param_groups:\n                    learning_rate = group['lr']\n\n                train_slice = self.corpus.get_next_train_slice()\n\n                train_data = self._batchify(train_slice, mini_batch_size)\n                log.info('\\t({:%H:%M:%S})'.format(datetime.datetime.now()))\n\n                # go into train mode\n                self.model.train()\n\n                # reset variables\n                epoch_start_time = time.time()\n                total_loss = 0\n                start_time = time.time()\n\n                hidden = self.model.init_hidden(mini_batch_size)\n\n                # not really sure what this does\n                ntokens = len(self.corpus.dictionary)\n\n                # do batches\n                for batch, i in enumerate(range(0, train_data.size(0) - 1, sequence_length)):\n\n                    data, targets = self._get_batch(train_data, i, sequence_length)\n\n                    # Starting each batch, we detach the hidden state from how it was previously produced.\n                    # If we didn't, the model would try backpropagating all the way to start of the dataset.\n                    hidden = self._repackage_hidden(hidden)\n\n                    self.model.zero_grad()\n                    optimizer.zero_grad()\n\n                    # do the forward pass in the model\n                    output, rnn_output, hidden = self.model.forward(data, hidden)\n\n                    # try to predict the targets\n                    loss = self.loss_function(output.view(-1, ntokens), targets)\n                    loss.backward()\n\n                    # `clip_grad_norm` helps prevent the exploding gradient problem in RNNs / LSTMs.\n                    torch.nn.utils.clip_grad_norm_(self.model.parameters(), clip)\n\n                    optimizer.step()\n\n                    total_loss += loss.data\n\n                    if batch % self.log_interval == 0 and batch > 0:\n                        cur_loss = total_loss.item() / self.log_interval\n                        elapsed = time.time() - start_time\n                        log.info('| split {:3d} /{:3d} | {:5d}/{:5d} batches | ms/batch {:5.2f} | '\n                              'loss {:5.2f} | ppl {:8.2f}'.format(\n                            split, number_of_splits, batch, len(train_data) // sequence_length,\n                                                            elapsed * 1000 / self.log_interval, cur_loss,\n                            math.exp(cur_loss)))\n                        total_loss = 0\n                        start_time = time.time()\n\n                log.info('training done! \\t({:%H:%M:%S})'.format(datetime.datetime.now()))\n\n                ###############################################################################\n                # TEST\n                ###############################################################################\n                self.model.eval()\n                val_loss = self.evaluate(val_data, mini_batch_size, sequence_length)\n                scheduler.step(val_loss)\n\n                log.info('best loss so far {:5.2f}'.format(best_val_loss))\n\n                # Save the model if the validation loss is the best we've seen so far.\n                if val_loss < best_val_loss:\n                    self.model.save(savefile)\n                    best_val_loss = val_loss\n\n                ###############################################################################\n                # print info\n                ###############################################################################\n                log.info('-' * 89)\n\n                local_split_number = split % number_of_splits\n                if local_split_number == 0: local_split_number = number_of_splits\n\n                summary = '| end of split {:3d} /{:3d} | epoch {:3d} | time: {:5.2f}s | valid loss {:5.2f} | ' \\\n                          'valid ppl {:8.2f} | learning rate {:3.2f}'.format(local_split_number,\n                                                                             number_of_splits,\n                                                                             epoch,\n                                                                             (time.time() - epoch_start_time),\n                                                                             val_loss,\n                                                                             math.exp(val_loss),\n                                                                             learning_rate)\n\n                with open(loss_txt, \"a\") as myfile:\n                    myfile.write('%s\\n' % summary)\n\n                log.info(summary)\n                log.info('-' * 89)\n\n        except KeyboardInterrupt:\n            log.info('-' * 89)\n            log.info('Exiting from training early')\n\n        ###############################################################################\n        # final testing\n        ###############################################################################\n        test_data = self._batchify(self.corpus.test, mini_batch_size)\n        test_loss = self.evaluate(test_data, mini_batch_size, sequence_length)\n\n        summary = 'TEST: valid loss {:5.2f} | valid ppl {:8.2f}'.format(test_loss, math.exp(test_loss))\n        with open(loss_txt, \"a\") as myfile:\n            myfile.write('%s\\n' % summary)\n\n        log.info(summary)\n        log.info('-' * 89)\n\n    def evaluate(self, data_source, eval_batch_size, sequence_length):\n        # Turn on evaluation mode which disables dropout.\n        self.model.eval()\n\n        with torch.no_grad():\n            total_loss = 0\n            ntokens = len(self.corpus.dictionary)\n\n            hidden = self.model.init_hidden(eval_batch_size)\n\n            for i in range(0, data_source.size(0) - 1, sequence_length):\n                data, targets = self._get_batch(data_source, i, sequence_length)\n                prediction, rnn_output, hidden = self.model.forward(data, hidden)\n                output_flat = prediction.view(-1, ntokens)\n                total_loss += len(data) * self.loss_function(output_flat, targets).data\n                hidden = self._repackage_hidden(hidden)\n            return total_loss.item() / len(data_source)\n\n    @staticmethod\n    def _batchify(data, batch_size):\n        # Work out how cleanly we can divide the dataset into bsz parts.\n        nbatch = data.size(0) // batch_size\n        # Trim off any extra elements that wouldn't cleanly fit (remainders).\n        data = data.narrow(0, 0, nbatch * batch_size)\n        # Evenly divide the data across the bsz batches.\n        data = data.view(batch_size, -1).t().contiguous()\n        if torch.cuda.is_available():\n            data = data.cuda()\n        return data\n\n    @staticmethod\n    def _get_batch(source, i, sequence_length):\n        seq_len = min(sequence_length, len(source) - 1 - i)\n        data = Variable(source[i:i + seq_len])\n        target = Variable(source[i + 1:i + 1 + seq_len].view(-1))\n        return data, target\n\n    @staticmethod\n    def _repackage_hidden(h):\n        \"\"\"Wraps hidden states in new Variables, to detach them from their history.\"\"\"\n        return tuple(Variable(v) for v in h)\n\n\n", "path": "flair/trainers/language_model_trainer.py"}]}