docstr: Tokenizes a text file.
docstr_tok: ["Tokenizes","a","text","file","."]
code: def tokenize(self, path):
      """Tokenizes a text file."""
      assert os.path.exists(path)
      # Add words to the dictionary
      with open(path, "r") as f:
          tokens = 0
          for line in f:
              words = line.split() + ["<eos>"]
              tokens += len(words)
              for word in words:
                  self.dictionary.add_word(word)
  
      # Tokenize file content
      with open(path, "r") as f:
          ids = t.LongTensor(tokens)
          token = 0
          for line in f:
              words = line.split() + ["<eos>"]
              for word in words:
                  ids[token] = self.dictionary.word2idx[word]
                  token += 1
  
      return ids
  
file_path: ENAS-pytorch/data/text.py
imports: import os
  import torch
  import collections
label_desc: Use image node_id 2 for calling the Corpus obj's underlying member method code with example usage: obj.tokenize(path) and returns: ids