docstr: Index sentences with a dictionary.
docstr_tok: ["Index","sentences","with","a","dictionary","."]
code: def index_data(path, bin_path, dico):
      """
      Index sentences with a dictionary.
      """
      if bin_path is not None and PathManager.isfile(bin_path):
          print("Loading data from %s ..." % bin_path)
          data = torch.load(bin_path)
          assert dico == data["dico"]
          return data
  
      positions = []
      sentences = []
      unk_words = {}
  
      # index sentences
      f = PathManager.open(path, "r", encoding="utf-8")
      for i, line in enumerate(f):
          if i % 1000000 == 0 and i > 0:
              print(i)
          s = line.rstrip().split()
          # skip empty sentences
          if len(s) == 0:
              print("Empty sentence in line %i." % i)
          # index sentence words
          count_unk = 0
          indexed = []
          for w in s:
              word_id = dico.index(w, no_unk=False)
              # if we find a special word which is not an unknown word,
              # skip the sentence
              if 0 <= word_id < 4 + SPECIAL_WORDS and word_id != 3:
                  logger.warning(
                      'Found unexpected special word "%s" (%i)!!'
                      % (w, word_id)
                  )
                  continue
              assert word_id >= 0
              indexed.append(word_id)
              if word_id == dico.unk_index:
                  unk_words[w] = unk_words.get(w, 0) + 1
                  count_unk += 1
          # add sentence
          positions.append(
              [len(sentences), len(sentences) + len(indexed)]
          )
          sentences.extend(indexed)
          sentences.append(1)  # EOS index
      f.close()
  
      # tensorize data
      positions = np.int64(positions)
      if len(dico) < 1 << 16:
          sentences = np.uint16(sentences)
      elif len(dico) < 1 << 31:
          sentences = np.int32(sentences)
      else:
          raise Exception("Dictionary is too big.")
      assert sentences.min() >= 0
      data = {
          "dico": dico,
          "positions": positions,
          "sentences": sentences,
          "unk_words": unk_words,
      }
      if bin_path is not None:
          print("Saving the data to %s ..." % bin_path)
          torch.save(data, bin_path, pickle_protocol=4)
  
      return data
  
file_path: pytext/pytext/data/xlm_dictionary.py
imports: import os
  from logging import getLogger
  import numpy
  import torch
  from pytext.utils.file_io import PathManager
label_desc: Use image node_id 11 for calling the Dictionary obj's underlying member method code with example usage: obj.index_data(path, bin_path, dico) and returns: data, data