class_docstr: analogous to fairseq.data.indexed_dataset.IndexedCachedDataset.
  Support loading .idx + .bin as fairseq does and also .npz. In self
  initializer if
  (1) path is passed in: call the initializer of parent class and loads
  idx + bin from the path.
  (2) path is None: Initialize an empty class. Call load(path) to load
  .npz data.
  When self.is_npz=True, use the implementation in this class which overrides
  parent methods. Otherwise use parent functions in fairseq.
class_docstr_tok: ["analogous","to","fairseq.data.indexed_dataset.IndexedCachedDataset",".","Support","loading",".idx","+",".bin","as","fairseq","does","and","also",".npz",".","In","self","initializer","if","(","1",")","path","is","passed","in",":","call","the","initializer","of","parent","class","and","loads","idx","+","bin","from","the","path",".","(","2",")","path","is","None",":","Initialize","an","empty","class",".","Call","load","(","path",")","to","load",".npz","data",".","When","self.is_npz=True",",","use","the","implementation","in","this","class","which","overrides","parent","methods",".","Otherwise","use","parent","functions","in","fairseq","."]
docstr: Add sentences from text files to the dataset.
  
  This method reads pairs of text files containing source and target
  sides of a bitext. Sentences are converted to integer sequences by
  tokenization and dictionary look-up. Note that this method removes all
  sentences which have been previously added to the data set.
  
  Example (single sentence):
      token_sequence = [123, 234, 345]
      dict.eos_idx = 2
      dialect_id = 10
      Result:
          reverse_order=False, append_eos=True, prepend_language_id=True:
              [10, 123, 234, 345, 2]
          reverse_order=False, append_eos=True, prepend_language_id=False:
              [123, 234, 345, 2, 10]
          reverse_order=True, append_eos=True, prepend_language_id=True:
              [10, 345, 234, 123, 2]
          reverse_order=True, append_eos=True, prepend_language_id=False:
              [345, 234, 123, 2, 10]
  
  Args:
      corpora: List of MultilingualCorpusConfig. If dialect_id is not
          None, it is added to the token sequence.
      reverse_order (bool): Whether to reverse the integer token sequence.
      append_eos (bool): Whether to add the end-of-sentence symbol to each
          sentence.
      prepend_language_id (bool): Only used if dialect_id is not None. If
          true, add ID at the begin of the token sequence. Otherwise, add
          it at the end of the token sequence.
      already_numberized (bool): If data_file contains lines of
          numberized tokens, then already_numberized should be set to True
          If data_file contains raw text sentences, then
          already_numberized should be False (default) -- in which case
          each line is tokenized with tokenizer then numberized with the
          dictionary before being added to the output buffer.
docstr_tok: ["Add","sentences","from","text","files","to","the","dataset",".","This","method","reads","pairs","of","text","files","containing","source","and","target","sides","of","a","bitext",".","Sentences","are","converted","to","integer","sequences","by","tokenization","and","dictionary","look-up",".","Note","that","this","method","removes","all","sentences","which","have","been","previously","added","to","the","data","set",".","Example","(","single","sentence",")",":","token_sequence","=","[","123",",","234",",","345","]","dict.eos_idx","=","2","dialect_id","=","10","Result",":","reverse_order=False",",","append_eos=True",",","prepend_language_id=True",":","[","10",",","123",",","234",",","345",",","2","]","reverse_order=False",",","append_eos=True",",","prepend_language_id=False",":","[","123",",","234",",","345",",","2",",","10","]","reverse_order=True",",","append_eos=True",",","prepend_language_id=True",":","[","10",",","345",",","234",",","123",",","2","]","reverse_order=True",",","append_eos=True",",","prepend_language_id=False",":","[","345",",","234",",","123",",","2",",","10","]","Args",":","corpora",":","List","of","MultilingualCorpusConfig",".","If","dialect_id","is","not","None",",","it","is","added","to","the","token","sequence",".","reverse_order","(","bool",")",":","Whether","to","reverse","the","integer","token","sequence",".","append_eos","(","bool",")",":","Whether","to","add","the","end-of-sentence","symbol","to","each","sentence",".","prepend_language_id","(","bool",")",":","Only","used","if","dialect_id","is","not","None",".","If","true",",","add","ID","at","the","begin","of","the","token","sequence",".","Otherwise",",","add","it","at","the","end","of","the","token","sequence",".","already_numberized","(","bool",")",":","If","data_file","contains","lines","of","numberized","tokens",",","then","already_numberized","should","be","set","to","True","If","data_file","contains","raw","text","sentences",",","then","already_numberized","should","be","False","(","default",")","--","in","which","case","each","line","is","tokenized","with","tokenizer","then","numberized","with","the","dictionary","before","being","added","to","the","output","buffer","."]
code: def parse_multilingual(
      self,
      corpora,
      reverse_order=False,
      append_eos=False,
      prepend_language_id=True,
      already_numberized=False,
  ):
      """Add sentences from text files to the dataset.
  
      This method reads pairs of text files containing source and target
      sides of a bitext. Sentences are converted to integer sequences by
      tokenization and dictionary look-up. Note that this method removes all
      sentences which have been previously added to the data set.
  
      Example (single sentence):
          token_sequence = [123, 234, 345]
          dict.eos_idx = 2
          dialect_id = 10
          Result:
              reverse_order=False, append_eos=True, prepend_language_id=True:
                  [10, 123, 234, 345, 2]
              reverse_order=False, append_eos=True, prepend_language_id=False:
                  [123, 234, 345, 2, 10]
              reverse_order=True, append_eos=True, prepend_language_id=True:
                  [10, 345, 234, 123, 2]
              reverse_order=True, append_eos=True, prepend_language_id=False:
                  [345, 234, 123, 2, 10]
  
      Args:
          corpora: List of MultilingualCorpusConfig. If dialect_id is not
              None, it is added to the token sequence.
          reverse_order (bool): Whether to reverse the integer token sequence.
          append_eos (bool): Whether to add the end-of-sentence symbol to each
              sentence.
          prepend_language_id (bool): Only used if dialect_id is not None. If
              true, add ID at the begin of the token sequence. Otherwise, add
              it at the end of the token sequence.
          already_numberized (bool): If data_file contains lines of
              numberized tokens, then already_numberized should be set to True
              If data_file contains raw text sentences, then
              already_numberized should be False (default) -- in which case
              each line is tokenized with tokenizer then numberized with the
              dictionary before being added to the output buffer.
  
      """
      array_list = []
      offsets = [0]
      sizes = []
      print(corpora)
      for corpus_config in corpora:
          print(corpus_config)
          print(corpus_config.data_file)
          prepend_inds = []
          append_inds = []
          if append_eos:
              append_inds.append(corpus_config.dict.eos_index)
          if corpus_config.dialect_id is not None:
              if prepend_language_id:
                  prepend_inds.append(corpus_config.dialect_id)
              else:
                  append_inds.append(corpus_config.dialect_id)
          with PathManager.open(corpus_config.data_file, "r") as f:
              for line in f:
                  if already_numberized:
                      inds = line.strip().split()
                      inds = [int(ind) for ind in inds]
                  else:
                      words = tokenizer.tokenize_line(line)
                      inds = [
                          corpus_config.dict.index(w) for w in words
                      ]
  
                  if reverse_order:
                      inds.reverse()
                  inds = prepend_inds + inds + append_inds
                  for _ in range(corpus_config.oversampling):
                      array_list.append(np.array(inds, dtype=np.int32))
                      offsets.append(offsets[-1] + len(inds))
                      sizes.append(len(inds))
  
      self.buffer = np.concatenate(array_list)
      self.offsets = np.array(offsets, dtype=np.int64)
      self.sizes = np.array(sizes, dtype=np.int32)
      del array_list
      del offsets
      del sizes
  
imports: import os
  import shutil
  import tempfile
  from typing import Dict, NamedTuple, Optional
  import numpy
  import torch
  from fairseq import data, tokenizer
  from fvcore.common.file_io import PathManager
  from pytorch_translate import constants
  from pytorch_translate.data import dictionary
label_desc: Use image node_id 9 for calling the InMemoryIndexedDataset obj's underlying member method code with example usage: obj.parse_multilingual(corpora, reverse_order, append_eos, prepend_language_id, already_numberized) without return types