class_docstr: Defines a dataset for language modeling.
class_docstr_tok: ["Defines","a","dataset","for","language","modeling","."]
docstr: Create a LanguageModelingDataset given a path and a field.
  
  Args:
      path: Path to the data file.
      text_field: The field that will be used for text data.
      newline_eos: Whether to add an <eos> token for every newline in the
          data file. Default: True.
      encoding: The encoding of the file.
      kwargs: Passed to the constructor of
          data.Dataset.
docstr_tok: ["Create","a","LanguageModelingDataset","given","a","path","and","a","field",".","Args",":","path",":","Path","to","the","data","file",".","text_field",":","The","field","that","will","be","used","for","text","data",".","newline_eos",":","Whether","to","add","an","<","eos",">","token","for","every","newline","in","the","data","file",".","Default",":","True",".","encoding",":","The","encoding","of","the","file",".","kwargs",":","Passed","to","the","constructor","of","data.Dataset","."]
code: def __init__(
      self,
      path,
      text_field,
      newline_eos=True,
      encoding="utf-8",
      **kwargs,
  ):
      """Create a LanguageModelingDataset given a path and a field.
  
      Args:
          path: Path to the data file.
          text_field: The field that will be used for text data.
          newline_eos: Whether to add an <eos> token for every newline in the
              data file. Default: True.
          encoding: The encoding of the file.
          kwargs: Passed to the constructor of
              data.Dataset.
      """
      fields = [("text", text_field)]
      text = []
      with io.open(path, encoding=encoding) as f:
          for line in f:
              text += text_field.preprocess(line)
              if newline_eos:
                  text.append("<eos>")
  
      examples = [data.Example.fromlist([text], fields)]
      super(LanguageModelingDataset, self).__init__(
          examples, fields, **kwargs
      )
  
file_path: pytext/pytext/legacy/datasets/language_modeling.py
imports: import io
  from ..None import data
label_desc: Use image node_id 1 to create a new LanguageModelingDataset object from inherited base classes: data with example: obj = LanguageModelingDataset(path, text_field, newline_eos, encoding)