class_docstr: The virtual base class for nlp datasets, such TextClassificationDataset, SeqLabelingDataset, and so on.
  The base class must be supered and re-implemented the method _read_file.
class_docstr_tok: ["The","virtual","base","class","for","nlp","datasets",",","such","TextClassificationDataset",",","SeqLabelingDataset",",","and","so","on",".","The","base","class","must","be","supered","and","re-implemented","the","method","_read_file","."]
docstr: Ags:
      base_path (:obj:`str`): The directory to the whole dataset.
      tokenizer (:obj:`PretrainedTokenizer` or :obj:`JiebaTokenizer`):
          It tokenizes the text and encodes the data as model needed.
      max_seq_len (:obj:`int`, `optional`, defaults to :128):
          If set to a number, will limit the total sequence returned so that it has a maximum length.
      mode (:obj:`str`, `optional`, defaults to `train`):
          It identifies the dataset mode (train, test or dev).
      data_file(:obj:`str`, `optional`, defaults to :obj:`None`):
          The data file name, which is relative to the base_path.
      label_file(:obj:`str`, `optional`, defaults to :obj:`None`):
          The label file name, which is relative to the base_path.
          It is all labels of the dataset, one line one label.
      label_list(:obj:`List[str]`, `optional`, defaults to :obj:`None`):
          The list of all labels of the dataset
docstr_tok: ["Ags",":","base_path","(",":","obj",":","`","str","`",")",":","The","directory","to","the","whole","dataset",".","tokenizer","(",":","obj",":","`","PretrainedTokenizer","`","or",":","obj",":","`","JiebaTokenizer","`",")",":","It","tokenizes","the","text","and","encodes","the","data","as","model","needed",".","max_seq_len","(",":","obj",":","`","int","`",",","`","optional","`",",","defaults","to",":128",")",":","If","set","to","a","number",",","will","limit","the","total","sequence","returned","so","that","it","has","a","maximum","length",".","mode","(",":","obj",":","`","str","`",",","`","optional","`",",","defaults","to","`","train","`",")",":","It","identifies","the","dataset","mode","(","train",",","test","or","dev",")",".","data_file","(",":","obj",":","`","str","`",",","`","optional","`",",","defaults","to",":","obj",":","`","None","`",")",":","The","data","file","name",",","which","is","relative","to","the","base_path",".","label_file","(",":","obj",":","`","str","`",",","`","optional","`",",","defaults","to",":","obj",":","`","None","`",")",":","The","label","file","name",",","which","is","relative","to","the","base_path",".","It","is","all","labels","of","the","dataset",",","one","line","one","label",".","label_list","(",":","obj",":","`","List","[","str","]","`",",","`","optional","`",",","defaults","to",":","obj",":","`","None","`",")",":","The","list","of","all","labels","of","the","dataset"]
code: def __init__(
      self,
      base_path: str,
      tokenizer: Union[PretrainedTokenizer, JiebaTokenizer],
      max_seq_len: Optional[int] = 128,
      mode: Optional[str] = "train",
      data_file: Optional[str] = None,
      label_file: Optional[str] = None,
      label_list: Optional[List[str]] = None,
  ):
      """
      Ags:
          base_path (:obj:`str`): The directory to the whole dataset.
          tokenizer (:obj:`PretrainedTokenizer` or :obj:`JiebaTokenizer`):
              It tokenizes the text and encodes the data as model needed.
          max_seq_len (:obj:`int`, `optional`, defaults to :128):
              If set to a number, will limit the total sequence returned so that it has a maximum length.
          mode (:obj:`str`, `optional`, defaults to `train`):
              It identifies the dataset mode (train, test or dev).
          data_file(:obj:`str`, `optional`, defaults to :obj:`None`):
              The data file name, which is relative to the base_path.
          label_file(:obj:`str`, `optional`, defaults to :obj:`None`):
              The label file name, which is relative to the base_path.
              It is all labels of the dataset, one line one label.
          label_list(:obj:`List[str]`, `optional`, defaults to :obj:`None`):
              The list of all labels of the dataset
      """
      self.data_file = os.path.join(base_path, data_file)
      self.label_list = label_list
  
      self.mode = mode
      self.tokenizer = tokenizer
      self.max_seq_len = max_seq_len
  
      if label_file:
          self.label_file = os.path.join(base_path, label_file)
          if not self.label_list:
              self.label_list = self._load_label_data()
          else:
              logger.warning(
                  "As label_list has been assigned, label_file is noneffective"
              )
      if self.label_list:
          self.label_map = {
              item: index for index, item in enumerate(self.label_list)
          }
  
imports: import csv
  import io
  import os
  from typing import Dict, List, Optional, Union, Tuple
  import numpy
  import paddle
  import paddlenlp
  from packaging.version import Version
  from paddlehub.env import DATA_HOME
  from paddlenlp.transformers import PretrainedTokenizer
  from paddlenlp.data import JiebaTokenizer
  from paddlehub.utils.log import logger
  from paddlehub.utils.utils import download, reseg_token_label, pad_sequence, trunc_sequence
  from paddlehub.utils.xarfile import is_xarfile, unarchive
label_desc: Use image node_id 1 to create a new BaseNLPDataset object from inherited base classes: object with example: obj = BaseNLPDataset(base_path, tokenizer, max_seq_len, mode, data_file, label_file, label_list)