class_docstr: 用于构建, 存储和使用 `str` 到 `int` 的一一映射::
  
      from fastNLP.core import Vocabulary
      vocab = Vocabulary()
      word_list = "this is a word list".split()
      # vocab更新自己的字典，输入为list列表
      vocab.update(word_list)
      vocab["word"] # str to int
      vocab.to_word(5) # int to str
  
  :param max_size: `Vocabulary` 的最大大小, 即能存储词的最大数量
      若为 ``None`` , 则不限制大小。
  :param min_freq: 能被记录下的词在文本中的最小出现频率, 应大于或等于 1。
      若小于该频率, 词语将被视为 `unknown`. 若为 ``None`` , 所有文本中的词都被记录。
  :param padding: padding的字符. 如果设置为 ``None`` ,
      则vocabulary中不考虑padding, 也不计入词表大小，为 ``None`` 的情况多在为 label 建立 Vocabulary 的情况。
  :param unknown: unknown的字符，所有未被记录的词在转为 :class:`int` 时将被视为 `unknown` 。
      如果设置为 ``None`` ,则 vocabulary 中不考虑 `unknown`, 也不计入词表大小。
      为 ``None`` 的情况多在为 labe l建立 Vocabulary 的情况
class_docstr_tok: ["\u7528\u4e8e\u6784\u5efa",",","\u5b58\u50a8\u548c\u4f7f\u7528","`","str","`","\u5230","`","int","`","\u7684\u4e00\u4e00\u6620\u5c04",":",":","from","fastNLP.core","import","Vocabulary","vocab","=","Vocabulary","(",")","word_list","=","``","this","is","a","word","list","''",".split","(",")","#","vocab\u66f4\u65b0\u81ea\u5df1\u7684\u5b57\u5178\uff0c\u8f93\u5165\u4e3alist\u5217\u8868","vocab.update","(","word_list",")","vocab","[","``","word","''","]","#","str","to","int","vocab.to_word","(","5",")","#","int","to","str",":","param","max_size",":","`","Vocabulary","`","\u7684\u6700\u5927\u5927\u5c0f",",","\u5373\u80fd\u5b58\u50a8\u8bcd\u7684\u6700\u5927\u6570\u91cf","\u82e5\u4e3a","``","None","``",",","\u5219\u4e0d\u9650\u5236\u5927\u5c0f\u3002",":","param","min_freq",":","\u80fd\u88ab\u8bb0\u5f55\u4e0b\u7684\u8bcd\u5728\u6587\u672c\u4e2d\u7684\u6700\u5c0f\u51fa\u73b0\u9891\u7387",",","\u5e94\u5927\u4e8e\u6216\u7b49\u4e8e","1\u3002","\u82e5\u5c0f\u4e8e\u8be5\u9891\u7387",",","\u8bcd\u8bed\u5c06\u88ab\u89c6\u4e3a","`","unknown","`",".","\u82e5\u4e3a","``","None","``",",","\u6240\u6709\u6587\u672c\u4e2d\u7684\u8bcd\u90fd\u88ab\u8bb0\u5f55\u3002",":","param","padding",":","padding\u7684\u5b57\u7b26",".","\u5982\u679c\u8bbe\u7f6e\u4e3a","``","None","``",",","\u5219vocabulary\u4e2d\u4e0d\u8003\u8651padding",",","\u4e5f\u4e0d\u8ba1\u5165\u8bcd\u8868\u5927\u5c0f\uff0c\u4e3a","``","None","``","\u7684\u60c5\u51b5\u591a\u5728\u4e3a","label","\u5efa\u7acb","Vocabulary","\u7684\u60c5\u51b5\u3002",":","param","unknown",":","unknown\u7684\u5b57\u7b26\uff0c\u6240\u6709\u672a\u88ab\u8bb0\u5f55\u7684\u8bcd\u5728\u8f6c\u4e3a",":","class",":","`","int","`","\u65f6\u5c06\u88ab\u89c6\u4e3a","`","unknown","`","\u3002","\u5982\u679c\u8bbe\u7f6e\u4e3a","``","None","``",",","\u5219","vocabulary","\u4e2d\u4e0d\u8003\u8651","`","unknown","`",",","\u4e5f\u4e0d\u8ba1\u5165\u8bcd\u8868\u5927\u5c0f\u3002","\u4e3a","``","None","``","\u7684\u60c5\u51b5\u591a\u5728\u4e3a","labe","l\u5efa\u7acb","Vocabulary","\u7684\u60c5\u51b5"]
docstr: 将 ``DataSet`` 中对应 field 的词转为数字，例如::
  
      # remember to use `field_name`
      vocab.index_dataset(train_data, dev_data, test_data, field_name='words')
  
  :param datasets: 其类型为 :class:`~fastNLP.core.dataset.DataSet` 或者 :class:`List` [ :class:`~fastNLP.core.dataset.DataSet` ]，
      即需要处理的一个或多个数据集
  :param field_name: 需要转为 index 的 field, 若有多个 DataSet, 每个 DataSet 都必须有此 field.
      目前支持 :class:`str` , :class:`List` [ :class:`str` ]
  :param new_field_name: 保存结果的 field_name。 若为 ``None`` , 将覆盖原 field。
docstr_tok: ["\u5c06","``","DataSet","``","\u4e2d\u5bf9\u5e94","field","\u7684\u8bcd\u8f6c\u4e3a\u6570\u5b57\uff0c\u4f8b\u5982",":",":","#","remember","to","use","`","field_name","`","vocab.index_dataset","(","train_data",",","dev_data",",","test_data",",","field_name='words","'",")",":","param","datasets",":","\u5176\u7c7b\u578b\u4e3a",":","class",":","`","~fastNLP.core.dataset.DataSet","`","\u6216\u8005",":","class",":","`","List","`","[",":","class",":","`","~fastNLP.core.dataset.DataSet","`","]","\uff0c","\u5373\u9700\u8981\u5904\u7406\u7684\u4e00\u4e2a\u6216\u591a\u4e2a\u6570\u636e\u96c6",":","param","field_name",":","\u9700\u8981\u8f6c\u4e3a","index","\u7684","field",",","\u82e5\u6709\u591a\u4e2a","DataSet",",","\u6bcf\u4e2a","DataSet","\u90fd\u5fc5\u987b\u6709\u6b64","field",".","\u76ee\u524d\u652f\u6301",":","class",":","`","str","`",",",":","class",":","`","List","`","[",":","class",":","`","str","`","]",":","param","new_field_name",":","\u4fdd\u5b58\u7ed3\u679c\u7684","field_name\u3002","\u82e5\u4e3a","``","None","``",",","\u5c06\u8986\u76d6\u539f","field\u3002"]
code: def index_dataset(
      self,
      *datasets,
      field_name: Union[List, str],
      new_field_name: Union[List, str, None] = None,
  ):
      r"""
      将 ``DataSet`` 中对应 field 的词转为数字，例如::
  
          # remember to use `field_name`
          vocab.index_dataset(train_data, dev_data, test_data, field_name='words')
  
      :param datasets: 其类型为 :class:`~fastNLP.core.dataset.DataSet` 或者 :class:`List` [ :class:`~fastNLP.core.dataset.DataSet` ]，
          即需要处理的一个或多个数据集
      :param field_name: 需要转为 index 的 field, 若有多个 DataSet, 每个 DataSet 都必须有此 field.
          目前支持 :class:`str` , :class:`List` [ :class:`str` ]
      :param new_field_name: 保存结果的 field_name。 若为 ``None`` , 将覆盖原 field。
      """
  
      def index_instance(field):
          r"""
          有几种情况, str, 1d-list, 2d-list
          :param ins:
          :return:
          """
          if isinstance(field, str) or not _is_iterable(field):
              return self.to_index(field)
          else:
              if isinstance(field[0], str) or not _is_iterable(
                  field[0]
              ):
                  return [self.to_index(w) for w in field]
              else:
                  if not isinstance(field[0][0], str) and _is_iterable(
                      field[0][0]
                  ):
                      raise RuntimeError(
                          "Only support field with 2 dimensions."
                      )
                  return [[self.to_index(c) for c in w] for w in field]
  
      new_field_name = new_field_name or field_name
  
      if type(new_field_name) == type(field_name):
          if isinstance(new_field_name, list):
              assert len(new_field_name) == len(field_name), (
                  "new_field_name should have same number elements with "
                  "field_name."
              )
          elif isinstance(new_field_name, str):
              field_name = [field_name]
              new_field_name = [new_field_name]
          else:
              raise TypeError(
                  "field_name and new_field_name can only be str or List[str]."
              )
  
      for idx, dataset in enumerate(datasets):
          if isinstance(dataset, DataSet):
              ds_lst = [dataset]
          elif _is_iterable(dataset):
              ds_lst = list(dataset)
          else:
              raise TypeError(
                  f"Only DataSet type is allowed, instead of {type(dataset)}."
              )
          try:
              for ds in ds_lst:
                  for f_n, n_f_n in zip(field_name, new_field_name):
                      ds.apply_field(
                          index_instance,
                          field_name=f_n,
                          new_field_name=n_f_n,
                          progress_bar=None,
                      )
          except Exception as e:
              logger.error(
                  "When processing the `{}` dataset, the following error occurred.".format(
                      idx
                  )
              )
              raise e
      return self
  
imports: from collections import Counter
  from functools import partial
  from functools import wraps
  from typing import List, Callable, Union
  from fastNLP.core.dataset import DataSet
  from fastNLP.core.utils.utils import Option
  from fastNLP.core.utils.utils import _is_iterable
  from .log import logger
  import io
label_desc: Use image node_id 17 for calling the Vocabulary obj's underlying member method code with example usage: obj.index_dataset() and returns: self, self, unknown, unknown