class_docstr: Indexing the text tokens.
  
  Parameters
  ----------
  tokens
      You may specify the input tokens as a python counter object or a list.
  
      If it's a counter
          It represents the text tokens + the frequencies of these tokens in the text data.
          Its keys will be indexed according to frequency thresholds such as `max_size` and `min_freq`.
      If it's a list
          It represents the list of tokens we will add to the vocabulary.
          We will follow the order of the tokens in the list to assign the indices.
          
      The special tokens (those specified in kwargs) that are not specified in `tokens`
      will be added after the tokens.
  max_size
      The maximum possible number of the most frequent tokens in the keys of `counter` that can be
      indexed. Note that this argument does not count only count tokens in counter and does not
      count the special tokens like the padding token and bos token. Suppose
      that there are different keys of `counter` whose counts are the same, if indexing all of
      them will exceed this argument value, such keys will be indexed one by one according to
      their __cmp__() order until the frequency threshold is met. If this argument is None or
      larger than its largest possible value restricted by `counter`, this
      argument has no effect.
  min_freq
      The minimum frequency required for a token in the keys of `counter` to be indexed.
      If it is None, all keys in `counter` will be used.
  unk_token
      The representation for any unknown token. If `unk_token` is not
      `None`, looking up any token that is not part of the vocabulary and
      thus considered unknown will return the index of `unk_token`. If
      None, looking up an unknown token will result in `KeyError`.
  `**kwargs`
      Keyword arguments of the format `xxx_token` can be used to specify
      further special tokens that will be exposed as attribute of the
      vocabulary and associated with an index.
      For example, specifying `mask_token='<mask>` as additional keyword
      argument when constructing a vocabulary `v` leads to `v.mask_token`
      exposing the value of the special token: `<mask>`.
      If the specified token is not part of the Vocabulary, it will be added to the vocabulary.
  
  
  Examples
  --------
  
  >>> import gluonnlp as nlp
  >>> # Case 1: Create vocab object from a list of tokens
  >>> vocab = nlp.data.Vocab(['hello', 'world'])
  >>> vocab
  Vocab(size=3, unk_token="<unk>")
  
  >>> vocab.all_tokens
  ['hello', 'world', '<unk>']
  
  >>> vocab.special_tokens
  ['<unk>']
  
  >>> vocab.non_special_tokens
  ['hello', 'world']
  
  >>> vocab['hello']
  0
  
  >>> vocab[['hello', 'world']]
  [0, 1]
  
  >>> vocab['hellohello']
  2
  
  >>> vocab.to_tokens([0, 1])
  ['hello', 'world']
  
  >>> # Case 2: Disabling the unknown token
  >>> vocab = nlp.data.Vocab(['hello', 'world'], unk_token=None)
  >>> vocab
  Vocab(size=2, unk_token=None)
  
  >>> vocab.special_tokens
  []
  
  >>> # Case 3: Customized special tokens
  >>> vocab = nlp.data.Vocab(['hello', 'world'], cls_token='<cls>', server_token='<server>')
  >>> vocab.all_tokens
  ['hello', 'world', '<unk>', '<cls>', '<server>']
  
  >>> vocab.cls_token
  '<cls>'
  
  >>> vocab.server_token
  '<server>'
  
  >>> # Case 4: Register special tokens
  >>> vocab = nlp.data.Vocab(['<bos>', '<cls>', 'hello'], bos_token='<bos>', cls_token='<cls>')
  >>> vocab.all_tokens
  ['<bos>', '<cls>', 'hello', '<unk>']
  
  >>> vocab.special_tokens
  ['<bos>', '<cls>', '<unk>']
  
  >>> # Case 5: Serialize and Load
  >>> vocab = nlp.data.Vocab(['hello', 'world'])
  >>> vocab.save('vocab.json')
  >>> new_vocab = nlp.data.Vocab.load('vocab.json')
  >>> print(new_vocab.all_tokens)
  ['hello', 'world', '<unk>']
  
  >>> # Case 6: Load from Counter. We will order the tokens based on frequency
  >>> import collections
  >>> text_data = ['hello', 'world', 'hello', 'nice', 'world', 'hi', 'world']
  >>> counter = collections.Counter(text_data)
  >>> vocab = nlp.data.Vocab(counter)
  >>> vocab
  Vocab(size=5, unk_token="<unk>")
  
  >>> vocab.all_tokens
  ['world', 'hello', 'nice', 'hi', '<unk>']
class_docstr_tok: ["Indexing","the","text","tokens",".","Parameters","--","--","--","--","--","tokens","You","may","specify","the","input","tokens","as","a","python","counter","object","or","a","list",".","If","it","'s","a","counter","It","represents","the","text","tokens","+","the","frequencies","of","these","tokens","in","the","text","data",".","Its","keys","will","be","indexed","according","to","frequency","thresholds","such","as","`","max_size","`","and","`","min_freq","`",".","If","it","'s","a","list","It","represents","the","list","of","tokens","we","will","add","to","the","vocabulary",".","We","will","follow","the","order","of","the","tokens","in","the","list","to","assign","the","indices",".","The","special","tokens","(","those","specified","in","kwargs",")","that","are","not","specified","in","`","tokens","`","will","be","added","after","the","tokens",".","max_size","The","maximum","possible","number","of","the","most","frequent","tokens","in","the","keys","of","`","counter","`","that","can","be","indexed",".","Note","that","this","argument","does","not","count","only","count","tokens","in","counter","and","does","not","count","the","special","tokens","like","the","padding","token","and","bos","token",".","Suppose","that","there","are","different","keys","of","`","counter","`","whose","counts","are","the","same",",","if","indexing","all","of","them","will","exceed","this","argument","value",",","such","keys","will","be","indexed","one","by","one","according","to","their","__cmp__","(",")","order","until","the","frequency","threshold","is","met",".","If","this","argument","is","None","or","larger","than","its","largest","possible","value","restricted","by","`","counter","`",",","this","argument","has","no","effect",".","min_freq","The","minimum","frequency","required","for","a","token","in","the","keys","of","`","counter","`","to","be","indexed",".","If","it","is","None",",","all","keys","in","`","counter","`","will","be","used",".","unk_token","The","representation","for","any","unknown","token",".","If","`","unk_token","`","is","not","`","None","`",",","looking","up","any","token","that","is","not","part","of","the","vocabulary","and","thus","considered","unknown","will","return","the","index","of","`","unk_token","`",".","If","None",",","looking","up","an","unknown","token","will","result","in","`","KeyError","`",".","`","*","*","kwargs","`","Keyword","arguments","of","the","format","`","xxx_token","`","can","be","used","to","specify","further","special","tokens","that","will","be","exposed","as","attribute","of","the","vocabulary","and","associated","with","an","index",".","For","example",",","specifying","`","mask_token=","'","<","mask",">","`","as","additional","keyword","argument","when","constructing","a","vocabulary","`","v","`","leads","to","`","v.mask_token","`","exposing","the","value","of","the","special","token",":","`","<","mask",">","`",".","If","the","specified","token","is","not","part","of","the","Vocabulary",",","it","will","be","added","to","the","vocabulary",".","Examples","--","--","--","--",">",">",">","import","gluonnlp","as","nlp",">",">",">","#","Case","1",":","Create","vocab","object","from","a","list","of","tokens",">",">",">","vocab","=","nlp.data.Vocab","(","[","'hello","'",",","'world","'","]",")",">",">",">","vocab","Vocab","(","size=3",",","unk_token=","''","<","unk",">","''",")",">",">",">","vocab.all_tokens","[","'hello","'",",","'world","'",",","'","<","unk",">","'","]",">",">",">","vocab.special_tokens","[","'","<","unk",">","'","]",">",">",">","vocab.non_special_tokens","[","'hello","'",",","'world","'","]",">",">",">","vocab","[","'hello","'","]","0",">",">",">","vocab","[","[","'hello","'",",","'world","'","]","]","[","0",",","1","]",">",">",">","vocab","[","'hellohello","'","]","2",">",">",">","vocab.to_tokens","(","[","0",",","1","]",")","[","'hello","'",",","'world","'","]",">",">",">","#","Case","2",":","Disabling","the","unknown","token",">",">",">","vocab","=","nlp.data.Vocab","(","[","'hello","'",",","'world","'","]",",","unk_token=None",")",">",">",">","vocab","Vocab","(","size=2",",","unk_token=None",")",">",">",">","vocab.special_tokens","[","]",">",">",">","#","Case","3",":","Customized","special","tokens",">",">",">","vocab","=","nlp.data.Vocab","(","[","'hello","'",",","'world","'","]",",","cls_token=","'","<","cls",">","'",",","server_token=","'","<","server",">","'",")",">",">",">","vocab.all_tokens","[","'hello","'",",","'world","'",",","'","<","unk",">","'",",","'","<","cls",">","'",",","'","<","server",">","'","]",">",">",">","vocab.cls_token","'","<","cls",">","'",">",">",">","vocab.server_token","'","<","server",">","'",">",">",">","#","Case","4",":","Register","special","tokens",">",">",">","vocab","=","nlp.data.Vocab","(","[","'","<","bos",">","'",",","'","<","cls",">","'",",","'hello","'","]",",","bos_token=","'","<","bos",">","'",",","cls_token=","'","<","cls",">","'",")",">",">",">","vocab.all_tokens","[","'","<","bos",">","'",",","'","<","cls",">","'",",","'hello","'",",","'","<","unk",">","'","]",">",">",">","vocab.special_tokens","[","'","<","bos",">","'",",","'","<","cls",">","'",",","'","<","unk",">","'","]",">",">",">","#","Case","5",":","Serialize","and","Load",">",">",">","vocab","=","nlp.data.Vocab","(","[","'hello","'",",","'world","'","]",")",">",">",">","vocab.save","(","'vocab.json","'",")",">",">",">","new_vocab","=","nlp.data.Vocab.load","(","'vocab.json","'",")",">",">",">","print","(","new_vocab.all_tokens",")","[","'hello","'",",","'world","'",",","'","<","unk",">","'","]",">",">",">","#","Case","6",":","Load","from","Counter",".","We","will","order","the","tokens","based","on","frequency",">",">",">","import","collections",">",">",">","text_data","=","[","'hello","'",",","'world","'",",","'hello","'",",","'nice","'",",","'world","'",",","'hi","'",",","'world","'","]",">",">",">","counter","=","collections.Counter","(","text_data",")",">",">",">","vocab","=","nlp.data.Vocab","(","counter",")",">",">",">","vocab","Vocab","(","size=5",",","unk_token=","''","<","unk",">","''",")",">",">",">","vocab.all_tokens","[","'world","'",",","'hello","'",",","'nice","'",",","'hi","'",",","'","<","unk",">","'","]"]
code: def __init__(
      self,
      tokens: Optional[Union[Counter, List]] = None,
      max_size: Optional[int] = None,
      min_freq: Optional[int] = None,
      *,
      unk_token: Optional[Hashable] = "<unk>",
      **kwargs,
  ):
      self._all_tokens = []
      self._token_to_idx = dict()
      self._special_token_kv = collections.OrderedDict()
      # Sanity checks.
      if not (min_freq is None or min_freq > 0):
          raise ValueError(
              "`min_freq` must be either a positive value or None."
          )
      # Add all tokens one by one, if the input is a python counter, we will sort the
      # (freq, token) pair in descending order to guarantee the insertion order.
      if isinstance(tokens, collections.Counter):
          if min_freq is None:
              valid_word_cnts = list(tokens.items())
          else:
              valid_word_cnts = [
                  ele for ele in tokens.items() if ele[1] >= min_freq
              ]
          valid_word_cnts.sort(
              key=lambda ele: (ele[1], ele[0]), reverse=True
          )
          if max_size is None or max_size >= len(valid_word_cnts):
              tokens = [ele[0] for ele in valid_word_cnts]
          else:
              tokens = [valid_word_cnts[i][0] for i in range(max_size)]
      else:
          if tokens is None:
              tokens = []
          if max_size is not None or min_freq is not None:
              warnings.warn(
                  "`max_size` and `min_freq` have no effect if the tokens is not"
                  " a python Counter."
              )
      for token in tokens:
          if token in self._token_to_idx:
              raise ValueError(
                  "Find duplicated token. {} is already added to the vocabulary. "
                  "Please check your input data.".format(token)
              )
          idx = len(self._all_tokens)
          self._all_tokens.append(token)
          self._token_to_idx[token] = idx
      for k, token in [("unk_token", unk_token)] + sorted(
          list(kwargs.items())
      ):
          _check_special_token_identifier(k)
          if token is None:
              continue
          if hasattr(self, k) or k in self._special_token_kv:
              raise ValueError(
                  'Duplicated keys! "{}" is already in the class. '
                  "Please consider to use another name as the identifier. "
                  'Received kwargs["{}"] = "{}"'.format(k, k, token)
              )
          if token in self.special_tokens:
              raise ValueError(
                  'Duplicate values! "{}" is already registered as a special token. '
                  "All registered special tokens={}".format(
                      token, self.special_tokens
                  )
              )
          setattr(self, k, token)
          self._special_token_kv[k] = token
          if token in self._token_to_idx:
              idx = self._token_to_idx[token]
          else:
              idx = len(self._all_tokens)
              self._all_tokens.append(token)
              self._token_to_idx[token] = idx
          # Add the {name}_idx properties to the object
          setattr(self, k[:(-6)] + "_id", idx)
      self._special_token_kv = collections.OrderedDict(
          sorted(
              self._special_token_kv.items(),
              key=lambda ele: self._token_to_idx[ele[1]],
          )
      )
      special_tokens_set = frozenset(self._special_token_kv.values())
      self._non_special_tokens = [
          ele
          for ele in self._all_tokens
          if ele not in special_tokens_set
      ]
  
imports: import collections
  import json
  import warnings
  import numpy
  from typing import Dict, Hashable, List, Optional, Counter, Union, Tuple
label_desc: Use image node_id 1 to create a new Vocab object  with example: obj = Vocab(tokens, max_size, min_freq)