class_docstr: Token embedding base class.
  
  
  To load token embeddings from an externally hosted pre-trained token embedding file, such as
  those of GloVe and FastText, use
  :func:`~mxnet.contrib.text.embedding.create(embedding_name, pretrained_file_name)`.
  To get all the available `embedding_name` and `pretrained_file_name`, use
  :func:`~mxnet.contrib.text.embedding.get_pretrained_file_names()`.
  
  Alternatively, to load embedding vectors from a custom pre-trained token embedding file, use
  :class:`~mxnet.contrib.text.embedding.CustomEmbedding`.
  
  Moreover, to load composite embedding vectors, such as to concatenate embedding vectors, use
  :class:`~mxnet.contrib.text.embedding.CompositeEmbedding`.
  
  For every unknown token, if its representation `self.unknown_token` is encountered in the
  pre-trained token embedding file, index 0 of `self.idx_to_vec` maps to the pre-trained token
  embedding vector loaded from the file; otherwise, index 0 of `self.idx_to_vec` maps to the
  token embedding vector initialized by `init_unknown_vec`.
  
  If a token is encountered multiple times in the pre-trained token embedding file, only the
  first-encountered token embedding vector will be loaded and the rest will be skipped.
  
  The indexed tokens in a text token embedding may come from a vocabulary or from the loaded
  embedding vectors. In the former case, only the indexed tokens in a vocabulary are associated
  with the loaded embedding vectors, such as loaded from a pre-trained token embedding file. In
  the later case, all the tokens from the loaded embedding vectors, such as loaded from a
  pre-trained token embedding file, are taken as the indexed tokens of the embedding.
  
  
  Attributes
  ----------
  token_to_idx : dict mapping str to int
      A dict mapping each token to its index integer.
  idx_to_token : list of strs
      A list of indexed tokens where the list indices and the token indices are aligned.
  unknown_token : hashable object
      The representation for any unknown token. In other words, any unknown token will be indexed
      as the same representation.
  reserved_tokens : list of strs or None
      A list of reserved tokens that will always be indexed.
  vec_len : int
      The length of the embedding vector for each token.
  idx_to_vec : mxnet.ndarray.NDArray
      For all the indexed tokens in this embedding, this NDArray maps each token's index to an
      embedding vector. The largest valid index maps to the initialized embedding vector for every
      reserved token, such as an unknown_token token and a padding token.
class_docstr_tok: ["Token","embedding","base","class",".","To","load","token","embeddings","from","an","externally","hosted","pre-trained","token","embedding","file",",","such","as","those","of","GloVe","and","FastText",",","use",":","func",":","`","~mxnet.contrib.text.embedding.create","(","embedding_name",",","pretrained_file_name",")","`",".","To","get","all","the","available","`","embedding_name","`","and","`","pretrained_file_name","`",",","use",":","func",":","`","~mxnet.contrib.text.embedding.get_pretrained_file_names","(",")","`",".","Alternatively",",","to","load","embedding","vectors","from","a","custom","pre-trained","token","embedding","file",",","use",":","class",":","`","~mxnet.contrib.text.embedding.CustomEmbedding","`",".","Moreover",",","to","load","composite","embedding","vectors",",","such","as","to","concatenate","embedding","vectors",",","use",":","class",":","`","~mxnet.contrib.text.embedding.CompositeEmbedding","`",".","For","every","unknown","token",",","if","its","representation","`","self.unknown_token","`","is","encountered","in","the","pre-trained","token","embedding","file",",","index","0","of","`","self.idx_to_vec","`","maps","to","the","pre-trained","token","embedding","vector","loaded","from","the","file",";","otherwise",",","index","0","of","`","self.idx_to_vec","`","maps","to","the","token","embedding","vector","initialized","by","`","init_unknown_vec","`",".","If","a","token","is","encountered","multiple","times","in","the","pre-trained","token","embedding","file",",","only","the","first-encountered","token","embedding","vector","will","be","loaded","and","the","rest","will","be","skipped",".","The","indexed","tokens","in","a","text","token","embedding","may","come","from","a","vocabulary","or","from","the","loaded","embedding","vectors",".","In","the","former","case",",","only","the","indexed","tokens","in","a","vocabulary","are","associated","with","the","loaded","embedding","vectors",",","such","as","loaded","from","a","pre-trained","token","embedding","file",".","In","the","later","case",",","all","the","tokens","from","the","loaded","embedding","vectors",",","such","as","loaded","from","a","pre-trained","token","embedding","file",",","are","taken","as","the","indexed","tokens","of","the","embedding",".","Attributes","--","--","--","--","--","token_to_idx",":","dict","mapping","str","to","int","A","dict","mapping","each","token","to","its","index","integer",".","idx_to_token",":","list","of","strs","A","list","of","indexed","tokens","where","the","list","indices","and","the","token","indices","are","aligned",".","unknown_token",":","hashable","object","The","representation","for","any","unknown","token",".","In","other","words",",","any","unknown","token","will","be","indexed","as","the","same","representation",".","reserved_tokens",":","list","of","strs","or","None","A","list","of","reserved","tokens","that","will","always","be","indexed",".","vec_len",":","int","The","length","of","the","embedding","vector","for","each","token",".","idx_to_vec",":","mxnet.ndarray.NDArray","For","all","the","indexed","tokens","in","this","embedding",",","this","NDArray","maps","each","token","'s","index","to","an","embedding","vector",".","The","largest","valid","index","maps","to","the","initialized","embedding","vector","for","every","reserved","token",",","such","as","an","unknown_token","token","and","a","padding","token","."]
docstr: Load embedding vectors from the pre-trained token embedding file.
  
  
  For every unknown token, if its representation `self.unknown_token` is encountered in the
  pre-trained token embedding file, index 0 of `self.idx_to_vec` maps to the pre-trained token
  embedding vector loaded from the file; otherwise, index 0 of `self.idx_to_vec` maps to the
  text embedding vector initialized by `init_unknown_vec`.
  
  If a token is encountered multiple times in the pre-trained text embedding file, only the
  first-encountered token embedding vector will be loaded and the rest will be skipped.
docstr_tok: ["Load","embedding","vectors","from","the","pre-trained","token","embedding","file",".","For","every","unknown","token",",","if","its","representation","`","self.unknown_token","`","is","encountered","in","the","pre-trained","token","embedding","file",",","index","0","of","`","self.idx_to_vec","`","maps","to","the","pre-trained","token","embedding","vector","loaded","from","the","file",";","otherwise",",","index","0","of","`","self.idx_to_vec","`","maps","to","the","text","embedding","vector","initialized","by","`","init_unknown_vec","`",".","If","a","token","is","encountered","multiple","times","in","the","pre-trained","text","embedding","file",",","only","the","first-encountered","token","embedding","vector","will","be","loaded","and","the","rest","will","be","skipped","."]
code: def _load_embedding(
      self,
      pretrained_file_path,
      elem_delim,
      init_unknown_vec,
      encoding="utf8",
  ):
      """Load embedding vectors from the pre-trained token embedding file.
  
  
      For every unknown token, if its representation `self.unknown_token` is encountered in the
      pre-trained token embedding file, index 0 of `self.idx_to_vec` maps to the pre-trained token
      embedding vector loaded from the file; otherwise, index 0 of `self.idx_to_vec` maps to the
      text embedding vector initialized by `init_unknown_vec`.
  
      If a token is encountered multiple times in the pre-trained text embedding file, only the
      first-encountered token embedding vector will be loaded and the rest will be skipped.
      """
  
      pretrained_file_path = os.path.expanduser(pretrained_file_path)
  
      if not os.path.isfile(pretrained_file_path):
          raise ValueError(
              "`pretrained_file_path` must be a valid path to "
              "the pre-trained token embedding file."
          )
  
      logging.info(
          "Loading pre-trained token embedding vectors from %s",
          pretrained_file_path,
      )
      vec_len = None
      all_elems = []
      tokens = set()
      loaded_unknown_vec = None
      line_num = 0
      with open(pretrained_file_path, "r", encoding=encoding) as f:
          for line in f:
              line_num += 1
              elems = line.rstrip().split(elem_delim)
  
              assert len(elems) > 1, (
                  f"At line {line_num} of the pre-trained text embedding file: the "
                  f"data format of the pre-trained token embedding file {pretrained_file_path} "
                  "is unexpected."
              )
  
              token, elems = elems[0], [float(i) for i in elems[1:]]
  
              if (
                  token == self.unknown_token
                  and loaded_unknown_vec is None
              ):
                  loaded_unknown_vec = elems
                  tokens.add(self.unknown_token)
              elif token in tokens:
                  warnings.warn(
                      f"At line {line_num} of the pre-trained token embedding file: the "
                      f"embedding vector for token {token} has been loaded and a duplicate "
                      "embedding for the  same token is seen and skipped."
                  )
              elif len(elems) == 1:
                  warnings.warn(
                      f"At line {line_num} of the pre-trained text embedding file: token {token} "
                      f"with 1-dimensional vector {elems} is likely a header and is "
                      "skipped."
                  )
              else:
                  if vec_len is None:
                      vec_len = len(elems)
                      # Reserve a vector slot for the unknown token at the very beggining because
                      # the unknown index is 0.
                      all_elems.extend([0] * vec_len)
                  else:
                      assert len(elems) == vec_len, (
                          f"At line {line_num} of the pre-trained token embedding file: the dimension "
                          f"of token {token} is {len(elems)} but the dimension of previous tokens is {vec_len}. "
                          "Dimensions of all the tokens must be the same."
                      )
                  all_elems.extend(elems)
                  self._idx_to_token.append(token)
                  self._token_to_idx[token] = (
                      len(self._idx_to_token) - 1
                  )
                  tokens.add(token)
  
      self._vec_len = vec_len
      array_fn = _mx_np.array if is_np_array() else nd.array
      self._idx_to_vec = array_fn(all_elems).reshape((-1, self.vec_len))
  
      if loaded_unknown_vec is None:
          init_val = init_unknown_vec(shape=self.vec_len)
          self._idx_to_vec[C.UNKNOWN_IDX] = (
              init_val.as_np_ndarray() if is_np_array() else init_val
          )
      else:
          self._idx_to_vec[C.UNKNOWN_IDX] = array_fn(loaded_unknown_vec)
  
imports: import logging
  import os
  import tarfile
  import warnings
  import zipfile
  from .None import _constants
  from .None import vocab
  from ...None import ndarray
  from ...None import registry
  from ...None import base
  from ...util import is_np_array
  from ...None import numpy
  from ...None import numpy_extension
label_desc: Use image node_id 5 for calling the _TokenEmbedding obj's underlying member method code with example usage: obj._load_embedding(pretrained_file_path, elem_delim, init_unknown_vec, encoding) without return types