class_docstr: A module holding the result of tokenization.
  
  ---
  name: Tokenization
  citation:
      author: Galkin
      year: 2021
      link: https://arxiv.org/abs/2106.12144
      github: https://github.com/migalkin/NodePiece
class_docstr_tok: ["A","module","holding","the","result","of","tokenization",".","--","-","name",":","Tokenization","citation",":","author",":","Galkin","year",":","2021","link",":","https",":","\/\/arxiv.org\/abs\/2106.12144","github",":","https",":","\/\/github.com\/migalkin\/NodePiece"]
docstr: Initialize the tokenization.
  
  :param assignment: shape: `(n, num_chosen_tokens)`
      the token assignment.
  :param token_representation: shape: `(num_total_tokens, *shape)`
      the token representations
  :param token_representation_kwargs:
      additional keyword-based parameters
  :param shape:
      The shape of an individual representation. If provided, has to match.
  :param kwargs:
      additional keyword-based parameters passed to :meth:`Representation.__init__`
  
  :raises ValueError: if there's a mismatch between the representation size
      and the vocabulary size
docstr_tok: ["Initialize","the","tokenization",".",":","param","assignment",":","shape",":","`","(","n",",","num_chosen_tokens",")","`","the","token","assignment",".",":","param","token_representation",":","shape",":","`","(","num_total_tokens",",","*","shape",")","`","the","token","representations",":","param","token_representation_kwargs",":","additional","keyword-based","parameters",":","param","shape",":","The","shape","of","an","individual","representation",".","If","provided",",","has","to","match",".",":","param","kwargs",":","additional","keyword-based","parameters","passed","to",":","meth",":","`","Representation.__init__","`",":","raises","ValueError",":","if","there","'s","a","mismatch","between","the","representation","size","and","the","vocabulary","size"]
code: def __init__(
      self,
      assignment: torch.LongTensor,
      token_representation: HintOrType[Representation] = None,
      token_representation_kwargs: OptionalKwargs = None,
      shape: Optional[OneOrSequence[int]] = None,
      **kwargs,
  ) -> None:
      """
      Initialize the tokenization.
  
      :param assignment: shape: `(n, num_chosen_tokens)`
          the token assignment.
      :param token_representation: shape: `(num_total_tokens, *shape)`
          the token representations
      :param token_representation_kwargs:
          additional keyword-based parameters
      :param shape:
          The shape of an individual representation. If provided, has to match.
      :param kwargs:
          additional keyword-based parameters passed to :meth:`Representation.__init__`
  
      :raises ValueError: if there's a mismatch between the representation size
          and the vocabulary size
      """
      # needs to be lazily imported to avoid cyclic imports
      from .. import representation_resolver
  
      # fill padding (nn.Embedding cannot deal with negative indices)
      padding = assignment < 0
      # sometimes, assignment.max() does not cover all relations (eg, inductive inference graphs
      # contain a subset of training relations) - for that, the padding index is the last index of the Representation
      self.vocabulary_size = (
          token_representation.max_id
          if isinstance(token_representation, Representation)
          else assignment.max().item()
          + 2  # exclusive (+1) and including padding (+1)
      )
  
      assignment[padding] = (
          self.vocabulary_size - 1
      )  # = assignment.max().item() + 1
      max_id, num_chosen_tokens = assignment.shape
  
      # resolve token representation
      token_representation = representation_resolver.make(
          token_representation,
          token_representation_kwargs,
          max_id=self.vocabulary_size,
      )
      shape = ShapeError.verify(
          shape=(num_chosen_tokens,) + token_representation.shape,
          reference=shape,
      )
      super().__init__(max_id=max_id, shape=shape, **kwargs)
  
      # input validation
      if token_representation.max_id < self.vocabulary_size:
          raise ValueError(
              f"The token representations only contain {token_representation.max_id} representations,"
              f"but there are {self.vocabulary_size} tokens in use.",
          )
      elif token_representation.max_id > self.vocabulary_size:
          logger.warning(
              f"Token representations do contain more representations ({token_representation.max_id}) "
              f"than tokens are used ({self.vocabulary_size}).",
          )
      # register as buffer
      self.register_buffer(name="assignment", tensor=assignment)
      # assign sub-module
      self.vocabulary = token_representation
  
imports: import logging
  import pathlib
  from typing import Callable, Iterable, List, NamedTuple, Optional, Union
  import torch
  from class_resolver import HintOrType, OneOrManyHintOrType, OneOrManyOptionalKwargs, OptionalKwargs
  from docdata import parse_docdata
  from .tokenization import Tokenizer, tokenizer_resolver
  from ..combination import ConcatAggregationCombination
  from ..perceptron import ConcatMLP
  from ..representation import CombinedRepresentation, Representation
  from ..utils import ShapeError
  from ...triples import CoreTriplesFactory
  from ...typing import MappedTriples, OneOrSequence
  from ...utils import broadcast_upgrade_to_sequences
label_desc: Use image node_id 1 to create a new TokenizationRepresentation object from inherited base classes: Representation with example: obj = TokenizationRepresentation(assignment, token_representation, token_representation_kwargs, shape)