docstr: Initializes a Lemmatizer model.
  
  The model consists of a decoder and an encoder. The encoder is either a RNN-cell (torch.nn.GRU)
  or a Token-Embedding from flair if an embedding is handed to the constructor (token_embedding).
  The output of the encoder is used as the initial hidden state to the decoder, which is an RNN-cell (GRU)
  that predicts the lemma of the given token one letter at a time.
  Note that one can use data in which only those words are annotated that differ from their lemma or data
  in which all words are annotated with a (maybe equal) lemma.
  
  Args:
      encode_characters: If True, use a character embedding to additionally encode tokens per character.
      start_symbol_for_encoding: If True, use a start symbol for encoding characters.
      end_symbol_for_encoding: If True, use an end symbol for encoding characters.
      bidirectional_encoding: If True, the character encoding is bidirectional.
      embeddings: Embedding used to encode sentence
      rnn_input_size: Input size of the RNN('s). Each letter of a token is represented by a hot-one-vector over
          the given character dictionary. This vector is transformed to a input_size vector with a linear layer.
      rnn_hidden_size: size of the hidden state of the RNN('s).
      rnn_layers: Number of stacked RNN cells
      beam_size: Number of hypothesis used when decoding the output of the RNN. Only used in prediction.
      char_dict: Dictionary of characters the model is able to process. The dictionary must contain <unk> for
          the handling of unknown characters. If None, a standard dictionary will be loaded. One can either hand
          over a path to a dictionary or the dictionary itself.
      label_type: Name of the gold labels to use.
      max_sequence_length_dependent_on_input: If set to True, the maximum length of a decoded sequence in
          the prediction depends on the sentences you want to lemmatize. To be precise the maximum length is
          computed as the length of the longest token in the sentences plus one.
      max_sequence_length: If set to True and max_sequence_length_dependend_on_input is False a fixed
          maximum length for the decoding will be used for all sentences.
      use_attention: whether to use attention. Only sensible if encoding via RNN
docstr_tok: ["Initializes","a","Lemmatizer","model",".","The","model","consists","of","a","decoder","and","an","encoder",".","The","encoder","is","either","a","RNN-cell","(","torch.nn.GRU",")","or","a","Token-Embedding","from","flair","if","an","embedding","is","handed","to","the","constructor","(","token_embedding",")",".","The","output","of","the","encoder","is","used","as","the","initial","hidden","state","to","the","decoder",",","which","is","an","RNN-cell","(","GRU",")","that","predicts","the","lemma","of","the","given","token","one","letter","at","a","time",".","Note","that","one","can","use","data","in","which","only","those","words","are","annotated","that","differ","from","their","lemma","or","data","in","which","all","words","are","annotated","with","a","(","maybe","equal",")","lemma",".","Args",":","encode_characters",":","If","True",",","use","a","character","embedding","to","additionally","encode","tokens","per","character",".","start_symbol_for_encoding",":","If","True",",","use","a","start","symbol","for","encoding","characters",".","end_symbol_for_encoding",":","If","True",",","use","an","end","symbol","for","encoding","characters",".","bidirectional_encoding",":","If","True",",","the","character","encoding","is","bidirectional",".","embeddings",":","Embedding","used","to","encode","sentence","rnn_input_size",":","Input","size","of","the","RNN","(","'s",")",".","Each","letter","of","a","token","is","represented","by","a","hot-one-vector","over","the","given","character","dictionary",".","This","vector","is","transformed","to","a","input_size","vector","with","a","linear","layer",".","rnn_hidden_size",":","size","of","the","hidden","state","of","the","RNN","(","'s",")",".","rnn_layers",":","Number","of","stacked","RNN","cells","beam_size",":","Number","of","hypothesis","used","when","decoding","the","output","of","the","RNN",".","Only","used","in","prediction",".","char_dict",":","Dictionary","of","characters","the","model","is","able","to","process",".","The","dictionary","must","contain","<","unk",">","for","the","handling","of","unknown","characters",".","If","None",",","a","standard","dictionary","will","be","loaded",".","One","can","either","hand","over","a","path","to","a","dictionary","or","the","dictionary","itself",".","label_type",":","Name","of","the","gold","labels","to","use",".","max_sequence_length_dependent_on_input",":","If","set","to","True",",","the","maximum","length","of","a","decoded","sequence","in","the","prediction","depends","on","the","sentences","you","want","to","lemmatize",".","To","be","precise","the","maximum","length","is","computed","as","the","length","of","the","longest","token","in","the","sentences","plus","one",".","max_sequence_length",":","If","set","to","True","and","max_sequence_length_dependend_on_input","is","False","a","fixed","maximum","length","for","the","decoding","will","be","used","for","all","sentences",".","use_attention",":","whether","to","use","attention",".","Only","sensible","if","encoding","via","RNN"]
code: def __init__(
      self,
      embeddings: Optional[flair.embeddings.TokenEmbeddings] = None,
      label_type: str = "lemma",
      rnn_input_size: int = 50,
      rnn_hidden_size: int = 256,
      rnn_layers: int = 2,
      encode_characters: bool = True,
      char_dict: Union[str, Dictionary] = "common-chars-lemmatizer",
      max_sequence_length_dependent_on_input: bool = True,
      max_sequence_length: int = 20,
      use_attention: bool = True,
      beam_size: int = 1,
      start_symbol_for_encoding: bool = True,
      end_symbol_for_encoding: bool = True,
      bidirectional_encoding: bool = True,
  ) -> None:
      """Initializes a Lemmatizer model.
  
      The model consists of a decoder and an encoder. The encoder is either a RNN-cell (torch.nn.GRU)
      or a Token-Embedding from flair if an embedding is handed to the constructor (token_embedding).
      The output of the encoder is used as the initial hidden state to the decoder, which is an RNN-cell (GRU)
      that predicts the lemma of the given token one letter at a time.
      Note that one can use data in which only those words are annotated that differ from their lemma or data
      in which all words are annotated with a (maybe equal) lemma.
  
      Args:
          encode_characters: If True, use a character embedding to additionally encode tokens per character.
          start_symbol_for_encoding: If True, use a start symbol for encoding characters.
          end_symbol_for_encoding: If True, use an end symbol for encoding characters.
          bidirectional_encoding: If True, the character encoding is bidirectional.
          embeddings: Embedding used to encode sentence
          rnn_input_size: Input size of the RNN('s). Each letter of a token is represented by a hot-one-vector over
              the given character dictionary. This vector is transformed to a input_size vector with a linear layer.
          rnn_hidden_size: size of the hidden state of the RNN('s).
          rnn_layers: Number of stacked RNN cells
          beam_size: Number of hypothesis used when decoding the output of the RNN. Only used in prediction.
          char_dict: Dictionary of characters the model is able to process. The dictionary must contain <unk> for
              the handling of unknown characters. If None, a standard dictionary will be loaded. One can either hand
              over a path to a dictionary or the dictionary itself.
          label_type: Name of the gold labels to use.
          max_sequence_length_dependent_on_input: If set to True, the maximum length of a decoded sequence in
              the prediction depends on the sentences you want to lemmatize. To be precise the maximum length is
              computed as the length of the longest token in the sentences plus one.
          max_sequence_length: If set to True and max_sequence_length_dependend_on_input is False a fixed
              maximum length for the decoding will be used for all sentences.
          use_attention: whether to use attention. Only sensible if encoding via RNN
      """
      super().__init__()
  
      self._label_type = label_type
      self.beam_size = beam_size
      self.max_sequence_length = max_sequence_length
      self.dependent_on_input = max_sequence_length_dependent_on_input
      self.start_symbol = start_symbol_for_encoding
      self.end_symbol = end_symbol_for_encoding
      self.bi_encoding = bidirectional_encoding
      self.rnn_hidden_size = rnn_hidden_size
  
      # whether to encode characters and whether to use attention (attention can only be used if chars are encoded)
      self.encode_characters = encode_characters
      self.use_attention = use_attention
      if not self.encode_characters:
          self.use_attention = False
  
      # character dictionary for decoding and encoding
      self.char_dictionary = (
          char_dict
          if isinstance(char_dict, Dictionary)
          else Dictionary.load(char_dict)
      )
  
      # make sure <unk> is in dictionary for handling of unknown characters
      if not self.char_dictionary.add_unk:
          raise KeyError("<unk> must be contained in char_dict")
  
      # add special symbols to dictionary if necessary and save respective indices
      self.dummy_index = self.char_dictionary.add_item("<>")
      self.start_index = self.char_dictionary.add_item("<S>")
      self.end_index = self.char_dictionary.add_item("<E>")
  
      # ---- ENCODER ----
      # encoder character embeddings
      self.encoder_character_embedding = nn.Embedding(
          len(self.char_dictionary), rnn_input_size
      )
  
      # encoder pre-trained embeddings
      self.encoder_embeddings = embeddings
  
      hidden_input_size = 0
      if embeddings:
          hidden_input_size += embeddings.embedding_length
      if encode_characters:
          hidden_input_size += rnn_hidden_size
      if encode_characters and bidirectional_encoding:
          hidden_input_size += rnn_hidden_size
      self.emb_to_hidden = nn.Linear(hidden_input_size, rnn_hidden_size)
  
      # encoder RNN
      self.encoder_rnn = nn.GRU(
          input_size=rnn_input_size,
          hidden_size=self.rnn_hidden_size,
          batch_first=True,
          num_layers=rnn_layers,
          bidirectional=self.bi_encoding,
      )
  
      # additional encoder linear layer if bidirectional encoding
      if self.bi_encoding:
          self.bi_hidden_states_to_hidden_size: Optional[
              nn.Linear
          ] = nn.Linear(
              2 * self.rnn_hidden_size, self.rnn_hidden_size, bias=False
          )
      else:
          self.bi_hidden_states_to_hidden_size = None
  
      # ---- DECODER ----
      # decoder: linear layers to transform vectors to and from alphabet_size
      self.decoder_character_embedding = nn.Embedding(
          len(self.char_dictionary), rnn_input_size
      )
  
      # when using attention we concatenate attention outcome and decoder hidden states
      self.character_decoder = nn.Linear(
          2 * self.rnn_hidden_size
          if self.use_attention
          else self.rnn_hidden_size,
          len(self.char_dictionary),
      )
  
      # decoder RNN
      self.rnn_input_size = rnn_input_size
      self.rnn_layers = rnn_layers
  
      self.decoder_rnn = nn.GRU(
          input_size=rnn_input_size,
          hidden_size=self.rnn_hidden_size,
          batch_first=True,
          num_layers=rnn_layers,
      )
  
      # loss and softmax
      self.loss = nn.CrossEntropyLoss(reduction="sum")
      # self.unreduced_loss = nn.CrossEntropyLoss(reduction='none')  # for prediction
      self.softmax = nn.Softmax(dim=2)
  
      self.to(flair.device)
  
imports: import logging
  from math import inf
  from typing import List, Optional, Tuple, Union
  import torch
  from torch import nn
  import flair.embeddings
  import flair.nn
  from flair.data import Dictionary, Sentence, Token
  from flair.datasets import DataLoader, FlairDatapointDataset
  from flair.training_utils import Result, store_embeddings
label_desc: Use image node_id 1 to create a new Lemmatizer object from inherited base classes: flair.nn.Classifier with example: obj = Lemmatizer(embeddings, label_type, rnn_input_size, rnn_hidden_size, rnn_layers, encode_characters, char_dict, max_sequence_length_dependent_on_input, max_sequence_length, use_attention, beam_size, start_symbol_for_encoding, end_symbol_for_encoding, bidirectional_encoding)