docstr: Initializes the trainer.
  
  Args:
      max_input_len (output): Maximum length of input sequences, longer sequences will be truncated.
      max_output_len (output): Maximum length of output sequences, longer sequences will be truncated.
      batch_size: Size of mini-batches for stochastic gradient descent.
      max_vocab_size_encoder: Maximum number of unique tokens to consider for encoder embeddings.
      max_vocab_size_decoder: Maximum number of unique tokens to consider for decoder embeddings.
      embedding_path_encoder: Path to embedding file for the encoder.
      embedding_path_decoder: Path to embedding file for the decoder.
      steps_per_epoch: Number of steps to train until callbacks are invoked.
      tensorboard_dir: Directory for saving tensorboard logs.
      model_save_path: Directory for saving the best model.
      shuffle_buffer_size: Size of the buffer for shuffling the files before batching.
      use_bucketing: Whether to bucket the sequences by length to reduce the amount of padding.
      bucketing_buffer_size_batches: Number of batches to buffer when bucketing sequences.
      bucketing_batches_to_bucket: Number of buffered batches from which sequences are collected for bucketing.
      logging_level: Level of logging to use, e.g. logging.INFO or logging.DEBUG.
      num_print_predictions: Number of sample predictions to print in each evaluation.
      steps_to_log: Number of steps to wait for logging output.
      preprocessor (optional): custom preprocessor, if None a standard preprocessor will be created.
docstr_tok: ["Initializes","the","trainer",".","Args",":","max_input_len","(","output",")",":","Maximum","length","of","input","sequences",",","longer","sequences","will","be","truncated",".","max_output_len","(","output",")",":","Maximum","length","of","output","sequences",",","longer","sequences","will","be","truncated",".","batch_size",":","Size","of","mini-batches","for","stochastic","gradient","descent",".","max_vocab_size_encoder",":","Maximum","number","of","unique","tokens","to","consider","for","encoder","embeddings",".","max_vocab_size_decoder",":","Maximum","number","of","unique","tokens","to","consider","for","decoder","embeddings",".","embedding_path_encoder",":","Path","to","embedding","file","for","the","encoder",".","embedding_path_decoder",":","Path","to","embedding","file","for","the","decoder",".","steps_per_epoch",":","Number","of","steps","to","train","until","callbacks","are","invoked",".","tensorboard_dir",":","Directory","for","saving","tensorboard","logs",".","model_save_path",":","Directory","for","saving","the","best","model",".","shuffle_buffer_size",":","Size","of","the","buffer","for","shuffling","the","files","before","batching",".","use_bucketing",":","Whether","to","bucket","the","sequences","by","length","to","reduce","the","amount","of","padding",".","bucketing_buffer_size_batches",":","Number","of","batches","to","buffer","when","bucketing","sequences",".","bucketing_batches_to_bucket",":","Number","of","buffered","batches","from","which","sequences","are","collected","for","bucketing",".","logging_level",":","Level","of","logging","to","use",",","e.g",".","logging.INFO","or","logging.DEBUG",".","num_print_predictions",":","Number","of","sample","predictions","to","print","in","each","evaluation",".","steps_to_log",":","Number","of","steps","to","wait","for","logging","output",".","preprocessor","(","optional",")",":","custom","preprocessor",",","if","None","a","standard","preprocessor","will","be","created","."]
code: def __init__(
      self,
      max_input_len=None,
      max_output_len=None,
      batch_size=16,
      max_vocab_size_encoder=200000,
      max_vocab_size_decoder=200000,
      embedding_path_encoder=None,
      embedding_path_decoder=None,
      steps_per_epoch=500,
      tensorboard_dir=None,
      model_save_path=None,
      shuffle_buffer_size=100000,
      use_bucketing=False,
      bucketing_buffer_size_batches=10000,
      bucketing_batches_to_bucket=100,
      logging_level=logging.INFO,
      num_print_predictions=5,
      steps_to_log=10,
      preprocessor: Union[Preprocessor, None] = None,
  ) -> None:
      """
      Initializes the trainer.
  
      Args:
          max_input_len (output): Maximum length of input sequences, longer sequences will be truncated.
          max_output_len (output): Maximum length of output sequences, longer sequences will be truncated.
          batch_size: Size of mini-batches for stochastic gradient descent.
          max_vocab_size_encoder: Maximum number of unique tokens to consider for encoder embeddings.
          max_vocab_size_decoder: Maximum number of unique tokens to consider for decoder embeddings.
          embedding_path_encoder: Path to embedding file for the encoder.
          embedding_path_decoder: Path to embedding file for the decoder.
          steps_per_epoch: Number of steps to train until callbacks are invoked.
          tensorboard_dir: Directory for saving tensorboard logs.
          model_save_path: Directory for saving the best model.
          shuffle_buffer_size: Size of the buffer for shuffling the files before batching.
          use_bucketing: Whether to bucket the sequences by length to reduce the amount of padding.
          bucketing_buffer_size_batches: Number of batches to buffer when bucketing sequences.
          bucketing_batches_to_bucket: Number of buffered batches from which sequences are collected for bucketing.
          logging_level: Level of logging to use, e.g. logging.INFO or logging.DEBUG.
          num_print_predictions: Number of sample predictions to print in each evaluation.
          steps_to_log: Number of steps to wait for logging output.
          preprocessor (optional): custom preprocessor, if None a standard preprocessor will be created.
      """
  
      self.max_input_len = max_input_len
      self.max_output_len = max_output_len
      self.batch_size = batch_size
      self.max_vocab_size_encoder = max_vocab_size_encoder
      self.max_vocab_size_decoder = max_vocab_size_decoder
      self.bucketing_buffer_size_batches = bucketing_buffer_size_batches
      self.bucketing_batches_to_bucket = bucketing_batches_to_bucket
      self.embedding_path_encoder = embedding_path_encoder
      self.embedding_path_decoder = embedding_path_decoder
      self.steps_per_epoch = steps_per_epoch
      self.tensorboard_dir = tensorboard_dir
      self.model_save_path = model_save_path
      self.loss_function = masked_crossentropy
      self.use_bucketing = use_bucketing
      self.shuffle_buffer_size = (
          None if use_bucketing else shuffle_buffer_size
      )
  
      self.bucket_generator = None
      if use_bucketing:
          self.bucket_generator = BucketGenerator(
              element_length_function=lambda vecs: len(vecs[0]),
              batch_size=self.batch_size,
              buffer_size_batches=self.bucketing_buffer_size_batches,
              batches_to_bucket=self.bucketing_batches_to_bucket,
              shuffle=True,
              seed=42,
          )
      self.logger = get_logger(__name__)
      self.logger.setLevel(logging_level)
      self.num_print_predictions = num_print_predictions
      self.steps_to_log = steps_to_log
      self.preprocessor = preprocessor or Preprocessor(
          start_token=START_TOKEN, end_token=END_TOKEN
      )
  
imports: import datetime
  import logging
  import os
  import tempfile
  from collections import Counter
  from typing import Tuple, List, Iterable, Callable, Dict, Union
  import tensorflow
  import yaml
  from headliner.callbacks.evaluation_callback import EvaluationCallback
  from headliner.callbacks.model_checkpoint_callback import ModelCheckpointCallback
  from headliner.callbacks.tensorboard_callback import TensorboardCallback
  from headliner.callbacks.validation_callback import ValidationCallback
  from headliner.embeddings import read_embedding, embedding_to_matrix
  from headliner.evaluation.scorer import Scorer
  from headliner.losses import masked_crossentropy
  from headliner.model.bert_summarizer import BertSummarizer
  from headliner.model.summarizer import Summarizer
  from headliner.preprocessing.bucket_generator import BucketGenerator
  from headliner.preprocessing.dataset_generator import DatasetGenerator
  from headliner.preprocessing.keras_tokenizer import KerasTokenizer
  from headliner.preprocessing.preprocessor import Preprocessor
  from headliner.preprocessing.vectorizer import Vectorizer
  from headliner.utils.logger import get_logger
label_desc: Use image node_id 1 to create a new Trainer object  with example: obj = Trainer(max_input_len, max_output_len, batch_size, max_vocab_size_encoder, max_vocab_size_decoder, embedding_path_encoder, embedding_path_decoder, steps_per_epoch, tensorboard_dir, model_save_path, shuffle_buffer_size, use_bucketing, bucketing_buffer_size_batches, bucketing_batches_to_bucket, logging_level, num_print_predictions, steps_to_log, preprocessor)