code: def get_dataset_from_generator(
      self, generator_fn, input_mode, update_hook=None
  ):
      def chunked_and_tokenized_dataset():
          for d in generator_fn():
              yield from self.text_to_tokens_mask(**d)
  
      types, shapes = self.feed_shape_type_def()
  
      if input_mode == InputMode.PREDICT:
          tqdm_mode = "predict"
      else:
          tqdm_mode = "train"
  
      if input_mode == InputMode.PREDICT or not has_targets(
          generator_fn
      ):
          types = types[0]
          shapes = shapes[0]
  
      raw_dataset = self.make_dataset_fn(
          data_fn=chunked_and_tokenized_dataset,
          tqdm_mode=tqdm_mode,
          update_hook=update_hook,
          types=types,
          shapes=shapes,
          skip_val=input_mode == InputMode.TRAIN,
      )
      if input_mode == InputMode.PREDICT:
          return {
              "predict_dataset": batch_dataset(
                  raw_dataset,
                  batch_size=self.config.predict_batch_size,
                  max_length=self.config.max_length,
                  table_batching=False,  # We cannot use table batching here because the order of the outputs is impacted.
                  shapes=shapes,
              )
          }
  
      if self.config.chunk_long_sequences:
          LOGGER.warning(
              "The dataset size is not adjusted for chunk long sequences when training from a generator"
          )
  
      if self.config.dataset_size is None:
          raise FinetuneError(
              "If you are using a callable as input you must provide config.dataset_size"
          )
  
      if self.config.class_weights is not None:
          raise FinetuneError(
              "Cannot use class weights in generator mode"
          )
  
      (
          self.config.val_size,
          self.config.val_interval,
      ) = validation_settings(
          dataset_size=self.config.dataset_size,
          batch_size=self.config.batch_size,
          val_size=self.config.val_size,
          val_interval=self.config.val_interval,
          keep_best_model=self.config.keep_best_model,
      )
  
      self.config.dataset_size -= self.config.val_size
  
      val_dataset = (
          lambda: raw_dataset()
          .shuffle(
              self.config.shuffle_buffer_size,
              seed=self.config.seed,
              reshuffle_each_iteration=False,
          )
          .take(self.config.val_size)
      )
      train_dataset = (
          lambda: raw_dataset()
          .shuffle(
              self.config.shuffle_buffer_size,
              seed=self.config.seed,
              reshuffle_each_iteration=False,
          )
          .skip(self.config.val_size)
      )
  
      return {
          "train_dataset": batch_dataset(
              train_dataset,
              batch_size=self.config.batch_size,
              max_length=self.config.max_length,
              shapes=shapes,
              n_epochs=self.config.n_epochs,
              shuffle=self.config.reshuffle_chunks,
              table_batching=self.config.table_batching,
              random_seed=self.config.seed,
          ),
          "val_dataset": batch_dataset(
              val_dataset,
              batch_size=self.config.batch_size,
              max_length=self.config.max_length,
              shapes=shapes,
              table_batching=self.config.table_batching,
          ),
      }
  
imports: import itertools
  import logging
  import sys
  import math
  import os
  import warnings
  from collections.abc import Iterable
  from collections import Counter
  from abc import ABCMeta, abstractmethod
  import tqdm
  import numpy
  import pandas
  import tensorflow
  from tensorflow.python.data import Dataset
  from sklearn.model_selection import train_test_split
  from sklearn.preprocessing import LabelBinarizer
  from sklearn.utils import shuffle
  import finetune
  from finetune.errors import FinetuneError
  from finetune.encoding.input_encoder import EncodedOutput, tokenize_context
  from finetune.util.imbalance import compute_class_weights
  from finetune.util.input_utils import InputMode, validation_settings, wrap_tqdm, Chunker, has_targets, batch_dataset
label_desc: Use image node_id 14 for calling the BasePipeline obj's underlying member method code with example usage: obj.get_dataset_from_generator(generator_fn, input_mode, update_hook) and returns: dict, dict