docstr: Create a dataset from a list of files
  
  Images or texts may be empty to create a dataset for prediction or evaluation only.
docstr_tok: ["Create","a","dataset","from","a","list","of","files","Images","or","texts","may","be","empty","to","create","a","dataset","for","prediction","or","evaluation","only","."]
code: def __init__(
      self,
      mode: PipelineMode,
      params: FileDataParams,
      skip_invalid=False,
      remove_invalid=True,
      non_existing_as_empty=False,
  ):
      """Create a dataset from a list of files
  
      Images or texts may be empty to create a dataset for prediction or evaluation only.
      """
      super().__init__(mode, params)
  
      if mode == PipelineMode.PREDICTION:
          texts = [None] * len(params.images)
      else:
          texts = params.texts
  
      if mode == PipelineMode.TARGETS:
          images = [None] * len(params.texts)
      else:
          images = params.images
  
      for image, text in zip(images, texts):
          try:
              if image is None and text is None:
                  raise Exception(
                      "An empty data point is not allowed. Both image and text file are None"
                  )
  
              img_bn, text_bn = None, None
              if image:
                  img_path, img_fn = os.path.split(image)
                  img_bn, img_ext = split_all_ext(img_fn)
  
                  if (
                      not self.params.non_existing_as_empty
                      and not os.path.exists(image)
                  ):
                      raise Exception(
                          "Image at '{}' must exist".format(image)
                      )
  
              if text:
                  if (
                      not self.params.non_existing_as_empty
                      and not os.path.exists(text)
                  ):
                      raise Exception(
                          "Text file at '{}' must exist".format(text)
                      )
  
                  text_path, text_fn = os.path.split(text)
                  text_bn, text_ext = split_all_ext(text_fn)
  
              if image and text and img_bn != text_bn:
                  raise Exception(
                      "Expected image base name equals text base name but got '{}' != '{}'".format(
                          img_bn, text_bn
                      )
                  )
          except Exception as e:
              logger.exception(e)
              if self.params.skip_invalid:
                  logger.warning(
                      "Exception raised. Invalid data. Skipping invalid example."
                  )
                  continue
              else:
                  raise e
  
          self.add_sample(
              {
                  "image_path": image,
                  "text_path": text,
                  "id": img_bn or text_bn,
                  "base_name": img_bn or text_bn,
              }
          )
  
imports: import codecs
  import os
  from copy import deepcopy
  from dataclasses import dataclass, field
  from typing import List
  import numpy
  import logging
  from paiargparse import pai_dataclass, pai_meta
  from tfaip.data.pipeline.definitions import PipelineMode, INPUT_PROCESSOR
  from calamari_ocr.ocr.dataset.datareader.base import CalamariDataGenerator, CalamariDataGeneratorParams, InputSample, SampleMeta
  from calamari_ocr.utils import split_all_ext, glob_all, keep_files_with_same_file_name
label_desc: Use image node_id 1 to create a new FileDataGenerator object from inherited base classes: CalamariDataGenerator with example: obj = FileDataGenerator(mode, params, skip_invalid, remove_invalid, non_existing_as_empty)