class_docstr: Data loader for models to ingest graph data.
  
  This class is internally used by the model to store the data passed by the user and batch over it during
  training and evaluation, and to obtain filters during evaluation.
  
  It can be used by advanced users to load custom datasets which are large, for performing partitioned training.
  The complete dataset will not get loaded in memory. It will load the data in chunks based on which partition
  is being trained.
  
  Example
  -------
  >>> from ampligraph.datasets import GraphDataLoader, BucketGraphPartitioner
  >>> from ampligraph.datasets.sqlite_adapter import SQLiteAdapter
  >>> from ampligraph.latent_features import ScoringBasedEmbeddingModel
  >>> AMPLIGRAPH_DATA_HOME='/your/path/to/datasets/'
  >>> # Graph loader - loads the data from the file, numpy array, etc and generates batches for iterating
  >>> path_to_training = AMPLIGRAPH_DATA_HOME + 'fb15k-237/train.txt'
  >>> dataset_loader = GraphDataLoader(path_to_training,
  >>>                                  backend=SQLiteAdapter, # type of backend to use
  >>>                                  batch_size=1000,       # batch size to use while iterating over this dataset
  >>>                                  dataset_type='train',  # dataset type
  >>>                                  use_filter=False,      # Whether to use filter or not
  >>>                                  use_indexer=True)      # indicates that the data needs to be mapped to index
  >>>
  >>> # Choose the partitioner - in this case we choose RandomEdges partitioner
  >>> partitioner = BucketGraphPartitioner(dataset_loader, k=3)
  >>> partitioned_model = ScoringBasedEmbeddingModel(eta=2,
  >>>                                                k=50,
  >>>                                                scoring_type='DistMult')
  >>> partitioned_model.compile(optimizer='adam', loss='multiclass_nll')
  >>> partitioned_model.fit(partitioner,            # pass the partitioner object as input to the fit function this will generate data for the model during training
  >>>                       epochs=10)              # number of epochs
  >>> indexer = partitioned_model.data_handler.get_mapper()    # get the mapper from the trained model
  >>> path_to_test = AMPLIGRAPH_DATA_HOME + 'fb15k-237/test.txt'
  >>> dataset_loader_test = GraphDataLoader(path_to_test,
  >>>                                       backend=SQLiteAdapter,                         # type of backend to use
  >>>                                       batch_size=400,                                # batch size to use while iterating over this dataset
  >>>                                       dataset_type='test',                           # dataset type
  >>>                                       use_indexer=indexer                            # mapper to map test concepts to the same indices used during training
  >>>                                       )
  >>> ranks = partitioned_model.evaluate(dataset_loader_test, # pass the dataloader object to generate data for the model during training
  >>>                                    batch_size=400)
  >>> print(ranks)
  [[  85    7]
   [  95    9]
   [1074   22]
   ...
   [ 546   95]
   [9961 7485]
   [1494    2]]
class_docstr_tok: ["Data","loader","for","models","to","ingest","graph","data",".","This","class","is","internally","used","by","the","model","to","store","the","data","passed","by","the","user","and","batch","over","it","during","training","and","evaluation",",","and","to","obtain","filters","during","evaluation",".","It","can","be","used","by","advanced","users","to","load","custom","datasets","which","are","large",",","for","performing","partitioned","training",".","The","complete","dataset","will","not","get","loaded","in","memory",".","It","will","load","the","data","in","chunks","based","on","which","partition","is","being","trained",".","Example","--","--","--","-",">",">",">","from","ampligraph.datasets","import","GraphDataLoader",",","BucketGraphPartitioner",">",">",">","from","ampligraph.datasets.sqlite_adapter","import","SQLiteAdapter",">",">",">","from","ampligraph.latent_features","import","ScoringBasedEmbeddingModel",">",">",">","AMPLIGRAPH_DATA_HOME='\/your\/path\/to\/datasets\/'",">",">",">","#","Graph","loader","-","loads","the","data","from","the","file",",","numpy","array",",","etc","and","generates","batches","for","iterating",">",">",">","path_to_training","=","AMPLIGRAPH_DATA_HOME","+","'fb15k-237\/train.txt'",">",">",">","dataset_loader","=","GraphDataLoader","(","path_to_training",",",">",">",">","backend=SQLiteAdapter",",","#","type","of","backend","to","use",">",">",">","batch_size=1000",",","#","batch","size","to","use","while","iterating","over","this","dataset",">",">",">","dataset_type='train","'",",","#","dataset","type",">",">",">","use_filter=False",",","#","Whether","to","use","filter","or","not",">",">",">","use_indexer=True",")","#","indicates","that","the","data","needs","to","be","mapped","to","index",">",">",">",">",">",">","#","Choose","the","partitioner","-","in","this","case","we","choose","RandomEdges","partitioner",">",">",">","partitioner","=","BucketGraphPartitioner","(","dataset_loader",",","k=3",")",">",">",">","partitioned_model","=","ScoringBasedEmbeddingModel","(","eta=2",",",">",">",">","k=50",",",">",">",">","scoring_type='DistMult","'",")",">",">",">","partitioned_model.compile","(","optimizer='adam","'",",","loss='multiclass_nll","'",")",">",">",">","partitioned_model.fit","(","partitioner",",","#","pass","the","partitioner","object","as","input","to","the","fit","function","this","will","generate","data","for","the","model","during","training",">",">",">","epochs=10",")","#","number","of","epochs",">",">",">","indexer","=","partitioned_model.data_handler.get_mapper","(",")","#","get","the","mapper","from","the","trained","model",">",">",">","path_to_test","=","AMPLIGRAPH_DATA_HOME","+","'fb15k-237\/test.txt'",">",">",">","dataset_loader_test","=","GraphDataLoader","(","path_to_test",",",">",">",">","backend=SQLiteAdapter",",","#","type","of","backend","to","use",">",">",">","batch_size=400",",","#","batch","size","to","use","while","iterating","over","this","dataset",">",">",">","dataset_type='test","'",",","#","dataset","type",">",">",">","use_indexer=indexer","#","mapper","to","map","test","concepts","to","the","same","indices","used","during","training",">",">",">",")",">",">",">","ranks","=","partitioned_model.evaluate","(","dataset_loader_test",",","#","pass","the","dataloader","object","to","generate","data","for","the","model","during","training",">",">",">","batch_size=400",")",">",">",">","print","(","ranks",")","[","[","85","7","]","[","95","9","]","[","1074","22","]","...","[","546","95","]","[","9961","7485","]","[","1494","2","]","]"]
docstr: Initialise persistent/in-memory data storage.
  
  Parameters
  ----------
  data_source: str or np.array or GraphDataLoader or AbstractGraphPartitioner
      File with data (e.g. CSV). Can be a path pointing to the file location, can be data loaded as numpy, a
      `GraphDataLoader` or an `AbstractGraphPartitioner` instance.
  batch_size: int
      Size of batch.
  dataset_type: str
      Kind of data provided (`"train"` | `"test"` | `"valid"`).
  backend: str
      Name of backend class (`NoBackend`, `SQLiteAdapter`) or already initialised backend.
      If `None`, `NoBackend` is used (in-memory processing).
  root_directory: str
       Path to a directory where the database will be created, and the data and mappings will be stored.
       If `None`, the root directory is obtained through the :meth:`tempfile.gettempdir()` method
       (default: `None`).
  use_indexer: bool or DataIndexer
      Flag to tell whether data should be indexed.
      If the DataIndexer object is passed, the mappings defined in the indexer will be reused
      to generate mappings for the current data.
  verbose: bool
      Verbosity.
  remap: bool
      Flag to be used by graph partitioner, indicates whether previously indexed data in partition has to
      be remapped to new indexes (0, <size_of_partition>). It has not to be used with ``use_indexer=True``.
      The new remappings will be persisted.
  name: str
      Name of the partition. This is internally used when the data is partitioned.
  parent: GraphDataLoader
      Parent dataloader. This is internally used when the data is partitioned.
  in_memory: bool
      Persist indexes or not.
  use_filter: bool or dict
      If `True`, current dataset will be used as filter.
      If `dict`, the datasets specified in the dict will be used for filtering.
      If `False`, the true positives will not be filtered from corruptions.
docstr_tok: ["Initialise","persistent\/in-memory","data","storage",".","Parameters","--","--","--","--","--","data_source",":","str","or","np.array","or","GraphDataLoader","or","AbstractGraphPartitioner","File","with","data","(","e.g",".","CSV",")",".","Can","be","a","path","pointing","to","the","file","location",",","can","be","data","loaded","as","numpy",",","a","`","GraphDataLoader","`","or","an","`","AbstractGraphPartitioner","`","instance",".","batch_size",":","int","Size","of","batch",".","dataset_type",":","str","Kind","of","data","provided","(","`","``","train","''","`","|","`","``","test","''","`","|","`","``","valid","''","`",")",".","backend",":","str","Name","of","backend","class","(","`","NoBackend","`",",","`","SQLiteAdapter","`",")","or","already","initialised","backend",".","If","`","None","`",",","`","NoBackend","`","is","used","(","in-memory","processing",")",".","root_directory",":","str","Path","to","a","directory","where","the","database","will","be","created",",","and","the","data","and","mappings","will","be","stored",".","If","`","None","`",",","the","root","directory","is","obtained","through","the",":","meth",":","`","tempfile.gettempdir","(",")","`","method","(","default",":","`","None","`",")",".","use_indexer",":","bool","or","DataIndexer","Flag","to","tell","whether","data","should","be","indexed",".","If","the","DataIndexer","object","is","passed",",","the","mappings","defined","in","the","indexer","will","be","reused","to","generate","mappings","for","the","current","data",".","verbose",":","bool","Verbosity",".","remap",":","bool","Flag","to","be","used","by","graph","partitioner",",","indicates","whether","previously","indexed","data","in","partition","has","to","be","remapped","to","new","indexes","(","0",",","<","size_of_partition",">",")",".","It","has","not","to","be","used","with","``","use_indexer=True","``",".","The","new","remappings","will","be","persisted",".","name",":","str","Name","of","the","partition",".","This","is","internally","used","when","the","data","is","partitioned",".","parent",":","GraphDataLoader","Parent","dataloader",".","This","is","internally","used","when","the","data","is","partitioned",".","in_memory",":","bool","Persist","indexes","or","not",".","use_filter",":","bool","or","dict","If","`","True","`",",","current","dataset","will","be","used","as","filter",".","If","`","dict","`",",","the","datasets","specified","in","the","dict","will","be","used","for","filtering",".","If","`","False","`",",","the","true","positives","will","not","be","filtered","from","corruptions","."]
code: def __init__(
      self,
      data_source,
      batch_size=1,
      dataset_type="train",
      backend=None,
      root_directory=None,
      use_indexer=True,
      verbose=False,
      remap=False,
      name="main_partition",
      parent=None,
      in_memory=False,
      use_filter=False,
  ):
      """Initialise persistent/in-memory data storage.
  
      Parameters
      ----------
      data_source: str or np.array or GraphDataLoader or AbstractGraphPartitioner
          File with data (e.g. CSV). Can be a path pointing to the file location, can be data loaded as numpy, a
          `GraphDataLoader` or an `AbstractGraphPartitioner` instance.
      batch_size: int
          Size of batch.
      dataset_type: str
          Kind of data provided (`"train"` | `"test"` | `"valid"`).
      backend: str
          Name of backend class (`NoBackend`, `SQLiteAdapter`) or already initialised backend.
          If `None`, `NoBackend` is used (in-memory processing).
      root_directory: str
           Path to a directory where the database will be created, and the data and mappings will be stored.
           If `None`, the root directory is obtained through the :meth:`tempfile.gettempdir()` method
           (default: `None`).
      use_indexer: bool or DataIndexer
          Flag to tell whether data should be indexed.
          If the DataIndexer object is passed, the mappings defined in the indexer will be reused
          to generate mappings for the current data.
      verbose: bool
          Verbosity.
      remap: bool
          Flag to be used by graph partitioner, indicates whether previously indexed data in partition has to
          be remapped to new indexes (0, <size_of_partition>). It has not to be used with ``use_indexer=True``.
          The new remappings will be persisted.
      name: str
          Name of the partition. This is internally used when the data is partitioned.
      parent: GraphDataLoader
          Parent dataloader. This is internally used when the data is partitioned.
      in_memory: bool
          Persist indexes or not.
      use_filter: bool or dict
          If `True`, current dataset will be used as filter.
          If `dict`, the datasets specified in the dict will be used for filtering.
          If `False`, the true positives will not be filtered from corruptions.
      """
      self.dataset_type = dataset_type
      self.data_source = data_source
      self.batch_size = batch_size
      if root_directory is None:
          self.root_directory = tempfile.gettempdir()
      else:
          self.root_directory = root_directory
      self.identifier = DataSourceIdentifier(self.data_source)
      self.use_indexer = use_indexer
      self.remap = remap
      self.in_memory = in_memory
      self.name = name
      self.parent = parent
      if use_filter is None or use_filter is True:
          self.use_filter = {"train": data_source}
      else:
          if isinstance(use_filter, dict) or use_filter is False:
              self.use_filter = use_filter
          else:
              msg = "use_filter should be a dictionary with keys as names of filters and \
                  values as data sources, instead got {}".format(
                  use_filter
              )
              logger.error(msg)
              raise Exception(msg)
      if bool(use_indexer) != (not remap):
          msg = "Either remap or Indexer should be specified at the same time."
          logger.error(msg)
          raise Exception(msg)
      if isinstance(backend, type) and backend != NoBackend:
          self.backend = backend(
              "database_{}_{}.db".format(
                  datetime.now().strftime("%d-%m-%Y_%I-%M-%S_%f_%p"),
                  str(uuid.uuid4()),
              ),
              identifier=self.identifier,
              root_directory=self.root_directory,
              use_indexer=self.use_indexer,
              remap=self.remap,
              name=self.name,
              parent=self.parent,
              in_memory=self.in_memory,
              verbose=verbose,
              use_filter=self.use_filter,
          )
          logger.debug(
              "Initialized Backend with database at: {}".format(
                  self.backend.db_path
              )
          )
  
      elif backend is None or backend == NoBackend:
          self.backend = NoBackend(
              self.identifier,
              use_indexer=self.use_indexer,
              remap=self.remap,
              name=self.name,
              parent=self.parent,
              in_memory=self.in_memory,
              use_filter=self.use_filter,
          )
      else:
          self.backend = backend
  
      self.backend._load(
          self.data_source, dataset_type=self.dataset_type
      )
      self.data_shape = self.backend.data_shape
      self.batch_iterator = self.get_batch_generator(
          use_filter=self.use_filter, dataset_type=self.dataset_type
      )
      self.metadata = self.backend.mapper.metadata
  
imports: import logging
  import tempfile
  import uuid
  from datetime import datetime
  import numpy
  import pandas
  import tensorflow
  from .data_indexer import DataIndexer
  from .source_identifier import DataSourceIdentifier
label_desc: Use image node_id 1 to create a new GraphDataLoader object  with example: obj = GraphDataLoader(data_source, batch_size, dataset_type, backend, root_directory, use_indexer, verbose, remap, name, parent, in_memory, use_filter)