files.new_contents: #!/bin/sh
  
  BASEDIR=`pwd`
  
  if [ ! -d "$BASEDIR" ]; then
      mkdir "$BASEDIR"
  fi
  
  
  # toy (testing dataset)
  if [ ! -d "$BASEDIR/toy" ]; then
      echo Downloading toy
      cd $BASEDIR
      curl -O https://www.uni-mannheim.de/media/Einrichtungen/dws/pi1/kge_datasets/toy.tar.gz
      tar xvf toy.tar.gz
      python preprocess.py toy
  else
      echo toy already present
  fi
  
  
  # fb15k
  if [ ! -d "$BASEDIR/fb15k" ]; then
    echo Downloading fb15k
    cd $BASEDIR
    curl -O https://www.uni-mannheim.de/media/Einrichtungen/dws/pi1/kge_datasets/fb15k.tar.gz
    tar xvf fb15k.tar.gz
    cd fb15k
    case "$(uname -s)" in
        CYGWIN*|MINGW32*|MSYS*)
            cmd.exe /c mklink train.txt freebase_mtr100_mte100-train.txt
            cmd.exe /c mklink valid.txt freebase_mtr100_mte100-valid.txt
            cmd.exe /c mklink test.txt freebase_mtr100_mte100-test.txt
            ;;
        *)
            ln -s freebase_mtr100_mte100-train.txt train.txt
            ln -s freebase_mtr100_mte100-valid.txt valid.txt
            ln -s freebase_mtr100_mte100-test.txt test.txt
            ;;
    esac
    cd ..
    python preprocess.py fb15k
  else
      echo fb15k already present
  fi
  
  # fb15k-237
  if [ ! -d "$BASEDIR/fb15k-237" ]; then
    echo Downloading fb15k-237
    cd $BASEDIR
    curl -O https://www.uni-mannheim.de/media/Einrichtungen/dws/pi1/kge_datasets/fb15k-237.tar.gz
    tar xvf fb15k-237.tar.gz
    python preprocess.py fb15k-237
  else
      echo fb15
  
  [... the rest is omitted ...]
files.new_path: data/download_all.sh
files.old_contents: #!/bin/sh
  
  BASEDIR=`pwd`
  
  if [ ! -d "$BASEDIR" ]; then
      mkdir "$BASEDIR"
  fi
  
  
  # toy (testing dataset)
  if [ ! -d "$BASEDIR/toy" ]; then
      echo Downloading toy
      cd $BASEDIR
      curl -O https://www.uni-mannheim.de/media/Einrichtungen/dws/pi1/kge_datasets/toy.tar.gz
      tar xvf toy.tar.gz
      python preprocess.py --folder toy
  else
      echo toy already present
  fi
  
  
  # fb15k
  if [ ! -d "$BASEDIR/fb15k" ]; then
    echo Downloading fb15k
    cd $BASEDIR
    curl -O https://www.uni-mannheim.de/media/Einrichtungen/dws/pi1/kge_datasets/fb15k.tar.gz
    tar xvf fb15k.tar.gz
    cd fb15k
    case "$(uname -s)" in
        CYGWIN*|MINGW32*|MSYS*)
            cmd.exe /c mklink train.txt freebase_mtr100_mte100-train.txt
            cmd.exe /c mklink valid.txt freebase_mtr100_mte100-valid.txt
            cmd.exe /c mklink test.txt freebase_mtr100_mte100-test.txt
            ;;
        *)
            ln -s freebase_mtr100_mte100-train.txt train.txt
            ln -s freebase_mtr100_mte100-valid.txt valid.txt
            ln -s freebase_mtr100_mte100-test.txt test.txt
            ;;
    esac
    cd ..
    python preprocess.py --folder fb15k
  else
      echo fb15k already present
  fi
  
  # fb15k-237
  if [ ! -d "$BASEDIR/fb15k-237" ]; then
    echo Downloading fb15k-237
    cd $BASEDIR
    curl -O https://www.uni-mannheim.de/media/Einrichtungen/dws/pi1/kge_datasets/fb15k-237.tar.gz
    tar xvf fb15k-237.tar.gz
    python preprocess.py --folder fb15k-237
  
  
  [... the rest is omitted ...]
files.old_path: data/download_all.sh
files.new_contents: #!/usr/bin/env python
  """Preprocess a KGE dataset into a the format expected by libkge.
  
  Call as `preprocess.py --folder <name>`. The original dataset should be stored in
  subfolder `name` and have files "train.txt", "valid.txt", and "test.txt". Each file
  contains one SPO triple per line, separated by tabs.
  
  During preprocessing, each distinct entity name and each distinct distinct relation name
  is assigned an index (dense). The index-to-object mapping is stored in files
  "entity_map.del" and "relation_map.del", resp. The triples (as indexes) are stored in
  files "train.del", "valid.del", and "test.del". Metadata information is stored in a file
  "dataset.yaml".
  
  """
  
  import argparse
  import yaml
  import os.path
  from collections import OrderedDict
  
  
  def store_map(symbol_map, filename):
      with open(filename, "w") as f:
          for symbol, index in symbol_map.items():
              f.write(f"{index}\t{symbol}\n")
  
  
  if __name__ == "__main__":
      parser = argparse.ArgumentParser()
      parser.add_argument("folder", type=str)
      parser.add_argument("--order_sop", action="store_true")
      args = parser.parse_args()
  
      print(f"Preprocessing {args.folder}...")
      raw_split_files = {"train": "train.txt", "valid": "valid.txt", "test": "test.txt"}
      split_files = {"train": "train.del", "valid": "valid.del", "test": "test.del"}
      split_sizes = {}
  
      if args.order_sop:
          S, P, O = 0, 2, 1
      else:
          S, P, O = 0, 1, 2
  
      # read data and collect entities and relations
      raw = {}
      entities = {}
      relations = {}
      ent_id = 0
      rel_id = 0
      for split, filename in raw_split_files.items():
          with open(args.folder + "/" + filename, "r") as f:
              raw[split] = list(map(lambda s: s.strip().split("\t"), f.readlines()))
              for t in raw[split]:
                  if t[S] not in entities:
                      entities[t[S]] =
  
  [... the rest is omitted ...]
files.new_path: data/preprocess.py
files.old_contents: import argparse
  
  
  def index(symbols, file):
      with open(file, "w") as f:
          for i, k in symbols.items():
              f.write(str(k) + "\t" + str(i) + "\n")
  
  
  if __name__ == "__main__":
      parser = argparse.ArgumentParser()
      parser.add_argument("--folder", type=str)
      parser.add_argument("--order_sop", action="store_true")
      args = parser.parse_args()
  
      print("Preprocessing " + args.folder)
      raw_split_files = {"train": "train.txt", "valid": "valid.txt", "test": "test.txt"}
      split_files = {"train": "train.del", "valid": "valid.del", "test": "test.del"}
  
      sub = 0
      rel = 1
      obj = 2
      if args.order_sop:
          obj = 1
          rel = 2
  
      # read data and collect entity and relation names
      raw = {}
      entities = {}
      relations = {}
      ent_id = 0
      rel_id = 0
      for k, file in raw_split_files.items():
          with open(args.folder + "/" + file, "r") as f:
              raw[k] = list(map(lambda s: s.strip().split("\t"), f.readlines()))
              for t in raw[k]:
                  if t[sub] not in entities:
                      entities[t[sub]] = ent_id
                      ent_id += 1
                  if t[rel] not in relations:
                      relations[t[rel]] = rel_id
                      rel_id += 1
                  if t[obj] not in entities:
                      entities[t[obj]] = ent_id
                      ent_id += 1
              print(str(len(raw[k])) + " triples in " + file)
  
      print(str(len(relations)) + " distinct relations")
      print(str(len(entities)) + " distinct entities")
      print("Writing indexes...")
      index(relations, args.folder + "/relation_map.del")
      index(entities, args.folder + "/entity_map.del")
  
      # write out
      print("Writing triples...")
      for k, file in split_files.items():
          with open(args.folder + "/" + file, "w") as f:
              for t in raw[k
  
  [... the rest is omitted ...]
files.old_path: data/preprocess.py
files.new_contents: # Job-level configuration options.
  job:
    # Type of job to run. Possible values are "train", "eval", and "search". See
    # the corresponding configuration keys for mode information.
    type: train
  
    # Main device to use for this job (e.g., 'cpu', 'cuda', 'cuda:0')
    device: 'cuda'  
  
  # The seeds of the PRNGs can be set manually for (increased) reproducability.
  # Use -1 to use default seed.
  random_seed:
    python: -1
    torch: -1
    numpy: -1
  
  
  ## DATASET #####################################################################
  
  dataset:
    # Specify a dataset here. There must be a folder of that name under "data/".
    # If this folder contains a dataset.yaml file, it overrides the defaults
    # specified below.
    name: 'toy'
  
    # Names of the training, validation, and test data files. Each file has the
    # following fields, all tab-separated:
    # - 0: subject index
    # - 1: relation index
    # - 2: object index
    # - 3-...: arbitrary metadata fields
    # Indexes are assumed to be dense throughout.
    train: train.del
    valid: valid.del
    test: test.del
  
    # Names of entity and relation map files, which store auxiliary information
    # about each entity/predicate. Fields are tab-separated:
    # - 0: entity/predicate index (as in train/valid/test)
    # - 1...: arbitrary metadata fields
    entity_map: entity_map.del
    relation_map: relation_map.del
  
    # other dataset specific keys
    +++: +++
  
  ## MODEL #######################################################################
  
  # Which KGE model to use. Examples include "rescal", "complex", "distmult",
  # "transe", or "conve". For available models, see the project homepage and/or
  # configuration files under kge/models and kge/models/experimental.
  model: ''
  
  
  ## TRAINING ####################################################################
  
  # Options of training jobs (job.type=="train")
  train:
    # Type of training job.
    # - KvsAll: scores each unique sp/po pair along with all possible objectes/subjects
  
  [... the rest is omitted ...]
files.new_path: kge/config-default.yaml
files.old_contents: # Job-level configuration options.
  job:
    # Type of job to run. Possible values are "train", "eval", and "search". See
    # the corresponding configuration keys for mode information.
    type: train
  
    # Main device to use for this job (e.g., 'cpu', 'cuda', 'cuda:0')
    device: 'cuda'  
  
  # The seeds of the PRNGs can be set manually for (increased) reproducability.
  # Use -1 to use default seed.
  random_seed:
    python: -1
    torch: -1
    numpy: -1
  
  
  ## DATASET #####################################################################
  
  dataset:
    # Specify a dataset here. There must be a folder of that name under "data/".
    name: 'toy'
  
    # Names of the training, validation, and test data files. Each file has the
    # following fields, all tab-separated:
    # - 0: subject index
    # - 1: relation index
    # - 2: object index
    # - 3-...: arbitrary metadata fields
    # Indexes are assumed to be dense throughout.
    train: train.del
    valid: valid.del
    test: test.del
  
    # Names of entity and relation map files, which store auxiliary information
    # about each entity/predicate. Fields are tab-separated:
    # - 0: entity/predicate index (as in train/valid/test)
    # - 1...: arbitrary metadata fields
    entity_map: entity_map.del
    relation_map: relation_map.del
  
  
  ## MODEL #######################################################################
  
  # Which KGE model to use. Examples include "rescal", "complex", "distmult",
  # "transe", or "conve". For available models, see the project homepage and/or
  # configuration files under kge/models and kge/models/experimental.
  model: ''
  
  
  ## TRAINING ####################################################################
  
  # Options of training jobs (job.type=="train")
  train:
    # Type of training job.
    # - KvsAll: scores each unique sp/po pair along with all possible objectes/subjects.
    # - negative_sampling: scores each unique spo triple along with sampled corrupted
    #   triples
    # - 1vsAll: scores each
  
  [... the rest is omitted ...]
files.old_path: kge/config-default.yaml
files.new_contents: import csv
  import os
  from collections import defaultdict, OrderedDict
  
  import torch
  import numpy as np
  
  from kge import Config, Configurable
  from kge.util.misc import kge_base_dir
  
  
  # TODO add support to pickle dataset (and indexes) and reload from there
  class Dataset(Configurable):
      def __init__(
          self,
          config,
          num_entities,
          entities,
          num_relations,
          relations,
          train,
          train_meta,
          valid,
          valid_meta,
          test,
          test_meta,
      ):
          super().__init__(config, "dataset")
          self.num_entities = num_entities
          self.entities = entities  # array: entity index -> metadata array of strings
          self.num_relations = num_relations
          self.relations = relations  # array: relation index -> metadata array of strings
          self.train = train  # (n,3) int32 tensor
          self.train_meta = (
              train_meta
          )  # array: triple row number -> metadata array of strings
          self.valid = valid  # (n,3) int32 tensor
          self.valid_meta = (
              valid_meta
          )  # array: triple row number -> metadata array of strings
          self.test = test  # (n,3) int32 tensor
          self.test_meta = (
              test_meta
          )  # array: triple row number -> metadata array of strings
          self.indexes = {}  # map: name of index -> index (used mainly by training jobs)
  
      @staticmethod
      def load(config: Config):
          name = config.get("dataset.name")
          base_dir = os.path.join(kge_base_dir(), "data", name)
          if os.path.isfile(os.path.join(base_dir, "dataset.yaml")):
              config.log("Loading configuration of dataset " + name + "...")
              config.load(os.path.join(base_dir, "dataset.yaml"))
  
          config.log("Loading dataset " + name + "...")
          num_entities, entities = Dataset._load_map(
              os.path.join(base_dir, config.get("dataset.entity_map"))
          )
          num_relations, relations = Dataset._load_map(
              os.path.join(base_dir, config.get("dataset.relation_map"))
          )
  
          train, train_meta = Dataset._load
  
  [... the rest is omitted ...]
files.new_path: kge/dataset.py
files.old_contents: import csv
  import os
  from collections import defaultdict, OrderedDict
  
  import torch
  import numpy as np
  
  from kge.util.misc import kge_base_dir
  
  
  # TODO add support to pickle dataset (and indexes) and reload from there
  class Dataset:
      def __init__(
          self,
          config,
          num_entities,
          entities,
          num_relations,
          relations,
          train,
          train_meta,
          valid,
          valid_meta,
          test,
          test_meta,
      ):
          self.config = config
          self.num_entities = num_entities
          self.entities = entities  # array: entity index -> metadata array of strings
          self.num_relations = num_relations
          self.relations = relations  # array: relation index -> metadata array of strings
          self.train = train  # (n,3) int32 tensor
          self.train_meta = (
              train_meta
          )  # array: triple row number -> metadata array of strings
          self.valid = valid  # (n,3) int32 tensor
          self.valid_meta = (
              valid_meta
          )  # array: triple row number -> metadata array of strings
          self.test = test  # (n,3) int32 tensor
          self.test_meta = (
              test_meta
          )  # array: triple row number -> metadata array of strings
          self.indexes = {}  # map: name of index -> index (used mainly by training jobs)
  
      @staticmethod
      def load(config):
          name = config.get("dataset.name")
          config.log("Loading dataset " + name + "...")
          base_dir = os.path.join(kge_base_dir(), "data/" + name)
  
          num_entities, entities = Dataset._load_map(
              os.path.join(base_dir, config.get("dataset.entity_map"))
          )
          num_relations, relations = Dataset._load_map(
              os.path.join(base_dir, config.get("dataset.relation_map"))
          )
  
          train, train_meta = Dataset._load_triples(
              os.path.join(base_dir, config.get("dataset.train"))
          )
  
          valid, valid_meta = Dataset._load_triples(
              os.path.join(base_dir, config.get("dataset.valid"))
          )
  
          test, test_meta = Dataset._load_triples
  
  [... the rest is omitted ...]
files.old_path: kge/dataset.py