"""
Loader settings:
we use the GMQL scala class CombineTextFileWithPathInputFormat in order to load the region and metadata files with the
same id_sample based on the hash of the file name
"""

inputFormatClass = 'it.polimi.genomics.spark.implementation.loaders.Loaders$CombineTextFileWithPathInputFormat'
keyFormatClass = 'org.apache.hadoop.io.LongWritable'
valueFormatClass = 'org.apache.hadoop.io.Text'
defaultCombineSize = 64

# configuration of the Hadoop loader used by spark
conf = {"textinputformat.record.delimiter": "\n",
        "mapreduce.input.fileinputformat.input.dir.recursive": "true",
        "mapred.max.split.size": str(defaultCombineSize*1024*1024)
        }


def generateHashKey(filename):
    return hash(filename)


def generateNameKey(filename):
    if filename.endswith(".meta"):
        return filename[:-5]
    else:
        return filename

import os

"""
Loader settings:
we use the GMQL scala class CombineTextFileWithPathInputFormat in order to load the region and metadata files with the
same id_sample based on the hash of the file name
"""

inputFormatClass = 'it.polimi.genomics.spark.implementation.loaders.Loaders$CombineTextFileWithPathInputFormat'
keyFormatClass = 'org.apache.hadoop.io.LongWritable'
valueFormatClass = 'org.apache.hadoop.io.Text'
defaultCombineSize = 64

# configuration of the Hadoop loader used by spark
conf = {"textinputformat.record.delimiter": "\n",
        "mapreduce.input.fileinputformat.input.dir.recursive": "true",
        "mapred.max.split.size": str(defaultCombineSize*1024*1024)
        }


"""
    Generation of the index of the pandas dataframe.
    This can be done in different ways:
    - hashing the complete file name
    - using directly the file as index (this is visually appealing :) )
"""

def generateHashKey(filename):
    return hash(filename)


def generateNameKey(filename):
    filename = os.path.basename(filename)
    if filename.endswith(".meta"):
        return filename[:-5]
    else:
        return filename

Use only the base file name for key
Use only the base file name for key


Former-commit-id: 3acbf9e93d3d501013e2a1b6aa9631bdcc663c66 [formerly eea949727d3693aa032033d84dcca3790d9072dd] [formerly ca065bc9f78b1416903179418d64b3273d437987]
Former-commit-id: 58c1d18e8960ad3236a34b8080b18ccff5684eaa
Former-commit-id: 6501068cea164df37ec055c3fb8baf8c89fc7823