#!/usr/bin/env python3

import os
import sys
import logging
import subprocess


logging.basicConfig(level=logging.INFO)


root_dir = 'submitted_data'
submitted_file_split = set()

for dir_, _, files in os.walk(root_dir):
    for file_name in files:
        rel_dir = os.path.relpath(dir_, root_dir)
        rel_file = os.path.join(root_dir, rel_dir, file_name)
        submitted_file_split.add(rel_file)


for submitted_file in submitted_file_split:

    if submitted_file.startswith('submitted_data'):
        dir_name, data_type, file_name = submitted_file.split('/')
        out_dir_name = 'processed_data'

        if not os.path.isdir(out_dir_name):
            os.makedirs(out_dir_name, exist_ok=True)
        if not os.path.isdir(out_dir_name + '/' + data_type):
            os.makedirs(out_dir_name + '/' + data_type, exist_ok=True)

        outfile = submitted_file.replace(dir_name, out_dir_name)

        if not os.path.isfile(outfile):
            if not data_type == 'tad':
                from cimr.processor.utils import Infiler
                infile = Infiler(
                    data_type, 
                    submitted_file, 
                    genome_build='b38', 
                    update_rsid=False, 
                    outfile=str(outfile),
                    chunksize=700000
                )
                infile.read_file()

                if data_type == 'eqtl':
                    from cimr.processor.query import Querier
                    genes = list(infile.list_genes())
                    queried = Querier(genes)
                    queried.form_query()

            else:
                logging.info(f' processed file already exists for {submitted_file}')
                logging.info(f' if reprocessing, delete {outfile} and file a new pull request')


[((80, 119), 'logging.basicConfig', 'logging.basicConfig', ([], {'level': 'logging.INFO'}), '(level=logging.INFO)\n', (99, 119), False, 'import logging\n'), ((202, 219), 'os.walk', 'os.walk', (['root_dir'], {}), '(root_dir)\n', (209, 219), False, 'import os\n'), ((267, 298), 'os.path.relpath', 'os.path.relpath', (['dir_', 'root_dir'], {}), '(dir_, root_dir)\n', (282, 298), False, 'import os\n'), ((318, 360), 'os.path.join', 'os.path.join', (['root_dir', 'rel_dir', 'file_name'], {}), '(root_dir, rel_dir, file_name)\n', (330, 360), False, 'import os\n'), ((626, 653), 'os.path.isdir', 'os.path.isdir', (['out_dir_name'], {}), '(out_dir_name)\n', (639, 653), False, 'import os\n'), ((667, 707), 'os.makedirs', 'os.makedirs', (['out_dir_name'], {'exist_ok': '(True)'}), '(out_dir_name, exist_ok=True)\n', (678, 707), False, 'import os\n'), ((723, 768), 'os.path.isdir', 'os.path.isdir', (["(out_dir_name + '/' + data_type)"], {}), "(out_dir_name + '/' + data_type)\n", (736, 768), False, 'import os\n'), ((782, 840), 'os.makedirs', 'os.makedirs', (["(out_dir_name + '/' + data_type)"], {'exist_ok': '(True)'}), "(out_dir_name + '/' + data_type, exist_ok=True)\n", (793, 840), False, 'import os\n'), ((923, 946), 'os.path.isfile', 'os.path.isfile', (['outfile'], {}), '(outfile)\n', (937, 946), False, 'import os\n'), ((1637, 1705), 'logging.info', 'logging.info', (['f""" processed file already exists for {submitted_file}"""'], {}), "(f' processed file already exists for {submitted_file}')\n", (1649, 1705), False, 'import logging\n'), ((1722, 1801), 'logging.info', 'logging.info', (['f""" if reprocessing, delete {outfile} and file a new pull request"""'], {}), "(f' if reprocessing, delete {outfile} and file a new pull request')\n", (1734, 1801), False, 'import logging\n'), ((1546, 1560), 'cimr.processor.query.Querier', 'Querier', (['genes'], {}), '(genes)\n', (1553, 1560), False, 'from cimr.processor.query import Querier\n')]