#!/usr/bin/env python3

import os
import sys
import logging
import subprocess


logging.basicConfig(level=logging.INFO)


root_dir = 'submitted_data'
submitted_file_split = set()

for dir_, _, files in os.walk(root_dir):
    for file_name in files:
        rel_dir = os.path.relpath(dir_, root_dir)
        rel_file = os.path.join(root_dir, rel_dir, file_name)
        submitted_file_split.add(rel_file)


for submitted_file in submitted_file_split:

    if submitted_file.startswith('submitted_data'):
        dir_name, data_type, file_name = submitted_file.split('/')
        out_dir_name = 'processed_data'

        if not os.path.isdir(out_dir_name):
            os.makedirs(out_dir_name, exist_ok=True)
        if not os.path.isdir(out_dir_name + '/' + data_type):
            os.makedirs(out_dir_name + '/' + data_type, exist_ok=True)

        outfile = submitted_file.replace(dir_name, out_dir_name)

        if not os.path.isfile(outfile):
            if not data_type == 'tad':
                from cimr.processor.utils import Infiler
                infile = Infiler(
                    data_type, 
                    submitted_file, 
                    genome_build='b38', 
                    update_rsid=False, 
                    outfile=str(outfile),
                    chunksize=700000
                )
                infile.read_file()

                if data_type == 'eqtl':
                    from cimr.processor.query import Querier
                    genes = list(infile.list_genes())
                    queried = Querier(genes)
                    queried.form_query()

            else:
                logging.info(f' processed file already exists for {submitted_file}')
                logging.info(f' if reprocessing, delete {outfile} and file a new pull request')


[((9, 0, 9, 39), 'logging.basicConfig', 'logging.basicConfig', (), '', False, 'import logging\n'), ((15, 22, 15, 39), 'os.walk', 'os.walk', ({(15, 30, 15, 38): 'root_dir'}, {}), '(root_dir)', False, 'import os\n'), ((17, 18, 17, 49), 'os.path.relpath', 'os.path.relpath', ({(17, 34, 17, 38): 'dir_', (17, 40, 17, 48): 'root_dir'}, {}), '(dir_, root_dir)', False, 'import os\n'), ((18, 19, 18, 61), 'os.path.join', 'os.path.join', ({(18, 32, 18, 40): 'root_dir', (18, 42, 18, 49): 'rel_dir', (18, 51, 18, 60): 'file_name'}, {}), '(root_dir, rel_dir, file_name)', False, 'import os\n'), ((28, 15, 28, 42), 'os.path.isdir', 'os.path.isdir', ({(28, 29, 28, 41): 'out_dir_name'}, {}), '(out_dir_name)', False, 'import os\n'), ((29, 12, 29, 52), 'os.makedirs', 'os.makedirs', (), '', False, 'import os\n'), ((30, 15, 30, 60), 'os.path.isdir', 'os.path.isdir', ({(30, 29, 30, 59): "(out_dir_name + '/' + data_type)"}, {}), "(out_dir_name + '/' + data_type)", False, 'import os\n'), ((31, 12, 31, 70), 'os.makedirs', 'os.makedirs', (), '', False, 'import os\n'), ((35, 15, 35, 38), 'os.path.isfile', 'os.path.isfile', ({(35, 30, 35, 37): 'outfile'}, {}), '(outfile)', False, 'import os\n'), ((55, 16, 55, 84), 'logging.info', 'logging.info', ({(55, 29, 55, 83): 'f""" processed file already exists for {submitted_file}"""'}, {}), "(f' processed file already exists for {submitted_file}')", False, 'import logging\n'), ((56, 16, 56, 95), 'logging.info', 'logging.info', ({(56, 29, 56, 94): 'f""" if reprocessing, delete {outfile} and file a new pull request"""'}, {}), "(f' if reprocessing, delete {outfile} and file a new pull request')", False, 'import logging\n'), ((51, 30, 51, 44), 'cimr.processor.query.Querier', 'Querier', ({(51, 38, 51, 43): 'genes'}, {}), '(genes)', False, 'from cimr.processor.query import Querier\n')]