scripts/parse_nagim.py
def _run_multiqc(
    samples: List[Sample],
    html_fpath: str,
    json_fpath: str,
    overwrite: bool = False,
) -> str:
    """
    Runs MultiQC on QC files from Picard and VerifyBAMID.

    Generates an HTML report and puts in into nagim web bucket.

    Generates a JSON with metrics, extracts useful metrics into another JSON
    indexed by sample, and returns path to this JSON.
    """
    tmp_bucket = f'gs://cpg-{NAGIM_PROJ_ID}-{NAMESPACE}-tmp/qc'

    row_by_sample_json_path = f'{tmp_bucket}/parsed-qc.json'
    if can_reuse(row_by_sample_json_path, overwrite):
        return row_by_sample_json_path

    b = setup_batch(
        title='Run MultiQC on NAGIM',
        keep_scratch=False,
        tmp_bucket=f'cpg-{NAGIM_PROJ_ID}-main-tmp',
        analysis_project_name=NAGIM_PROJ_ID,
    )

    if not can_reuse([json_fpath, html_fpath], overwrite):
        j = b.new_job('Run MultiQC')
        j.image(DRIVER_IMAGE)

        qc_endings = set()
        qc_paths = []
        for s in samples:
            for qc_ending, qc_path in s.qc_files.items():
                qc_paths.append(qc_path)
                qc_endings.add(qc_ending)

        file_list_path = f'{tmp_bucket}/multiqc-file-list.txt'
        df = pd.DataFrame({'_': path} for path in qc_paths)
        df.to_csv(file_list_path, header=None, index=None)
        file_list = b.read_input(file_list_path)

        j.env('GOOGLE_APPLICATION_CREDENTIALS', '/gsa-key/key.json')
        j.command(f'pip install multiqc')
        j.cpu(16)
        j.storage('100G')
        j.command(
            f'gcloud -q auth activate-service-account --key-file=$GOOGLE_APPLICATION_CREDENTIALS'
        )
        j.command(f'mkdir inputs')
        j.command(f'cat {file_list} | gsutil -m cp -I inputs/')

        ending_list = ', '.join(f'.{ending}' for ending in qc_endings)
        mqc_conf = f'extra_fn_clean_exts: [{ending_list}]'
        j.command(
            f'multiqc inputs -o output -f --fn_as_s_name --cl_config "{mqc_conf}"'
        )
        j.command(f'cp output/multiqc_report.html {j.report_html}')
        j.command(f'cp output/multiqc_data/multiqc_data.json {j.json}')

        b.write_output(j.report_html, html_fpath)
        b.write_output(j.json, json_fpath)
        logger.info(f'Written MultiQC reports to {html_fpath}')
        multiqc_json = j.json
    else:
        multiqc_json = b.read_input(json_fpath)

    def _parse_multiqc_json(json_fpath) -> Dict:
        with open(json_fpath) as f:
            d = json.load(f)
        row_by_sample = {}
        for tool_d in d['report_general_stats_data']:
            for sample, val_by_metric in tool_d.items():
                if sample not in row_by_sample:
                    row_by_sample[sample] = dict(s=sample)
                row = row_by_sample[sample]
                for metric, multiqc_metric in QC_METRICS:
                    if multiqc_metric in val_by_metric:
                        row[metric] = val_by_metric[multiqc_metric]
        return row_by_sample

    parse_j = b.new_python_job('Parse MultiQC JSON')
    row_by_sample_resource = parse_j.call(_parse_multiqc_json, multiqc_json)
    b.write_output(row_by_sample_resource.as_json(), row_by_sample_json_path)
    b.run(wait=True)
    return row_by_sample_json_path

    Runs MultiQC on QC files from Picard and VerifyBAMID.

    Generates an HTML report and puts in into nagim web bucket.

    Generates a JSON with metrics, extracts useful metrics into another JSON
    indexed by sample, and returns path to this JSON.
    
Runs MultiQC on QC files from Picard and VerifyBAMID.
Generates an HTML report and puts in into nagim web bucket.

Generates a JSON with metrics, extracts useful metrics into another JSON
indexed by sample, and returns path to this JSON.
Runs MultiQC on QC files from Picard and VerifyBAMID . Generates an HTML report and puts in into nagim web bucket . Generates a JSON with metrics extracts useful metrics into another JSON indexed by sample and returns path to this JSON .
def _run_multiqc(
    samples: List[Sample],
    html_fpath: str,
    json_fpath: str,
    overwrite: bool = False,
) -> str:
    tmp_bucket = f'gs://cpg-{NAGIM_PROJ_ID}-{NAMESPACE}-tmp/qc'
    row_by_sample_json_path = f'{tmp_bucket}/parsed-qc.json'
    if can_reuse(row_by_sample_json_path, overwrite):
        return row_by_sample_json_path
    b = setup_batch(
        title='Run MultiQC on NAGIM',
        keep_scratch=False,
        tmp_bucket=f'cpg-{NAGIM_PROJ_ID}-main-tmp',
        analysis_project_name=NAGIM_PROJ_ID,
    )
    if not can_reuse([json_fpath, html_fpath], overwrite):
        j = b.new_job('Run MultiQC')
        j.image(DRIVER_IMAGE)
        qc_endings = set()
        qc_paths = []
        for s in samples:
            for qc_ending, qc_path in s.qc_files.items():
                qc_paths.append(qc_path)
                qc_endings.add(qc_ending)
        file_list_path = f'{tmp_bucket}/multiqc-file-list.txt'
        df = pd.DataFrame({'_': path} for path in qc_paths)
        df.to_csv(file_list_path, header=None, index=None)
        file_list = b.read_input(file_list_path)
        j.env('GOOGLE_APPLICATION_CREDENTIALS', '/gsa-key/key.json')
        j.command(f'pip install multiqc')
        j.cpu(16)
        j.storage('100G')
        j.command(
            f'gcloud -q auth activate-service-account --key-file=$GOOGLE_APPLICATION_CREDENTIALS'
        )
        j.command(f'mkdir inputs')
        j.command(f'cat {file_list} | gsutil -m cp -I inputs/')
        ending_list = ', '.join(f'.{ending}' for ending in qc_endings)
        mqc_conf = f'extra_fn_clean_exts: [{ending_list}]'
        j.command(
            f'multiqc inputs -o output -f --fn_as_s_name --cl_config "{mqc_conf}"'
        )
        j.command(f'cp output/multiqc_report.html {j.report_html}')
        j.command(f'cp output/multiqc_data/multiqc_data.json {j.json}')
        b.write_output(j.report_html, html_fpath)
        b.write_output(j.json, json_fpath)
        logger.info(f'Written MultiQC reports to {html_fpath}')
        multiqc_json = j.json
    else:
        multiqc_json = b.read_input(json_fpath)
    def _parse_multiqc_json(json_fpath) -> Dict:
        with open(json_fpath) as f:
            d = json.load(f)
        row_by_sample = {}
        for tool_d in d['report_general_stats_data']:
            for sample, val_by_metric in tool_d.items():
                if sample not in row_by_sample:
                    row_by_sample[sample] = dict(s=sample)
                row = row_by_sample[sample]
                for metric, multiqc_metric in QC_METRICS:
                    if multiqc_metric in val_by_metric:
                        row[metric] = val_by_metric[multiqc_metric]
        return row_by_sample
    parse_j = b.new_python_job('Parse MultiQC JSON')
    row_by_sample_resource = parse_j.call(_parse_multiqc_json, multiqc_json)
    b.write_output(row_by_sample_resource.as_json(), row_by_sample_json_path)
    b.run(wait=True)
    return row_by_sample_json_path
def _run_multiqc ( samples : List [ Sample ] , html_fpath : str , json_fpath : str , overwrite : bool = False , ) -> str : tmp_bucket = f'gs://cpg-{NAGIM_PROJ_ID}-{NAMESPACE}-tmp/qc' row_by_sample_json_path = f'{tmp_bucket}/parsed-qc.json' if can_reuse ( row_by_sample_json_path , overwrite ) : return row_by_sample_json_path b = setup_batch ( title = 'Run MultiQC on NAGIM' , keep_scratch = False , tmp_bucket = f'cpg-{NAGIM_PROJ_ID}-main-tmp' , analysis_project_name = NAGIM_PROJ_ID , ) if not can_reuse ( [ json_fpath , html_fpath ] , overwrite ) : j = b . new_job ( 'Run MultiQC' ) j . image ( DRIVER_IMAGE ) qc_endings = set ( ) qc_paths = [ ] for s in samples : for qc_ending , qc_path in s . qc_files . items ( ) : qc_paths . append ( qc_path ) qc_endings . add ( qc_ending ) file_list_path = f'{tmp_bucket}/multiqc-file-list.txt' df = pd . DataFrame ( { '_' : path } for path in qc_paths ) df . to_csv ( file_list_path , header = None , index = None ) file_list = b . read_input ( file_list_path ) j . env ( 'GOOGLE_APPLICATION_CREDENTIALS' , '/gsa-key/key.json' ) j . command ( f'pip install multiqc' ) j . cpu ( 16 ) j . storage ( '100G' ) j . command ( f'gcloud -q auth activate-service-account --key-file=$GOOGLE_APPLICATION_CREDENTIALS' ) j . command ( f'mkdir inputs' ) j . command ( f'cat {file_list} | gsutil -m cp -I inputs/' ) ending_list = ', ' . join ( f'.{ending}' for ending in qc_endings ) mqc_conf = f'extra_fn_clean_exts: [{ending_list}]' j . command ( f'multiqc inputs -o output -f --fn_as_s_name --cl_config "{mqc_conf}"' ) j . command ( f'cp output/multiqc_report.html {j.report_html}' ) j . command ( f'cp output/multiqc_data/multiqc_data.json {j.json}' ) b . write_output ( j . report_html , html_fpath ) b . write_output ( j . json , json_fpath ) logger . info ( f'Written MultiQC reports to {html_fpath}' ) multiqc_json = j . json else : multiqc_json = b . read_input ( json_fpath ) def _parse_multiqc_json ( json_fpath ) -> Dict : with open ( json_fpath ) as f : d = json . load ( f ) row_by_sample = { } for tool_d in d [ 'report_general_stats_data' ] : for sample , val_by_metric in tool_d . items ( ) : if sample not in row_by_sample : row_by_sample [ sample ] = dict ( s = sample ) row = row_by_sample [ sample ] for metric , multiqc_metric in QC_METRICS : if multiqc_metric in val_by_metric : row [ metric ] = val_by_metric [ multiqc_metric ] return row_by_sample parse_j = b . new_python_job ( 'Parse MultiQC JSON' ) row_by_sample_resource = parse_j . call ( _parse_multiqc_json , multiqc_json ) b . write_output ( row_by_sample_resource . as_json ( ) , row_by_sample_json_path ) b . run ( wait = True ) return row_by_sample_json_path
Runs MultiQC on QC files from Picard and VerifyBAMID.
Runs MultiQC on QC files from Picard and VerifyBAMID .
"""
    Runs MultiQC on QC files from Picard and VerifyBAMID.

    Generates an HTML report and puts in into nagim web bucket.

    Generates a JSON with metrics, extracts useful metrics into another JSON
    indexed by sample, and returns path to this JSON.
    """
samples
List[Sample]
html_fpath
str
json_fpath
str
overwrite
bool