scripts/parse_nagim.py
def parse(
    tmp_dir,
    confirm: bool,
    dry_run: bool,
    overwrite_multiqc: bool,
    skip_checking_objects: bool,
):
    """
    Assuming the data is transferred to the CPG bucket, populate the SM projects.
    """
    if not tmp_dir:
        tmp_dir = tempfile.gettempdir()

    samples = _parse_sample_project_map(SAMPLE_TO_PROJECT_TSV_PATH)

    # Find GVCFs, CRAMs and other files after transferring, and checks that all
    # of them have corresponding tbi/crai/md5.
    _find_upload_files(samples, tmp_dir)

    # Some samples processed with Terra use CPG IDs, checking if we already
    # have them in the SMDB and fixing the external IDs.
    _fix_sample_ids(samples)

    multiqc_html_path = join(
        f'gs://cpg-{NAGIM_PROJ_ID}-{NAMESPACE}-web/qc/multiqc.html'
    )
    multiqc_json_path = join(
        f'gs://cpg-{NAGIM_PROJ_ID}-{NAMESPACE}-analysis/qc/multiqc_data.json'
    )
    if 'QC' in SOURCES_TO_PROCESS:
        logger.info('Running MultiQC on QC files')
        parsed_json_fpath = _run_multiqc(
            samples, multiqc_html_path, multiqc_json_path, overwrite=overwrite_multiqc
        )
        gfs = gcsfs.GCSFileSystem()
        with gfs.open(parsed_json_fpath) as f:
            row_by_sample = json.load(f)
        for s in samples:
            if s.nagim_id in row_by_sample:
                s.qc_values = row_by_sample[s.nagim_id]

    # Creating a parser for each project separately, because `sample_metadata_project`
    # is an initialization parameter, and we want to write to multiple projects.
    for proj in PROJECT_ID_MAP.values():
        sm_proj = _get_sm_proj_id(proj)
        sample_tsv_file = join(tmp_dir, f'sm-nagim-parser-samples-{sm_proj}.csv')

        rows = []
        for s in samples:
            if s.project_id != proj:
                continue

            row = dict(
                cpg_id=s.cpg_id,
                ext_id=s.ext_id,
                gvcf=s.gvcf,
                cram=s.cram,
                project=s.project_id,
            )
            for metric, val in s.qc_values.items():
                row[f'qc_value_{metric}'] = val
            rows.append(row)

        if len(rows) == 0:
            logger.info(f'No samples for project {sm_proj} found, skipping')
            continue

        df = pd.DataFrame(rows)
        df.to_csv(sample_tsv_file, index=False)
        logger.info(
            f'Processing {len(df)} samples for project {sm_proj}, '
            f'sample manifest: {sample_tsv_file}'
        )

        parser = NagimParser(
            path_prefix=None,
            sample_metadata_project=sm_proj,
            skip_checking_gcs_objects=skip_checking_objects,
            verbose=False,
            multiqc_html_path=multiqc_html_path,
            multiqc_json_path=multiqc_json_path,
        )
        with open(sample_tsv_file) as f:
            parser.parse_manifest(f, dry_run=dry_run, confirm=confirm)

    Assuming the data is transferred to the CPG bucket, populate the SM projects.
    
Assuming the data is transferred to the CPG bucket, populate the SM projects.
Assuming the data is transferred to the CPG bucket populate the SM projects .
def parse(
    tmp_dir,
    confirm: bool,
    dry_run: bool,
    overwrite_multiqc: bool,
    skip_checking_objects: bool,
):
    if not tmp_dir:
        tmp_dir = tempfile.gettempdir()
    samples = _parse_sample_project_map(SAMPLE_TO_PROJECT_TSV_PATH)
    _find_upload_files(samples, tmp_dir)
    _fix_sample_ids(samples)
    multiqc_html_path = join(
        f'gs://cpg-{NAGIM_PROJ_ID}-{NAMESPACE}-web/qc/multiqc.html'
    )
    multiqc_json_path = join(
        f'gs://cpg-{NAGIM_PROJ_ID}-{NAMESPACE}-analysis/qc/multiqc_data.json'
    )
    if 'QC' in SOURCES_TO_PROCESS:
        logger.info('Running MultiQC on QC files')
        parsed_json_fpath = _run_multiqc(
            samples, multiqc_html_path, multiqc_json_path, overwrite=overwrite_multiqc
        )
        gfs = gcsfs.GCSFileSystem()
        with gfs.open(parsed_json_fpath) as f:
            row_by_sample = json.load(f)
        for s in samples:
            if s.nagim_id in row_by_sample:
                s.qc_values = row_by_sample[s.nagim_id]
    for proj in PROJECT_ID_MAP.values():
        sm_proj = _get_sm_proj_id(proj)
        sample_tsv_file = join(tmp_dir, f'sm-nagim-parser-samples-{sm_proj}.csv')
        rows = []
        for s in samples:
            if s.project_id != proj:
                continue
            row = dict(
                cpg_id=s.cpg_id,
                ext_id=s.ext_id,
                gvcf=s.gvcf,
                cram=s.cram,
                project=s.project_id,
            )
            for metric, val in s.qc_values.items():
                row[f'qc_value_{metric}'] = val
            rows.append(row)
        if len(rows) == 0:
            logger.info(f'No samples for project {sm_proj} found, skipping')
            continue
        df = pd.DataFrame(rows)
        df.to_csv(sample_tsv_file, index=False)
        logger.info(
            f'Processing {len(df)} samples for project {sm_proj}, '
            f'sample manifest: {sample_tsv_file}'
        )
        parser = NagimParser(
            path_prefix=None,
            sample_metadata_project=sm_proj,
            skip_checking_gcs_objects=skip_checking_objects,
            verbose=False,
            multiqc_html_path=multiqc_html_path,
            multiqc_json_path=multiqc_json_path,
        )
        with open(sample_tsv_file) as f:
            parser.parse_manifest(f, dry_run=dry_run, confirm=confirm)
def parse ( tmp_dir , confirm : bool , dry_run : bool , overwrite_multiqc : bool , skip_checking_objects : bool , ) : if not tmp_dir : tmp_dir = tempfile . gettempdir ( ) samples = _parse_sample_project_map ( SAMPLE_TO_PROJECT_TSV_PATH ) _find_upload_files ( samples , tmp_dir ) _fix_sample_ids ( samples ) multiqc_html_path = join ( f'gs://cpg-{NAGIM_PROJ_ID}-{NAMESPACE}-web/qc/multiqc.html' ) multiqc_json_path = join ( f'gs://cpg-{NAGIM_PROJ_ID}-{NAMESPACE}-analysis/qc/multiqc_data.json' ) if 'QC' in SOURCES_TO_PROCESS : logger . info ( 'Running MultiQC on QC files' ) parsed_json_fpath = _run_multiqc ( samples , multiqc_html_path , multiqc_json_path , overwrite = overwrite_multiqc ) gfs = gcsfs . GCSFileSystem ( ) with gfs . open ( parsed_json_fpath ) as f : row_by_sample = json . load ( f ) for s in samples : if s . nagim_id in row_by_sample : s . qc_values = row_by_sample [ s . nagim_id ] for proj in PROJECT_ID_MAP . values ( ) : sm_proj = _get_sm_proj_id ( proj ) sample_tsv_file = join ( tmp_dir , f'sm-nagim-parser-samples-{sm_proj}.csv' ) rows = [ ] for s in samples : if s . project_id != proj : continue row = dict ( cpg_id = s . cpg_id , ext_id = s . ext_id , gvcf = s . gvcf , cram = s . cram , project = s . project_id , ) for metric , val in s . qc_values . items ( ) : row [ f'qc_value_{metric}' ] = val rows . append ( row ) if len ( rows ) == 0 : logger . info ( f'No samples for project {sm_proj} found, skipping' ) continue df = pd . DataFrame ( rows ) df . to_csv ( sample_tsv_file , index = False ) logger . info ( f'Processing {len(df)} samples for project {sm_proj}, ' f'sample manifest: {sample_tsv_file}' ) parser = NagimParser ( path_prefix = None , sample_metadata_project = sm_proj , skip_checking_gcs_objects = skip_checking_objects , verbose = False , multiqc_html_path = multiqc_html_path , multiqc_json_path = multiqc_json_path , ) with open ( sample_tsv_file ) as f : parser . parse_manifest ( f , dry_run = dry_run , confirm = confirm )
Assuming the data is transferred to the CPG bucket, populate the SM projects.
Assuming the data is transferred to the CPG bucket populate the SM projects .
"""
    Assuming the data is transferred to the CPG bucket, populate the SM projects.
    """ # Find GVCFs, CRAMs and other files after transferring, and checks that all # of them have corresponding tbi/crai/md5. # Some samples processed with Terra use CPG IDs, checking if we already # have them in the SMDB and fixing the external IDs. # Creating a parser for each project separately, because `sample_metadata_project` # is an initialization parameter, and we want to write to multiple projects.
tmp_dir
confirm
bool
dry_run
bool
overwrite_multiqc
bool
skip_checking_objects
bool