scripts/parse_nagim.py
def _find_upload_files(samples: List[Sample], tmp_dir, overwrite=False):
    """
    Populate fields for each sample and verify that every sample has an expected
    set of files.
    """
    sample_by_sid = {s.nagim_id: s for s in samples}

    # Find files
    for source_name in SOURCES_TO_PROCESS:
        source = SOURCES[source_name]
        for ending in source.search_pattern_by_ending:
            paths = _cache_bucket_ls(
                ending_to_search=ending,
                source_bucket=source.get_upload_bucket(ending),
                tmp_dir=tmp_dir,
                overwrite=overwrite,
            )
            for path in paths:
                assert path.endswith(f'.{ending}')
                sid = basename(path)[: -len(f'.{ending}')]
                if sid not in sample_by_sid:
                    continue
                sample_by_sid[sid].files[(source.name, ending)] = path

    # Tally found files
    for source_name in SOURCES_TO_PROCESS:
        source = SOURCES[source_name]
        for ending in source.search_pattern_by_ending:
            found_samples = len(
                [s for s in sample_by_sid.values() if (source.name, ending) in s.files]
            )
            logger.info(
                f'Found {found_samples}/{len(sample_by_sid)} '
                f'{source.name}/{ending} files'
            )

    # For each sample, verify that the set of found files is consistent
    for sample in sample_by_sid.values():
        if 'GVCF' in SOURCES_TO_PROCESS:
            sample.gvcf = sample.files.get(('GVCF', 'hard-filtered.g.vcf.gz'))
            sample.tbi = sample.files.get(('GVCF', 'hard-filtered.g.vcf.gz.tbi'))

            if sample.gvcf and not sample.tbi:
                logger.warning(f'Found GVCF without TBI: {sample.nagim_id}')
            elif sample.tbi and not sample.gvcf:
                logger.warning(f'Found TBI without GVCF: {sample.nagim_id}')
            elif not sample.gvcf:
                logger.warning(f'Not found GVCF: {sample.nagim_id}')

        if 'CRAM' in SOURCES_TO_PROCESS:
            sample.cram = sample.files.get((SOURCES['CRAM'].name, 'cram'))
            sample.crai = sample.files.get((SOURCES['CRAM'].name, 'cram.crai'))
            sample.cram_md5 = sample.files.get((SOURCES['CRAM'].name, 'cram.md5'))

            if sample.cram and not sample.crai:
                logger.warning(f'Found CRAM without CRAI: {sample.nagim_id}')
            if sample.cram and not sample.cram_md5:
                logger.warning(f'Found CRAM without md5: {sample.nagim_id}')
            if sample.crai and not sample.cram:
                logger.warning(f'Found CRAI without CRAM: {sample.nagim_id}')

        if 'QC' in SOURCES_TO_PROCESS:
            for qc_ending in [
                'alignment_summary_metrics',
                'duplicate_metrics',
                'insert_size_metrics',
                'preBqsr.selfSM',
                'wgs_metrics',
            ]:
                no_qc = 0
                key = (SOURCES['QC'].name, qc_ending)
                if not sample.files.get(key):
                    if sample.gvcf:
                        logger.warning(
                            f'Found GVCF without QC {qc_ending}: {sample.nagim_id}'
                        )
                    no_qc += 1
                    continue
                if no_qc:
                    logger.warning(f'Not found QC {qc_ending} for {no_qc} samples')
                sample.qc_files[qc_ending] = sample.files[key]

    Populate fields for each sample and verify that every sample has an expected
    set of files.
    
Populate fields for each sample and verify that every sample has an expected
set of files.
Populate fields for each sample and verify that every sample has an expected set of files .
def _find_upload_files(samples: List[Sample], tmp_dir, overwrite=False):
    sample_by_sid = {s.nagim_id: s for s in samples}
    for source_name in SOURCES_TO_PROCESS:
        source = SOURCES[source_name]
        for ending in source.search_pattern_by_ending:
            paths = _cache_bucket_ls(
                ending_to_search=ending,
                source_bucket=source.get_upload_bucket(ending),
                tmp_dir=tmp_dir,
                overwrite=overwrite,
            )
            for path in paths:
                assert path.endswith(f'.{ending}')
                sid = basename(path)[: -len(f'.{ending}')]
                if sid not in sample_by_sid:
                    continue
                sample_by_sid[sid].files[(source.name, ending)] = path
    for source_name in SOURCES_TO_PROCESS:
        source = SOURCES[source_name]
        for ending in source.search_pattern_by_ending:
            found_samples = len(
                [s for s in sample_by_sid.values() if (source.name, ending) in s.files]
            )
            logger.info(
                f'Found {found_samples}/{len(sample_by_sid)} '
                f'{source.name}/{ending} files'
            )
    for sample in sample_by_sid.values():
        if 'GVCF' in SOURCES_TO_PROCESS:
            sample.gvcf = sample.files.get(('GVCF', 'hard-filtered.g.vcf.gz'))
            sample.tbi = sample.files.get(('GVCF', 'hard-filtered.g.vcf.gz.tbi'))
            if sample.gvcf and not sample.tbi:
                logger.warning(f'Found GVCF without TBI: {sample.nagim_id}')
            elif sample.tbi and not sample.gvcf:
                logger.warning(f'Found TBI without GVCF: {sample.nagim_id}')
            elif not sample.gvcf:
                logger.warning(f'Not found GVCF: {sample.nagim_id}')
        if 'CRAM' in SOURCES_TO_PROCESS:
            sample.cram = sample.files.get((SOURCES['CRAM'].name, 'cram'))
            sample.crai = sample.files.get((SOURCES['CRAM'].name, 'cram.crai'))
            sample.cram_md5 = sample.files.get((SOURCES['CRAM'].name, 'cram.md5'))
            if sample.cram and not sample.crai:
                logger.warning(f'Found CRAM without CRAI: {sample.nagim_id}')
            if sample.cram and not sample.cram_md5:
                logger.warning(f'Found CRAM without md5: {sample.nagim_id}')
            if sample.crai and not sample.cram:
                logger.warning(f'Found CRAI without CRAM: {sample.nagim_id}')
        if 'QC' in SOURCES_TO_PROCESS:
            for qc_ending in [
                'alignment_summary_metrics',
                'duplicate_metrics',
                'insert_size_metrics',
                'preBqsr.selfSM',
                'wgs_metrics',
            ]:
                no_qc = 0
                key = (SOURCES['QC'].name, qc_ending)
                if not sample.files.get(key):
                    if sample.gvcf:
                        logger.warning(
                            f'Found GVCF without QC {qc_ending}: {sample.nagim_id}'
                        )
                    no_qc += 1
                    continue
                if no_qc:
                    logger.warning(f'Not found QC {qc_ending} for {no_qc} samples')
                sample.qc_files[qc_ending] = sample.files[key]
def _find_upload_files ( samples : List [ Sample ] , tmp_dir , overwrite = False ) : sample_by_sid = { s . nagim_id : s for s in samples } for source_name in SOURCES_TO_PROCESS : source = SOURCES [ source_name ] for ending in source . search_pattern_by_ending : paths = _cache_bucket_ls ( ending_to_search = ending , source_bucket = source . get_upload_bucket ( ending ) , tmp_dir = tmp_dir , overwrite = overwrite , ) for path in paths : assert path . endswith ( f'.{ending}' ) sid = basename ( path ) [ : - len ( f'.{ending}' ) ] if sid not in sample_by_sid : continue sample_by_sid [ sid ] . files [ ( source . name , ending ) ] = path for source_name in SOURCES_TO_PROCESS : source = SOURCES [ source_name ] for ending in source . search_pattern_by_ending : found_samples = len ( [ s for s in sample_by_sid . values ( ) if ( source . name , ending ) in s . files ] ) logger . info ( f'Found {found_samples}/{len(sample_by_sid)} ' f'{source.name}/{ending} files' ) for sample in sample_by_sid . values ( ) : if 'GVCF' in SOURCES_TO_PROCESS : sample . gvcf = sample . files . get ( ( 'GVCF' , 'hard-filtered.g.vcf.gz' ) ) sample . tbi = sample . files . get ( ( 'GVCF' , 'hard-filtered.g.vcf.gz.tbi' ) ) if sample . gvcf and not sample . tbi : logger . warning ( f'Found GVCF without TBI: {sample.nagim_id}' ) elif sample . tbi and not sample . gvcf : logger . warning ( f'Found TBI without GVCF: {sample.nagim_id}' ) elif not sample . gvcf : logger . warning ( f'Not found GVCF: {sample.nagim_id}' ) if 'CRAM' in SOURCES_TO_PROCESS : sample . cram = sample . files . get ( ( SOURCES [ 'CRAM' ] . name , 'cram' ) ) sample . crai = sample . files . get ( ( SOURCES [ 'CRAM' ] . name , 'cram.crai' ) ) sample . cram_md5 = sample . files . get ( ( SOURCES [ 'CRAM' ] . name , 'cram.md5' ) ) if sample . cram and not sample . crai : logger . warning ( f'Found CRAM without CRAI: {sample.nagim_id}' ) if sample . cram and not sample . cram_md5 : logger . warning ( f'Found CRAM without md5: {sample.nagim_id}' ) if sample . crai and not sample . cram : logger . warning ( f'Found CRAI without CRAM: {sample.nagim_id}' ) if 'QC' in SOURCES_TO_PROCESS : for qc_ending in [ 'alignment_summary_metrics' , 'duplicate_metrics' , 'insert_size_metrics' , 'preBqsr.selfSM' , 'wgs_metrics' , ] : no_qc = 0 key = ( SOURCES [ 'QC' ] . name , qc_ending ) if not sample . files . get ( key ) : if sample . gvcf : logger . warning ( f'Found GVCF without QC {qc_ending}: {sample.nagim_id}' ) no_qc += 1 continue if no_qc : logger . warning ( f'Not found QC {qc_ending} for {no_qc} samples' ) sample . qc_files [ qc_ending ] = sample . files [ key ]
Populate fields for each sample and verify that every sample has an expected
set of files.
Populate fields for each sample and verify that every sample has an expected set of files .
"""
    Populate fields for each sample and verify that every sample has an expected
    set of files.
    """ # Find files # Tally found files # For each sample, verify that the set of found files is consistent
samples
List[Sample]
tmp_dir
overwrite