src/utilities/rna_velocity/run_10x_velocyto.py
def main(logger):
    """ Download reference genome, run velocyto jobs, and upload results to S3.

        logger - Logger object that exposes the interface the code directly uses
    """

    parser = get_parser()

    args = parser.parse_args()

    args.root_dir = pathlib.Path(args.root_dir)

    if os.environ.get("AWS_BATCH_JOB_ID"):
        args.root_dir = args.root_dir / os.environ["AWS_BATCH_JOB_ID"]

    # local directories
    if args.s3_input_path.endswith("/"):
        args.s3_input_path = args.s3_input_path[:-1]

    sample_id = os.path.basename(args.s3_input_path)
    result_path = args.root_dir / "data" / sample_id
    if args.legacy:
        fastq_path = result_path / "fastqs"
    else:
        fastq_path = result_path
    fastq_path.mkdir(parents=True, exist_ok=True)

    genome_base_dir = args.root_dir / "genome" / "STAR-2.7.9a"
    genome_base_dir.mkdir(parents=True, exist_ok=True)

    barcode_base_dir = args.root_dir / "barcodes"
    barcode_base_dir.mkdir(parents=True, exist_ok=True)

    # check if the input genome and region are valid
    if args.taxon in reference_genomes:
        genome_name = reference_genomes[args.taxon]
    else:
        raise ValueError(f"unknown taxon {args.taxon}")

    # check if the input genome and region are valid
    if args.version_10x in barcodes_10x:
        barcode_name = barcodes_10x[args.version_10x]
    else:
        raise ValueError(f"unknown 10x version {args.version_10x}")

    genome_dir = genome_base_dir / genome_name
    ref_genome_10x_file = f"STAR-2.7.9a/{genome_name}.tgz"

    barcode_dir = barcode_base_dir / barcode_name
    barcode_10x_file = f"STAR-2.7.9a/10x_barcode_whitelists/{barcode_name}"


    logger.info(
        f"""Run Info: partition {args.partition_id} out of {args.num_partitions}
            taxon:\t{args.taxon}
                ref_genome_10x_file:\t{ref_genome_10x_file}   
                genome_dir:\t{genome_dir}   
            10x version:\t{args.version_10x}
                barcode_10x_file:\t{barcode_10x_file}
                barcode_dir:\t{barcode_dir}
                s3_input_path:\t{args.s3_input_path}
                s3_output_path:\t{args.s3_output_path}"""
    )


    s3 = boto3.resource("s3")


    # download the reference barcode & genome data
    
    logger.info(f"Downloading and extracting genome data {genome_name}")
    s3_genome_object = s3.Object(S3_REFERENCE[args.region], ref_genome_10x_file)

    with tarfile.open(fileobj=s3_genome_object.get()["Body"], mode="r|gz") as tf:
        tf.extractall(path=genome_base_dir)

    sys.stdout.flush()

    

    # download the fastq files
    command = [
        "aws",
        "s3",
        "cp",
        "--no-progress",
        "--recursive",
        "--exclude",
        "'*'",
        "--include",
        f"'*{args.sample_prefix}*'",
        "--force-glacier-transfer" if args.glacier else "",
        args.s3_input_path,
        f"{fastq_path}",
    ]
    log_command(logger, command, shell=True)

    logger.info(f"Running partition {args.partition_id} of {args.num_partitions}")

    # Set variables
    # Read in R1 files
    R1_files = glob.glob(fastq_path + '/*R[1]*.fastq.gz')
    R1_files.sort()

    R1_list = ",".join(R1_files)

    # Read in R2 files, sort to ensure order
    R2_files = glob.glob(fastq_path + '/*R[2]*.fastq.gz')
    R2_files.sort()
    
    R2_list = ",".join(R2_files)

    # Run cellranger
    os.chdir(result_path)

    sys.exit(0)



    if args.by_folder:
        
        command = [
        CELLRANGER,
        "count",
        "--localmem=240", # By default, will use 90% of mem
        "--nosecondary",
        "--disable-ui",
        f"--expect-cells={args.cell_count}",
        f"--id={args.sample_prefix}",
        f"--fastqs={fastq_path}",
        f"--transcriptome={genome_dir}" # no sample_prefix: run all samples in folder
        ]

    else:

        command = [
            CELLRANGER,
            "count",
            "--localmem=240", # By default, will use 90% of mem
            "--nosecondary",
            "--disable-ui",
            f"--expect-cells={args.cell_count}",
            f"--id={args.sample_prefix}",
            f"--fastqs={fastq_path}",
            f"--transcriptome={genome_dir}",
            f"--sample={args.sample_prefix}",
        ]

    failed = log_command(
        logger,
        command,
        shell=True,
        stdout=subprocess.PIPE,
        stderr=subprocess.STDOUT,
        universal_newlines=True,
    )

    if failed:
        raise RuntimeError("cellranger count failed")

    # Move outs folder to S3
    command = [
        "aws",
        "s3",
        "sync",
        "--no-progress",
        os.path.join(result_path, args.sample_prefix, "outs"),
        posixpath.join(args.s3_output_path, args.sample_prefix),
    ]
    for i in range(S3_RETRY):
        if not log_command(logger, command, shell=True):
            break
        logger.info(f"retrying sync")
    else:
        raise RuntimeError(f"couldn't sync output")
 Download reference genome, run velocyto jobs, and upload results to S3.

        logger - Logger object that exposes the interface the code directly uses
    
Download reference genome, run velocyto jobs, and upload results to S3.
logger - Logger object that exposes the interface the code directly uses
Download reference genome run velocyto jobs and upload results to S3 . logger - Logger object that exposes the interface the code directly uses
def main(logger):
    parser = get_parser()
    args = parser.parse_args()
    args.root_dir = pathlib.Path(args.root_dir)
    if os.environ.get("AWS_BATCH_JOB_ID"):
        args.root_dir = args.root_dir / os.environ["AWS_BATCH_JOB_ID"]
    if args.s3_input_path.endswith("/"):
        args.s3_input_path = args.s3_input_path[:-1]
    sample_id = os.path.basename(args.s3_input_path)
    result_path = args.root_dir / "data" / sample_id
    if args.legacy:
        fastq_path = result_path / "fastqs"
    else:
        fastq_path = result_path
    fastq_path.mkdir(parents=True, exist_ok=True)
    genome_base_dir = args.root_dir / "genome" / "STAR-2.7.9a"
    genome_base_dir.mkdir(parents=True, exist_ok=True)
    barcode_base_dir = args.root_dir / "barcodes"
    barcode_base_dir.mkdir(parents=True, exist_ok=True)
    if args.taxon in reference_genomes:
        genome_name = reference_genomes[args.taxon]
    else:
        raise ValueError(f"unknown taxon {args.taxon}")
    if args.version_10x in barcodes_10x:
        barcode_name = barcodes_10x[args.version_10x]
    else:
        raise ValueError(f"unknown 10x version {args.version_10x}")
    genome_dir = genome_base_dir / genome_name
    ref_genome_10x_file = f"STAR-2.7.9a/{genome_name}.tgz"
    barcode_dir = barcode_base_dir / barcode_name
    barcode_10x_file = f"STAR-2.7.9a/10x_barcode_whitelists/{barcode_name}"
    logger.info(
        f"""Run Info: partition {args.partition_id} out of {args.num_partitions}
            taxon:\t{args.taxon}
                ref_genome_10x_file:\t{ref_genome_10x_file}   
                genome_dir:\t{genome_dir}   
            10x version:\t{args.version_10x}
                barcode_10x_file:\t{barcode_10x_file}
                barcode_dir:\t{barcode_dir}
                s3_input_path:\t{args.s3_input_path}
                s3_output_path:\t{args.s3_output_path}"""
    )
    s3 = boto3.resource("s3")
    logger.info(f"Downloading and extracting genome data {genome_name}")
    s3_genome_object = s3.Object(S3_REFERENCE[args.region], ref_genome_10x_file)
    with tarfile.open(fileobj=s3_genome_object.get()["Body"], mode="r|gz") as tf:
        tf.extractall(path=genome_base_dir)
    sys.stdout.flush()
    command = [
        "aws",
        "s3",
        "cp",
        "--no-progress",
        "--recursive",
        "--exclude",
        "'*'",
        "--include",
        f"'*{args.sample_prefix}*'",
        "--force-glacier-transfer" if args.glacier else "",
        args.s3_input_path,
        f"{fastq_path}",
    ]
    log_command(logger, command, shell=True)
    logger.info(f"Running partition {args.partition_id} of {args.num_partitions}")
    R1_files = glob.glob(fastq_path + '/*R[1]*.fastq.gz')
    R1_files.sort()
    R1_list = ",".join(R1_files)
    R2_files = glob.glob(fastq_path + '/*R[2]*.fastq.gz')
    R2_files.sort()
    R2_list = ",".join(R2_files)
    os.chdir(result_path)
    sys.exit(0)
    if args.by_folder:
        command = [
        CELLRANGER,
        "count",
        "--localmem=240", 
        "--nosecondary",
        "--disable-ui",
        f"--expect-cells={args.cell_count}",
        f"--id={args.sample_prefix}",
        f"--fastqs={fastq_path}",
        f"--transcriptome={genome_dir}" 
        ]
    else:
        command = [
            CELLRANGER,
            "count",
            "--localmem=240", 
            "--nosecondary",
            "--disable-ui",
            f"--expect-cells={args.cell_count}",
            f"--id={args.sample_prefix}",
            f"--fastqs={fastq_path}",
            f"--transcriptome={genome_dir}",
            f"--sample={args.sample_prefix}",
        ]
    failed = log_command(
        logger,
        command,
        shell=True,
        stdout=subprocess.PIPE,
        stderr=subprocess.STDOUT,
        universal_newlines=True,
    )
    if failed:
        raise RuntimeError("cellranger count failed")
    command = [
        "aws",
        "s3",
        "sync",
        "--no-progress",
        os.path.join(result_path, args.sample_prefix, "outs"),
        posixpath.join(args.s3_output_path, args.sample_prefix),
    ]
    for i in range(S3_RETRY):
        if not log_command(logger, command, shell=True):
            break
        logger.info(f"retrying sync")
    else:
        raise RuntimeError(f"couldn't sync output")
def main ( logger ) : parser = get_parser ( ) args = parser . parse_args ( ) args . root_dir = pathlib . Path ( args . root_dir ) if os . environ . get ( "AWS_BATCH_JOB_ID" ) : args . root_dir = args . root_dir / os . environ [ "AWS_BATCH_JOB_ID" ] if args . s3_input_path . endswith ( "/" ) : args . s3_input_path = args . s3_input_path [ : - 1 ] sample_id = os . path . basename ( args . s3_input_path ) result_path = args . root_dir / "data" / sample_id if args . legacy : fastq_path = result_path / "fastqs" else : fastq_path = result_path fastq_path . mkdir ( parents = True , exist_ok = True ) genome_base_dir = args . root_dir / "genome" / "STAR-2.7.9a" genome_base_dir . mkdir ( parents = True , exist_ok = True ) barcode_base_dir = args . root_dir / "barcodes" barcode_base_dir . mkdir ( parents = True , exist_ok = True ) if args . taxon in reference_genomes : genome_name = reference_genomes [ args . taxon ] else : raise ValueError ( f"unknown taxon {args.taxon}" ) if args . version_10x in barcodes_10x : barcode_name = barcodes_10x [ args . version_10x ] else : raise ValueError ( f"unknown 10x version {args.version_10x}" ) genome_dir = genome_base_dir / genome_name ref_genome_10x_file = f"STAR-2.7.9a/{genome_name}.tgz" barcode_dir = barcode_base_dir / barcode_name barcode_10x_file = f"STAR-2.7.9a/10x_barcode_whitelists/{barcode_name}" logger . info ( f"""Run Info: partition {args.partition_id} out of {args.num_partitions}
            taxon:\t{args.taxon}
                ref_genome_10x_file:\t{ref_genome_10x_file}   
                genome_dir:\t{genome_dir}   
            10x version:\t{args.version_10x}
                barcode_10x_file:\t{barcode_10x_file}
                barcode_dir:\t{barcode_dir}
                s3_input_path:\t{args.s3_input_path}
                s3_output_path:\t{args.s3_output_path}""" ) s3 = boto3 . resource ( "s3" ) logger . info ( f"Downloading and extracting genome data {genome_name}" ) s3_genome_object = s3 . Object ( S3_REFERENCE [ args . region ] , ref_genome_10x_file ) with tarfile . open ( fileobj = s3_genome_object . get ( ) [ "Body" ] , mode = "r|gz" ) as tf : tf . extractall ( path = genome_base_dir ) sys . stdout . flush ( ) command = [ "aws" , "s3" , "cp" , "--no-progress" , "--recursive" , "--exclude" , "'*'" , "--include" , f"'*{args.sample_prefix}*'" , "--force-glacier-transfer" if args . glacier else "" , args . s3_input_path , f"{fastq_path}" , ] log_command ( logger , command , shell = True ) logger . info ( f"Running partition {args.partition_id} of {args.num_partitions}" ) R1_files = glob . glob ( fastq_path + '/*R[1]*.fastq.gz' ) R1_files . sort ( ) R1_list = "," . join ( R1_files ) R2_files = glob . glob ( fastq_path + '/*R[2]*.fastq.gz' ) R2_files . sort ( ) R2_list = "," . join ( R2_files ) os . chdir ( result_path ) sys . exit ( 0 ) if args . by_folder : command = [ CELLRANGER , "count" , "--localmem=240" , "--nosecondary" , "--disable-ui" , f"--expect-cells={args.cell_count}" , f"--id={args.sample_prefix}" , f"--fastqs={fastq_path}" , f"--transcriptome={genome_dir}" ] else : command = [ CELLRANGER , "count" , "--localmem=240" , "--nosecondary" , "--disable-ui" , f"--expect-cells={args.cell_count}" , f"--id={args.sample_prefix}" , f"--fastqs={fastq_path}" , f"--transcriptome={genome_dir}" , f"--sample={args.sample_prefix}" , ] failed = log_command ( logger , command , shell = True , stdout = subprocess . PIPE , stderr = subprocess . STDOUT , universal_newlines = True , ) if failed : raise RuntimeError ( "cellranger count failed" ) command = [ "aws" , "s3" , "sync" , "--no-progress" , os . path . join ( result_path , args . sample_prefix , "outs" ) , posixpath . join ( args . s3_output_path , args . sample_prefix ) , ] for i in range ( S3_RETRY ) : if not log_command ( logger , command , shell = True ) : break logger . info ( f"retrying sync" ) else : raise RuntimeError ( f"couldn't sync output" )
Download reference genome, run velocyto jobs, and upload results to S3.
Download reference genome run velocyto jobs and upload results to S3 .
""" Download reference genome, run velocyto jobs, and upload results to S3.

        logger - Logger object that exposes the interface the code directly uses
    """ # local directories # check if the input genome and region are valid # check if the input genome and region are valid # download the reference barcode & genome data # download the fastq files # Set variables # Read in R1 files # Read in R2 files, sort to ensure order # Run cellranger # By default, will use 90% of mem # no sample_prefix: run all samples in folder # By default, will use 90% of mem # Move outs folder to S3
logger