src/utilities/rna_velocity/run_kb_python.py
def parse_ref(args, run_dir, logger):
    """Parser for the `ref` command. Build kallisto index files.
    
    Keyword Arguments:
    args -- Command-line arguments dictionary, as parsed by argparse (type: dict)
    run_dir -- Path on the EC2 instance under which jobs are run (type: pathlib.Path)
    logger -- Logger object that exposes the interface the code directly uses (type: logging.Logger)
    """
    # Build paths on the EC2 instance to store inputs and outputs of kallisto index building. Download kallisto index building files from the AWS S3 bucket to the EC2 instance, if not using the in-built kallisto indices
    kallisto_index_dir = run_dir / "kallisto_index"
    kallisto_index_inputs = kallisto_index_dir / "inputs"
    kallisto_index_outputs = kallisto_index_dir / "outputs"
    kallisto_index_inputs.mkdir(parents=True)
    kallisto_index_outputs.mkdir(parents=True)
    kb_ref_paths = dict()
    s3_kb_ref = dict()
    kb_ref_output_to_s3 = dict()

    if "-d" not in sys.argv:
        if "--workflow" in sys.argv and "kite" in sys.argv:
            kb_ref_paths["feature_path"] = kallisto_index_inputs / os.path.basename(
                args.feature
            )
            (
                s3_kb_ref["s3_feature_bucket"],
                s3_kb_ref["s3_feature_prefix"],
            ) = s3u.s3_bucket_and_key(args.feature)
            s3c.download_file(
                Bucket=s3_kb_ref["s3_feature_bucket"],
                Key=s3_kb_ref["s3_feature_prefix"],
                Filename=str(kb_ref_paths["feature_path"]),
            )

        for arg in ["fasta", "gtf"]:
            kb_ref_paths[arg + "_path"] = kallisto_index_inputs / os.path.basename(
                getattr(args, arg)
            )
            (
                s3_kb_ref["s3_" + arg + "_bucket"],
                s3_kb_ref["s3_" + arg + "_prefix"],
            ) = s3u.s3_bucket_and_key(getattr(args, arg))
            s3c.download_file(
                Bucket=s3_kb_ref["s3_" + arg + "_bucket"],
                Key=s3_kb_ref["s3_" + arg + "_prefix"],
                Filename=str(kb_ref_paths[arg + "_path"]),
            )

    for arg in ["-i", "-g", "-f1", "-f2", "-c1", "-c2", "--tmp"]:
        if arg in sys.argv:
            arg = arg[2:] if arg == "--tmp" else arg[1:]
            print(
                f"testing purpose - see if getattr(args, arg) output all argument names from '-i', '-g', '-f1', '-f2', '-c1', '-c2', '--tmp': {getattr(args, arg)}"
            )  # testing purpose
            if arg == "tmp":
                kb_ref_paths[arg + "_path"] = kallisto_index_dir / "alter_tmp"
                s3_kb_ref["s3_" + arg + "_bucket"] = s3u.s3_bucket_and_key(
                    getattr(args, arg)
                )[0]
                s3_kb_ref["s3_" + arg + "_prefix"] = posixpath.join(
                    s3u.s3_bucket_and_key(getattr(args, arg))[1], "alter_tmp"
                )
            else:
                kb_ref_paths[arg + "_path"] = kallisto_index_outputs / os.path.basename(
                    getattr(args, arg)
                )
                (
                    s3_kb_ref["s3_" + arg + "_bucket"],
                    s3_kb_ref["s3_" + arg + "_prefix"],
                ) = s3u.s3_bucket_and_key(getattr(args, arg))

    # Build the command of running `kb ref` to generate kallisto index files
    ref_input_boolean = ["--lamanno", "--overwrite", "--keep-tmp", "--verbose"]
    ref_input_upload_required = ["-i", "-g", "-f1", "-f2", "-c1", "-c2", "--tmp"]
    ref_input_left_args = ["-d", "-n", "-k", "--workflow"]

    kb_ref_command = ["kb", "ref"]
    for input in ["fasta", "gtf"]:
        if "-d" not in sys.argv:
            print(
                f"testing purpose: `ref` positional argument: {getattr(args, input)}"
            )  # testing purpose
            if "--workflow" in sys.argv and "kite" in sys.argv:
                print(
                    f"testing purpose: `ref` positional argument: {getattr(args, input)}"
                )  # testing purpose
                kb_ref_command += [str(kb_ref_paths["feature_path"])]
            kb_ref_command += [str(kb_ref_paths[input + "_path"])]
    for input in ref_input_boolean:
        if input in sys.argv:
            kb_ref_command += [input]
    for input in ref_input_upload_required:
        if input in sys.argv:
            kb_ref_command += [
                input,
                str(kb_ref_paths[input[2:] + "_path"])
                if input == "--tmp"
                else str(kb_ref_paths[input[1:] + "_path"]),
            ]
    for input in ref_input_left_args:
        if input in sys.argv:
            kb_ref_command += [input, getattr(args, input)]
    print(
        f"testing purpose - check if kb_ref_command is of correct format: {kb_ref_command}"
    )  # testing purpose

    # Run the command to generate kallisto index files, and upload the output files on the EC2 instance back to AWS S3
    kb_ref_failed = ut_log.log_command(
        logger,
        kb_ref_command,
        shell=True,
        stdout=subprocess.PIPE,
        stderr=subprocess.STDOUT,
        universal_newlines=True,
    )
    kb_ref_t_config = TransferConfig(
        use_threads=False
    )  # testing purpose: comment this line if this runs into error. kallisto indices are not pretty big and don't necessarily need a transfer config
    if kb_ref_failed:
        raise RuntimeError("kallisto index building failed")
    else:
        kb_ref_upload_files = glob.glob(str(kallisto_index_outputs / "*"))
        print(
            f"testing purpose - check kallisto index files: {kb_ref_upload_files}"
        )  # testing purpose
        for file in kb_ref_upload_files:
            logger.info(
                f"Uploading {os.path.basename(file)} from the EC2 instance to AWS S3"
            )
            s3c.upload_file(
                Filename=file,
                Bucket=kb_ref_output_to_s3[file][0],
                Key=kb_ref_output_to_s3[file][1],
                Config=kb_ref_t_config,
            )
            time.sleep(30)
            print(
                f"testing purpose - see if kb_ref upload_file function intakes correct bucket and prefix names for kallisto index output files: {kb_ref_output_to_s3[file][0]}, {kb_ref_output_to_s3[file][1]}"
            )  # testing purpose
    return
Parser for the `ref` command. Build kallisto index files.
    
    Keyword Arguments:
    args -- Command-line arguments dictionary, as parsed by argparse (type: dict)
    run_dir -- Path on the EC2 instance under which jobs are run (type: pathlib.Path)
    logger -- Logger object that exposes the interface the code directly uses (type: logging.Logger)
    
Parser for the `ref` command. Build kallisto index files.
Parser for the ` ref ` command . Build kallisto index files .
def parse_ref(args, run_dir, logger):
    kallisto_index_dir = run_dir / "kallisto_index"
    kallisto_index_inputs = kallisto_index_dir / "inputs"
    kallisto_index_outputs = kallisto_index_dir / "outputs"
    kallisto_index_inputs.mkdir(parents=True)
    kallisto_index_outputs.mkdir(parents=True)
    kb_ref_paths = dict()
    s3_kb_ref = dict()
    kb_ref_output_to_s3 = dict()
    if "-d" not in sys.argv:
        if "--workflow" in sys.argv and "kite" in sys.argv:
            kb_ref_paths["feature_path"] = kallisto_index_inputs / os.path.basename(
                args.feature
            )
            (
                s3_kb_ref["s3_feature_bucket"],
                s3_kb_ref["s3_feature_prefix"],
            ) = s3u.s3_bucket_and_key(args.feature)
            s3c.download_file(
                Bucket=s3_kb_ref["s3_feature_bucket"],
                Key=s3_kb_ref["s3_feature_prefix"],
                Filename=str(kb_ref_paths["feature_path"]),
            )
        for arg in ["fasta", "gtf"]:
            kb_ref_paths[arg + "_path"] = kallisto_index_inputs / os.path.basename(
                getattr(args, arg)
            )
            (
                s3_kb_ref["s3_" + arg + "_bucket"],
                s3_kb_ref["s3_" + arg + "_prefix"],
            ) = s3u.s3_bucket_and_key(getattr(args, arg))
            s3c.download_file(
                Bucket=s3_kb_ref["s3_" + arg + "_bucket"],
                Key=s3_kb_ref["s3_" + arg + "_prefix"],
                Filename=str(kb_ref_paths[arg + "_path"]),
            )
    for arg in ["-i", "-g", "-f1", "-f2", "-c1", "-c2", "--tmp"]:
        if arg in sys.argv:
            arg = arg[2:] if arg == "--tmp" else arg[1:]
            print(
                f"testing purpose - see if getattr(args, arg) output all argument names from '-i', '-g', '-f1', '-f2', '-c1', '-c2', '--tmp': {getattr(args, arg)}"
            )  
            if arg == "tmp":
                kb_ref_paths[arg + "_path"] = kallisto_index_dir / "alter_tmp"
                s3_kb_ref["s3_" + arg + "_bucket"] = s3u.s3_bucket_and_key(
                    getattr(args, arg)
                )[0]
                s3_kb_ref["s3_" + arg + "_prefix"] = posixpath.join(
                    s3u.s3_bucket_and_key(getattr(args, arg))[1], "alter_tmp"
                )
            else:
                kb_ref_paths[arg + "_path"] = kallisto_index_outputs / os.path.basename(
                    getattr(args, arg)
                )
                (
                    s3_kb_ref["s3_" + arg + "_bucket"],
                    s3_kb_ref["s3_" + arg + "_prefix"],
                ) = s3u.s3_bucket_and_key(getattr(args, arg))
    ref_input_boolean = ["--lamanno", "--overwrite", "--keep-tmp", "--verbose"]
    ref_input_upload_required = ["-i", "-g", "-f1", "-f2", "-c1", "-c2", "--tmp"]
    ref_input_left_args = ["-d", "-n", "-k", "--workflow"]
    kb_ref_command = ["kb", "ref"]
    for input in ["fasta", "gtf"]:
        if "-d" not in sys.argv:
            print(
                f"testing purpose: `ref` positional argument: {getattr(args, input)}"
            )  
            if "--workflow" in sys.argv and "kite" in sys.argv:
                print(
                    f"testing purpose: `ref` positional argument: {getattr(args, input)}"
                )  
                kb_ref_command += [str(kb_ref_paths["feature_path"])]
            kb_ref_command += [str(kb_ref_paths[input + "_path"])]
    for input in ref_input_boolean:
        if input in sys.argv:
            kb_ref_command += [input]
    for input in ref_input_upload_required:
        if input in sys.argv:
            kb_ref_command += [
                input,
                str(kb_ref_paths[input[2:] + "_path"])
                if input == "--tmp"
                else str(kb_ref_paths[input[1:] + "_path"]),
            ]
    for input in ref_input_left_args:
        if input in sys.argv:
            kb_ref_command += [input, getattr(args, input)]
    print(
        f"testing purpose - check if kb_ref_command is of correct format: {kb_ref_command}"
    )  
    kb_ref_failed = ut_log.log_command(
        logger,
        kb_ref_command,
        shell=True,
        stdout=subprocess.PIPE,
        stderr=subprocess.STDOUT,
        universal_newlines=True,
    )
    kb_ref_t_config = TransferConfig(
        use_threads=False
    )  : comment this line if this runs into error. kallisto indices are not pretty big and don't necessarily need a transfer config
    if kb_ref_failed:
        raise RuntimeError("kallisto index building failed")
    else:
        kb_ref_upload_files = glob.glob(str(kallisto_index_outputs / "*"))
        print(
            f"testing purpose - check kallisto index files: {kb_ref_upload_files}"
        )  
        for file in kb_ref_upload_files:
            logger.info(
                f"Uploading {os.path.basename(file)} from the EC2 instance to AWS S3"
            )
            s3c.upload_file(
                Filename=file,
                Bucket=kb_ref_output_to_s3[file][0],
                Key=kb_ref_output_to_s3[file][1],
                Config=kb_ref_t_config,
            )
            time.sleep(30)
            print(
                f"testing purpose - see if kb_ref upload_file function intakes correct bucket and prefix names for kallisto index output files: {kb_ref_output_to_s3[file][0]}, {kb_ref_output_to_s3[file][1]}"
            )  
    return
def parse_ref ( args , run_dir , logger ) : kallisto_index_dir = run_dir / "kallisto_index" kallisto_index_inputs = kallisto_index_dir / "inputs" kallisto_index_outputs = kallisto_index_dir / "outputs" kallisto_index_inputs . mkdir ( parents = True ) kallisto_index_outputs . mkdir ( parents = True ) kb_ref_paths = dict ( ) s3_kb_ref = dict ( ) kb_ref_output_to_s3 = dict ( ) if "-d" not in sys . argv : if "--workflow" in sys . argv and "kite" in sys . argv : kb_ref_paths [ "feature_path" ] = kallisto_index_inputs / os . path . basename ( args . feature ) ( s3_kb_ref [ "s3_feature_bucket" ] , s3_kb_ref [ "s3_feature_prefix" ] , ) = s3u . s3_bucket_and_key ( args . feature ) s3c . download_file ( Bucket = s3_kb_ref [ "s3_feature_bucket" ] , Key = s3_kb_ref [ "s3_feature_prefix" ] , Filename = str ( kb_ref_paths [ "feature_path" ] ) , ) for arg in [ "fasta" , "gtf" ] : kb_ref_paths [ arg + "_path" ] = kallisto_index_inputs / os . path . basename ( getattr ( args , arg ) ) ( s3_kb_ref [ "s3_" + arg + "_bucket" ] , s3_kb_ref [ "s3_" + arg + "_prefix" ] , ) = s3u . s3_bucket_and_key ( getattr ( args , arg ) ) s3c . download_file ( Bucket = s3_kb_ref [ "s3_" + arg + "_bucket" ] , Key = s3_kb_ref [ "s3_" + arg + "_prefix" ] , Filename = str ( kb_ref_paths [ arg + "_path" ] ) , ) for arg in [ "-i" , "-g" , "-f1" , "-f2" , "-c1" , "-c2" , "--tmp" ] : if arg in sys . argv : arg = arg [ 2 : ] if arg == "--tmp" else arg [ 1 : ] print ( f"testing purpose - see if getattr(args, arg) output all argument names from '-i', '-g', '-f1', '-f2', '-c1', '-c2', '--tmp': {getattr(args, arg)}" ) if arg == "tmp" : kb_ref_paths [ arg + "_path" ] = kallisto_index_dir / "alter_tmp" s3_kb_ref [ "s3_" + arg + "_bucket" ] = s3u . s3_bucket_and_key ( getattr ( args , arg ) ) [ 0 ] s3_kb_ref [ "s3_" + arg + "_prefix" ] = posixpath . join ( s3u . s3_bucket_and_key ( getattr ( args , arg ) ) [ 1 ] , "alter_tmp" ) else : kb_ref_paths [ arg + "_path" ] = kallisto_index_outputs / os . path . basename ( getattr ( args , arg ) ) ( s3_kb_ref [ "s3_" + arg + "_bucket" ] , s3_kb_ref [ "s3_" + arg + "_prefix" ] , ) = s3u . s3_bucket_and_key ( getattr ( args , arg ) ) ref_input_boolean = [ "--lamanno" , "--overwrite" , "--keep-tmp" , "--verbose" ] ref_input_upload_required = [ "-i" , "-g" , "-f1" , "-f2" , "-c1" , "-c2" , "--tmp" ] ref_input_left_args = [ "-d" , "-n" , "-k" , "--workflow" ] kb_ref_command = [ "kb" , "ref" ] for input in [ "fasta" , "gtf" ] : if "-d" not in sys . argv : print ( f"testing purpose: `ref` positional argument: {getattr(args, input)}" ) if "--workflow" in sys . argv and "kite" in sys . argv : print ( f"testing purpose: `ref` positional argument: {getattr(args, input)}" ) kb_ref_command += [ str ( kb_ref_paths [ "feature_path" ] ) ] kb_ref_command += [ str ( kb_ref_paths [ input + "_path" ] ) ] for input in ref_input_boolean : if input in sys . argv : kb_ref_command += [ input ] for input in ref_input_upload_required : if input in sys . argv : kb_ref_command += [ input , str ( kb_ref_paths [ input [ 2 : ] + "_path" ] ) if input == "--tmp" else str ( kb_ref_paths [ input [ 1 : ] + "_path" ] ) , ] for input in ref_input_left_args : if input in sys . argv : kb_ref_command += [ input , getattr ( args , input ) ] print ( f"testing purpose - check if kb_ref_command is of correct format: {kb_ref_command}" ) kb_ref_failed = ut_log . log_command ( logger , kb_ref_command , shell = True , stdout = subprocess . PIPE , stderr = subprocess . STDOUT , universal_newlines = True , ) kb_ref_t_config = TransferConfig ( use_threads = False ) if kb_ref_failed : raise RuntimeError ( "kallisto index building failed" ) else : kb_ref_upload_files = glob . glob ( str ( kallisto_index_outputs / "*" ) ) print ( f"testing purpose - check kallisto index files: {kb_ref_upload_files}" ) for file in kb_ref_upload_files : logger . info ( f"Uploading {os.path.basename(file)} from the EC2 instance to AWS S3" ) s3c . upload_file ( Filename = file , Bucket = kb_ref_output_to_s3 [ file ] [ 0 ] , Key = kb_ref_output_to_s3 [ file ] [ 1 ] , Config = kb_ref_t_config , ) time . sleep ( 30 ) print ( f"testing purpose - see if kb_ref upload_file function intakes correct bucket and prefix names for kallisto index output files: {kb_ref_output_to_s3[file][0]}, {kb_ref_output_to_s3[file][1]}" ) return
Parser for the `ref` command.
Parser for the ` ref ` command .
"""Parser for the `ref` command. Build kallisto index files.
    
    Keyword Arguments:
    args -- Command-line arguments dictionary, as parsed by argparse (type: dict)
    run_dir -- Path on the EC2 instance under which jobs are run (type: pathlib.Path)
    logger -- Logger object that exposes the interface the code directly uses (type: logging.Logger)
    """ # Build paths on the EC2 instance to store inputs and outputs of kallisto index building. Download kallisto index building files from the AWS S3 bucket to the EC2 instance, if not using the in-built kallisto indices # testing purpose # Build the command of running `kb ref` to generate kallisto index files # testing purpose # testing purpose # testing purpose # Run the command to generate kallisto index files, and upload the output files on the EC2 instance back to AWS S3 # testing purpose: comment this line if this runs into error. kallisto indices are not pretty big and don't necessarily need a transfer config # testing purpose # testing purpose
args
run_dir
logger