def run_steps(args: argparse.Namespace):
    """Run all steps required to complete task.  Called directly from main."""

    logging.basicConfig(level=logging.INFO, format="sockeye.autopilot: %(message)s")

    # (1) Establish task

    logging.info("=== Start Autopilot ===")
    # Listed task
    if args.task:
        task = TASKS[args.task]
        logging.info("Task: %s", task.description)
        logging.info("URL: %s", task.url)

        def report_data(file_sets):
            for file_set in file_sets:
                for fname in file_set[:2]:
                    logging.info("    %s", fname)

        logging.info("  Train:")
        report_data(task.train)
        logging.info("  Dev:")
        report_data(task.dev)
        logging.info("  Test:")
        report_data(task.test)
    # Custom task
    else:
        logging.info("Task: custom")
    # Source and target language codes
    lang_codes = (task.src_lang, task.trg_lang) if args.task else args.custom_lang

    # (2) Establish workspace and task directories

    logging.info("=== Establish working directories ===")
    logging.info("Workspace: %s", args.workspace)
    special_fname = os.path.join(args.workspace, FILE_WORKSPACE)
    if not os.path.exists(args.workspace):
        logging.info("Create: %s", args.workspace)
        os.makedirs(args.workspace)
        touch_file(special_fname)
    else:
        if not os.path.exists(special_fname):
            raise RuntimeError("Directory %s exists but %s does not, stopping to avoid overwriting files in non-workspace directory"
                            % (args.workspace, special_fname))

    dir_third_party = os.path.join(args.workspace, third_party.DIR_THIRD_PARTY)
    dir_cache = os.path.join(args.workspace, DIR_CACHE)
    dir_logs = os.path.join(args.workspace, DIR_LOGS)
    dir_systems = os.path.join(args.workspace, DIR_SYSTEMS)
    task_name = args.task if args.task else args.custom_task
    if args.test:
        task_name += SUFFIX_TEST
    dir_task = os.path.join(dir_systems, task_name)
    for dirname in (dir_third_party, dir_cache, dir_logs, dir_systems, dir_task):
        if os.path.exists(dirname):
            logging.info("Exists: %s", dirname)
        else:
            logging.info("Create: %s", dirname)
            os.makedirs(dirname)

    # (3) Checkout necessary tools

    logging.info("=== Checkout third-party tools ===")
    # Requires tokenization?
    if args.task or args.custom_text_type == CUSTOM_UTF8_RAW:
        third_party.checkout_moses_tokenizer(args.workspace)
    # Requires byte-pair encoding?
    if args.task or args.custom_text_type in (CUSTOM_UTF8_RAW, CUSTOM_UTF8_TOK):
        third_party.checkout_subword_nmt(args.workspace)

    # (4) Populate train/dev/test data

    # This step also normalizes whitespace on data population or copy, ensuring
    # that for all input data, only ASCII newlines are considered line breaks.
    logging.info("=== Populate train/dev/test data ===")
    step_dir_raw = os.path.join(dir_task, DIR_DATA, DIR_RAW)
    complete_fname = os.path.join(step_dir_raw, FILE_COMPLETE)
    if os.path.exists(complete_fname):
        logging.info("Re-use completed step: %s", step_dir_raw)
    else:
        # Listed task
        if args.task:
            raw_files = identify_raw_files(task, test_mode=args.test)
            with tempfile.TemporaryDirectory(prefix="raw.", dir=dir_task) as raw_dir:
                # Download (or locate in cache) and extract raw files to temp directory
                logging.info("=== Download and extract raw files ===")
                download_extract_raw_files(raw_files, dir_cache, raw_dir)
                # Copy required files to train/dev/test
                logging.info("=== Create input data files ===")
                renew_step_dir(step_dir_raw)
                # Test mode uses the full test set as training data and the
                # first line of the test set as dev and test data
                populate_parallel_text(raw_dir,
                                       task.test if args.test else task.train,
                                       os.path.join(step_dir_raw, PREFIX_TRAIN),
                                       False)
                populate_parallel_text(raw_dir,
                                       task.test if args.test else task.dev,
                                       os.path.join(step_dir_raw, PREFIX_DEV),
                                       False,
                                       head_n=1 if args.test else 0)
                populate_parallel_text(raw_dir,
                                       task.test,
                                       os.path.join(step_dir_raw, PREFIX_TEST),
                                       True,
                                       head_n=1 if args.test else 0)
        # Custom task
        else:
            logging.info("=== Copy input data files ===")
            renew_step_dir(step_dir_raw)
            copy_parallel_text(args.custom_train, os.path.join(step_dir_raw, PREFIX_TRAIN))
            copy_parallel_text(args.custom_dev, os.path.join(step_dir_raw, PREFIX_DEV))
            copy_parallel_text(args.custom_test, os.path.join(step_dir_raw, PREFIX_TEST))
        # Record success
        touch_file(complete_fname)
        logging.info("Step complete: %s", step_dir_raw)

    # (5) Tokenize train/dev/test data

    # Task requires tokenization if _any_ raw file is not already tokenized
    requires_tokenization = False
    if args.task:
        for file_sets in (task.train, task.dev, task.test):
            for _, _, text_type in file_sets:
                if text_type in TEXT_REQUIRES_TOKENIZATION:
                    requires_tokenization = True
    else:
        if args.custom_text_type == CUSTOM_UTF8_RAW:
            requires_tokenization = True
    logging.info("=== Tokenize train/dev/test data ===")
    step_dir_tok = os.path.join(dir_task, DIR_DATA, DIR_TOK)
    complete_fname = os.path.join(step_dir_tok, FILE_COMPLETE)
    if os.path.exists(complete_fname):
        logging.info("Re-use completed step: %s", step_dir_tok)
    else:
        renew_step_dir(step_dir_tok)

        # Tokenize each data file using the appropriate language code OR link
        # raw file if already tokenized.
        for fname in os.listdir(step_dir_raw):
            if fname.startswith("."):
                continue
            input_fname = os.path.join(step_dir_raw, fname)
            output_fname = os.path.join(step_dir_tok, fname)
            if requires_tokenization:
                lang_code = lang_codes[0] if fname.endswith(SUFFIX_SRC_GZ) else lang_codes[1]
                logging.info("Tokenize (%s): %s -> %s", lang_code, input_fname, output_fname)
                third_party.call_moses_tokenizer(workspace_dir=args.workspace,
                                                 input_fname=input_fname,
                                                 output_fname=output_fname,
                                                 lang_code=lang_code)
            else:
                logging.info("Link pre-tokenized: %s -> %s", input_fname, output_fname)
                os.symlink(os.path.join("..", DIR_RAW, fname), output_fname)
        # Record success
        touch_file(complete_fname)
        logging.info("Step complete: %s", step_dir_tok)

    # (6) Learn byte-pair encoding model

    # Task requires byte-pair encoding unless using pre-encoded custom data
    skip_bpe = (not args.task) and args.custom_text_type == CUSTOM_UTF8_BPE
    logging.info("=== Learn byte-pair encoding model ===")
    step_dir_bpe_model = os.path.join(dir_task, DIR_BPE_MODEL)
    complete_fname = os.path.join(step_dir_bpe_model, FILE_COMPLETE)
    if os.path.exists(complete_fname):
        logging.info("Re-use completed step: %s", step_dir_bpe_model)
    else:
        renew_step_dir(step_dir_bpe_model)
        if skip_bpe:
            logging.info("BPE model not required for pre-encoded data")
        else:
            source_fname = os.path.join(step_dir_tok, PREFIX_TRAIN + SUFFIX_SRC_GZ)
            target_fname = os.path.join(step_dir_tok, PREFIX_TRAIN + SUFFIX_TRG_GZ)
            codes_fname = os.path.join(step_dir_bpe_model, FILE_BPE_CODES)
            num_ops = task.bpe_op if args.task else args.custom_bpe_op
            if args.test:
                num_ops = TEST_BPE_OPS
            logging.info("BPE Learn (%s): %s + %s -> %s", num_ops, source_fname, target_fname, codes_fname)
            third_party.call_learn_bpe(workspace_dir=args.workspace,
                                       source_fname=source_fname,
                                       target_fname=target_fname,
                                       model_fname=codes_fname,
                                       num_ops=num_ops)
        # Record success
        touch_file(complete_fname)
        logging.info("Step complete: %s", step_dir_bpe_model)

    # (7) Byte-pair encode data

    logging.info("=== Byte-pair encode train/dev/test data ===")
    step_dir_bpe = os.path.join(dir_task, DIR_DATA, DIR_BPE)
    complete_fname = os.path.join(step_dir_bpe, FILE_COMPLETE)
    if os.path.exists(complete_fname):
        logging.info("Re-use completed step: %s", step_dir_bpe)
    else:
        renew_step_dir(step_dir_bpe)
        # Encode each data file
        for fname in os.listdir(step_dir_tok):
            if fname.startswith("."):
                continue
            input_fname = os.path.join(step_dir_tok, fname)
            output_fname = os.path.join(step_dir_bpe, fname)
            if skip_bpe:
                logging.info("Link pre-encoded: %s -> %s", input_fname, output_fname)
                os.symlink(os.path.join("..", DIR_TOK, fname), output_fname)
            else:
                codes_fname = os.path.join(step_dir_bpe_model, FILE_BPE_CODES)
                logging.info("BPE: %s -> %s", input_fname, output_fname)
                third_party.call_apply_bpe(workspace_dir=args.workspace,
                                           input_fname=input_fname,
                                           output_fname=output_fname,
                                           model_fname=codes_fname)
        # Record success
        touch_file(complete_fname)
        logging.info("Step complete: %s", step_dir_bpe)

    # Done if only running data preparation steps
    if args.model == MODEL_NONE:
        return

    # (8) Run Sockeye training

    logging.info("=== Train translation model ===")
    logging.info("Model: %s", args.model)
    if args.model == MODEL_GNMT:
        logging.info("NOTE: This is an 8 layer LSTM model similar (but not exactly identical) to the 'GNMT' architecture.")
    step_dir_model = os.path.join(dir_task, DIR_PREFIX_MODEL + args.model)
    complete_fname = os.path.join(step_dir_model, FILE_COMPLETE)
    if os.path.exists(complete_fname):
        logging.info("Re-use completed step: %s", step_dir_model)
    else:
        log_fname = os.path.join(args.workspace,
                                 DIR_LOGS,
                                 "sockeye.{{}}.{}.{}.{}.log".format(task_name, args.model, os.getpid()))
        call_sockeye_train(args.model,
                           step_dir_bpe,
                           step_dir_model,
                           log_fname.format("train"),
                           args.gpus,
                           test_mode=args.test)
        call_sockeye_average(step_dir_model, log_fname.format("average"))
        # Record success
        touch_file(complete_fname)
        logging.info("Step complete: %s", step_dir_model)

    # (9) Decode test sets

    logging.info("=== Decode test sets ===")
    logging.info("Settings: %s", args.decode_settings)
    step_dir_results = os.path.join(dir_task, DIR_RESULTS)
    if not os.path.exists(step_dir_results):
        logging.info("Create: %s", step_dir_results)
        os.makedirs(step_dir_results)
    # To collect BPE output names
    output_fnames_bpe = []
    # For each test file
    for fname in os.listdir(step_dir_bpe):
        if fname.startswith(PREFIX_TEST) and fname.endswith(SUFFIX_SRC_GZ):
            input_fname = os.path.join(step_dir_bpe, fname)
            # /path/to/results/test[.N].<model>.<settings>
            output_fname = os.path.join(step_dir_results, "{}.{}.{}.{}".format(args.model,
                                                                               args.decode_settings,
                                                                               fname[:-len(SUFFIX_SRC_GZ) - 1],
                                                                               SUFFIX_BPE))
            output_fnames_bpe.append(output_fname)
            # For the shared results directory, a command file indicates that
            # the step has completed successfully.
            command_fname = output_fname + "." + SUFFIX_COMMAND
            if os.path.exists(command_fname):
                logging.info("Re-use output: %s", output_fname)
            else:
                log_fname = os.path.join(args.workspace,
                                 DIR_LOGS,
                                 "sockeye.translate.{}.{}.{}.{}.log".format(task_name,
                                                                            args.model,
                                                                            fname[:-len(SUFFIX_SRC_GZ) - 1],
                                                                            os.getpid()))
                call_sockeye_translate(args=DECODE_ARGS[args.decode_settings],
                                       input_fname=input_fname,
                                       output_fname=output_fname,
                                       model_dir=step_dir_model,
                                       log_fname=log_fname,
                                       use_cpu=(args.gpus == 0))

    # (10) Evaluate test sets (bpe/tok/detok)

    lang_code = lang_codes[1] if lang_codes else None
    logging.info("=== Score outputs ===")
    # For each output file
    for fname_bpe in output_fnames_bpe:
        # Score byte-pair encoded
        fname_base = os.path.basename(fname_bpe)[:-len(SUFFIX_BPE)].split(".", 2)[2]
        fname_ref_bpe = os.path.join(step_dir_bpe, fname_base + SUFFIX_TRG_GZ)
        fname_bleu_bpe = fname_bpe + "." + SUFFIX_BLEU
        if os.path.exists(fname_bleu_bpe):
            logging.info("Re-use output: %s", fname_bleu_bpe)
        else:
            fname_log = os.path.join(args.workspace,
                                     DIR_LOGS,
                                     "sacrebleu.sacrebleu.{}.{}.{}.{}.log".format(task_name,
                                                                                  args.model,
                                                                                  fname_base + SUFFIX_BPE,
                                                                                  os.getpid()))
            call_sacrebleu(input_fname=fname_bpe,
                           ref_fname=fname_ref_bpe,
                           output_fname=fname_bleu_bpe,
                           log_fname=fname_log,
                           tokenized=True)
        # Score tokenized
        fname_tok = fname_bpe[:-len(SUFFIX_BPE)] + SUFFIX_TOK
        fname_ref_tok = os.path.join(step_dir_tok, fname_base + SUFFIX_TRG_GZ)
        fname_bleu_tok = fname_tok + "." + SUFFIX_BLEU
        if os.path.exists(fname_bleu_tok):
            logging.info("Re-use output: %s", fname_bleu_tok)
        else:
            # Merge BPE
            logging.info("Merge BPE: %s -> %s", fname_bpe, fname_tok)
            third_party.merge_bpe(input_fname=fname_bpe, output_fname=fname_tok)
            fname_log = os.path.join(args.workspace,
                                     DIR_LOGS,
                                     "sacrebleu.sacrebleu.{}.{}.{}.{}.log".format(task_name,
                                                                                  args.model,
                                                                                  fname_base + SUFFIX_TOK,
                                                                                  os.getpid()))
            call_sacrebleu(input_fname=fname_tok,
                           ref_fname=fname_ref_tok,
                           output_fname=fname_bleu_tok,
                           log_fname=fname_log,
                           tokenized=True)
        # Score detokenized (WMT-compatible BLEU)
        fname_detok = fname_bpe[:-len(SUFFIX_BPE)] + SUFFIX_DETOK
        fname_ref_raw = os.path.join(step_dir_raw, fname_base + SUFFIX_TRG_GZ)
        fname_bleu_detok = fname_detok + "." + SUFFIX_SACREBLEU
        if os.path.exists(fname_bleu_detok):
            logging.info("Re-use output: %s", fname_bleu_detok)
        else:
            if not requires_tokenization:
                logging.info(
                    "WARNING: Task uses pre-tokenized data, cannot reliably detokenize to compute WMT-compatible scores")
                continue
            # Detokenize
            logging.info("Detokenize (%s): %s -> %s", lang_code, fname_tok, fname_detok)
            third_party.call_moses_detokenizer(workspace_dir=args.workspace,
                                               input_fname=fname_tok,
                                               output_fname=fname_detok,
                                               lang_code=lang_code)
            fname_log = os.path.join(args.workspace,
                                     DIR_LOGS,
                                     "sacrebleu.sacrebleu.{}.{}.{}.{}.log".format(task_name,
                                                                                  args.model,
                                                                                  fname_base + SUFFIX_DETOK,
                                                                                  os.getpid()))
            call_sacrebleu(input_fname=fname_detok,
                           ref_fname=fname_ref_raw,
                           output_fname=fname_bleu_detok,
                           log_fname=fname_log,
                           tokenized=False)
Run all steps required to complete task.  Called directly from main.