def run_test(args, i, t):
    out = {}
    outdir = None
    if args.force_test_tool:
        t["tool"] = args.force_test_tool

    if t["tool"].split("/")[-1] != re.sub("\-test\.yaml$", ".cwl",args.test.split("/")[-1]):
        raise CompareFail("%s != %s" % (t["tool"].split("/")[-1],re.sub("\-test\.yaml$",".cwl",args.test.split("/")[-1])))

    try:
        if "output" in t and not args.conformance_test:
            test_command = [args.tool]

            if "outdir" in t:
                outdir = t["outdir"]
            else:
                # Add prefixes if running on MacOSX so that boot2docker writes to /Users
                if 'darwin' in sys.platform:
                    outdir = tempfile.mkdtemp(prefix=os.path.abspath(os.path.curdir))
                else:
                    outdir = tempfile.mkdtemp()
            if args.tmp:
                test_command.extend(["--tmp-outdir-prefix={}".format(outdir), "--tmpdir-prefix={}".format(outdir)])

            if args.push_image:
                test_command.extend(["--push-image"])

            test_command.extend(["--outdir={}".format(outdir),
                                 t["tool"],
                                 t["job"]])
            outstr = subprocess.check_output(test_command)
            out = {"output": json.loads(outstr)}
        else:
            test_command = [args.tool,
                            "--conformance-test",
                            "--basedir=" + args.basedir,
                            "--no-container",
                            "--quiet",
                            t["tool"],
                            t["job"]]
            outstr = subprocess.check_output(test_command)
            outstr = re.sub("^\[.*\n", "", outstr)
            out = yaml.load(outstr)
    except __HOLE__ as v:
        _logger.error(v)
        _logger.error(outstr)
    except subprocess.CalledProcessError as err:
        if err.returncode == UNSUPPORTED_FEATURE:
            return UNSUPPORTED_FEATURE
        else:
            _logger.error("""Test failed: %s""", " ".join([pipes.quote(tc) for tc in test_command]))
            _logger.error(t.get("doc"))
            _logger.error("Returned non-zero")
            return 1
    except yaml.scanner.ScannerError as e:
        _logger.error("""Test failed: %s""", " ".join([pipes.quote(tc) for tc in test_command]))
        _logger.error(outstr)
        _logger.error("Parse error %s", str(e))

    pwd = os.path.abspath(os.path.dirname(t["job"]))
    # t["args"] = map(lambda x: x.replace("$PWD", pwd), t["args"])
    # if "stdin" in t:
    #     t["stdin"] = t["stdin"].replace("$PWD", pwd)

    failed = False
    if "output" in t and not args.conformance_test:
        checkkeys = ["output"]
    else:
        checkkeys = ["args", "stdin", "stdout", "createfiles"]
    ignore = None
    if "ignore_keys" in t:
        ignore = t["ignore_keys"]
    for key in checkkeys:
        try:
            compare(t.get(key), out.get(key),ignore)
        except CompareFail as ex:
            _logger.warn("""Test failed: %s""", " ".join([pipes.quote(tc) for tc in test_command]))
            _logger.warn(t.get("doc"))
            _logger.warn("%s expected %s\n got %s", key,
                         json.dumps(t.get(key), indent=4, sort_keys=True),
                         json.dumps(out.get(key), indent=4, sort_keys=True))
            _logger.warn("Compare failure %s", ex)
            failed = True

    # if outdir:
    #     shutil.rmtree(outdir, True)

    if failed:
        return 1
    else:
        return 0
dataset/ETHPy150Open common-workflow-language/workflows/test/cwltest.py/run_test