clearml_agent/commands/worker.py
def run_custom_build_script(self, script, task, execution, venv_folder, git_root):
        # type: (str, tasks_api.Task, ExecutionInfo, Path, str)-> Tuple[Path, Path, Path]
        """
        Run a custom env build script
        :param script:
        :return: A tuple containing:
            - a full path to a python executable
            - a new task entry_point (replacing the entry_point in the task's script section)
            - a new working directory (replacing the working_dir in the task's script section)
            - a requirements manager instance
        """
        os.environ["CLEARML_TASK_SCRIPT_ENTRY"] = execution.entry_point
        os.environ["CLEARML_TASK_WORKING_DIR"] = execution.working_dir
        os.environ["CLEARML_VENV_PATH"] = str(venv_folder)
        os.environ["CLEARML_GIT_ROOT"] = git_root

        script = os.path.expanduser(os.path.expandvars(script))

        try:
            if not os.path.isfile(script):
                raise SkippedCustomBuildScript("Build script {} is not found".format(script))
        except OSError as ex:
            raise SkippedCustomBuildScript(str(ex))

        print("Running custom build script {}".format(script))

        script_output_file = NamedTemporaryFile(prefix="custom_build_script", suffix=".json", mode="wt", delete=False)

        os.environ["CLEARML_AGENT_CUSTOM_BUILD_SCRIPT"] = script
        os.environ["CLEARML_CUSTOM_BUILD_TASK_CONFIG_JSON"] = json.dumps(
            task.to_dict(), separators=(',', ':'), default=str
        )
        os.environ["CLEARML_CUSTOM_BUILD_OUTPUT"] = script_output_file.name

        try:
            subprocess.check_call([script])
        except subprocess.CalledProcessError as ex:
            raise CustomBuildScriptFailed(
                message="Custom build script failed with return code {}".format(ex.returncode),
                errno=ex.returncode
            )

        output = Path(script_output_file.name).read_text()
        if not output:
            raise SkippedCustomBuildScript("Build script {} did not return any output".format(script))

        try:
            output = json.loads(output)
            binary = Path(output["binary"])
            entry_point = Path(output["entry_point"])
            working_dir = Path(output["working_dir"])
        except ValueError as ex:
            raise SkippedCustomBuildScript(
                "Failed parsing build script output JSON ({}): {}".format(script_output_file.name, ex)
            )
        except KeyError as ex:
            raise SkippedCustomBuildScript("Build script output missing {} field".format(ex.args[0]))

        try:
            if not binary.is_file():
                raise SkippedCustomBuildScript(
                    "Invalid binary path returned from custom build script: {}".format(binary)
                )
            if not entry_point.is_file():
                raise SkippedCustomBuildScript(
                    "Invalid entrypoint path returned from custom build script: {}".format(entry_point)
                )
            if not working_dir.is_dir():
                raise SkippedCustomBuildScript(
                    "Invalid working dir returned from custom build script: {}".format(working_dir)
                )
        except OSError as ex:
            raise SkippedCustomBuildScript(str(ex))

        return binary, entry_point, working_dir

        Run a custom env build script
        :param script:
        :return: A tuple containing:
            - a full path to a python executable
            - a new task entry_point (replacing the entry_point in the task's script section)
            - a new working directory (replacing the working_dir in the task's script section)
            - a requirements manager instance
        
Run a custom env build script
Run a custom env build script
def run_custom_build_script(self, script, task, execution, venv_folder, git_root):
        os.environ["CLEARML_TASK_SCRIPT_ENTRY"] = execution.entry_point
        os.environ["CLEARML_TASK_WORKING_DIR"] = execution.working_dir
        os.environ["CLEARML_VENV_PATH"] = str(venv_folder)
        os.environ["CLEARML_GIT_ROOT"] = git_root
        script = os.path.expanduser(os.path.expandvars(script))
        try:
            if not os.path.isfile(script):
                raise SkippedCustomBuildScript("Build script {} is not found".format(script))
        except OSError as ex:
            raise SkippedCustomBuildScript(str(ex))
        print("Running custom build script {}".format(script))
        script_output_file = NamedTemporaryFile(prefix="custom_build_script", suffix=".json", mode="wt", delete=False)
        os.environ["CLEARML_AGENT_CUSTOM_BUILD_SCRIPT"] = script
        os.environ["CLEARML_CUSTOM_BUILD_TASK_CONFIG_JSON"] = json.dumps(
            task.to_dict(), separators=(',', ':'), default=str
        )
        os.environ["CLEARML_CUSTOM_BUILD_OUTPUT"] = script_output_file.name
        try:
            subprocess.check_call([script])
        except subprocess.CalledProcessError as ex:
            raise CustomBuildScriptFailed(
                message="Custom build script failed with return code {}".format(ex.returncode),
                errno=ex.returncode
            )
        output = Path(script_output_file.name).read_text()
        if not output:
            raise SkippedCustomBuildScript("Build script {} did not return any output".format(script))
        try:
            output = json.loads(output)
            binary = Path(output["binary"])
            entry_point = Path(output["entry_point"])
            working_dir = Path(output["working_dir"])
        except ValueError as ex:
            raise SkippedCustomBuildScript(
                "Failed parsing build script output JSON ({}): {}".format(script_output_file.name, ex)
            )
        except KeyError as ex:
            raise SkippedCustomBuildScript("Build script output missing {} field".format(ex.args[0]))
        try:
            if not binary.is_file():
                raise SkippedCustomBuildScript(
                    "Invalid binary path returned from custom build script: {}".format(binary)
                )
            if not entry_point.is_file():
                raise SkippedCustomBuildScript(
                    "Invalid entrypoint path returned from custom build script: {}".format(entry_point)
                )
            if not working_dir.is_dir():
                raise SkippedCustomBuildScript(
                    "Invalid working dir returned from custom build script: {}".format(working_dir)
                )
        except OSError as ex:
            raise SkippedCustomBuildScript(str(ex))
        return binary, entry_point, working_dir
def run_custom_build_script ( self , script , task , execution , venv_folder , git_root ) : os . environ [ "CLEARML_TASK_SCRIPT_ENTRY" ] = execution . entry_point os . environ [ "CLEARML_TASK_WORKING_DIR" ] = execution . working_dir os . environ [ "CLEARML_VENV_PATH" ] = str ( venv_folder ) os . environ [ "CLEARML_GIT_ROOT" ] = git_root script = os . path . expanduser ( os . path . expandvars ( script ) ) try : if not os . path . isfile ( script ) : raise SkippedCustomBuildScript ( "Build script {} is not found" . format ( script ) ) except OSError as ex : raise SkippedCustomBuildScript ( str ( ex ) ) print ( "Running custom build script {}" . format ( script ) ) script_output_file = NamedTemporaryFile ( prefix = "custom_build_script" , suffix = ".json" , mode = "wt" , delete = False ) os . environ [ "CLEARML_AGENT_CUSTOM_BUILD_SCRIPT" ] = script os . environ [ "CLEARML_CUSTOM_BUILD_TASK_CONFIG_JSON" ] = json . dumps ( task . to_dict ( ) , separators = ( ',' , ':' ) , default = str ) os . environ [ "CLEARML_CUSTOM_BUILD_OUTPUT" ] = script_output_file . name try : subprocess . check_call ( [ script ] ) except subprocess . CalledProcessError as ex : raise CustomBuildScriptFailed ( message = "Custom build script failed with return code {}" . format ( ex . returncode ) , errno = ex . returncode ) output = Path ( script_output_file . name ) . read_text ( ) if not output : raise SkippedCustomBuildScript ( "Build script {} did not return any output" . format ( script ) ) try : output = json . loads ( output ) binary = Path ( output [ "binary" ] ) entry_point = Path ( output [ "entry_point" ] ) working_dir = Path ( output [ "working_dir" ] ) except ValueError as ex : raise SkippedCustomBuildScript ( "Failed parsing build script output JSON ({}): {}" . format ( script_output_file . name , ex ) ) except KeyError as ex : raise SkippedCustomBuildScript ( "Build script output missing {} field" . format ( ex . args [ 0 ] ) ) try : if not binary . is_file ( ) : raise SkippedCustomBuildScript ( "Invalid binary path returned from custom build script: {}" . format ( binary ) ) if not entry_point . is_file ( ) : raise SkippedCustomBuildScript ( "Invalid entrypoint path returned from custom build script: {}" . format ( entry_point ) ) if not working_dir . is_dir ( ) : raise SkippedCustomBuildScript ( "Invalid working dir returned from custom build script: {}" . format ( working_dir ) ) except OSError as ex : raise SkippedCustomBuildScript ( str ( ex ) ) return binary , entry_point , working_dir
Run a custom env build script
Run a custom env build script
# type: (str, tasks_api.Task, ExecutionInfo, Path, str)-> Tuple[Path, Path, Path] """
        Run a custom env build script
        :param script:
        :return: A tuple containing:
            - a full path to a python executable
            - a new task entry_point (replacing the entry_point in the task's script section)
            - a new working directory (replacing the working_dir in the task's script section)
            - a requirements manager instance
        """
A tuple containing:
a full path to a python executable
a new task entry_point (replacing the entry_point in the task's script section)
a new working directory (replacing the working_dir in the task's script section)
a requirements manager instance
A tuple containing : a full path to a python executable a new task entry_point ( replacing the entry_point in the task ' s script section ) a new working directory ( replacing the working_dir in the task ' s script section ) a requirements manager instance
self
script
None
task
execution
venv_folder
git_root