clearml_agent/commands/worker.py
def install_virtualenv(
        self,
        venv_dir=None,
        requested_python_version=None,
        standalone_mode=False,
        execution_info=None,
        cached_requirements=None,
    ):
        # type: (str, str, bool, ExecutionInfo, dict) -> Tuple[Path, RequirementsManager, bool]
        """
        Install a new python virtual environment, removing the old one if exists
        If skip_pip_venv_install is True or contains a string (or if CLEARML_SKIP_PIP_VENV_INSTALL is set)
        then an emtpy virtual env folder is created and package manager is configured to work with the global python
        interpreter (or using a custom interpreter if an interpreter path is passed in this variable)
        :return: virtualenv directory, requirements manager to use with task, True if there is a cached venv entry
        """
        skip_pip_venv_install = self._get_skip_pip_venv_install()

        if self._session.config.get("agent.ignore_requested_python_version", None):
            requested_python_version = ''

        requested_python_version = \
            requested_python_version or \
            Text(self._session.config.get("agent.python_binary", None)) or \
            Text(self._session.config.get("agent.default_python", None)) or \
            '{}.{}'.format(sys.version_info.major, sys.version_info.minor)

        if self.is_conda:
            executable_version_suffix = \
                requested_python_version[max(requested_python_version.find('python'), 0):].replace('python', '')
            executable_name = 'python'
        else:
            override_interpreter_path = None
            if skip_pip_venv_install and isinstance(skip_pip_venv_install, str):
                if find_executable(skip_pip_venv_install):
                    override_interpreter_path = skip_pip_venv_install
                else:
                    print(
                        "Warning: interpreter {} could not be found. Reverting to the default interpreter resolution".format(
                            skip_pip_venv_install
                        )
                    )
            if override_interpreter_path:
                print("Python interpreter {} is set from environment var".format(override_interpreter_path))
                executable_name = override_interpreter_path
                executable_version_suffix = self._get_python_version_suffix(executable_name)
            else:
                try:
                    executable_version, executable_version_suffix, executable_name = \
                        self.find_python_executable_for_version(requested_python_version)
                except Exception:
                    def_python_version = Text(self._session.config.get("agent.python_binary", None)) or \
                                         Text(self._session.config.get("agent.default_python", None)) or \
                                         '{}.{}'.format(sys.version_info.major, sys.version_info.minor)
                    print('Warning: could not locate requested Python version {}, reverting to version {}'.format(
                        requested_python_version, def_python_version))
                    executable_version, executable_version_suffix, executable_name = \
                        self.find_python_executable_for_version(def_python_version)

                self._session.config.put("agent.default_python", executable_version_suffix)
                self._session.config.put("agent.python_binary", executable_name)

        venv_dir = Path(venv_dir) if venv_dir else \
            Path(self._session.config["agent.venvs_dir"], executable_version_suffix)
        venv_dir = Path(os.path.expanduser(os.path.expandvars(venv_dir.as_posix())))

        first_time = not standalone_mode and (
            is_windows_platform()
            or self.is_conda
            or not venv_dir.is_dir()
            or not self.is_venv_update
        )

        if not standalone_mode:
            rm_tree(normalize_path(venv_dir, WORKING_REPOSITORY_DIR))
            rm_tree(normalize_path(venv_dir, WORKING_STANDALONE_DIR))

        call_package_manager_create, requirements_manager = self._setup_package_api(
            executable_name=executable_name,
            executable_version_suffix=executable_version_suffix,
            venv_dir=venv_dir,
            execution_info=execution_info,
            standalone_mode=standalone_mode,
            skip_pip_venv_install=skip_pip_venv_install,
            first_time=first_time,
        )

        # check if we have a cached folder
        if cached_requirements and not skip_pip_venv_install and self.package_api.get_cached_venv(
            requirements=cached_requirements,
            docker_cmd=execution_info.docker_cmd if execution_info else None,
            python_version=self.package_api.python,
            cuda_version=self._session.config.get("agent.cuda_version"),
            destination_folder=Path(venv_dir)
        ):
            print('::: Using Cached environment {} :::'.format(self.package_api.get_last_used_entry_cache()))
            return venv_dir, requirements_manager, True

        # create the initial venv
        if not skip_pip_venv_install:
            if call_package_manager_create:
                self.package_api.create()
        else:
            if not venv_dir.exists():
                venv_dir.mkdir(parents=True, exist_ok=True)

        return venv_dir, requirements_manager, False

        Install a new python virtual environment, removing the old one if exists
        If skip_pip_venv_install is True or contains a string (or if CLEARML_SKIP_PIP_VENV_INSTALL is set)
        then an emtpy virtual env folder is created and package manager is configured to work with the global python
        interpreter (or using a custom interpreter if an interpreter path is passed in this variable)
        :return: virtualenv directory, requirements manager to use with task, True if there is a cached venv entry
        
Install a new python virtual environment, removing the old one if exists
If skip_pip_venv_install is True or contains a string (or if CLEARML_SKIP_PIP_VENV_INSTALL is set)
then an emtpy virtual env folder is created and package manager is configured to work with the global python
interpreter (or using a custom interpreter if an interpreter path is passed in this variable)
Install a new python virtual environment removing the old one if exists If skip_pip_venv_install is True or contains a string ( or if CLEARML_SKIP_PIP_VENV_INSTALL is set ) then an emtpy virtual env folder is created and package manager is configured to work with the global python interpreter ( or using a custom interpreter if an interpreter path is passed in this variable )
def install_virtualenv(
        self,
        venv_dir=None,
        requested_python_version=None,
        standalone_mode=False,
        execution_info=None,
        cached_requirements=None,
    ):
        skip_pip_venv_install = self._get_skip_pip_venv_install()
        if self._session.config.get("agent.ignore_requested_python_version", None):
            requested_python_version = ''
        requested_python_version = \
            requested_python_version or \
            Text(self._session.config.get("agent.python_binary", None)) or \
            Text(self._session.config.get("agent.default_python", None)) or \
            '{}.{}'.format(sys.version_info.major, sys.version_info.minor)
        if self.is_conda:
            executable_version_suffix = \
                requested_python_version[max(requested_python_version.find('python'), 0):].replace('python', '')
            executable_name = 'python'
        else:
            override_interpreter_path = None
            if skip_pip_venv_install and isinstance(skip_pip_venv_install, str):
                if find_executable(skip_pip_venv_install):
                    override_interpreter_path = skip_pip_venv_install
                else:
                    print(
                        "Warning: interpreter {} could not be found. Reverting to the default interpreter resolution".format(
                            skip_pip_venv_install
                        )
                    )
            if override_interpreter_path:
                print("Python interpreter {} is set from environment var".format(override_interpreter_path))
                executable_name = override_interpreter_path
                executable_version_suffix = self._get_python_version_suffix(executable_name)
            else:
                try:
                    executable_version, executable_version_suffix, executable_name = \
                        self.find_python_executable_for_version(requested_python_version)
                except Exception:
                    def_python_version = Text(self._session.config.get("agent.python_binary", None)) or \
                                         Text(self._session.config.get("agent.default_python", None)) or \
                                         '{}.{}'.format(sys.version_info.major, sys.version_info.minor)
                    print('Warning: could not locate requested Python version {}, reverting to version {}'.format(
                        requested_python_version, def_python_version))
                    executable_version, executable_version_suffix, executable_name = \
                        self.find_python_executable_for_version(def_python_version)
                self._session.config.put("agent.default_python", executable_version_suffix)
                self._session.config.put("agent.python_binary", executable_name)
        venv_dir = Path(venv_dir) if venv_dir else \
            Path(self._session.config["agent.venvs_dir"], executable_version_suffix)
        venv_dir = Path(os.path.expanduser(os.path.expandvars(venv_dir.as_posix())))
        first_time = not standalone_mode and (
            is_windows_platform()
            or self.is_conda
            or not venv_dir.is_dir()
            or not self.is_venv_update
        )
        if not standalone_mode:
            rm_tree(normalize_path(venv_dir, WORKING_REPOSITORY_DIR))
            rm_tree(normalize_path(venv_dir, WORKING_STANDALONE_DIR))
        call_package_manager_create, requirements_manager = self._setup_package_api(
            executable_name=executable_name,
            executable_version_suffix=executable_version_suffix,
            venv_dir=venv_dir,
            execution_info=execution_info,
            standalone_mode=standalone_mode,
            skip_pip_venv_install=skip_pip_venv_install,
            first_time=first_time,
        )
        if cached_requirements and not skip_pip_venv_install and self.package_api.get_cached_venv(
            requirements=cached_requirements,
            docker_cmd=execution_info.docker_cmd if execution_info else None,
            python_version=self.package_api.python,
            cuda_version=self._session.config.get("agent.cuda_version"),
            destination_folder=Path(venv_dir)
        ):
            print('::: Using Cached environment {} :::'.format(self.package_api.get_last_used_entry_cache()))
            return venv_dir, requirements_manager, True
        if not skip_pip_venv_install:
            if call_package_manager_create:
                self.package_api.create()
        else:
            if not venv_dir.exists():
                venv_dir.mkdir(parents=True, exist_ok=True)
        return venv_dir, requirements_manager, False
def install_virtualenv ( self , venv_dir = None , requested_python_version = None , standalone_mode = False , execution_info = None , cached_requirements = None , ) : skip_pip_venv_install = self . _get_skip_pip_venv_install ( ) if self . _session . config . get ( "agent.ignore_requested_python_version" , None ) : requested_python_version = '' requested_python_version = requested_python_version or Text ( self . _session . config . get ( "agent.python_binary" , None ) ) or Text ( self . _session . config . get ( "agent.default_python" , None ) ) or '{}.{}' . format ( sys . version_info . major , sys . version_info . minor ) if self . is_conda : executable_version_suffix = requested_python_version [ max ( requested_python_version . find ( 'python' ) , 0 ) : ] . replace ( 'python' , '' ) executable_name = 'python' else : override_interpreter_path = None if skip_pip_venv_install and isinstance ( skip_pip_venv_install , str ) : if find_executable ( skip_pip_venv_install ) : override_interpreter_path = skip_pip_venv_install else : print ( "Warning: interpreter {} could not be found. Reverting to the default interpreter resolution" . format ( skip_pip_venv_install ) ) if override_interpreter_path : print ( "Python interpreter {} is set from environment var" . format ( override_interpreter_path ) ) executable_name = override_interpreter_path executable_version_suffix = self . _get_python_version_suffix ( executable_name ) else : try : executable_version , executable_version_suffix , executable_name = self . find_python_executable_for_version ( requested_python_version ) except Exception : def_python_version = Text ( self . _session . config . get ( "agent.python_binary" , None ) ) or Text ( self . _session . config . get ( "agent.default_python" , None ) ) or '{}.{}' . format ( sys . version_info . major , sys . version_info . minor ) print ( 'Warning: could not locate requested Python version {}, reverting to version {}' . format ( requested_python_version , def_python_version ) ) executable_version , executable_version_suffix , executable_name = self . find_python_executable_for_version ( def_python_version ) self . _session . config . put ( "agent.default_python" , executable_version_suffix ) self . _session . config . put ( "agent.python_binary" , executable_name ) venv_dir = Path ( venv_dir ) if venv_dir else Path ( self . _session . config [ "agent.venvs_dir" ] , executable_version_suffix ) venv_dir = Path ( os . path . expanduser ( os . path . expandvars ( venv_dir . as_posix ( ) ) ) ) first_time = not standalone_mode and ( is_windows_platform ( ) or self . is_conda or not venv_dir . is_dir ( ) or not self . is_venv_update ) if not standalone_mode : rm_tree ( normalize_path ( venv_dir , WORKING_REPOSITORY_DIR ) ) rm_tree ( normalize_path ( venv_dir , WORKING_STANDALONE_DIR ) ) call_package_manager_create , requirements_manager = self . _setup_package_api ( executable_name = executable_name , executable_version_suffix = executable_version_suffix , venv_dir = venv_dir , execution_info = execution_info , standalone_mode = standalone_mode , skip_pip_venv_install = skip_pip_venv_install , first_time = first_time , ) if cached_requirements and not skip_pip_venv_install and self . package_api . get_cached_venv ( requirements = cached_requirements , docker_cmd = execution_info . docker_cmd if execution_info else None , python_version = self . package_api . python , cuda_version = self . _session . config . get ( "agent.cuda_version" ) , destination_folder = Path ( venv_dir ) ) : print ( '::: Using Cached environment {} :::' . format ( self . package_api . get_last_used_entry_cache ( ) ) ) return venv_dir , requirements_manager , True if not skip_pip_venv_install : if call_package_manager_create : self . package_api . create ( ) else : if not venv_dir . exists ( ) : venv_dir . mkdir ( parents = True , exist_ok = True ) return venv_dir , requirements_manager , False
Install a new python virtual environment, removing the old one if exists
If skip_pip_venv_install is True or contains a string (or if CLEARML_SKIP_PIP_VENV_INSTALL is set)
then an emtpy virtual env folder is created and package manager is configured to work with the global python
interpreter (or using a custom interpreter if an interpreter path is passed in this variable)
Install a new python virtual environment removing the old one if exists If skip_pip_venv_install is True or contains a string ( or if CLEARML_SKIP_PIP_VENV_INSTALL is set ) then an emtpy virtual env folder is created and package manager is configured to work with the global python interpreter ( or using a custom interpreter if an interpreter path is passed in this variable )
# type: (str, str, bool, ExecutionInfo, dict) -> Tuple[Path, RequirementsManager, bool] """
        Install a new python virtual environment, removing the old one if exists
        If skip_pip_venv_install is True or contains a string (or if CLEARML_SKIP_PIP_VENV_INSTALL is set)
        then an emtpy virtual env folder is created and package manager is configured to work with the global python
        interpreter (or using a custom interpreter if an interpreter path is passed in this variable)
        :return: virtualenv directory, requirements manager to use with task, True if there is a cached venv entry
        """ # check if we have a cached folder # create the initial venv
virtualenv directory, requirements manager to use with task, True if there is a cached venv entry
virtualenv directory requirements manager to use with task True if there is a cached venv entry
self
venv_dir
requested_python_version
standalone_mode
execution_info
cached_requirements