python/ray/_private/runtime_env/packaging.py
def download_and_unpack_package(
    pkg_uri: str,
    base_directory: str,
    logger: Optional[logging.Logger] = default_logger,
) -> str:
    """Download the package corresponding to this URI and unpack it if zipped.

    Will be written to a file or directory named {base_directory}/{uri}.
    Returns the path to this file or directory.
    """
    pkg_file = Path(_get_local_path(base_directory, pkg_uri))
    with FileLock(str(pkg_file) + ".lock"):
        if logger is None:
            logger = default_logger

        logger.debug(f"Fetching package for URI: {pkg_uri}")

        local_dir = get_local_dir_from_uri(pkg_uri, base_directory)
        assert local_dir != pkg_file, "Invalid pkg_file!"
        if local_dir.exists():
            assert local_dir.is_dir(), f"{local_dir} is not a directory"
        else:
            protocol, pkg_name = parse_uri(pkg_uri)
            if protocol == Protocol.GCS:
                # Download package from the GCS.
                code = _internal_kv_get(pkg_uri)
                if code is None:
                    raise IOError(f"Failed to fetch URI {pkg_uri} from GCS.")
                code = code or b""
                pkg_file.write_bytes(code)

                if is_zip_uri(pkg_uri):
                    unzip_package(
                        package_path=pkg_file,
                        target_dir=local_dir,
                        remove_top_level_directory=False,
                        unlink_zip=True,
                        logger=logger,
                    )
                else:
                    return str(pkg_file)
            elif protocol in Protocol.remote_protocols():
                # Download package from remote URI
                tp = None

                if protocol == Protocol.S3:
                    try:
                        from smart_open import open
                        import boto3
                    except ImportError:
                        raise ImportError(
                            "You must `pip install smart_open` and "
                            "`pip install boto3` to fetch URIs in s3 "
                            "bucket."
                        )
                    tp = {"client": boto3.client("s3")}
                elif protocol == Protocol.GS:
                    try:
                        from smart_open import open
                        from google.cloud import storage  # noqa: F401
                    except ImportError:
                        raise ImportError(
                            "You must `pip install smart_open` and "
                            "`pip install google-cloud-storage` "
                            "to fetch URIs in Google Cloud Storage bucket."
                        )
                else:
                    try:
                        from smart_open import open
                    except ImportError:
                        raise ImportError(
                            "You must `pip install smart_open` "
                            f"to fetch {protocol.value.upper()} URIs."
                        )

                with open(pkg_uri, "rb", transport_params=tp) as package_zip:
                    with open(pkg_file, "wb") as fin:
                        fin.write(package_zip.read())

                unzip_package(
                    package_path=pkg_file,
                    target_dir=local_dir,
                    remove_top_level_directory=True,
                    unlink_zip=True,
                    logger=logger,
                )
            else:
                raise NotImplementedError(f"Protocol {protocol} is not supported")

        return str(local_dir)
Download the package corresponding to this URI and unpack it if zipped.

    Will be written to a file or directory named {base_directory}/{uri}.
    Returns the path to this file or directory.
    
Download the package corresponding to this URI and unpack it if zipped.
Will be written to a file or directory named {base_directory}/{uri}.
Returns the path to this file or directory.
Download the package corresponding to this URI and unpack it if zipped . Will be written to a file or directory named { base_directory } / { uri } . Returns the path to this file or directory .
def download_and_unpack_package(
    pkg_uri: str,
    base_directory: str,
    logger: Optional[logging.Logger] = default_logger,
) -> str:
    pkg_file = Path(_get_local_path(base_directory, pkg_uri))
    with FileLock(str(pkg_file) + ".lock"):
        if logger is None:
            logger = default_logger
        logger.debug(f"Fetching package for URI: {pkg_uri}")
        local_dir = get_local_dir_from_uri(pkg_uri, base_directory)
        assert local_dir != pkg_file, "Invalid pkg_file!"
        if local_dir.exists():
            assert local_dir.is_dir(), f"{local_dir} is not a directory"
        else:
            protocol, pkg_name = parse_uri(pkg_uri)
            if protocol == Protocol.GCS:
                code = _internal_kv_get(pkg_uri)
                if code is None:
                    raise IOError(f"Failed to fetch URI {pkg_uri} from GCS.")
                code = code or b""
                pkg_file.write_bytes(code)
                if is_zip_uri(pkg_uri):
                    unzip_package(
                        package_path=pkg_file,
                        target_dir=local_dir,
                        remove_top_level_directory=False,
                        unlink_zip=True,
                        logger=logger,
                    )
                else:
                    return str(pkg_file)
            elif protocol in Protocol.remote_protocols():
                tp = None
                if protocol == Protocol.S3:
                    try:
                        from smart_open import open
                        import boto3
                    except ImportError:
                        raise ImportError(
                            "You must `pip install smart_open` and "
                            "`pip install boto3` to fetch URIs in s3 "
                            "bucket."
                        )
                    tp = {"client": boto3.client("s3")}
                elif protocol == Protocol.GS:
                    try:
                        from smart_open import open
                        from google.cloud import storage  
                    except ImportError:
                        raise ImportError(
                            "You must `pip install smart_open` and "
                            "`pip install google-cloud-storage` "
                            "to fetch URIs in Google Cloud Storage bucket."
                        )
                else:
                    try:
                        from smart_open import open
                    except ImportError:
                        raise ImportError(
                            "You must `pip install smart_open` "
                            f"to fetch {protocol.value.upper()} URIs."
                        )
                with open(pkg_uri, "rb", transport_params=tp) as package_zip:
                    with open(pkg_file, "wb") as fin:
                        fin.write(package_zip.read())
                unzip_package(
                    package_path=pkg_file,
                    target_dir=local_dir,
                    remove_top_level_directory=True,
                    unlink_zip=True,
                    logger=logger,
                )
            else:
                raise NotImplementedError(f"Protocol {protocol} is not supported")
        return str(local_dir)
def download_and_unpack_package ( pkg_uri : str , base_directory : str , logger : Optional [ logging . Logger ] = default_logger , ) -> str : pkg_file = Path ( _get_local_path ( base_directory , pkg_uri ) ) with FileLock ( str ( pkg_file ) + ".lock" ) : if logger is None : logger = default_logger logger . debug ( f"Fetching package for URI: {pkg_uri}" ) local_dir = get_local_dir_from_uri ( pkg_uri , base_directory ) assert local_dir != pkg_file , "Invalid pkg_file!" if local_dir . exists ( ) : assert local_dir . is_dir ( ) , f"{local_dir} is not a directory" else : protocol , pkg_name = parse_uri ( pkg_uri ) if protocol == Protocol . GCS : code = _internal_kv_get ( pkg_uri ) if code is None : raise IOError ( f"Failed to fetch URI {pkg_uri} from GCS." ) code = code or b"" pkg_file . write_bytes ( code ) if is_zip_uri ( pkg_uri ) : unzip_package ( package_path = pkg_file , target_dir = local_dir , remove_top_level_directory = False , unlink_zip = True , logger = logger , ) else : return str ( pkg_file ) elif protocol in Protocol . remote_protocols ( ) : tp = None if protocol == Protocol . S3 : try : from smart_open import open import boto3 except ImportError : raise ImportError ( "You must `pip install smart_open` and " "`pip install boto3` to fetch URIs in s3 " "bucket." ) tp = { "client" : boto3 . client ( "s3" ) } elif protocol == Protocol . GS : try : from smart_open import open from google . cloud import storage except ImportError : raise ImportError ( "You must `pip install smart_open` and " "`pip install google-cloud-storage` " "to fetch URIs in Google Cloud Storage bucket." ) else : try : from smart_open import open except ImportError : raise ImportError ( "You must `pip install smart_open` " f"to fetch {protocol.value.upper()} URIs." ) with open ( pkg_uri , "rb" , transport_params = tp ) as package_zip : with open ( pkg_file , "wb" ) as fin : fin . write ( package_zip . read ( ) ) unzip_package ( package_path = pkg_file , target_dir = local_dir , remove_top_level_directory = True , unlink_zip = True , logger = logger , ) else : raise NotImplementedError ( f"Protocol {protocol} is not supported" ) return str ( local_dir )
Download the package corresponding to this URI and unpack it if zipped.
Download the package corresponding to this URI and unpack it if zipped .
"""Download the package corresponding to this URI and unpack it if zipped.

    Will be written to a file or directory named {base_directory}/{uri}.
    Returns the path to this file or directory.
    """ # Download package from the GCS. # Download package from remote URI # noqa: F401
pkg_uri
str
base_directory
str
logger
Optional[logging.Logger]