python/ray/tune/utils/file_transfer.py
def sync_dir_between_nodes(
    source_ip: str,
    source_path: str,
    target_ip: str,
    target_path: str,
    force_all: bool = False,
    chunk_size_bytes: int = _DEFAULT_CHUNK_SIZE_BYTES,
    max_size_bytes: Optional[int] = _DEFAULT_MAX_SIZE_BYTES,
    return_futures: bool = False,
) -> Union[None, Tuple[ray.ObjectRef, ray.ActorID, ray.ObjectRef]]:
    """Synchronize directory on source node to directory on target node.

    Per default, this function will collect information about already existing
    files in the target directory. Only files that differ in either mtime or
    filesize will be transferred, unless ``force_all=True``.

    Args:
        source_ip: IP of source node.
        source_path: Path to directory on source node.
        target_ip: IP of target node.
        target_path: Path to directory on target node.
        force_all: If True, all files will be transferred (not just differing files).
        chunk_size_bytes: Chunk size for data transfer.
        max_size_bytes: If packed data exceeds this value, raise an error before
            transfer. If ``None``, no limit is enforced.
        return_futures: If True, returns a tuple of the unpack future,
            the pack actor, and the files_stats future. If False (default) will
            block until synchronization finished and return None.

    Returns:
        None, or Tuple of unpack future, pack actor, and files_stats future.

    """
    pack_actor_on_source_node = _PackActor.options(
        num_cpus=0, resources={f"node:{source_ip}": 0.01}, placement_group=None
    )
    unpack_on_target_node = _unpack_from_actor.options(
        num_cpus=0, resources={f"node:{target_ip}": 0.01}, placement_group=None
    )

    if force_all:
        files_stats = None
    else:
        files_stats = _remote_get_recursive_files_and_stats.options(
            num_cpus=0, resources={f"node:{target_ip}": 0.01}, placement_group=None
        ).remote(target_path)

    pack_actor = pack_actor_on_source_node.remote(
        source_dir=source_path,
        files_stats=files_stats,
        chunk_size_bytes=chunk_size_bytes,
        max_size_bytes=max_size_bytes,
    )
    unpack_future = unpack_on_target_node.remote(pack_actor, target_path)

    if return_futures:
        return unpack_future, pack_actor, files_stats

    return ray.get(unpack_future)
Synchronize directory on source node to directory on target node.

    Per default, this function will collect information about already existing
    files in the target directory. Only files that differ in either mtime or
    filesize will be transferred, unless ``force_all=True``.

    Args:
        source_ip: IP of source node.
        source_path: Path to directory on source node.
        target_ip: IP of target node.
        target_path: Path to directory on target node.
        force_all: If True, all files will be transferred (not just differing files).
        chunk_size_bytes: Chunk size for data transfer.
        max_size_bytes: If packed data exceeds this value, raise an error before
            transfer. If ``None``, no limit is enforced.
        return_futures: If True, returns a tuple of the unpack future,
            the pack actor, and the files_stats future. If False (default) will
            block until synchronization finished and return None.

    Returns:
        None, or Tuple of unpack future, pack actor, and files_stats future.

    
Synchronize directory on source node to directory on target node.
Per default, this function will collect information about already existing
files in the target directory. Only files that differ in either mtime or
filesize will be transferred, unless ``force_all=True``.
Synchronize directory on source node to directory on target node . Per default this function will collect information about already existing files in the target directory . Only files that differ in either mtime or filesize will be transferred unless ` ` force_all = True ` ` .
def sync_dir_between_nodes(
    source_ip: str,
    source_path: str,
    target_ip: str,
    target_path: str,
    force_all: bool = False,
    chunk_size_bytes: int = _DEFAULT_CHUNK_SIZE_BYTES,
    max_size_bytes: Optional[int] = _DEFAULT_MAX_SIZE_BYTES,
    return_futures: bool = False,
) -> Union[None, Tuple[ray.ObjectRef, ray.ActorID, ray.ObjectRef]]:
    pack_actor_on_source_node = _PackActor.options(
        num_cpus=0, resources={f"node:{source_ip}": 0.01}, placement_group=None
    )
    unpack_on_target_node = _unpack_from_actor.options(
        num_cpus=0, resources={f"node:{target_ip}": 0.01}, placement_group=None
    )
    if force_all:
        files_stats = None
    else:
        files_stats = _remote_get_recursive_files_and_stats.options(
            num_cpus=0, resources={f"node:{target_ip}": 0.01}, placement_group=None
        ).remote(target_path)
    pack_actor = pack_actor_on_source_node.remote(
        source_dir=source_path,
        files_stats=files_stats,
        chunk_size_bytes=chunk_size_bytes,
        max_size_bytes=max_size_bytes,
    )
    unpack_future = unpack_on_target_node.remote(pack_actor, target_path)
    if return_futures:
        return unpack_future, pack_actor, files_stats
    return ray.get(unpack_future)
def sync_dir_between_nodes ( source_ip : str , source_path : str , target_ip : str , target_path : str , force_all : bool = False , chunk_size_bytes : int = _DEFAULT_CHUNK_SIZE_BYTES , max_size_bytes : Optional [ int ] = _DEFAULT_MAX_SIZE_BYTES , return_futures : bool = False , ) -> Union [ None , Tuple [ ray . ObjectRef , ray . ActorID , ray . ObjectRef ] ] : pack_actor_on_source_node = _PackActor . options ( num_cpus = 0 , resources = { f"node:{source_ip}" : 0.01 } , placement_group = None ) unpack_on_target_node = _unpack_from_actor . options ( num_cpus = 0 , resources = { f"node:{target_ip}" : 0.01 } , placement_group = None ) if force_all : files_stats = None else : files_stats = _remote_get_recursive_files_and_stats . options ( num_cpus = 0 , resources = { f"node:{target_ip}" : 0.01 } , placement_group = None ) . remote ( target_path ) pack_actor = pack_actor_on_source_node . remote ( source_dir = source_path , files_stats = files_stats , chunk_size_bytes = chunk_size_bytes , max_size_bytes = max_size_bytes , ) unpack_future = unpack_on_target_node . remote ( pack_actor , target_path ) if return_futures : return unpack_future , pack_actor , files_stats return ray . get ( unpack_future )
Synchronize directory on source node to directory on target node.
Synchronize directory on source node to directory on target node .
"""Synchronize directory on source node to directory on target node.

    Per default, this function will collect information about already existing
    files in the target directory. Only files that differ in either mtime or
    filesize will be transferred, unless ``force_all=True``.

    Args:
        source_ip: IP of source node.
        source_path: Path to directory on source node.
        target_ip: IP of target node.
        target_path: Path to directory on target node.
        force_all: If True, all files will be transferred (not just differing files).
        chunk_size_bytes: Chunk size for data transfer.
        max_size_bytes: If packed data exceeds this value, raise an error before
            transfer. If ``None``, no limit is enforced.
        return_futures: If True, returns a tuple of the unpack future,
            the pack actor, and the files_stats future. If False (default) will
            block until synchronization finished and return None.

    Returns:
        None, or Tuple of unpack future, pack actor, and files_stats future.

    """
None, or Tuple of unpack future, pack actor, and files_stats future.
None or Tuple of unpack future pack actor and files_stats future .
source_ip
str
IP of source node.
IP of source node .
source_path
str
Path to directory on source node.
Path to directory on source node .
target_ip
str
IP of target node.
IP of target node .
target_path
str
Path to directory on target node.
Path to directory on target node .
force_all
bool
If True, all files will be transferred (not just differing files).
If True all files will be transferred ( not just differing files ) .
chunk_size_bytes
int
Chunk size for data transfer.
Chunk size for data transfer .
max_size_bytes
Optional[int]
If packed data exceeds this value, raise an error before
transfer. If ``None``, no limit is enforced.
If packed data exceeds this value raise an error before transfer . If ` ` None ` ` no limit is enforced .
return_futures
bool
If True, returns a tuple of the unpack future,
the pack actor, and the files_stats future. If False (default) will
block until synchronization finished and return None.
If True returns a tuple of the unpack future the pack actor and the files_stats future . If False ( default ) will block until synchronization finished and return None .