python/ray/util/client/dataclient.py
def chunk_put(req: ray_client_pb2.DataRequest):
    """
    Chunks a put request. Doing this lazily is important for large objects,
    since taking slices of bytes objects does a copy. This means if we
    immediately materialized every chunk of a large object and inserted them
    into the result_queue, we would effectively double the memory needed
    on the client to handle the put.
    """
    total_size = len(req.put.data)
    assert total_size > 0, "Cannot chunk object with missing data"
    if total_size >= OBJECT_TRANSFER_WARNING_SIZE and log_once(
        "client_object_put_size_warning"
    ):
        size_gb = total_size / 2 ** 30
        warnings.warn(
            "Ray Client is attempting to send a "
            f"{size_gb:.2f} GiB object over the network, which may "
            "be slow. Consider serializing the object and using a remote "
            "URI to transfer via S3 or Google Cloud Storage instead. "
            "Documentation for doing this can be found here: "
            "https://docs.ray.io/en/latest/handling-dependencies.html#remote-uris",
            UserWarning,
        )
    total_chunks = math.ceil(total_size / OBJECT_TRANSFER_CHUNK_SIZE)
    for chunk_id in range(0, total_chunks):
        start = chunk_id * OBJECT_TRANSFER_CHUNK_SIZE
        end = min(total_size, (chunk_id + 1) * OBJECT_TRANSFER_CHUNK_SIZE)
        chunk = ray_client_pb2.PutRequest(
            client_ref_id=req.put.client_ref_id,
            data=req.put.data[start:end],
            chunk_id=chunk_id,
            total_chunks=total_chunks,
            total_size=total_size,
        )
        yield ray_client_pb2.DataRequest(req_id=req.req_id, put=chunk)

    Chunks a put request. Doing this lazily is important for large objects,
    since taking slices of bytes objects does a copy. This means if we
    immediately materialized every chunk of a large object and inserted them
    into the result_queue, we would effectively double the memory needed
    on the client to handle the put.
    
Chunks a put request. Doing this lazily is important for large objects,
since taking slices of bytes objects does a copy. This means if we
immediately materialized every chunk of a large object and inserted them
into the result_queue, we would effectively double the memory needed
on the client to handle the put.
Chunks a put request . Doing this lazily is important for large objects since taking slices of bytes objects does a copy . This means if we immediately materialized every chunk of a large object and inserted them into the result_queue we would effectively double the memory needed on the client to handle the put .
def chunk_put(req: ray_client_pb2.DataRequest):
    total_size = len(req.put.data)
    assert total_size > 0, "Cannot chunk object with missing data"
    if total_size >= OBJECT_TRANSFER_WARNING_SIZE and log_once(
        "client_object_put_size_warning"
    ):
        size_gb = total_size / 2 ** 30
        warnings.warn(
            "Ray Client is attempting to send a "
            f"{size_gb:.2f} GiB object over the network, which may "
            "be slow. Consider serializing the object and using a remote "
            "URI to transfer via S3 or Google Cloud Storage instead. "
            "Documentation for doing this can be found here: "
            "https://docs.ray.io/en/latest/handling-dependencies.html#remote-uris",
            UserWarning,
        )
    total_chunks = math.ceil(total_size / OBJECT_TRANSFER_CHUNK_SIZE)
    for chunk_id in range(0, total_chunks):
        start = chunk_id * OBJECT_TRANSFER_CHUNK_SIZE
        end = min(total_size, (chunk_id + 1) * OBJECT_TRANSFER_CHUNK_SIZE)
        chunk = ray_client_pb2.PutRequest(
            client_ref_id=req.put.client_ref_id,
            data=req.put.data[start:end],
            chunk_id=chunk_id,
            total_chunks=total_chunks,
            total_size=total_size,
        )
        yield ray_client_pb2.DataRequest(req_id=req.req_id, put=chunk)
def chunk_put ( req : ray_client_pb2 . DataRequest ) : total_size = len ( req . put . data ) assert total_size > 0 , "Cannot chunk object with missing data" if total_size >= OBJECT_TRANSFER_WARNING_SIZE and log_once ( "client_object_put_size_warning" ) : size_gb = total_size / 2 ** 30 warnings . warn ( "Ray Client is attempting to send a " f"{size_gb:.2f} GiB object over the network, which may " "be slow. Consider serializing the object and using a remote " "URI to transfer via S3 or Google Cloud Storage instead. " "Documentation for doing this can be found here: " "https://docs.ray.io/en/latest/handling-dependencies.html#remote-uris" , UserWarning , ) total_chunks = math . ceil ( total_size / OBJECT_TRANSFER_CHUNK_SIZE ) for chunk_id in range ( 0 , total_chunks ) : start = chunk_id * OBJECT_TRANSFER_CHUNK_SIZE end = min ( total_size , ( chunk_id + 1 ) * OBJECT_TRANSFER_CHUNK_SIZE ) chunk = ray_client_pb2 . PutRequest ( client_ref_id = req . put . client_ref_id , data = req . put . data [ start : end ] , chunk_id = chunk_id , total_chunks = total_chunks , total_size = total_size , ) yield ray_client_pb2 . DataRequest ( req_id = req . req_id , put = chunk )
Chunks a put request.
Chunks a put request .
"""
    Chunks a put request. Doing this lazily is important for large objects,
    since taking slices of bytes objects does a copy. This means if we
    immediately materialized every chunk of a large object and inserted them
    into the result_queue, we would effectively double the memory needed
    on the client to handle the put.
    """
req
ray_client_pb2.DataRequest