python/ray/serve/api.py
def start(
    detached: bool = False,
    http_options: Optional[Union[dict, HTTPOptions]] = None,
    dedicated_cpu: bool = False,
    _checkpoint_path: str = DEFAULT_CHECKPOINT_PATH,
    _override_controller_namespace: Optional[str] = None,
    **kwargs,
) -> ServeControllerClient:
    """Initialize a serve instance.

    By default, the instance will be scoped to the lifetime of the returned
    Client object (or when the script exits). If detached is set to True, the
    instance will instead persist until serve.shutdown() is called. This is
    only relevant if connecting to a long-running Ray cluster (e.g., with
    ray.init(address="auto") or ray.init("ray://<remote_addr>")).

    Args:
        detached (bool): Whether not the instance should be detached from this
          script. If set, the instance will live on the Ray cluster until it is
          explicitly stopped with serve.shutdown().
        http_options (Optional[Dict, serve.HTTPOptions]): Configuration options
          for HTTP proxy. You can pass in a dictionary or HTTPOptions object
          with fields:

            - host(str, None): Host for HTTP servers to listen on. Defaults to
              "127.0.0.1". To expose Serve publicly, you probably want to set
              this to "0.0.0.0".
            - port(int): Port for HTTP server. Defaults to 8000.
            - root_path(str): Root path to mount the serve application
              (for example, "/serve"). All deployment routes will be prefixed
              with this path. Defaults to "".
            - middlewares(list): A list of Starlette middlewares that will be
              applied to the HTTP servers in the cluster. Defaults to [].
            - location(str, serve.config.DeploymentMode): The deployment
              location of HTTP servers:

                - "HeadOnly": start one HTTP server on the head node. Serve
                  assumes the head node is the node you executed serve.start
                  on. This is the default.
                - "EveryNode": start one HTTP server per node.
                - "NoServer" or None: disable HTTP server.
            - num_cpus (int): The number of CPU cores to reserve for each
              internal Serve HTTP proxy actor.  Defaults to 0.
        dedicated_cpu (bool): Whether to reserve a CPU core for the internal
          Serve controller actor.  Defaults to False.
    """
    http_deprecated_args = ["http_host", "http_port", "http_middlewares"]
    for key in http_deprecated_args:
        if key in kwargs:
            raise ValueError(
                f"{key} is deprecated, please use serve.start(http_options="
                f'{{"{key}": {kwargs[key]}}}) instead.'
            )
    # Initialize ray if needed.
    ray.worker.global_worker.filter_logs_by_job = False
    if not ray.is_initialized():
        ray.init(namespace="serve")

    controller_namespace = get_controller_namespace(
        detached, _override_controller_namespace=_override_controller_namespace
    )

    try:
        client = get_global_client(
            _override_controller_namespace=_override_controller_namespace,
            _health_check_controller=True,
        )
        logger.info(
            "Connecting to existing Serve instance in namespace "
            f"'{controller_namespace}'."
        )

        _check_http_and_checkpoint_options(client, http_options, _checkpoint_path)
        return client
    except RayServeException:
        pass

    if detached:
        controller_name = SERVE_CONTROLLER_NAME
    else:
        controller_name = format_actor_name(get_random_letters(), SERVE_CONTROLLER_NAME)

    if isinstance(http_options, dict):
        http_options = HTTPOptions.parse_obj(http_options)
    if http_options is None:
        http_options = HTTPOptions()

    controller = ServeController.options(
        num_cpus=1 if dedicated_cpu else 0,
        name=controller_name,
        lifetime="detached" if detached else None,
        max_restarts=-1,
        max_task_retries=-1,
        # Pin Serve controller on the head node.
        resources={get_current_node_resource_key(): 0.01},
        namespace=controller_namespace,
        max_concurrency=CONTROLLER_MAX_CONCURRENCY,
    ).remote(
        controller_name,
        http_options,
        _checkpoint_path,
        detached=detached,
        _override_controller_namespace=_override_controller_namespace,
    )

    proxy_handles = ray.get(controller.get_http_proxies.remote())
    if len(proxy_handles) > 0:
        try:
            ray.get(
                [handle.ready.remote() for handle in proxy_handles.values()],
                timeout=HTTP_PROXY_TIMEOUT,
            )
        except ray.exceptions.GetTimeoutError:
            raise TimeoutError(
                "HTTP proxies not available after {HTTP_PROXY_TIMEOUT}s."
            )

    client = ServeControllerClient(
        controller,
        controller_name,
        detached=detached,
        _override_controller_namespace=_override_controller_namespace,
    )
    set_global_client(client)
    logger.info(
        f"Started{' detached ' if detached else ' '}Serve instance in "
        f"namespace '{controller_namespace}'."
    )
    return client
Initialize a serve instance.

    By default, the instance will be scoped to the lifetime of the returned
    Client object (or when the script exits). If detached is set to True, the
    instance will instead persist until serve.shutdown() is called. This is
    only relevant if connecting to a long-running Ray cluster (e.g., with
    ray.init(address="auto") or ray.init("ray://<remote_addr>")).

    Args:
        detached (bool): Whether not the instance should be detached from this
          script. If set, the instance will live on the Ray cluster until it is
          explicitly stopped with serve.shutdown().
        http_options (Optional[Dict, serve.HTTPOptions]): Configuration options
          for HTTP proxy. You can pass in a dictionary or HTTPOptions object
          with fields:

            - host(str, None): Host for HTTP servers to listen on. Defaults to
              "127.0.0.1". To expose Serve publicly, you probably want to set
              this to "0.0.0.0".
            - port(int): Port for HTTP server. Defaults to 8000.
            - root_path(str): Root path to mount the serve application
              (for example, "/serve"). All deployment routes will be prefixed
              with this path. Defaults to "".
            - middlewares(list): A list of Starlette middlewares that will be
              applied to the HTTP servers in the cluster. Defaults to [].
            - location(str, serve.config.DeploymentMode): The deployment
              location of HTTP servers:

                - "HeadOnly": start one HTTP server on the head node. Serve
                  assumes the head node is the node you executed serve.start
                  on. This is the default.
                - "EveryNode": start one HTTP server per node.
                - "NoServer" or None: disable HTTP server.
            - num_cpus (int): The number of CPU cores to reserve for each
              internal Serve HTTP proxy actor.  Defaults to 0.
        dedicated_cpu (bool): Whether to reserve a CPU core for the internal
          Serve controller actor.  Defaults to False.
    
Initialize a serve instance.
By default, the instance will be scoped to the lifetime of the returned
Client object (or when the script exits). If detached is set to True, the
instance will instead persist until serve.shutdown() is called. This is
only relevant if connecting to a long-running Ray cluster  or ray.init("ray://")).
Initialize a serve instance . By default the instance will be scoped to the lifetime of the returned Client object ( or when the script exits ) . If detached is set to True the instance will instead persist until serve . shutdown () is called . This is only relevant if connecting to a long - running Ray cluster or ray . init ( " ray : // " )) .
def start(
    detached: bool = False,
    http_options: Optional[Union[dict, HTTPOptions]] = None,
    dedicated_cpu: bool = False,
    _checkpoint_path: str = DEFAULT_CHECKPOINT_PATH,
    _override_controller_namespace: Optional[str] = None,
    **kwargs,
) -> ServeControllerClient:
    http_deprecated_args = ["http_host", "http_port", "http_middlewares"]
    for key in http_deprecated_args:
        if key in kwargs:
            raise ValueError(
                f"{key} is deprecated, please use serve.start(http_options="
                f'{{"{key}": {kwargs[key]}}}) instead.'
            )
    ray.worker.global_worker.filter_logs_by_job = False
    if not ray.is_initialized():
        ray.init(namespace="serve")
    controller_namespace = get_controller_namespace(
        detached, _override_controller_namespace=_override_controller_namespace
    )
    try:
        client = get_global_client(
            _override_controller_namespace=_override_controller_namespace,
            _health_check_controller=True,
        )
        logger.info(
            "Connecting to existing Serve instance in namespace "
            f"'{controller_namespace}'."
        )
        _check_http_and_checkpoint_options(client, http_options, _checkpoint_path)
        return client
    except RayServeException:
        pass
    if detached:
        controller_name = SERVE_CONTROLLER_NAME
    else:
        controller_name = format_actor_name(get_random_letters(), SERVE_CONTROLLER_NAME)
    if isinstance(http_options, dict):
        http_options = HTTPOptions.parse_obj(http_options)
    if http_options is None:
        http_options = HTTPOptions()
    controller = ServeController.options(
        num_cpus=1 if dedicated_cpu else 0,
        name=controller_name,
        lifetime="detached" if detached else None,
        max_restarts=-1,
        max_task_retries=-1,
        resources={get_current_node_resource_key(): 0.01},
        namespace=controller_namespace,
        max_concurrency=CONTROLLER_MAX_CONCURRENCY,
    ).remote(
        controller_name,
        http_options,
        _checkpoint_path,
        detached=detached,
        _override_controller_namespace=_override_controller_namespace,
    )
    proxy_handles = ray.get(controller.get_http_proxies.remote())
    if len(proxy_handles) > 0:
        try:
            ray.get(
                [handle.ready.remote() for handle in proxy_handles.values()],
                timeout=HTTP_PROXY_TIMEOUT,
            )
        except ray.exceptions.GetTimeoutError:
            raise TimeoutError(
                "HTTP proxies not available after {HTTP_PROXY_TIMEOUT}s."
            )
    client = ServeControllerClient(
        controller,
        controller_name,
        detached=detached,
        _override_controller_namespace=_override_controller_namespace,
    )
    set_global_client(client)
    logger.info(
        f"Started{' detached ' if detached else ' '}Serve instance in "
        f"namespace '{controller_namespace}'."
    )
    return client
def start ( detached : bool = False , http_options : Optional [ Union [ dict , HTTPOptions ] ] = None , dedicated_cpu : bool = False , _checkpoint_path : str = DEFAULT_CHECKPOINT_PATH , _override_controller_namespace : Optional [ str ] = None , ** kwargs , ) -> ServeControllerClient : http_deprecated_args = [ "http_host" , "http_port" , "http_middlewares" ] for key in http_deprecated_args : if key in kwargs : raise ValueError ( f"{key} is deprecated, please use serve.start(http_options=" f'{{"{key}": {kwargs[key]}}}) instead.' ) ray . worker . global_worker . filter_logs_by_job = False if not ray . is_initialized ( ) : ray . init ( namespace = "serve" ) controller_namespace = get_controller_namespace ( detached , _override_controller_namespace = _override_controller_namespace ) try : client = get_global_client ( _override_controller_namespace = _override_controller_namespace , _health_check_controller = True , ) logger . info ( "Connecting to existing Serve instance in namespace " f"'{controller_namespace}'." ) _check_http_and_checkpoint_options ( client , http_options , _checkpoint_path ) return client except RayServeException : pass if detached : controller_name = SERVE_CONTROLLER_NAME else : controller_name = format_actor_name ( get_random_letters ( ) , SERVE_CONTROLLER_NAME ) if isinstance ( http_options , dict ) : http_options = HTTPOptions . parse_obj ( http_options ) if http_options is None : http_options = HTTPOptions ( ) controller = ServeController . options ( num_cpus = 1 if dedicated_cpu else 0 , name = controller_name , lifetime = "detached" if detached else None , max_restarts = - 1 , max_task_retries = - 1 , resources = { get_current_node_resource_key ( ) : 0.01 } , namespace = controller_namespace , max_concurrency = CONTROLLER_MAX_CONCURRENCY , ) . remote ( controller_name , http_options , _checkpoint_path , detached = detached , _override_controller_namespace = _override_controller_namespace , ) proxy_handles = ray . get ( controller . get_http_proxies . remote ( ) ) if len ( proxy_handles ) > 0 : try : ray . get ( [ handle . ready . remote ( ) for handle in proxy_handles . values ( ) ] , timeout = HTTP_PROXY_TIMEOUT , ) except ray . exceptions . GetTimeoutError : raise TimeoutError ( "HTTP proxies not available after {HTTP_PROXY_TIMEOUT}s." ) client = ServeControllerClient ( controller , controller_name , detached = detached , _override_controller_namespace = _override_controller_namespace , ) set_global_client ( client ) logger . info ( f"Started{' detached ' if detached else ' '}Serve instance in " f"namespace '{controller_namespace}'." ) return client
Initialize a serve instance.
Initialize a serve instance .
"""Initialize a serve instance.

    By default, the instance will be scoped to the lifetime of the returned
    Client object (or when the script exits). If detached is set to True, the
    instance will instead persist until serve.shutdown() is called. This is
    only relevant if connecting to a long-running Ray cluster (e.g., with
    ray.init(address="auto") or ray.init("ray://<remote_addr>")).

    Args:
        detached (bool): Whether not the instance should be detached from this
          script. If set, the instance will live on the Ray cluster until it is
          explicitly stopped with serve.shutdown().
        http_options (Optional[Dict, serve.HTTPOptions]): Configuration options
          for HTTP proxy. You can pass in a dictionary or HTTPOptions object
          with fields:

            - host(str, None): Host for HTTP servers to listen on. Defaults to
              "127.0.0.1". To expose Serve publicly, you probably want to set
              this to "0.0.0.0".
            - port(int): Port for HTTP server. Defaults to 8000.
            - root_path(str): Root path to mount the serve application
              (for example, "/serve"). All deployment routes will be prefixed
              with this path. Defaults to "".
            - middlewares(list): A list of Starlette middlewares that will be
              applied to the HTTP servers in the cluster. Defaults to [].
            - location(str, serve.config.DeploymentMode): The deployment
              location of HTTP servers:

                - "HeadOnly": start one HTTP server on the head node. Serve
                  assumes the head node is the node you executed serve.start
                  on. This is the default.
                - "EveryNode": start one HTTP server per node.
                - "NoServer" or None: disable HTTP server.
            - num_cpus (int): The number of CPU cores to reserve for each
              internal Serve HTTP proxy actor.  Defaults to 0.
        dedicated_cpu (bool): Whether to reserve a CPU core for the internal
          Serve controller actor.  Defaults to False.
    """ # Initialize ray if needed. # Pin Serve controller on the head node.
detached
bool
Whether not the instance should be detached from this
script. If set, the instance will live on the Ray cluster until it is
explicitly stopped with serve.shutdown().
Whether not the instance should be detached from this script . If set the instance will live on the Ray cluster until it is explicitly stopped with serve . shutdown () .
http_options
Optional[Union[dict, HTTPOptions]]
Configuration options
for HTTP proxy. You can pass in a dictionary or HTTPOptions object
with fields.

host(str, None): Host for HTTP servers to listen on.

"HeadOnly": start one HTTP server on the head node. Serve
assumes the head node is the node you executed serve.start
on. This is the default.
"EveryNode": start one HTTP server per node.
"NoServer" or None: disable HTTP server.
num_cpus (int): The number of CPU cores to reserve for each
internal Serve HTTP proxy actor.  Defaults to 0.
Configuration options for HTTP proxy . You can pass in a dictionary or HTTPOptions object with fields . host ( str None ) : Host for HTTP servers to listen on . " HeadOnly " : start one HTTP server on the head node . Serve assumes the head node is the node you executed serve . start on . This is the default . " EveryNode " : start one HTTP server per node . " NoServer " or None : disable HTTP server . num_cpus ( int ) : The number of CPU cores to reserve for each internal Serve HTTP proxy actor . Defaults to 0 .
dedicated_cpu
bool
Whether to reserve a CPU core for the internal
Serve controller actor.  Defaults to False.
Whether to reserve a CPU core for the internal Serve controller actor . Defaults to False .
_checkpoint_path
str
_override_controller_namespace
Optional[str]