python/ray/autoscaler/_private/aws/cloudwatch/cloudwatch_helper.py
def _ssm_command_waiter(self, document_name, parameters):
        """ wait for SSM command to complete on all cluster nodes """

        # This waiter differs from the built-in SSM.Waiter by
        # optimistically waiting for the command invocation to
        # exist instead of failing immediately, and by resubmitting
        # any failed command until all retry attempts are exhausted.
        response = self._send_command_to_all_nodes(
            document_name,
            parameters,
        )
        command_id = response["Command"]["CommandId"]

        cloudwatch_config = self.provider_config["cloudwatch"]
        agent_retryer_config = cloudwatch_config \
            .get("agent") \
            .get("retryer", {})
        max_attempts = agent_retryer_config.get("max_attempts", 120)
        delay_seconds = agent_retryer_config.get("delay_seconds", 30)
        num_attempts = 0
        for node_id in self.node_ids:
            while True:
                num_attempts += 1
                logger.debug("Listing SSM command ID {} invocations on node {}"
                             .format(command_id, node_id))
                response = self.ssm_client.list_command_invocations(
                    CommandId=command_id,
                    InstanceId=node_id,
                )
                cmd_invocations = response["CommandInvocations"]
                if not cmd_invocations:
                    logger.debug(
                        "SSM Command ID {} invocation does not exist. If "
                        "the command was just started, it may take a "
                        "few seconds to register.".format(command_id))
                else:
                    if len(cmd_invocations) > 1:
                        logger.warning(
                            "Expected to find 1 SSM command invocation with "
                            "ID {} on node {} but found {}: {}".format(
                                command_id,
                                node_id,
                                len(cmd_invocations),
                                cmd_invocations,
                            ))
                    cmd_invocation = cmd_invocations[0]
                    if cmd_invocation["Status"] == "Success":
                        logger.debug(
                            "SSM Command ID {} completed successfully."
                            .format(command_id))
                        break
                    if num_attempts >= max_attempts:
                        logger.error(
                            "Max attempts for command {} exceeded on node {}"
                            .format(command_id, node_id))
                        raise botocore.exceptions.WaiterError(
                            name="ssm_waiter",
                            reason="Max attempts exceeded",
                            last_response=cmd_invocation,
                        )
                    if cmd_invocation["Status"] == "Failed":
                        logger.debug(
                            "SSM Command ID {} failed. Retrying in {} seconds."
                            .format(command_id, delay_seconds))
                        response = self._send_command_to_nodes(
                            document_name, parameters, node_id)
                        command_id = response["Command"]["CommandId"]
                        logger.debug("Sent SSM command ID {} to node {}"
                                     .format(command_id, node_id))
                time.sleep(delay_seconds)
 wait for SSM command to complete on all cluster nodes 
wait for SSM command to complete on all cluster nodes
wait for SSM command to complete on all cluster nodes
def _ssm_command_waiter(self, document_name, parameters):
        response = self._send_command_to_all_nodes(
            document_name,
            parameters,
        )
        command_id = response["Command"]["CommandId"]
        cloudwatch_config = self.provider_config["cloudwatch"]
        agent_retryer_config = cloudwatch_config \
            .get("agent") \
            .get("retryer", {})
        max_attempts = agent_retryer_config.get("max_attempts", 120)
        delay_seconds = agent_retryer_config.get("delay_seconds", 30)
        num_attempts = 0
        for node_id in self.node_ids:
            while True:
                num_attempts += 1
                logger.debug("Listing SSM command ID {} invocations on node {}"
                             .format(command_id, node_id))
                response = self.ssm_client.list_command_invocations(
                    CommandId=command_id,
                    InstanceId=node_id,
                )
                cmd_invocations = response["CommandInvocations"]
                if not cmd_invocations:
                    logger.debug(
                        "SSM Command ID {} invocation does not exist. If "
                        "the command was just started, it may take a "
                        "few seconds to register.".format(command_id))
                else:
                    if len(cmd_invocations) > 1:
                        logger.warning(
                            "Expected to find 1 SSM command invocation with "
                            "ID {} on node {} but found {}: {}".format(
                                command_id,
                                node_id,
                                len(cmd_invocations),
                                cmd_invocations,
                            ))
                    cmd_invocation = cmd_invocations[0]
                    if cmd_invocation["Status"] == "Success":
                        logger.debug(
                            "SSM Command ID {} completed successfully."
                            .format(command_id))
                        break
                    if num_attempts >= max_attempts:
                        logger.error(
                            "Max attempts for command {} exceeded on node {}"
                            .format(command_id, node_id))
                        raise botocore.exceptions.WaiterError(
                            name="ssm_waiter",
                            reason="Max attempts exceeded",
                            last_response=cmd_invocation,
                        )
                    if cmd_invocation["Status"] == "Failed":
                        logger.debug(
                            "SSM Command ID {} failed. Retrying in {} seconds."
                            .format(command_id, delay_seconds))
                        response = self._send_command_to_nodes(
                            document_name, parameters, node_id)
                        command_id = response["Command"]["CommandId"]
                        logger.debug("Sent SSM command ID {} to node {}"
                                     .format(command_id, node_id))
                time.sleep(delay_seconds)
def _ssm_command_waiter ( self , document_name , parameters ) : response = self . _send_command_to_all_nodes ( document_name , parameters , ) command_id = response [ "Command" ] [ "CommandId" ] cloudwatch_config = self . provider_config [ "cloudwatch" ] agent_retryer_config = cloudwatch_config . get ( "agent" ) . get ( "retryer" , { } ) max_attempts = agent_retryer_config . get ( "max_attempts" , 120 ) delay_seconds = agent_retryer_config . get ( "delay_seconds" , 30 ) num_attempts = 0 for node_id in self . node_ids : while True : num_attempts += 1 logger . debug ( "Listing SSM command ID {} invocations on node {}" . format ( command_id , node_id ) ) response = self . ssm_client . list_command_invocations ( CommandId = command_id , InstanceId = node_id , ) cmd_invocations = response [ "CommandInvocations" ] if not cmd_invocations : logger . debug ( "SSM Command ID {} invocation does not exist. If " "the command was just started, it may take a " "few seconds to register." . format ( command_id ) ) else : if len ( cmd_invocations ) > 1 : logger . warning ( "Expected to find 1 SSM command invocation with " "ID {} on node {} but found {}: {}" . format ( command_id , node_id , len ( cmd_invocations ) , cmd_invocations , ) ) cmd_invocation = cmd_invocations [ 0 ] if cmd_invocation [ "Status" ] == "Success" : logger . debug ( "SSM Command ID {} completed successfully." . format ( command_id ) ) break if num_attempts >= max_attempts : logger . error ( "Max attempts for command {} exceeded on node {}" . format ( command_id , node_id ) ) raise botocore . exceptions . WaiterError ( name = "ssm_waiter" , reason = "Max attempts exceeded" , last_response = cmd_invocation , ) if cmd_invocation [ "Status" ] == "Failed" : logger . debug ( "SSM Command ID {} failed. Retrying in {} seconds." . format ( command_id , delay_seconds ) ) response = self . _send_command_to_nodes ( document_name , parameters , node_id ) command_id = response [ "Command" ] [ "CommandId" ] logger . debug ( "Sent SSM command ID {} to node {}" . format ( command_id , node_id ) ) time . sleep ( delay_seconds )
wait for SSM command to complete on all cluster nodes
wait for SSM command to complete on all cluster nodes
""" wait for SSM command to complete on all cluster nodes """ # This waiter differs from the built-in SSM.Waiter by # optimistically waiting for the command invocation to # exist instead of failing immediately, and by resubmitting # any failed command until all retry attempts are exhausted.
self
document_name
parameters