spark_plugin.py
def _get_executor_metrics(self, apps):
        """
        Get executor metrics corresponding to a running application

        Args:
        apps (dict): Mapping of application id to app name, user,
                     and url endpoint for API call
        """
        for app_id, (app_name, app_user, tracking_url) in apps.items():
            resp = self.spark_agent.request_metrics(tracking_url, APPS_ENDPOINT, app_id, "executors")
            if not resp:
                continue

            dim = {"app_name": str(app_name), "user": str(app_user)}
            dim.update(self.global_dimensions)

            exec_metrics = self.metrics[(app_name, app_user)]

            if len(resp):
                exec_metrics["spark.executor.count"] = MetricRecord("spark.executor.count", "gauge", len(resp), dim)

            for executor in resp:
                if executor.get("id") == "driver":
                    self._update_driver_metrics(executor, exec_metrics, dim)
                else:
                    self._update_executor_metrics(executor, exec_metrics, dim)

        Get executor metrics corresponding to a running application

        Args:
        apps (dict): Mapping of application id to app name, user,
                     and url endpoint for API call
        
Get executor metrics corresponding to a running application
Args:
apps (dict): Mapping of application id to app name, user,
and url endpoint for API call
Get executor metrics corresponding to a running application Args : apps ( dict ) : Mapping of application id to app name user and url endpoint for API call
def _get_executor_metrics(self, apps):
        for app_id, (app_name, app_user, tracking_url) in apps.items():
            resp = self.spark_agent.request_metrics(tracking_url, APPS_ENDPOINT, app_id, "executors")
            if not resp:
                continue
            dim = {"app_name": str(app_name), "user": str(app_user)}
            dim.update(self.global_dimensions)
            exec_metrics = self.metrics[(app_name, app_user)]
            if len(resp):
                exec_metrics["spark.executor.count"] = MetricRecord("spark.executor.count", "gauge", len(resp), dim)
            for executor in resp:
                if executor.get("id") == "driver":
                    self._update_driver_metrics(executor, exec_metrics, dim)
                else:
                    self._update_executor_metrics(executor, exec_metrics, dim)
def _get_executor_metrics ( self , apps ) : for app_id , ( app_name , app_user , tracking_url ) in apps . items ( ) : resp = self . spark_agent . request_metrics ( tracking_url , APPS_ENDPOINT , app_id , "executors" ) if not resp : continue dim = { "app_name" : str ( app_name ) , "user" : str ( app_user ) } dim . update ( self . global_dimensions ) exec_metrics = self . metrics [ ( app_name , app_user ) ] if len ( resp ) : exec_metrics [ "spark.executor.count" ] = MetricRecord ( "spark.executor.count" , "gauge" , len ( resp ) , dim ) for executor in resp : if executor . get ( "id" ) == "driver" : self . _update_driver_metrics ( executor , exec_metrics , dim ) else : self . _update_executor_metrics ( executor , exec_metrics , dim )
Get executor metrics corresponding to a running application
Args:
apps (dict): Mapping of application id to app name, user,
and url endpoint for API call
Get executor metrics corresponding to a running application Args : apps ( dict ) : Mapping of application id to app name user and url endpoint for API call
"""
        Get executor metrics corresponding to a running application

        Args:
        apps (dict): Mapping of application id to app name, user,
                     and url endpoint for API call
        """
self
apps