spark_plugin.py
def _get_job_metrics(self, apps):
        """
        Get job metrics corresponding to a running application

        Args:
        apps (dict): Mapping of application id to app name, user,
                     and url endpoint for API call
        """
        for app_id, (app_name, app_user, tracking_url) in apps.items():
            resp = self.spark_agent.request_metrics(tracking_url, APPS_ENDPOINT, app_id, "jobs", status="RUNNING")
            if not resp:
                continue

            dim = {"app_name": str(app_name), "user": str(app_user)}
            dim.update(self.global_dimensions)

            job_metrics = self.metrics[(app_name, app_user)]
            if len(resp):
                job_metrics["spark.num_running_jobs"] = MetricRecord("spark.num_running_jobs", "gauge", len(resp), dim)

            for job in resp:
                for key, (metric_type, metric_name) in metrics.SPARK_JOB_METRICS.items():

                    if key not in job:
                        continue
                    metric_value = job[key]

                    mr = job_metrics.get(key, MetricRecord(metric_name, metric_type, 0, dim))
                    mr.value += metric_value
                    job_metrics[key] = mr

        Get job metrics corresponding to a running application

        Args:
        apps (dict): Mapping of application id to app name, user,
                     and url endpoint for API call
        
Get job metrics corresponding to a running application
Args:
apps (dict): Mapping of application id to app name, user,
and url endpoint for API call
Get job metrics corresponding to a running application Args : apps ( dict ) : Mapping of application id to app name user and url endpoint for API call
def _get_job_metrics(self, apps):
        for app_id, (app_name, app_user, tracking_url) in apps.items():
            resp = self.spark_agent.request_metrics(tracking_url, APPS_ENDPOINT, app_id, "jobs", status="RUNNING")
            if not resp:
                continue
            dim = {"app_name": str(app_name), "user": str(app_user)}
            dim.update(self.global_dimensions)
            job_metrics = self.metrics[(app_name, app_user)]
            if len(resp):
                job_metrics["spark.num_running_jobs"] = MetricRecord("spark.num_running_jobs", "gauge", len(resp), dim)
            for job in resp:
                for key, (metric_type, metric_name) in metrics.SPARK_JOB_METRICS.items():
                    if key not in job:
                        continue
                    metric_value = job[key]
                    mr = job_metrics.get(key, MetricRecord(metric_name, metric_type, 0, dim))
                    mr.value += metric_value
                    job_metrics[key] = mr
def _get_job_metrics ( self , apps ) : for app_id , ( app_name , app_user , tracking_url ) in apps . items ( ) : resp = self . spark_agent . request_metrics ( tracking_url , APPS_ENDPOINT , app_id , "jobs" , status = "RUNNING" ) if not resp : continue dim = { "app_name" : str ( app_name ) , "user" : str ( app_user ) } dim . update ( self . global_dimensions ) job_metrics = self . metrics [ ( app_name , app_user ) ] if len ( resp ) : job_metrics [ "spark.num_running_jobs" ] = MetricRecord ( "spark.num_running_jobs" , "gauge" , len ( resp ) , dim ) for job in resp : for key , ( metric_type , metric_name ) in metrics . SPARK_JOB_METRICS . items ( ) : if key not in job : continue metric_value = job [ key ] mr = job_metrics . get ( key , MetricRecord ( metric_name , metric_type , 0 , dim ) ) mr . value += metric_value job_metrics [ key ] = mr
Get job metrics corresponding to a running application
Args:
apps (dict): Mapping of application id to app name, user,
and url endpoint for API call
Get job metrics corresponding to a running application Args : apps ( dict ) : Mapping of application id to app name user and url endpoint for API call
"""
        Get job metrics corresponding to a running application

        Args:
        apps (dict): Mapping of application id to app name, user,
                     and url endpoint for API call
        """
self
apps