spark_plugin.py
def _update_executor_metrics(self, executor, exec_metrics, dim):
        """
        Helper method to get executor metrics

        Args:
        executor (dict): JSON response containing executor metrics
        exec_metrics (dict): Mapping of metric name (key in JSON response) to
                             MetricRecord object to be sent by collectd
        dim (dict): Dictionary representing dimensions
        """
        for key, (metric_type, metric_name) in metrics.SPARK_EXECUTOR_METRICS.items():

            if key not in executor:
                continue
            metric_value = executor[key]

            mr = exec_metrics.get(metric_name, MetricRecord(metric_name, metric_type, 0, dim))
            mr.value += metric_value
            exec_metrics[metric_name] = mr

        if not self.enhanced_flag and len(self.include) == 0:
            return

        for key, (metric_type, metric_name) in metrics.SPARK_EXECUTOR_METRICS_ENHANCED.items():

            if key not in executor:
                continue
            if metric_name in self.exclude:
                continue
            if len(self.include) > 0 and metric_name not in self.include:
                continue

            metric_value = executor[key]

            mr = exec_metrics.get(metric_name, MetricRecord(metric_name, metric_type, 0, dim))
            mr.value += metric_value
            exec_metrics[metric_name] = mr

        Helper method to get executor metrics

        Args:
        executor (dict): JSON response containing executor metrics
        exec_metrics (dict): Mapping of metric name (key in JSON response) to
                             MetricRecord object to be sent by collectd
        dim (dict): Dictionary representing dimensions
        
Helper method to get executor metrics
Args:
executor (dict): JSON response containing executor metrics
exec_metrics (dict): Mapping of metric name (key in JSON response) to
MetricRecord object to be sent by collectd
dim (dict): Dictionary representing dimensions
Helper method to get executor metrics Args : executor ( dict ) : JSON response containing executor metrics exec_metrics ( dict ) : Mapping of metric name ( key in JSON response ) to MetricRecord object to be sent by collectd dim ( dict ) : Dictionary representing dimensions
def _update_executor_metrics(self, executor, exec_metrics, dim):
        for key, (metric_type, metric_name) in metrics.SPARK_EXECUTOR_METRICS.items():
            if key not in executor:
                continue
            metric_value = executor[key]
            mr = exec_metrics.get(metric_name, MetricRecord(metric_name, metric_type, 0, dim))
            mr.value += metric_value
            exec_metrics[metric_name] = mr
        if not self.enhanced_flag and len(self.include) == 0:
            return
        for key, (metric_type, metric_name) in metrics.SPARK_EXECUTOR_METRICS_ENHANCED.items():
            if key not in executor:
                continue
            if metric_name in self.exclude:
                continue
            if len(self.include) > 0 and metric_name not in self.include:
                continue
            metric_value = executor[key]
            mr = exec_metrics.get(metric_name, MetricRecord(metric_name, metric_type, 0, dim))
            mr.value += metric_value
            exec_metrics[metric_name] = mr
def _update_executor_metrics ( self , executor , exec_metrics , dim ) : for key , ( metric_type , metric_name ) in metrics . SPARK_EXECUTOR_METRICS . items ( ) : if key not in executor : continue metric_value = executor [ key ] mr = exec_metrics . get ( metric_name , MetricRecord ( metric_name , metric_type , 0 , dim ) ) mr . value += metric_value exec_metrics [ metric_name ] = mr if not self . enhanced_flag and len ( self . include ) == 0 : return for key , ( metric_type , metric_name ) in metrics . SPARK_EXECUTOR_METRICS_ENHANCED . items ( ) : if key not in executor : continue if metric_name in self . exclude : continue if len ( self . include ) > 0 and metric_name not in self . include : continue metric_value = executor [ key ] mr = exec_metrics . get ( metric_name , MetricRecord ( metric_name , metric_type , 0 , dim ) ) mr . value += metric_value exec_metrics [ metric_name ] = mr
Helper method to get executor metrics
Args:
executor (dict): JSON response containing executor metrics
exec_metrics (dict): Mapping of metric name (key in JSON response) to
MetricRecord object to be sent by collectd
dim (dict): Dictionary representing dimensions
Helper method to get executor metrics Args : executor ( dict ) : JSON response containing executor metrics exec_metrics ( dict ) : Mapping of metric name ( key in JSON response ) to MetricRecord object to be sent by collectd dim ( dict ) : Dictionary representing dimensions
"""
        Helper method to get executor metrics

        Args:
        executor (dict): JSON response containing executor metrics
        exec_metrics (dict): Mapping of metric name (key in JSON response) to
                             MetricRecord object to be sent by collectd
        dim (dict): Dictionary representing dimensions
        """
self
executor
exec_metrics
dim