def compute_dataset_statistics(dataset_path,
                               output_path,
                               config):
    """
    Compute the statistics of fields of a TensorDataset

    Parameters
    ----------
    dataset_path : str
        path to the dataset
    output_dir : str
        where to save the data
    config : :obj:`YamlConfig`
        parameters for the analysis
    """
    # parse config
    analysis_fields = config['analysis_fields']
    num_percentiles = config['num_percentiles']
    thresholds = config['thresholds']
    log_rate = config['log_rate']

    num_bins = config['num_bins']
    font_size = config['font_size']
    line_width = config['line_width']
    dpi = config['dpi']
    
    # create dataset for the aggregated results
    dataset = TensorDataset.open(dataset_path)
    num_datapoints = dataset.num_datapoints

    # allocate buffers
    analysis_data = {}
    for field in analysis_fields:
        analysis_data[field] = []

    # loop through dataset
    for i in range(num_datapoints):
        if i % log_rate == 0:
            logging.info('Reading datapoint %d of %d' %(i+1, num_datapoints))

        # read datapoint
        datapoint = dataset.datapoint(i, analysis_fields)
        for key, value in datapoint.iteritems():
            analysis_data[key].append(value)

    # create output CSV
    stats_headers = {
        'name': 'str',
        'mean': 'float',
        'median': 'float',
        'std': 'float'
    }    
    for i in range(num_percentiles):
        pctile = int((100.0 / num_percentiles) * i)
        field = '%d_pctile' %(pctile)
        stats_headers[field] = 'float'
    for t in thresholds:
        field = 'pct_above_%.3f' %(t)
        stats_headers[field] = 'float'
    
    # analyze statistics
    for field, data in analysis_data.iteritems():
        # init arrays
        data = np.array(data)

        # init filename
        stats_filename = os.path.join(output_path, '%s_stats.json' %(field))
        if os.path.exists(stats_filename):
            logging.warning('Statistics file %s exists!' %(stats_filename))
        
        # stats
        mean = np.mean(data)
        median = np.median(data)
        std = np.std(data)
        stats = {
            'name': str(field),
            'mean': float(mean),
            'median': float(median),
            'std': float(std),
        }
        for i in range(num_percentiles):
            pctile = int((100.0 / num_percentiles) * i)
            pctile_field = '%d_pctile' %(pctile)
            stats[pctile_field] = float(np.percentile(data, pctile))
        for t in thresholds:
            t_field = 'pct_above_%.3f' %(t)
            stats[t_field] = float(np.mean(1 * (data > t)))
        json.dump(stats,
                  open(stats_filename, 'w'),
                  indent=2,
                  sort_keys=True)
                  
        # histogram
        num_unique = np.unique(data).shape[0]
        nb = min(num_bins, data.shape[0], num_unique)
        bounds = (np.min(data), np.max(data))
        vis2d.figure()
        utils.histogram(data,
                        nb,
                        bounds,
                        normalized=False,
                        plot=True)
        vis2d.xlabel(field, fontsize=font_size)
        vis2d.ylabel('Count', fontsize=font_size)
        data_filename = os.path.join(output_path, 'histogram_%s.pdf' %(field))
        vis2d.show(data_filename, dpi=dpi)
Compute the statistics of fields of a TensorDataset

    Parameters
    ----------
    dataset_path : str
        path to the dataset
    output_dir : str
        where to save the data
    config : :obj:`YamlConfig`
        parameters for the analysis