def produce_report(self):
    'Produce a report of the batch jobs.'
    s3_prefix = ('reading_results/%s/logs/%s/' % (self.basename, self._job_queue))
    logger.info(('Producing batch report for %s, from prefix %s.' % (self.basename, s3_prefix)))
    s3 = boto3.client('s3')
    file_tree = self._get_results_file_tree(s3, s3_prefix)
    logger.info(('Found %d relevant files.' % len(file_tree)))
    stat_files = {'git_info.txt': (self._handle_git_info, self._report_git_info), 'timing.txt': (self._handle_timing, self._report_timing), 'raw_tuples.pkl': (None, None), 'hist_data.pkl': (self._handle_hist_data, self._report_hist_data), 'sum_data.pkl': (self._handle_sum_data, self._report_sum_data)}
    stat_aggs = {}
    for (stat_file, (handle_stats, report_stats)) in stat_files.items():
        logger.info(('Aggregating %s...' % stat_file))
        my_agg = {}
        file_paths = file_tree.get_paths(stat_file)
        logger.info(('Found %d files for %s.' % (len(file_paths), stat_file)))
        for (sub_path, file_entry) in file_paths:
            s3_key = file_entry['key']
            ref = sub_path[0]
            file = s3.get_object(Bucket=bucket_name, Key=s3_key)
            file_bytes = file['Body'].read()
            if (handle_stats is not None):
                handle_stats(ref, my_agg, file_bytes)
        if ((report_stats is not None) and len(my_agg)):
            report_stats(my_agg)
        stat_aggs[stat_file] = my_agg
    for (end_type, jobs) in self.run_record.items():
        self.reporter.add_text(('Jobs %s: %d' % (end_type, len(jobs))), section='Totals')
    s3_prefix = ('reading_results/%s/' % self.basename)
    fname = self.reporter.make_report()
    with open(fname, 'rb') as f:
        s3.put_object(Bucket=bucket_name, Key=(s3_prefix + fname), Body=f.read())
    s3.put_object(Bucket=bucket_name, Key=(s3_prefix + ('stat_aggregates_%s.pkl' % self.time_tag)), Body=pickle.dumps(stat_aggs))
    return (file_tree, stat_aggs)
Produce a report of the batch jobs.
def produce_report(self):
    
    s3_prefix = ('reading_results/%s/logs/%s/' % (self.basename, self._job_queue))
    logger.info(('Producing batch report for %s, from prefix %s.' % (self.basename, s3_prefix)))
    s3 = boto3.client('s3')
    file_tree = self._get_results_file_tree(s3, s3_prefix)
    logger.info(('Found %d relevant files.' % len(file_tree)))
    stat_files = {'git_info.txt': (self._handle_git_info, self._report_git_info), 'timing.txt': (self._handle_timing, self._report_timing), 'raw_tuples.pkl': (None, None), 'hist_data.pkl': (self._handle_hist_data, self._report_hist_data), 'sum_data.pkl': (self._handle_sum_data, self._report_sum_data)}
    stat_aggs = {}
    for (stat_file, (handle_stats, report_stats)) in stat_files.items():
        logger.info(('Aggregating %s...' % stat_file))
        my_agg = {}
        file_paths = file_tree.get_paths(stat_file)
        logger.info(('Found %d files for %s.' % (len(file_paths), stat_file)))
        for (sub_path, file_entry) in file_paths:
            s3_key = file_entry['key']
            ref = sub_path[0]
            file = s3.get_object(Bucket=bucket_name, Key=s3_key)
            file_bytes = file['Body'].read()
            if (handle_stats is not None):
                handle_stats(ref, my_agg, file_bytes)
        if ((report_stats is not None) and len(my_agg)):
            report_stats(my_agg)
        stat_aggs[stat_file] = my_agg
    for (end_type, jobs) in self.run_record.items():
        self.reporter.add_text(('Jobs %s: %d' % (end_type, len(jobs))), section='Totals')
    s3_prefix = ('reading_results/%s/' % self.basename)
    fname = self.reporter.make_report()
    with open(fname, 'rb') as f:
        s3.put_object(Bucket=bucket_name, Key=(s3_prefix + fname), Body=f.read())
    s3.put_object(Bucket=bucket_name, Key=(s3_prefix + ('stat_aggregates_%s.pkl' % self.time_tag)), Body=pickle.dumps(stat_aggs))
    return (file_tree, stat_aggs)