analyze_my_system.py
def run_all(args):
    "Run all metrics on the data and save JSON files with the results."
    # Annotate generated data.
    annotated = annotate_data(args.source_file,
                              args.annotations_file,
                              tag=True,
                              compounds=True)
    
    # Load training data. (For computing novelty.)
    train_data = load_json('./Data/COCO/Processed/tokenized_train2014.json')
    train_descriptions = [entry['caption'] for entry in train_data['annotations']]
    
    # Load annotated data.
    sentences = sentences_from_file(args.annotations_file)
    
    # Analyze the data.
    stats = system_stats(sentences)
    
    # Get raw descriptions.
    gen_descriptions = [entry['caption'] for entry in load_json(args.source_file)]
    extra_stats = sentence_stats(train_descriptions, gen_descriptions)
    stats.update(extra_stats)
    
    # Save statistics data.
    save_json(stats, args.stats_file)
    
    ################################
    # Global recall
    
    train_stats = load_json('./Data/COCO/Processed/train_stats.json')
    val_stats = load_json('./Data/COCO/Processed/val_stats.json')

    train     = set(train_stats['types'])
    val       = set(val_stats['types'])
    learnable = train & val
    
    gen = set(stats['types'])
    recalled = gen & val
    
    coverage = {"recalled": recalled,
                "score": len(recalled)/len(learnable),
                "not_in_val": gen - learnable}
    
    coverage['omissions'] = most_frequent_omissions(coverage['recalled'],
                                                    val_stats,         # Use validation set as reference.
                                                    n=None)
    val_count_list = get_count_list(val_stats)
    coverage['percentiles'] = percentiles(val_count_list, recalled)
    save_json(coverage, args.global_coverage_file)
    
    ####################################
    # Local recall
    
    val_index = index_from_file('./Data/COCO/Processed/tagged_val2014.json', tagged=True, lower=True)
    generated = {entry['image_id']: entry['tokenized'] for entry in annotated}
    local_recall_res = dict(scores = local_recall_scores(generated, val_index),
                            counts = local_recall_counts(generated, val_index))
    save_json(local_recall_res, args.local_coverage_file)
    
    ##################################
    # Nouns pps
    npdata = {'pp_data': pp_stats(annotated), 'compound_data': compound_stats(annotated)}
    save_json(npdata, args.noun_pp_file)
Run all metrics on the data and save JSON files with the results.
Run all metrics on the data and save JSON files with the results.
Run all metrics on the data and save JSON files with the results .
def run_all(args):
    annotated = annotate_data(args.source_file,
                              args.annotations_file,
                              tag=True,
                              compounds=True)
    train_data = load_json('./Data/COCO/Processed/tokenized_train2014.json')
    train_descriptions = [entry['caption'] for entry in train_data['annotations']]
    sentences = sentences_from_file(args.annotations_file)
    stats = system_stats(sentences)
    gen_descriptions = [entry['caption'] for entry in load_json(args.source_file)]
    extra_stats = sentence_stats(train_descriptions, gen_descriptions)
    stats.update(extra_stats)
    save_json(stats, args.stats_file)
    train_stats = load_json('./Data/COCO/Processed/train_stats.json')
    val_stats = load_json('./Data/COCO/Processed/val_stats.json')
    train     = set(train_stats['types'])
    val       = set(val_stats['types'])
    learnable = train & val
    gen = set(stats['types'])
    recalled = gen & val
    coverage = {"recalled": recalled,
                "score": len(recalled)/len(learnable),
                "not_in_val": gen - learnable}
    coverage['omissions'] = most_frequent_omissions(coverage['recalled'],
                                                    val_stats,         
                                                    n=None)
    val_count_list = get_count_list(val_stats)
    coverage['percentiles'] = percentiles(val_count_list, recalled)
    save_json(coverage, args.global_coverage_file)
    ####
    val_index = index_from_file('./Data/COCO/Processed/tagged_val2014.json', tagged=True, lower=True)
    generated = {entry['image_id']: entry['tokenized'] for entry in annotated}
    local_recall_res = dict(scores = local_recall_scores(generated, val_index),
                            counts = local_recall_counts(generated, val_index))
    save_json(local_recall_res, args.local_coverage_file)
    ##
    npdata = {'pp_data': pp_stats(annotated), 'compound_data': compound_stats(annotated)}
    save_json(npdata, args.noun_pp_file)
def run_all ( args ) : annotated = annotate_data ( args . source_file , args . annotations_file , tag = True , compounds = True ) train_data = load_json ( './Data/COCO/Processed/tokenized_train2014.json' ) train_descriptions = [ entry [ 'caption' ] for entry in train_data [ 'annotations' ] ] sentences = sentences_from_file ( args . annotations_file ) stats = system_stats ( sentences ) gen_descriptions = [ entry [ 'caption' ] for entry in load_json ( args . source_file ) ] extra_stats = sentence_stats ( train_descriptions , gen_descriptions ) stats . update ( extra_stats ) save_json ( stats , args . stats_file ) train_stats = load_json ( './Data/COCO/Processed/train_stats.json' ) val_stats = load_json ( './Data/COCO/Processed/val_stats.json' ) train = set ( train_stats [ 'types' ] ) val = set ( val_stats [ 'types' ] ) learnable = train & val gen = set ( stats [ 'types' ] ) recalled = gen & val coverage = { "recalled" : recalled , "score" : len ( recalled ) / len ( learnable ) , "not_in_val" : gen - learnable } coverage [ 'omissions' ] = most_frequent_omissions ( coverage [ 'recalled' ] , val_stats , n = None ) val_count_list = get_count_list ( val_stats ) coverage [ 'percentiles' ] = percentiles ( val_count_list , recalled ) save_json ( coverage , args . global_coverage_file ) val_index = index_from_file ( './Data/COCO/Processed/tagged_val2014.json' , tagged = True , lower = True ) generated = { entry [ 'image_id' ] : entry [ 'tokenized' ] for entry in annotated } local_recall_res = dict ( scores = local_recall_scores ( generated , val_index ) , counts = local_recall_counts ( generated , val_index ) ) save_json ( local_recall_res , args . local_coverage_file ) npdata = { 'pp_data' : pp_stats ( annotated ) , 'compound_data' : compound_stats ( annotated ) } save_json ( npdata , args . noun_pp_file )
Run all metrics on the data and save JSON files with the results.
Run all metrics on the data and save JSON files with the results .
"Run all metrics on the data and save JSON files with the results." # Annotate generated data. # Load training data. (For computing novelty.) # Load annotated data. # Analyze the data. # Get raw descriptions. # Save statistics data. ################################ # Global recall # Use validation set as reference. #################################### # Local recall ################################## # Nouns pps
args