Skip to content
Snippets Groups Projects
Select Git revision
  • resnet_50_train_val
  • acc_vs_qa_rarity
  • acc_vs_ratio
  • rel_bin_feats
  • extract_word_vectors
  • no_single_feats_pretrained
  • no_single_feats_discon
  • single_features
  • no_single_feats
  • simple_avg_reg_feat
  • add_test_eval
  • use_margin_mil_loss
  • backup_simple_avg_reg_feat
  • bn_before_feat_add
  • use_detector_scores
  • discon_wordvec_classifiers
  • avg_reg_feats
  • fix_genome_split
  • GenVQA_obj_atr_ans_mil
  • inner_product_selection
20 results
Blame Permalink
freq_acc_plot.py 2.96 KiB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108 






import os
import ujson
import pprint
import math
from collections import namedtuple
from operator import attrgetter
import numpy as np
import matplotlib
matplotlib.use('pdf')
import matplotlib.pyplot as plt

import constants

pp = pprint.PrettyPrinter(indent=4)
if __name__=='__main__':
    print 'Reading {}'.format(constants.regions_json)
    with open(constants.regions_json) as file:
        all_regions_data = ujson.load(file)

    print 'Reading {}'.format(constants.genome_train_region_ids)
    with open(constants.genome_train_region_ids) as file:
        train_region_ids = ujson.load(file)

    obj_pred_json = os.path.join(
        constants.region_object_scores_dirname,
        'object_predictions.json')
    
    print 'Reading {}'.format(obj_pred_json)
    with open(obj_pred_json) as file:
        obj_pred_data = ujson.load(file)

    obj_train_freq_dict = dict()
    obj_test_freq_dict = dict()
    obj_hit_dict = dict()
    
    for region_id in train_region_ids:
        region_data = all_regions_data[region_id]
        for obj in region_data['object_names']:
            if obj not in obj_train_freq_dict:
                obj_train_freq_dict[obj] = 0

            obj_train_freq_dict[obj] += 1



    for region_id, pred_dict in obj_pred_data.items():
        for obj in pred_dict['gt']:
            if obj not in obj_hit_dict:
                obj_hit_dict[obj] = 0
            
            if obj in pred_dict['pred']:
                obj_hit_dict[obj] += 1

            if obj not in obj_test_freq_dict:
                obj_test_freq_dict[obj] = 1
            else:
                obj_test_freq_dict[obj] += 1


    AccFreqData = namedtuple('AccFreqData',['label','acc','train_freq','test_freq'])
    records = []
    for obj, hits in obj_hit_dict.items():
        if obj not in obj_train_freq_dict:
            obj_train_freq_dict[obj] = 0

        records += [AccFreqData(
            label=obj, 
            acc=float(hits)/float(obj_test_freq_dict[obj]), 
            test_freq=obj_test_freq_dict[obj],
            train_freq=obj_train_freq_dict[obj])]

    sorted_records = sorted(records, key=attrgetter('train_freq'))
    pp.pprint(sorted_records)
        # print '{} \t {} \t {}'.format(
        #     obj,
        #     hits/float(obj_test_freq_dict[obj]),
        #     obj_test_freq_dict[obj],
        # )

    freq = []
    acc = []
    for record in sorted_records:
        freq.append(record.train_freq)
        acc.append(record.acc)
        
    acc_by_freq = {
        'freq': freq,
        'acc': acc,
    }
    print len(freq)
    acc_by_freq_filename = os.path.join(
        constants.region_object_scores_dirname,
        'acc_by_freq.json')
    with open(acc_by_freq_filename,'w') as file:
        ujson.dump(acc_by_freq,file)

    plt.plot(freq[200:800],acc[200:800])
    plot_filename = os.path.join(
        constants.region_object_scores_dirname,
        'obj_acc_vs_freq.pdf')
    plt.savefig(plot_filename)
    
    
        
    print(len(obj_pred_data))