obj atr eval script now can write html files to display results

answer_classifier_cached_features/train.py

@@ -15,7 +15,7 @@ import numpy as np
 import pdb
 from itertools import izip
 import tensorflow as tf
-
+import gc
 
 class graph_creator():
     def __init__(
@@ -895,7 +895,10 @@ def train(
                 zip(vars_to_eval_names, eval_vars)}
             logger.log(iter, False, eval_vars_dict)
             iter+=1
-            
+
+            if iter%8000==0:
+                gc.collect()
+
         logger.log(iter-1, True, eval_vars_dict)
 
 

constants_crunchy.py

@@ -5,7 +5,7 @@ def mkdir_if_not_exists(dir_name):
     if not os.path.exists(dir_name):
         os.mkdir(dir_name)
         
-experiment_name = 'class_through_vqa_mil_ans' #'QA_joint_pretrain_genome_split'
+experiment_name = 'obj_atr_through_none' #'QA_joint_pretrain_genome_split'
 
 # Global output directory (all subexperiments will be saved here)
 global_output_dir = '/home/tanmay/Code/GenVQA/Exp_Results/VQA'
@@ -131,6 +131,12 @@ region_model_accuracies_txt = os.path.join(
 region_eval_on = 'train_held_out' # One of {'test','train_held_out','train_subset'}
 region_model_to_eval = region_model + '-' + '102000'
 
+region_pred_vis_dirname = os.path.join(
+    region_output_dir,
+    'region_pred_vis_'+region_eval_on)
+
+mkdir_if_not_exists(region_pred_vis_dirname)
+
 region_attribute_scores_dirname = os.path.join(
     region_output_dir,
     'attribute_scores')
@@ -192,8 +198,8 @@ answer_batch_size = 25
 answer_num_epochs = 20
 answer_offset = 0
 answer_obj_atr_loss_wt = 1.0
-answer_ans_loss_wt = 0.1
-answer_mil_loss_wt = 0.2
+answer_ans_loss_wt = 0.0
+answer_mil_loss_wt = 0.0
 answer_regularization_coeff = 1e-5
 answer_queue_size = 500
 answer_embedding_dim = 600
@@ -223,7 +229,7 @@ answer_fine_tune_from = answer_model + '-' + str(answer_fine_tune_from_iter)
 
 # Answer eval params
 answer_eval_on = 'val'
-answer_model_to_eval = answer_model + '-28000'
+answer_model_to_eval = answer_model + '-98000'
 
 vqa_results_dir = os.path.join(
     answer_output_dir,

data/cropped_regions_cached_features.py

@@ -71,6 +71,8 @@ class data():
         try:
             batch = dict()
             batch['region_id'] = self.sample_to_region_dict[sample]
+            batch['image_id'] = self.regions[batch['region_id']]['image_id']
+
             batch['region_feats'], read_success = self.get_region_feats(sample)
             if read_success:
                 batch['object_label'], batch['object_label_words'] = \
@@ -112,7 +114,8 @@ class data():
 
         batch_size = len(samples)
         batch = dict()
-        batch['region_ids'] = dict()
+        batch['region_ids'] = [None]*batch_size
+        batch['image_ids'] = [None]*batch_size
         batch['region_feats'] = np.zeros(
             [batch_size, self.resnet_feat_dim], np.float32)
         batch['object_labels']= np.zeros(
@@ -122,6 +125,7 @@ class data():
 
         for index, single_batch in enumerate(batch_list):
             batch['region_ids'][index] = single_batch['region_id']
+            batch['image_ids'][index] = single_batch['image_id']
             batch['region_feats'][index, :] = single_batch['region_feats']
             batch['object_labels'][index, :] = single_batch['object_label']
             batch['attribute_labels'][index,:] = single_batch['attribute_label']

object_attribute_classifier_cached_features/eval.py

@@ -3,6 +3,8 @@ import os
 import ujson
 import numpy as np
 
+import image_io
+from visual_util import html_writer
 import data.cropped_regions_cached_features as cropped_regions
 import tftools.data
 from tftools import var_collect, placeholder_management
@@ -72,7 +74,17 @@ def create_batch_generator(region_ids_json):
 
 
 class eval_mgr():
-    def __init__(self, scores_dirname):
+    def __init__(
+            self, 
+            scores_dirname, 
+            inv_object_labels_dict,
+            vis_dirname,
+            genome_region_dir):
+
+        self.scores_dirname = scores_dirname
+        self.inv_object_labels_dict = inv_object_labels_dict
+        self.vis_dirname = vis_dirname
+        self.genome_region_dir = genome_region_dir
         self.epsilon = 0.00001
         self.num_iter = 0.0
         self.object_accuracy = 0.0
@@ -82,15 +94,27 @@ class eval_mgr():
         self.scores_dict = dict()
         self.labels_dict = dict()
         self.attribute_ids = np.arange(10)*10
+        self.html_filename = os.path.join(
+            vis_dirname,
+            'index.html')
+        self.html_writer = html_writer.HtmlWriter(self.html_filename)
+        col_dict = {
+            0: 'Image',
+            1: 'Ground Truth',
+            2: 'Prediction'
+        }
+        self.html_writer.add_element(col_dict)
         for i in xrange(10):
             self.scores_dict[i] = []
             self.labels_dict[i] = []
-        self.scores_dirname = scores_dirname
-
+        
+            
     def eval(self,
              iter,
              eval_vars_dict,
-             labels):
+             labels,
+             image_ids,
+             region_ids):
         self.num_iter += 1.0
        
         self.eval_object_accuracy(
@@ -105,9 +129,59 @@ class eval_mgr():
             eval_vars_dict['attribute_prob'],
             labels['attributes'])
 
+        pred_obj_labels = self.get_labels(eval_vars_dict['object_prob'])[0:10]
+        gt_obj_labels = self.get_labels(labels['objects'])[0:10]
+        region_paths = self.get_region_paths(image_ids, region_ids)
+        
+        self.save_image_pred(
+            pred_obj_labels,
+            gt_obj_labels,
+            region_ids,
+            region_paths)
+
         if iter%500 == 0:
             self.write_scores()
 
+    def save_image_pred(
+            self,
+            pred_obj_labels,
+            gt_obj_labels,
+            region_ids,
+            region_paths):
+
+        for i in xrange(len(pred_obj_labels)):
+            img = image_io.imread(region_paths[i])
+            filename = str(region_ids[i]) + '.jpg'
+            filename = os.path.join(
+                self.vis_dirname, 
+                str(region_ids[i]) + '.jpg')
+            image_io.imwrite(img, filename)        
+            
+            col_dict = {
+                0: self.html_writer.image_tag(str(region_ids[i]) + '.jpg', 224, 224),
+                1: gt_obj_labels[i],
+                2: pred_obj_labels[i],
+            }
+            
+            self.html_writer.add_element(col_dict)
+            
+
+    def get_region_paths(self, image_ids, region_ids):
+        region_paths = [None]*len(image_ids)
+        for i, (image_id, region_id) in enumerate(zip(image_ids, region_ids)):
+            region_paths[i] = os.path.join(
+                self.genome_region_dir,
+                str(image_id) + '/' + str(region_id) + '.jpg')
+            
+        return region_paths
+                
+    def get_labels(self, object_prob):
+        label_ids = np.argmax(object_prob,1).tolist()
+        labels = [None]*len(label_ids)
+        for i, label_id in enumerate(label_ids):
+            labels[i] = self.inv_object_labels_dict[label_id]
+        return labels
+
     def append_to_scores_labels_list(self, prob, labels):
         for i in xrange(10):
             self.scores_dict[i].append(
@@ -240,7 +314,10 @@ def eval(
             labels = dict()
             labels['objects'] = batch['object_labels']
             labels['attributes'] = batch['attribute_labels']
-            evaluator.eval(iter, eval_vars_dict, labels)
+            image_ids = batch['image_ids']
+            region_ids = batch['region_ids']
+            #print zip(image_ids, region_ids)
+            evaluator.eval(iter, eval_vars_dict, labels, image_ids, region_ids)
             print 'Object accuracy: {}'.format(evaluator.get_object_accuracy())
             # print 'Recall: {}'.format(evaluator.get_recall())
             # print 'Precision: {}'.format(evaluator.get_precision())
@@ -304,8 +381,18 @@ if __name__=='__main__':
         'attribute_prob': graph.obj_atr_inference.attribute_prob,
     }
 
+    # Read inverse object labels
+    with open(constants.object_labels_json,'r') as file:
+        object_labels_dict = ujson.load(file)
+
+    inv_object_labels_dict = {int(v): k for k,v in object_labels_dict.items()}
+
     print 'Creating evaluator...'
-    evaluator = eval_mgr(constants.region_attribute_scores_dirname)
+    evaluator = eval_mgr(
+        constants.region_attribute_scores_dirname,
+        inv_object_labels_dict,
+        constants.region_pred_vis_dirname,
+        constants.image_dir)
 
     print 'Start evaluating...'
     eval(

vqa_parser.py

@@ -196,7 +196,8 @@ def list_of_train_question_ids(
 
     with open(json_anno,'r') as file:
         anno_data = ujson.load(file)
-    
+
+    # map image ids to list of question ids
     imageid_to_qid = dict()
 
     for qid, data in anno_data.items():

word2vec/word_vector_management.py

@@ -98,13 +98,13 @@ class word_vector_manager():
         return phrase_word_vector
                 
     def object_label_word_vectors(self):
-        inv_object_labels = {v: k for k, v in self.object_labels.items()}
-        num_object_labels = len(inv_object_labels)
+        self.inv_object_labels = {v: k for k, v in self.object_labels.items()}
+        num_object_labels = len(self.inv_object_labels)
 
         object_label_vector_list = [None]*num_object_labels
         for i in xrange(num_object_labels):
             object_label_vector_list[i] = self.create_phrase_word_vectors(
-                inv_object_labels[i],
+                self.inv_object_labels[i],
                 'object_label_' + str(i))
 
         self.object_label_vectors = tf.concat(