eval and visualize script for the interpreted evaluation

answer_classifier_cached_features/eval_interpret.py

@@ -182,7 +182,7 @@ class eval_mgr():
             
             dict_entry['relevance_scores'] = eval_vars_dict['relevance_prob_' + str(j)].tolist()
 
-            selected_region = np.argmax(dict_entry['relevance_scores'][0,:])
+            selected_region = np.argmax(eval_vars_dict['relevance_prob_' + str(j)][0,:])
 
             question_id = batch['question_id'][j]
             pred_answer, pred_score = self.get_pred_answer(
@@ -197,7 +197,7 @@ class eval_mgr():
                 'pred_obj_labels': pred_obj_labels[selected_region + j*constants.num_region_proposals],
                 'pred_atr_labels': pred_atr_labels[selected_region + j*constants.num_region_proposals],
             }
-            
+
             if question_id not in self.seen_qids:
                 self.seen_qids.add(question_id)
                 self.results.append(result_entry)

visual_util/visualize_relevance_interpret.py

+import ujson
+import os
+import csv
+import numpy as np
+import random
+from matplotlib import cm
+
+import image_io
+from html_writer import HtmlWriter
+import pdb
+
+class RelevanceVisualizer():
+    def __init__(
+            self,
+            eval_data_json,
+            eval_results_json,
+            anno_data_json,
+            image_dir,
+            region_dir,
+            output_dir,
+            data_type):
+        self.image_dir = image_dir
+        self.region_dir = region_dir
+        self.output_dir = output_dir
+        self.data_type = data_type
+        self.eval_data = self.read_json_file(eval_data_json)
+        self.eval_results_ = self.read_json_file(eval_results_json)
+        self.eval_results = dict()
+        for item in self.eval_results_:
+            self.eval_results[item['question_id']] = item
+        self.anno_data = self.read_json_file(anno_data_json)
+        self.html_filename = os.path.join(output_dir, 'index.html')
+        self.html_writer = HtmlWriter(self.html_filename)
+
+    def read_json_file(self, filename):
+        print 'Reading {} ...'.format(filename)
+        with open(filename, 'r') as file:
+            return ujson.load(file)
+
+    def get_image_name(self,qid):
+        image_id = self.anno_data[qid]['image_id']
+        image_name = 'COCO_' + self.data_type + '_' + str(image_id).zfill(12)
+        image_path = os.path.join(image_dir, image_name + '.jpg')
+        return image_name, image_path
+
+    def get_image(self, key):
+        image_name, image_path = self.get_image_name(key)
+        im = image_io.imread(image_path)
+        return im
+
+    def get_bboxes(self, key):
+        image_name, image_path = self.get_image_name(key)
+        bbox_dir = os.path.join(self.region_dir, image_name)
+        bbox_csv = os.path.join(bbox_dir, 'edge_boxes.csv')
+        bboxes = []
+        with open(bbox_csv, 'r') as csvfile:
+            bbox_reader = csv.DictReader(
+                csvfile, 
+                delimiter=',', 
+                fieldnames=['x', 'y', 'w', 'h', 'score'])
+            for bbox in bbox_reader:
+                bboxes.append(bbox)
+        return bboxes
+        
+    def get_pos_ans_rel_scores(self, key):
+        ans, score = self.eval_data[key]['positive_answer'].items()[0]
+        return ans, float(score), self.eval_data[key]['relevance_scores'][0]
+        
+    def get_pred_ans_rel_scores(self, key):
+        ans, score = self.eval_data[key]['positive_answer'].items()[0]
+        score = float(score)
+        pred_ans_score = (ans, score)
+        pred_ans_id = 0
+        count = 1
+        for ans, score in self.eval_data[key]['negative_answers'].items():
+            score = float(score)
+            if score > pred_ans_score[1]:
+                pred_ans_score = (ans, score)
+                pred_ans_id = count
+            count += 1
+
+        return pred_ans_score[0], pred_ans_score[1], \
+            self.eval_data[key]['relevance_scores'][pred_ans_id]
+
+    def get_box_score_pairs(self, bboxes, scores):
+        pairs = []
+        for i, bbox in enumerate(bboxes):
+            pairs.append((bbox, scores[i]))
+        return pairs
+
+    def create_relevance_map(self, key, mode='pos'):
+        image_name, image_path = self.get_image_name(key)
+        im = image_io.imread(image_path)
+        if len(im.shape)==2:
+            im_h, im_w = im.shape
+            im_ = im
+            im = np.zeros([im_h, im_w, 3])
+            for i in xrange(3):
+                im[:,:,i] = im_
+
+        bboxes = self.get_bboxes(key)
+        if mode=='pos':
+            ans, ans_score, rel_scores = self.get_pos_ans_rel_scores(key)
+        elif mode=='pred':
+            ans, ans_score, rel_scores = self.get_pred_ans_rel_scores(key)
+        else:
+            print 'mode can only take values {\'pred\' or \'pos\'}'
+            raise
+        box_score_pairs = self.get_box_score_pairs(bboxes, rel_scores)
+        rel_map = np.zeros(im.shape[0:2])
+        for box, score in box_score_pairs:
+            box_map = self.make_boxmap(box, im.shape[0:2])
+            rel_map = rel_map + score*box_map
+            # gauss_map = self.make_gaussian(box, im.shape[0:2])
+            # rel_map = np.maximum(rel_map, score*box_map)
+        rel_map_ =cm.jet(np.uint8(rel_map*255))[:,:,0:3]*255
+        # im_rel_map = np.uint8(0.5*im+0.5*rel_map_)
+        im_rel_map = np.uint8(0.0*im + 1.0*np.tile(rel_map[:,:,None], [1,1,3])*im)
+        return rel_map, im_rel_map, im, ans, ans_score
+
+    def make_gaussian(self, box, im_size):
+        im_h, im_w = im_size
+        x = np.arange(im_w)
+        y = np.arange(im_h)
+        xx, yy = np.meshgrid(x, y)
+        sigma_x = float(box['w'])/4.0
+        sigma_y = float(box['h'])/4.0
+        cx = float(box['x']) + float(box['w'])/2.0
+        cy = float(box['y']) + float(box['w'])/2.0
+        g = np.exp(-((xx-cx)**2/(2*sigma_x**2)) - ((yy-cy)**2/(2*sigma_y**2)))
+        return g
+
+    def make_boxmap(self, box, im_size):
+        im_h, im_w = im_size
+        x = int(box['x'])
+        y = int(box['y'])
+        w = int(box['w'])
+        h = int(box['h'])
+        map = np.zeros(im_size)
+        map[y-1:y+h-1, x-1:x+w-1] = 1.0
+        return map
+
+    def write_html(self):
+        col_dict = {
+            0: 'Question',
+            1: 'Pos. Answer',
+            2: 'Pred. Answer',
+            3: 'Pred. Relevance',
+            4: 'Pred. Objects',
+            5: 'Pred. Attributes',
+            6: 'Question Id',
+            7: 'Image'
+        }
+        self.html_writer.add_element(col_dict)
+        random.seed(0)
+        qids = sorted(rel_vis.eval_data.keys())
+        random.shuffle(qids)
+        for qid in qids[0:5000]:
+            question = rel_vis.anno_data[qid]['question']
+
+            pred_rel, pred_im_rel, im, pred_ans, pred_score = rel_vis.create_relevance_map(
+                qid,
+                mode='pred')
+            pos_rel, pos_im_rel, im, pos_ans, pos_score = rel_vis.create_relevance_map(
+                qid,
+                mode='pos')
+
+            if np.max(pred_rel) < 0.5:
+                continue
+
+            pred_im_name = 'pred_rel_' + qid + '.jpg'
+            pos_im_name = 'pos_rel_' + qid + '.jpg'
+            im_name = 'im_' + qid + '.jpg'
+
+            pred_rel_filename = os.path.join(self.output_dir, pred_im_name)
+            pos_rel_filename = os.path.join(self.output_dir, pos_im_name)
+            im_filename = os.path.join(self.output_dir, im_name)
+
+            image_io.imwrite(pred_im_rel, pred_rel_filename)
+            image_io.imwrite(pos_im_rel, pos_rel_filename)
+            image_io.imwrite(im, im_filename)
+
+            im_h, im_w = pred_rel.shape[0:2]
+            col_dict = {
+                0 : question,
+                1 : pos_ans + ': ' + str(pos_score),      
+                2 : pred_ans + ': ' + str(pred_score),               
+                3 : self.html_writer.image_tag(pred_im_name, im_h, im_w),
+                4 : self.eval_results[int(qid)]['pred_obj_labels'], 
+                5 : self.eval_results[int(qid)]['pred_atr_labels'], 
+                6 : qid,
+                7 : self.html_writer.image_tag(im_name, im_h, im_w),
+            }
+            
+            self.html_writer.add_element(col_dict)
+
+        self.html_writer.close_file()
+
+
+if __name__=='__main__':
+    data_type = 'val2014'
+    data_dir = '/home/ssd/VQA'
+
+    image_dir = os.path.join(data_dir, data_type)
+
+    region_dir = os.path.join(
+        data_dir,
+        data_type + '_cropped_large')
+
+    anno_data_json = os.path.join(
+        data_dir, 
+        'mscoco_val2014_annotations_with_parsed_questions.json')
+
+    exp_dir = '/home/tanmay/Code/GenVQA/Exp_Results/models_cvpr/' + \
+              'ans_through_obj_atr_rel_bin_feats/'
+
+    eval_data_json = os.path.join(
+        exp_dir,
+        'answer_classifiers/Results/eval_val_data.json')
+
+    eval_results_json = os.path.join(
+        exp_dir,
+        'answer_classifiers/Results/eval_val_results.json')
+
+    output_dir = os.path.join(exp_dir, 'qual_results_val_interpret')
+    if not os.path.exists(output_dir):
+        os.mkdir(output_dir)
+
+    rel_vis = RelevanceVisualizer(
+        eval_data_json,
+        eval_results_json,
+        anno_data_json,
+        image_dir,
+        region_dir,
+        output_dir,
+        data_type)
+
+    rel_vis.write_html()
+