Merge branch 'use_detector_scores' of...

Merge branch 'use_detector_scores' of gitlab-beta.engr.illinois.edu:Vision/GenVQA into use_detector_scores

answer_classifier_cached_features/select_best_model.py

@@ -237,13 +237,14 @@ def eval_model(model_to_eval, results_json):
         constants.answer_batch_size,
         constants.image_size,
         constants.num_negative_answers,
-        constants.answer_embedding_dim,
         constants.answer_regularization_coeff,
         constants.answer_batch_size*constants.num_region_proposals,
         0,
         0,
         0,
         constants.answer_obj_atr_loss_wt,
+        constants.answer_ans_loss_wt,
+        constants.answer_mil_loss_wt,
         resnet_feat_dim=constants.resnet_feat_dim,
         training=False)
 

object_attribute_classifier_cached_features/select_best_model.py

@@ -93,9 +93,13 @@ class eval_mgr():
              labels):
         self.num_iter += 1.0
        
-        self.eval_object_accuracy(
+        # self.eval_object_accuracy(
+        #     eval_vars_dict['object_prob'],
+        #     labels['objects'])
+        self.top_k_accuracy(
             eval_vars_dict['object_prob'],
-            labels['objects'])
+            labels['objects'],
+            5)
 
         self.eval_attribute_pr(
             eval_vars_dict['attribute_prob'],
@@ -129,6 +133,25 @@ class eval_mgr():
             with open(filename, 'w') as file:
                 ujson.dump(self.labels_dict[i], file, indent=4)
                 
+    def top_k_accuracy(
+            self,
+            prob,
+            labels,
+            k):
+
+        num_samples, num_classes = prob.shape
+        ids = np.arange(num_classes)
+        accuracy = 0.0
+        for i in xrange(num_samples):
+            gt_ids = set(np.where(labels[i,:]>0.5)[0].tolist())
+            top_k = set(np.argsort(prob[i,:]).tolist()[-1:-1-k:-1])
+            count = 0.0
+            for idx in gt_ids:
+                if idx in top_k:
+                    count += 1.0
+            accuracy += count/max(len(gt_ids),1)
+        self.object_accuracy += accuracy/num_samples
+
     def eval_object_accuracy(
             self,
             prob,
@@ -316,7 +339,8 @@ def model_path_generator(models_dir, start_model, step_size):
 
 if __name__=='__main__':
     model_paths = model_path_generator(
-        constants.region_output_dir,
+        constants.answer_output_dir,
+        # constants.region_output_dir,
         constants.region_start_model,
         constants.region_step_size)