diff --git a/metrics.py b/metrics.py index a7211f4dfeb610e8887936be3ba4353e41a22fcc..0e30f4bdfa2c9bb1487d1ea0dbaf0c4d3ae62473 100644 --- a/metrics.py +++ b/metrics.py @@ -1,3 +1,5 @@ +import tensorflow as tf + def fraction_in_top_k(y, y_pred): y_list = tf.unpack(y) y_pred_list = tf.unpack(y_pred) @@ -18,8 +20,8 @@ def fraction_in_top_k(y, y_pred): return accuracy def presence_in_top_k(y, y_pred, k): - max_ids = tf.argmax(y,1) - is_present = tf.nn.in_top_k(y_pred, max_ids, k) + gt_ids = tf.argmax(y,1) + is_present = tf.nn.in_top_k(y_pred, gt_ids, k) is_present = tf.cast(is_present, tf.float32) return tf.reduce_mean(is_present) diff --git a/object_attribute_classifier_cached_features/eval.py b/object_attribute_classifier_cached_features/eval.py index 6c87d0493586a52a5d1a0e9e9a9be76894ed89f0..df3fe43798e678c28691299b44b73a6854edac78 100644 --- a/object_attribute_classifier_cached_features/eval.py +++ b/object_attribute_classifier_cached_features/eval.py @@ -117,9 +117,13 @@ class eval_mgr(): region_ids): self.num_iter += 1.0 - self.eval_object_accuracy( + # self.eval_object_accuracy( + # eval_vars_dict['object_prob'], + # labels['objects']) + self.top_k_accuracy( eval_vars_dict['object_prob'], - labels['objects']) + labels['objects'], + 5) self.eval_attribute_pr( eval_vars_dict['attribute_prob'], @@ -129,8 +133,8 @@ class eval_mgr(): eval_vars_dict['attribute_prob'], labels['attributes']) - pred_obj_labels = self.get_labels(eval_vars_dict['object_prob'])[0:10] - gt_obj_labels = self.get_labels(labels['objects'])[0:10] + pred_obj_labels = self.get_top_k_labels(eval_vars_dict['object_prob'],5)[0:10] + gt_obj_labels = self.get_gt_labels(labels['objects'])[0:10] region_paths = self.get_region_paths(image_ids, region_ids) if constants.visualize_object_predictions: @@ -183,6 +187,26 @@ class eval_mgr(): labels[i] = self.inv_object_labels_dict[label_id] return labels + def get_gt_labels(self, labels): + num_samples, num_classes = labels.shape + gt_labels = [None]*num_samples + for i in xrange(num_samples): + gt_ids = np.where(labels[i,:]>0.5)[0].tolist() + gt_labels[i] = [] + for idx in gt_ids: + gt_labels[i] += [self.inv_object_labels_dict[idx]] + return gt_labels + + def get_top_k_labels(self, obj_prob, k): + num_samples, num_classes = obj_prob.shape + top_k_labels = [None]*num_samples + for i in xrange(num_samples): + top_k = np.argsort(obj_prob[i,:]).tolist()[-1:-1-k:-1] + top_k_labels[i] = [] + for idx in top_k: + top_k_labels[i] += [self.inv_object_labels_dict[idx]] + return top_k_labels + def append_to_scores_labels_list(self, prob, labels): for i in xrange(10): self.scores_dict[i].append( @@ -216,6 +240,25 @@ class eval_mgr(): current_object_accuracy = np.sum(matches)/matches.shape[0] self.object_accuracy += current_object_accuracy + def top_k_accuracy( + self, + prob, + labels, + k): + + num_samples, num_classes = prob.shape + ids = np.arange(num_classes) + accuracy = 0.0 + for i in xrange(num_samples): + gt_ids = set(np.where(labels[i,:]>0.5)[0].tolist()) + top_k = set(np.argsort(prob[i,:]).tolist()[-1:-1-k:-1]) + count = 0.0 + for idx in gt_ids: + if idx in top_k: + count += 1.0 + accuracy += count/max(len(gt_ids),1) + self.object_accuracy += accuracy/num_samples + def eval_attribute_pr( self, prob,