Something went wrong on our end
eval.py 14.37 KiB
import pdb
import os
import ujson
import numpy as np
import image_io
from visual_util import html_writer
import data.cropped_regions_cached_features as cropped_regions
import tftools.data
from tftools import var_collect, placeholder_management
from object_attribute_classifier_cached_features import inference
from word2vec.word_vector_management import word_vector_manager
import train
import losses
import constants
import tensorflow as tf
def create_initializer(graph, sess, model_to_eval):
class initializer():
def __init__(self):
with graph.tf_graph.as_default():
model_restorer = tf.train.Saver(graph.vars_to_save)
print 'Restoring model {}'.format(model_to_eval)
model_restorer.restore(sess, model_to_eval)
all_vars = tf.all_variables()
other_vars = [
var for var in all_vars
if var not in graph.vars_to_save]
var_collect.print_var_list(
graph.vars_to_save,
'Restored Variables')
var_collect.print_var_list(
other_vars,
'Unrestored Variables')
def initialize(self):
pass
return initializer()
def create_batch_generator(region_ids_json):
data_mgr = cropped_regions.data(
constants.genome_resnet_feat_dir,
constants.image_dir,
constants.object_labels_json,
constants.attribute_labels_json,
constants.regions_json,
region_ids_json,
constants.image_size,
channels=3,
resnet_feat_dim = constants.resnet_feat_dim,
mean_image_filename=None)
num_regions = len(data_mgr.region_ids)
print num_regions
index_generator = tftools.data.random(
constants.region_batch_size,
num_regions,
1,
0)
batch_generator = tftools.data.async_batch_generator(
data_mgr,
index_generator,
constants.region_queue_size)
return batch_generator
class eval_mgr():
def __init__(
self,
scores_dirname,
inv_object_labels_dict,
vis_dirname,
genome_region_dir):
self.scores_dirname = scores_dirname
self.inv_object_labels_dict = inv_object_labels_dict
self.vis_dirname = vis_dirname
self.genome_region_dir = genome_region_dir
self.epsilon = 0.00001
self.num_iter = 0.0
self.object_accuracy = 0.0
self.precision = np.zeros([11], np.float32)
self.recall = np.zeros([11], np.float32)
self.fall_out = np.zeros([11], np.float32)
self.scores_dict = dict()
self.labels_dict = dict()
self.attribute_ids = np.arange(10)*10
self.html_filename = os.path.join(
vis_dirname,
'index.html')
self.html_writer = html_writer.HtmlWriter(self.html_filename)
col_dict = {
0: 'Image',
1: 'Ground Truth',
2: 'Prediction'
}
self.html_writer.add_element(col_dict)
for i in xrange(10):
self.scores_dict[i] = []
self.labels_dict[i] = []
def eval(self,
iter,
eval_vars_dict,
labels,
image_ids,
region_ids):
self.num_iter += 1.0
# self.eval_object_accuracy(
# eval_vars_dict['object_prob'],
# labels['objects'])
self.top_k_accuracy(
eval_vars_dict['object_prob'],
labels['objects'],
5)
self.eval_attribute_pr(
eval_vars_dict['attribute_prob'],
labels['attributes'])
self.append_to_scores_labels_list(
eval_vars_dict['attribute_prob'],
labels['attributes'])
pred_obj_labels = self.get_top_k_labels(eval_vars_dict['object_prob'],5)[0:10]
gt_obj_labels = self.get_gt_labels(labels['objects'])[0:10]
region_paths = self.get_region_paths(image_ids, region_ids)
if constants.visualize_object_predictions:
self.save_image_pred(
pred_obj_labels,
gt_obj_labels,
region_ids,
region_paths)
if iter%500 == 0:
self.write_scores()
def save_image_pred(
self,
pred_obj_labels,
gt_obj_labels,
region_ids,
region_paths):
for i in xrange(len(pred_obj_labels)):
img = image_io.imread(region_paths[i])
filename = str(region_ids[i]) + '.jpg'
filename = os.path.join(
self.vis_dirname,
str(region_ids[i]) + '.jpg')
image_io.imwrite(img, filename)
col_dict = {
0: self.html_writer.image_tag(str(region_ids[i]) + '.jpg', 224, 224),
1: gt_obj_labels[i],
2: pred_obj_labels[i],
}
self.html_writer.add_element(col_dict)
def get_region_paths(self, image_ids, region_ids):
region_paths = [None]*len(image_ids)
for i, (image_id, region_id) in enumerate(zip(image_ids, region_ids)):
region_paths[i] = os.path.join(
self.genome_region_dir,
str(image_id) + '/' + str(region_id) + '.jpg')
return region_paths
def get_labels(self, object_prob):
label_ids = np.argmax(object_prob,1).tolist()
labels = [None]*len(label_ids)
for i, label_id in enumerate(label_ids):
labels[i] = self.inv_object_labels_dict[label_id]
return labels
def get_gt_labels(self, labels):
num_samples, num_classes = labels.shape
gt_labels = [None]*num_samples
for i in xrange(num_samples):
gt_ids = np.where(labels[i,:]>0.5)[0].tolist()
gt_labels[i] = []
for idx in gt_ids:
gt_labels[i] += [self.inv_object_labels_dict[idx]]
return gt_labels
def get_top_k_labels(self, obj_prob, k):
num_samples, num_classes = obj_prob.shape
top_k_labels = [None]*num_samples
for i in xrange(num_samples):
top_k = np.argsort(obj_prob[i,:]).tolist()[-1:-1-k:-1]
top_k_labels[i] = []
for idx in top_k:
top_k_labels[i] += [self.inv_object_labels_dict[idx]]
return top_k_labels
def append_to_scores_labels_list(self, prob, labels):
for i in xrange(10):
self.scores_dict[i].append(
prob[:,self.attribute_ids[i]].tolist())
self.labels_dict[i].append(
labels[:,self.attribute_ids[i]].tolist())
def write_scores(self):
for i in xrange(10):
filename = os.path.join(
self.scores_dirname,
'scores_' + str(i) + '.json')
with open(filename, 'w') as file:
ujson.dump(self.scores_dict[i], file, indent=4)
filename = os.path.join(
self.scores_dirname,
'labels_' + str(i) + '.json')
with open(filename, 'w') as file:
ujson.dump(self.labels_dict[i], file, indent=4)
def eval_object_accuracy(
self,
prob,
labels):
matches = np.equal(
np.argmax(prob, 1),
np.argmax(labels, 1)).astype(np.float32)
current_object_accuracy = np.sum(matches)/matches.shape[0]
self.object_accuracy += current_object_accuracy
def top_k_accuracy(
self,
prob,
labels,
k):
num_samples, num_classes = prob.shape
ids = np.arange(num_classes)
accuracy = 0.0
for i in xrange(num_samples):
gt_ids = set(np.where(labels[i,:]>0.5)[0].tolist())
top_k = set(np.argsort(prob[i,:]).tolist()[-1:-1-k:-1])
count = 0.0
for idx in gt_ids:
if idx in top_k:
count += 1.0
accuracy += count/max(len(gt_ids),1)
self.object_accuracy += accuracy/num_samples
def eval_attribute_pr(
self,
prob,
labels):
thresholds = np.arange(0.0,1.1,0.1).tolist()
current_recall = np.zeros([11], dtype=np.float32)
current_precision = np.zeros([11], dtype=np.float32)
current_fall_out = np.zeros([11], dtype=np.float32)
for i, threshold in enumerate(thresholds):
matches = np.equal(
prob > threshold,
labels == 1).astype(np.int32)
thresholded = (prob > threshold).astype(np.int32)
correct_attributes = np.sum(matches, 0)
tp_attributes = np.sum(labels*thresholded, 0)
tn_attributes = np.sum((1-labels)*(1-thresholded), 0)
fp_attributes = np.sum((1-labels)*thresholded, 0)
positive_attributes = np.sum(labels, 0)
negative_attributes = np.sum(1-labels, 0)
num_attribute_samples = matches.shape[0]
current_recall[i] = np.mean(
(tp_attributes + self.epsilon) / \
(positive_attributes + self.epsilon))
current_precision[i] = np.mean(
(tp_attributes + self.epsilon) / \
(tp_attributes + fp_attributes + self.epsilon))
current_fall_out[i] = np.mean(
(fp_attributes + self.epsilon) / \
(tn_attributes + fp_attributes + self.epsilon))
self.recall += current_recall
self.precision += current_precision
self.fall_out += current_fall_out
def get_object_accuracy(self):
return self.object_accuracy/self.num_iter
def get_precision(self):
return self.precision/self.num_iter
def get_recall(self):
return self.recall/self.num_iter
def get_fall_out(self):
return self.fall_out/self.num_iter
def get_ap(self):
precision = self.get_precision()
recall = self.get_recall()
slots = precision.size-1
ap = 0.0
for i in xrange(slots):
area = (precision[i+1] + precision[i]) * \
(recall[i] - recall[i+1]) / 2
ap += area
area = (1.0 + precision[slots]) * \
(recall[slots] - 0.0) / 2
ap += area
return ap
def eval(
batch_generator,
sess,
initializer,
vars_to_eval_dict,
feed_dict_creator,
evaluator):
vars_to_eval_names = []
vars_to_eval = []
for var_name, var in vars_to_eval_dict.items():
vars_to_eval_names += [var_name]
vars_to_eval += [var]
with sess.as_default():
initializer.initialize()
iter = 0
for batch in batch_generator:
print iter
feed_dict = feed_dict_creator(batch)
eval_vars = sess.run(
vars_to_eval,
feed_dict = feed_dict)
eval_vars_dict = {
var_name: eval_var for var_name, eval_var in
zip(vars_to_eval_names, eval_vars)}
# print batch['region_ids']
labels = dict()
labels['objects'] = batch['object_labels']
labels['attributes'] = batch['attribute_labels']
image_ids = batch['image_ids']
region_ids = batch['region_ids']
#print zip(image_ids, region_ids)
evaluator.eval(iter, eval_vars_dict, labels, image_ids, region_ids)
print 'Object accuracy: {}'.format(evaluator.get_object_accuracy())
# print 'Recall: {}'.format(evaluator.get_recall())
# print 'Precision: {}'.format(evaluator.get_precision())
# print 'Fall_out: {}'.format(evaluator.get_fall_out())
# print 'AP: {}'.format(evaluator.get_ap())
iter+=1
# print 'Object accuracy: {}'.format(evaluator.get_object_accuracy())
# print 'Recall: {}'.format(evaluator.get_recall())
# print 'Precision: {}'.format(evaluator.get_precision())
# print 'AP: {}'.format(evaluator.get_ap())
if __name__=='__main__':
num_epochs = 1
if constants.region_eval_on=='train':
region_ids_json = constants.genome_train_region_ids
elif constants.region_eval_on=='val':
region_ids_json = constants.genome_val_region_ids
elif constants.region_eval_on=='test':
region_ids_json = constants.genome_test_region_ids
else:
print "eval_on can only be 'test' or 'val' or 'train'"
raise
print 'Creating batch generator...'
batch_generator = create_batch_generator(region_ids_json)
print 'Creating computation graph...'
graph = train.graph_creator(
constants.region_batch_size,
constants.tb_log_dir,
constants.image_size,
constants.num_object_labels,
constants.num_attribute_labels,
constants.region_regularization_coeff,
resnet_feat_dim=constants.resnet_feat_dim,
training=False)
print 'Starting a session...'
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
config.gpu_options.per_process_gpu_memory_fraction = 0.8
sess = tf.Session(config=config, graph=graph.tf_graph)
print 'Creating initializer...'
initializer = create_initializer(
graph,
sess,
#constants.pretrained_model)
constants.answer_model_to_eval)
#constants.region_model_to_eval)
print 'Creating feed dict creator...'
feed_dict_creator = train.create_feed_dict_creator(graph.plh)
print 'Creating dict of vars to be evaluated...'
vars_to_eval_dict = {
'object_prob': graph.obj_atr_inference.object_prob,
'attribute_prob': graph.obj_atr_inference.attribute_prob,
}
# Read inverse object labels
with open(constants.object_labels_json,'r') as file:
object_labels_dict = ujson.load(file)
inv_object_labels_dict = {int(v): k for k,v in object_labels_dict.items()}
print 'Creating evaluator...'
evaluator = eval_mgr(
constants.region_attribute_scores_dirname,
inv_object_labels_dict,
constants.region_pred_vis_dirname,
constants.image_dir)
print 'Start evaluating...'
eval(
batch_generator,
sess,
initializer,
vars_to_eval_dict,
feed_dict_creator,
evaluator)