Something went wrong on our end
eval.py 10.15 KiB
# from word2vec.word_vector_management import word_vector_manager
# import object_attribute_classifier.inference as feature_graph
# import region_relevance_network.inference as relevance_graph
# import answer_classifier.inference as answer_graph
from tftools import var_collect, placeholder_management
import tftools.data
import losses
import constants
import tftools.var_collect as var_collect
import data.vqa_cached_features as vqa_data
import answer_classifier_cached_features.train as train
import numpy as np
import pdb
import ujson
import tensorflow as tf
def create_initializer(graph, sess, model):
class initializer():
def __init__(self):
with graph.tf_graph.as_default():
model_vars = graph.vars_to_save
model_restorer = tf.train.Saver(model_vars)
model_restorer.restore(sess, model)
not_to_init = model_vars
all_vars = tf.all_variables()
other_vars = [var for var in all_vars
if var not in not_to_init]
var_collect.print_var_list(
other_vars,
'vars_to_init')
self.init = tf.initialize_variables(other_vars)
def initialize(self):
sess.run(self.init)
return initializer()
def create_batch_generator(mode):
if mode=='val':
vqa_resnet_feat_dir = constants.vqa_val_resnet_feat_dir
vqa_anno = constants.vqa_val_anno
qids_json = constants.vqa_val_qids
else:
print "mode needs to be one of {'val'}, found " + mode
data_mgr = vqa_data.data(
vqa_resnet_feat_dir,
vqa_anno,
qids_json,
constants.vocab_json,
constants.vqa_answer_vocab_json,
constants.object_labels_json,
constants.attribute_labels_json,
constants.image_size,
constants.num_region_proposals,
constants.num_negative_answers,
resnet_feat_dim=constants.resnet_feat_dim)
num_questions = len(data_mgr.qids)
index_generator = tftools.data.sequential(
constants.answer_batch_size,
num_questions,
1,
0)
batch_generator = tftools.data.async_batch_generator(
data_mgr,
index_generator,
constants.answer_queue_size)
return batch_generator
def create_feed_dict_creator(plh, num_neg_answers):
def feed_dict_creator(batch):
vqa_batch = batch
batch_size = len(vqa_batch['question'])
# Create vqa inputs
inputs = {
'region_feats': np.concatenate(vqa_batch['region_feats'], axis=0),
'positive_answer': vqa_batch['positive_answer'],
}
for i in xrange(4):
bin_name = 'bin_' + str(i)
inputs[bin_name] = [
vqa_batch['question'][j][bin_name] for j in xrange(batch_size)]
for i in xrange(num_neg_answers):
answer_name = 'negative_answer_' + str(i)
inputs[answer_name] = [
vqa_batch['negative_answers'][j][i] for j in xrange(batch_size)]
inputs['positive_nouns'] = [
a + b for a, b in zip(
vqa_batch['question_nouns'],
vqa_batch['positive_answer_nouns'])]
inputs['positive_adjectives'] = [
a + b for a, b in zip(
vqa_batch['question_adjectives'],
vqa_batch['positive_answer_adjectives'])]
for i in xrange(num_neg_answers):
name = 'negative_nouns_' + str(i)
list_ith_negative_answer_nouns = [
vqa_batch['negative_answers_nouns'][j][i]
for j in xrange(batch_size)]
inputs[name] = [
a + b for a, b in zip(
vqa_batch['question_nouns'],
list_ith_negative_answer_nouns)]
name = 'negative_adjectives_' + str(i)
list_ith_negative_answer_adjectives = [
vqa_batch['negative_answers_adjectives'][j][i]
for j in xrange(batch_size)]
inputs[name] = [
a + b for a, b in zip(
vqa_batch['question_adjectives'],
list_ith_negative_answer_adjectives)]
inputs['keep_prob'] = 1.0
return plh.get_feed_dict(inputs)
return feed_dict_creator
class eval_mgr():
def __init__(self, eval_data_json, results_json):
self.eval_data_json= eval_data_json
self.results_json = results_json
self.eval_data = dict()
self.correct = 0
self.total = 0
self.results = []
self.seen_qids = set()
def eval(self, iter, eval_vars_dict, batch):
batch_size = len(batch['question_unencoded'])
for j in xrange(batch_size):
dict_entry = dict()
dict_entry['question'] = batch['question_unencoded'][j]
dict_entry['positive_answer'] = {
batch['positive_answer_unencoded'][j]:
str(eval_vars_dict['answer_score_' + str(j)][0,0])}
dict_entry['negative_answers'] = dict()
for i in xrange(len(batch['negative_answers_unencoded'][j])):
answer = batch['negative_answers_unencoded'][j][i]
dict_entry['negative_answers'][answer] = \
str(eval_vars_dict['answer_score_' + str(j)][0,i+1])
dict_entry['relevance_scores'] = eval_vars_dict['relevance_prob_' + str(j)].tolist()
question_id = batch['question_id'][j]
pred_answer, pred_score = self.get_pred_answer(
[batch['positive_answer_unencoded'][j]] + \
batch['negative_answers_unencoded'][j],
eval_vars_dict['answer_score_' + str(j)][0,:].tolist()
)
result_entry = {
'question_id': int(question_id),
'answer': pred_answer
}
if question_id not in self.seen_qids:
self.seen_qids.add(question_id)
self.results.append(result_entry)
else:
print 'Already evaluated on this sample'
self.eval_data[str(question_id)] = dict_entry
# print dict_entry
self.total += batch_size
self.correct += eval_vars_dict['accuracy']*batch_size
self.print_accuracy()
if iter%100==0:
self.write_data()
def get_pred_answer(self, answers, scores):
pred_answer = ''
pred_score = -1e5
for answer, score in zip(answers, scores):
if score > pred_score:
pred_score = score
pred_answer = answer
return pred_answer, pred_score
def is_correct(self, answer_scores):
max_id = np.argmax(answer_scores, 1)
if max_id[0]==0:
return True
def print_accuracy(self):
print 'Total: {} Correct: {} Accuracy: {}'.format(
self.total,
self.correct,
self.correct/float(self.total))
def write_data(self):
with open(self.eval_data_json, 'w') as file:
ujson.dump(self.eval_data, file, indent=4, sort_keys=True)
with open(self.results_json, 'w') as file:
ujson.dump(self.results, file, indent=4, sort_keys=True)
def eval(
batch_generator,
sess,
initializer,
vars_to_eval_dict,
feed_dict_creator,
evaluator):
vars_to_eval_names = []
vars_to_eval = []
for var_name, var in vars_to_eval_dict.items():
vars_to_eval_names += [var_name]
vars_to_eval += [var]
with sess.as_default():
initializer.initialize()
iter = 0
for batch in batch_generator:
print '---'
print 'Iter: {}'.format(iter)
feed_dict = feed_dict_creator(batch)
eval_vars = sess.run(
vars_to_eval,
feed_dict = feed_dict)
eval_vars_dict = {
var_name: eval_var for var_name, eval_var in
zip(vars_to_eval_names, eval_vars)}
evaluator.eval(iter, eval_vars_dict, batch)
iter+=1
evaluator.write_data()
if __name__=='__main__':
print 'Creating batch generator...'
batch_generator = create_batch_generator(constants.answer_eval_on)
print 'Creating computation graph...'
graph = train.graph_creator(
constants.tb_log_dir,
constants.answer_batch_size,
constants.image_size,
constants.num_negative_answers,
constants.answer_embedding_dim,
constants.answer_regularization_coeff,
constants.answer_batch_size*constants.num_region_proposals,
0,
0,
0,
constants.answer_obj_atr_loss_wt,
constants.answer_ans_loss_wt,
constants.answer_mil_loss_wt,
resnet_feat_dim=constants.resnet_feat_dim,
training=False)
print 'Starting a session...'
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
config.gpu_options.per_process_gpu_memory_fraction = 0.9
sess = tf.Session(config=config, graph=graph.tf_graph)
print 'Creating initializer...'
initializer = create_initializer(
graph,
sess,
constants.answer_model_to_eval)
print 'Creating feed dict creator...'
feed_dict_creator = create_feed_dict_creator(
graph.plh,
constants.num_negative_answers)
print 'Creating dict of vars to be evaluated...'
vars_to_eval_dict = {
'accuracy': graph.answer_accuracy,
}
for j in xrange(constants.answer_batch_size):
vars_to_eval_dict['answer_score_'+str(j)] = \
graph.answer_inference.answer_score[j]
vars_to_eval_dict['relevance_prob_'+str(j)] = \
graph.relevance_inference.answer_region_prob[j]
print 'Creating evaluation manager...'
evaluator = eval_mgr(
constants.answer_eval_data_json,
constants.answer_eval_results_json)
print 'Start training...'
eval(
batch_generator,
sess,
initializer,
vars_to_eval_dict,
feed_dict_creator,
evaluator)