Something went wrong on our end
eval.py 8.37 KiB
# from word2vec.word_vector_management import word_vector_manager
# import object_attribute_classifier.inference as feature_graph
# import region_relevance_network.inference as relevance_graph
# import answer_classifier.inference as answer_graph
from tftools import var_collect, placeholder_management
import tftools.data
import losses
import constants
import tftools.var_collect as var_collect
import data.vqa_cached_features as vqa_data
import answer_classifier_cached_features.train as train
import numpy as np
import pdb
import ujson
import tensorflow as tf
def create_initializer(graph, sess, model):
class initializer():
def __init__(self):
with graph.tf_graph.as_default():
model_vars = graph.vars_to_save
model_restorer = tf.train.Saver(model_vars)
model_restorer.restore(sess, model)
not_to_init = model_vars
all_vars = tf.all_variables()
other_vars = [var for var in all_vars
if var not in not_to_init]
var_collect.print_var_list(
other_vars,
'vars_to_init')
self.init = tf.initialize_variables(other_vars)
def initialize(self):
sess.run(self.init)
return initializer()
def create_batch_generator(mode):
if mode=='val':
vqa_image_dir = constants.vqa_val_image_dir
vqa_anno = constants.vqa_val_anno
num_questions = constants.num_val_questions
elif mode=='train':
vqa_image_dir = constants.vqa_train_image_dir
vqa_anno = constants.vqa_train_anno
num_questions = constants.num_train_questions
else:
print "mode needs to be one of {'train','test','val'}, found " + mode
data_mgr = vqa_data.data(
vqa_image_dir,
vqa_anno,
constants.vocab_json,
constants.vqa_answer_vocab_json,
constants.image_size,
constants.num_region_proposals,
constants.num_negative_answers)
index_generator = tftools.data.random(
constants.answer_batch_size,
num_questions,
1,
0)
batch_generator = tftools.data.async_batch_generator(
data_mgr,
index_generator,
constants.answer_queue_size)
return batch_generator
def create_feed_dict_creator(plh, num_neg_answers):
def feed_dict_creator(batch):
vqa_batch = batch
batch_size = len(vqa_batch['question'])
# Create vqa inputs
inputs = {
'region_feats': np.concatenate(vqa_batch['region_feats'], axis=0),
'positive_answer': vqa_batch['positive_answer'],
}
for i in xrange(4):
bin_name = 'bin_' + str(i)
inputs[bin_name] = [
vqa_batch['question'][j][bin_name] for j in xrange(batch_size)]
for i in xrange(num_neg_answers):
answer_name = 'negative_answer_' + str(i)
inputs[answer_name] = [
vqa_batch['negative_answers'][j][i] for j in xrange(batch_size)]
inputs['positive_nouns'] = [
a + b for a, b in zip(
vqa_batch['question_nouns'],
vqa_batch['positive_answer_nouns'])]
inputs['positive_adjectives'] = [
a + b for a, b in zip(
vqa_batch['question_adjectives'],
vqa_batch['positive_answer_adjectives'])]
for i in xrange(num_neg_answers):
name = 'negative_nouns_' + str(i)
list_ith_negative_answer_nouns = [
vqa_batch['negative_answers_nouns'][j][i]
for j in xrange(batch_size)]
inputs[name] = [
a + b for a, b in zip(
vqa_batch['question_nouns'],
list_ith_negative_answer_nouns)]
name = 'negative_adjectives_' + str(i)
list_ith_negative_answer_adjectives = [
vqa_batch['negative_answers_adjectives'][j][i]
for j in xrange(batch_size)]
inputs[name] = [
a + b for a, b in zip(
vqa_batch['question_adjectives'],
list_ith_negative_answer_adjectives)]
inputs['keep_prob'] = 1.0
return plh.get_feed_dict(inputs)
return feed_dict_creator
class eval_mgr():
def __init__(self, eval_data_json):
self.eval_data_json= eval_data_json
self.eval_data = dict()
self.correct = 0
self.total = 0
def eval(self, iter, eval_vars_dict, batch):
batch_size = len(batch['question_unencoded'])
for j in xrange(batch_size):
dict_entry = dict()
dict_entry['question'] = batch['question_unencoded'][j]
dict_entry['positive_answer'] = {
batch['positive_answer_unencoded'][j]:
str(eval_vars_dict['answer_scores'][j][0,0])}
dict_entry['negative_answers'] = dict()
for i in xrange(len(batch['negative_answers_unencoded'])):
answer = batch['negative_answers_unencoded'][j][i]
dict_entry['negative_answers'][answer] = \
str(eval_vars_dict['answer_scores'][j][0,i+1])
question_id = batch['question_id'][j]
self.eval_data[str(question_id)] = dict_entry
print dict_entry
self.total += batch_size
self.correct += eval_vars_dict['accuracy']*batch_size
self.print_accuracy()
if iter%100==0:
self.write_data()
def is_correct(self, answer_scores):
max_id = np.argmax(answer_scores, 1)
if max_id[0]==0:
return True
def print_accuracy(self):
print 'Total: {} Correct: {} Accuracy: {}'.format(
self.total,
self.correct,
self.correct/float(self.total))
def write_data(self):
with open(self.eval_data_json, 'w') as file:
ujson.dump(self.eval_data, file, indent=4, sort_keys=True)
def eval(
batch_generator,
sess,
initializer,
vars_to_eval_dict,
feed_dict_creator,
evaluator):
vars_to_eval_names = []
vars_to_eval = []
for var_name, var in vars_to_eval_dict.items():
vars_to_eval_names += [var_name]
vars_to_eval += [var]
with sess.as_default():
initializer.initialize()
iter = 0
for batch in batch_generator:
print '---'
print 'Iter: {}'.format(iter)
feed_dict = feed_dict_creator(batch)
eval_vars = sess.run(
vars_to_eval,
feed_dict = feed_dict)
eval_vars_dict = {
var_name: eval_var for var_name, eval_var in
zip(vars_to_eval_names, eval_vars)}
evaluator.eval(iter, eval_vars_dict, batch)
iter+=1
evaluator.write_data()
if __name__=='__main__':
print 'Creating batch generator...'
batch_generator = create_batch_generator(constants.answer_eval_on)
print 'Creating computation graph...'
graph = train.graph_creator(
constants.tb_log_dir,
constants.answer_batch_size,
constants.image_size,
constants.num_negative_answers,
constants.answer_embedding_dim,
constants.answer_regularization_coeff,
constants.answer_batch_size*constants.num_region_proposals,
0,
0,
0,
resnet_feat_dim=constants.resnet_feat_dim,
training=False)
print 'Starting a session...'
sess = tf.Session(graph=graph.tf_graph)
print 'Creating initializer...'
initializer = create_initializer(
graph,
sess,
constants.answer_model_to_eval)
print 'Creating feed dict creator...'
feed_dict_creator = create_feed_dict_creator(
graph.plh,
constants.num_negative_answers)
print 'Creating dict of vars to be evaluated...'
vars_to_eval_dict = {
'accuracy': graph.answer_accuracy,
'answer_scores': graph.answer_inference.answer_score,
}
print 'Creating evaluation manager...'
evaluator = eval_mgr(
constants.answer_eval_data_json)
print 'Start training...'
eval(
batch_generator,
sess,
initializer,
vars_to_eval_dict,
feed_dict_creator,
evaluator)