diff --git a/answer_classifier_cached_features/eval.py b/answer_classifier_cached_features/eval.py
index 72a394b1306d14f7e4b3f89abef3ef585dd165ce..e5b2a4ec2205b80bbe0cabb80aca9dd3ec9ced1f 100644
--- a/answer_classifier_cached_features/eval.py
+++ b/answer_classifier_cached_features/eval.py
@@ -39,24 +39,25 @@ def create_initializer(graph, sess, model):
def create_batch_generator(mode):
if mode=='val':
- vqa_image_dir = constants.vqa_val_image_dir
+ vqa_resnet_feat_dir = constants.vqa_val_resnet_feat_dir
vqa_anno = constants.vqa_val_anno
num_questions = constants.num_val_questions
elif mode=='train':
- vqa_image_dir = constants.vqa_train_image_dir
+ vqa_resnet_feat_dir = constants.vqa_train_resnet_feat_dir
vqa_anno = constants.vqa_train_anno
num_questions = constants.num_train_questions
else:
print "mode needs to be one of {'train','test','val'}, found " + mode
data_mgr = vqa_data.data(
- vqa_image_dir,
+ vqa_resnet_feat_dir,
vqa_anno,
constants.vocab_json,
constants.vqa_answer_vocab_json,
constants.image_size,
constants.num_region_proposals,
- constants.num_negative_answers)
+ constants.num_negative_answers,
+ resnet_feat_dim=constants.resnet_feat_dim)
index_generator = tftools.data.random(
constants.answer_batch_size,
@@ -128,28 +129,43 @@ def create_feed_dict_creator(plh, num_neg_answers):
class eval_mgr():
- def __init__(self, eval_data_json):
+ def __init__(self, eval_data_json, results_json):
self.eval_data_json= eval_data_json
+ self.results_json = results_json
self.eval_data = dict()
self.correct = 0
self.total = 0
+ self.results = []
def eval(self, iter, eval_vars_dict, batch):
batch_size = len(batch['question_unencoded'])
+
for j in xrange(batch_size):
dict_entry = dict()
dict_entry['question'] = batch['question_unencoded'][j]
dict_entry['positive_answer'] = {
batch['positive_answer_unencoded'][j]:
- str(eval_vars_dict['answer_scores'][j][0,0])}
+ str(eval_vars_dict['answer_score_' + str(j)][0,0])}
dict_entry['negative_answers'] = dict()
- for i in xrange(len(batch['negative_answers_unencoded'])):
+ for i in xrange(len(batch['negative_answers_unencoded'][j])):
answer = batch['negative_answers_unencoded'][j][i]
dict_entry['negative_answers'][answer] = \
- str(eval_vars_dict['answer_scores'][j][0,i+1])
+ str(eval_vars_dict['answer_score_' + str(j)][0,i+1])
question_id = batch['question_id'][j]
+ pred_answer, pred_score = self.get_pred_answer(
+ [batch['positive_answer_unencoded'][j]] + \
+ batch['negative_answers_unencoded'][j],
+ eval_vars_dict['answer_score_' + str(j)][0,:].tolist()
+ )
+
+ result_entry = {
+ 'question_id': int(question_id),
+ 'answer': pred_answer
+ }
+ self.results.append(result_entry)
+
self.eval_data[str(question_id)] = dict_entry
print dict_entry
@@ -164,6 +180,16 @@ class eval_mgr():
if iter%100==0:
self.write_data()
+ def get_pred_answer(self, answers, scores):
+ pred_answer = ''
+ pred_score = -1e5
+ for answer, score in zip(answers, scores):
+ if score > pred_score:
+ pred_score = score
+ pred_answer = answer
+
+ return pred_answer, pred_score
+
def is_correct(self, answer_scores):
max_id = np.argmax(answer_scores, 1)
if max_id[0]==0:
@@ -178,6 +204,9 @@ class eval_mgr():
def write_data(self):
with open(self.eval_data_json, 'w') as file:
ujson.dump(self.eval_data, file, indent=4, sort_keys=True)
+
+ with open(self.results_json, 'w') as file:
+ ujson.dump(self.results, file, indent=4, sort_keys=True)
def eval(
@@ -250,12 +279,15 @@ if __name__=='__main__':
print 'Creating dict of vars to be evaluated...'
vars_to_eval_dict = {
'accuracy': graph.answer_accuracy,
- 'answer_scores': graph.answer_inference.answer_score,
}
+ for j in xrange(constants.answer_batch_size):
+ vars_to_eval_dict['answer_score_'+str(j)] = \
+ graph.answer_inference.answer_score[j]
print 'Creating evaluation manager...'
evaluator = eval_mgr(
- constants.answer_eval_data_json)
+ constants.answer_eval_data_json,
+ constants.answer_results_json)
print 'Start training...'
eval(
diff --git a/answer_classifier_cached_features/select_best_model.py b/answer_classifier_cached_features/select_best_model.py
new file mode 100644
index 0000000000000000000000000000000000000000..e5b2a4ec2205b80bbe0cabb80aca9dd3ec9ced1f
--- /dev/null
+++ b/answer_classifier_cached_features/select_best_model.py
@@ -0,0 +1,305 @@
+# from word2vec.word_vector_management import word_vector_manager
+# import object_attribute_classifier.inference as feature_graph
+# import region_relevance_network.inference as relevance_graph
+# import answer_classifier.inference as answer_graph
+from tftools import var_collect, placeholder_management
+import tftools.data
+import losses
+import constants
+import tftools.var_collect as var_collect
+import data.vqa_cached_features as vqa_data
+import answer_classifier_cached_features.train as train
+
+import numpy as np
+import pdb
+import ujson
+import tensorflow as tf
+
+
+def create_initializer(graph, sess, model):
+ class initializer():
+ def __init__(self):
+ with graph.tf_graph.as_default():
+ model_vars = graph.vars_to_save
+ model_restorer = tf.train.Saver(model_vars)
+ model_restorer.restore(sess, model)
+ not_to_init = model_vars
+ all_vars = tf.all_variables()
+ other_vars = [var for var in all_vars
+ if var not in not_to_init]
+ var_collect.print_var_list(
+ other_vars,
+ 'vars_to_init')
+ self.init = tf.initialize_variables(other_vars)
+
+ def initialize(self):
+ sess.run(self.init)
+
+ return initializer()
+
+def create_batch_generator(mode):
+ if mode=='val':
+ vqa_resnet_feat_dir = constants.vqa_val_resnet_feat_dir
+ vqa_anno = constants.vqa_val_anno
+ num_questions = constants.num_val_questions
+ elif mode=='train':
+ vqa_resnet_feat_dir = constants.vqa_train_resnet_feat_dir
+ vqa_anno = constants.vqa_train_anno
+ num_questions = constants.num_train_questions
+ else:
+ print "mode needs to be one of {'train','test','val'}, found " + mode
+
+ data_mgr = vqa_data.data(
+ vqa_resnet_feat_dir,
+ vqa_anno,
+ constants.vocab_json,
+ constants.vqa_answer_vocab_json,
+ constants.image_size,
+ constants.num_region_proposals,
+ constants.num_negative_answers,
+ resnet_feat_dim=constants.resnet_feat_dim)
+
+ index_generator = tftools.data.random(
+ constants.answer_batch_size,
+ num_questions,
+ 1,
+ 0)
+
+ batch_generator = tftools.data.async_batch_generator(
+ data_mgr,
+ index_generator,
+ constants.answer_queue_size)
+
+ return batch_generator
+
+
+def create_feed_dict_creator(plh, num_neg_answers):
+ def feed_dict_creator(batch):
+ vqa_batch = batch
+ batch_size = len(vqa_batch['question'])
+ # Create vqa inputs
+ inputs = {
+ 'region_feats': np.concatenate(vqa_batch['region_feats'], axis=0),
+ 'positive_answer': vqa_batch['positive_answer'],
+ }
+ for i in xrange(4):
+ bin_name = 'bin_' + str(i)
+ inputs[bin_name] = [
+ vqa_batch['question'][j][bin_name] for j in xrange(batch_size)]
+
+ for i in xrange(num_neg_answers):
+ answer_name = 'negative_answer_' + str(i)
+ inputs[answer_name] = [
+ vqa_batch['negative_answers'][j][i] for j in xrange(batch_size)]
+
+ inputs['positive_nouns'] = [
+ a + b for a, b in zip(
+ vqa_batch['question_nouns'],
+ vqa_batch['positive_answer_nouns'])]
+
+ inputs['positive_adjectives'] = [
+ a + b for a, b in zip(
+ vqa_batch['question_adjectives'],
+ vqa_batch['positive_answer_adjectives'])]
+
+ for i in xrange(num_neg_answers):
+ name = 'negative_nouns_' + str(i)
+ list_ith_negative_answer_nouns = [
+ vqa_batch['negative_answers_nouns'][j][i]
+ for j in xrange(batch_size)]
+ inputs[name] = [
+ a + b for a, b in zip(
+ vqa_batch['question_nouns'],
+ list_ith_negative_answer_nouns)]
+
+ name = 'negative_adjectives_' + str(i)
+ list_ith_negative_answer_adjectives = [
+ vqa_batch['negative_answers_adjectives'][j][i]
+ for j in xrange(batch_size)]
+ inputs[name] = [
+ a + b for a, b in zip(
+ vqa_batch['question_adjectives'],
+ list_ith_negative_answer_adjectives)]
+
+ inputs['keep_prob'] = 1.0
+
+ return plh.get_feed_dict(inputs)
+
+ return feed_dict_creator
+
+
+class eval_mgr():
+ def __init__(self, eval_data_json, results_json):
+ self.eval_data_json= eval_data_json
+ self.results_json = results_json
+ self.eval_data = dict()
+ self.correct = 0
+ self.total = 0
+ self.results = []
+
+ def eval(self, iter, eval_vars_dict, batch):
+ batch_size = len(batch['question_unencoded'])
+
+ for j in xrange(batch_size):
+ dict_entry = dict()
+ dict_entry['question'] = batch['question_unencoded'][j]
+ dict_entry['positive_answer'] = {
+ batch['positive_answer_unencoded'][j]:
+ str(eval_vars_dict['answer_score_' + str(j)][0,0])}
+
+ dict_entry['negative_answers'] = dict()
+ for i in xrange(len(batch['negative_answers_unencoded'][j])):
+ answer = batch['negative_answers_unencoded'][j][i]
+ dict_entry['negative_answers'][answer] = \
+ str(eval_vars_dict['answer_score_' + str(j)][0,i+1])
+
+ question_id = batch['question_id'][j]
+ pred_answer, pred_score = self.get_pred_answer(
+ [batch['positive_answer_unencoded'][j]] + \
+ batch['negative_answers_unencoded'][j],
+ eval_vars_dict['answer_score_' + str(j)][0,:].tolist()
+ )
+
+ result_entry = {
+ 'question_id': int(question_id),
+ 'answer': pred_answer
+ }
+ self.results.append(result_entry)
+
+ self.eval_data[str(question_id)] = dict_entry
+
+ print dict_entry
+
+ self.total += batch_size
+
+ self.correct += eval_vars_dict['accuracy']*batch_size
+
+ self.print_accuracy()
+
+
+ if iter%100==0:
+ self.write_data()
+
+ def get_pred_answer(self, answers, scores):
+ pred_answer = ''
+ pred_score = -1e5
+ for answer, score in zip(answers, scores):
+ if score > pred_score:
+ pred_score = score
+ pred_answer = answer
+
+ return pred_answer, pred_score
+
+ def is_correct(self, answer_scores):
+ max_id = np.argmax(answer_scores, 1)
+ if max_id[0]==0:
+ return True
+
+ def print_accuracy(self):
+ print 'Total: {} Correct: {} Accuracy: {}'.format(
+ self.total,
+ self.correct,
+ self.correct/float(self.total))
+
+ def write_data(self):
+ with open(self.eval_data_json, 'w') as file:
+ ujson.dump(self.eval_data, file, indent=4, sort_keys=True)
+
+ with open(self.results_json, 'w') as file:
+ ujson.dump(self.results, file, indent=4, sort_keys=True)
+
+
+def eval(
+ batch_generator,
+ sess,
+ initializer,
+ vars_to_eval_dict,
+ feed_dict_creator,
+ evaluator):
+
+ vars_to_eval_names = []
+ vars_to_eval = []
+ for var_name, var in vars_to_eval_dict.items():
+ vars_to_eval_names += [var_name]
+ vars_to_eval += [var]
+
+ with sess.as_default():
+ initializer.initialize()
+
+ iter = 0
+ for batch in batch_generator:
+ print '---'
+ print 'Iter: {}'.format(iter)
+ feed_dict = feed_dict_creator(batch)
+ eval_vars = sess.run(
+ vars_to_eval,
+ feed_dict = feed_dict)
+ eval_vars_dict = {
+ var_name: eval_var for var_name, eval_var in
+ zip(vars_to_eval_names, eval_vars)}
+ evaluator.eval(iter, eval_vars_dict, batch)
+ iter+=1
+
+ evaluator.write_data()
+
+
+if __name__=='__main__':
+ print 'Creating batch generator...'
+ batch_generator = create_batch_generator(constants.answer_eval_on)
+
+ print 'Creating computation graph...'
+ graph = train.graph_creator(
+ constants.tb_log_dir,
+ constants.answer_batch_size,
+ constants.image_size,
+ constants.num_negative_answers,
+ constants.answer_embedding_dim,
+ constants.answer_regularization_coeff,
+ constants.answer_batch_size*constants.num_region_proposals,
+ 0,
+ 0,
+ 0,
+ resnet_feat_dim=constants.resnet_feat_dim,
+ training=False)
+
+ print 'Starting a session...'
+ sess = tf.Session(graph=graph.tf_graph)
+
+ print 'Creating initializer...'
+ initializer = create_initializer(
+ graph,
+ sess,
+ constants.answer_model_to_eval)
+
+ print 'Creating feed dict creator...'
+ feed_dict_creator = create_feed_dict_creator(
+ graph.plh,
+ constants.num_negative_answers)
+
+ print 'Creating dict of vars to be evaluated...'
+ vars_to_eval_dict = {
+ 'accuracy': graph.answer_accuracy,
+ }
+ for j in xrange(constants.answer_batch_size):
+ vars_to_eval_dict['answer_score_'+str(j)] = \
+ graph.answer_inference.answer_score[j]
+
+ print 'Creating evaluation manager...'
+ evaluator = eval_mgr(
+ constants.answer_eval_data_json,
+ constants.answer_results_json)
+
+ print 'Start training...'
+ eval(
+ batch_generator,
+ sess,
+ initializer,
+ vars_to_eval_dict,
+ feed_dict_creator,
+ evaluator)
+
+
+
+
+
+
diff --git a/constants.py b/constants.py
index 47443cc308e07a806c7bcf89c8e13139c765d6e8..7c96051665f4d068c5edd14526cb3e872030a71a 100644
--- a/constants.py
+++ b/constants.py
@@ -7,3 +7,4 @@ elif hostname=='vision-gpu-2':
from constants_vision_gpu_2 import *
elif hostname=='crunchy':
from constants_crunchy import *
+
diff --git a/data/vqa_cached_features.py b/data/vqa_cached_features.py
index 9fdf937089987d277f3dcddffe00ae497a49b556..913654e8a2ac981bfeab64f2a1a73a1cfe8cad72 100644
--- a/data/vqa_cached_features.py
+++ b/data/vqa_cached_features.py
@@ -32,6 +32,10 @@ class data():
resnet_feat_dim=2048,
mean_image_filename=None):
self.feat_dir = feat_dir
+ # data_split = re.split(
+ # '_',
+ # os.path.split(self.feat_dir)[1])[0]
+ # pdb.set_trace()
self.h = image_size[0]
self.w = image_size[1]
self.c = channels
@@ -124,9 +128,13 @@ class data():
def get_region_feats(self, sample):
question_id = self.sample_to_question_dict[sample]
image_id = self.anno[str(question_id)]['image_id']
+ data_split = re.split(
+ '_',
+ os.path.split(self.feat_dir)[1])[0]
+
feat_path = os.path.join(
self.feat_dir,
- 'COCO_train2014_' + str(image_id).zfill(12) + '.npy')
+ 'COCO_' + data_split + '_' + str(image_id).zfill(12) + '.npy')
return np.load(feat_path)
def get_single_image(self, sample, region_number, batch_list, worker_id):
diff --git a/tftools/placeholder_management.py b/tftools/placeholder_management.py
index 2f7c13f77fcf2f93cdacd0fc96795b45843ee07c..45958deb02c1a2bf05815e0fcfd10f1e94956d68 100644
--- a/tftools/placeholder_management.py
+++ b/tftools/placeholder_management.py
@@ -223,12 +223,13 @@ class PlaceholderManager():
if __name__ == '__main__':
+ sess = tf.InteractiveSession()
plh = PlaceholderManager()
plh.add_list_placeholder(
'list_of_ints',
tf.int32,
3,
- [1,2],
+ [2],
)
inputs = {
@@ -238,7 +239,12 @@ if __name__ == '__main__':
[3,4]]
}
+ list_of_ints = plh['list_of_ints']
+ z = list_of_ints[0] + list_of_ints[1]
feed_dict = plh.get_feed_dict(inputs)
+ print z.eval(feed_dict)
print plh.feed_dict_debug_string(feed_dict)
+
+
pdb.set_trace()