From db4a039501efd54a7899c5ff9d2079145da5d31b Mon Sep 17 00:00:00 2001
From: tgupta6 <tgupta6@illinois.edu>
Date: Tue, 13 Sep 2016 16:07:14 -0500
Subject: [PATCH] model with elementwise multiplication and linear classifier
working
---
.../inference.py | 73 +++++++++++--------
answer_classifier_cached_features/train.py | 2 +-
constants_crunchy.py | 10 +--
3 files changed, 49 insertions(+), 36 deletions(-)
diff --git a/answer_classifier_cached_features/inference.py b/answer_classifier_cached_features/inference.py
index fc9ec26..ff11c42 100644
--- a/answer_classifier_cached_features/inference.py
+++ b/answer_classifier_cached_features/inference.py
@@ -23,40 +23,34 @@ class AnswerInference():
self.num_answers = num_answers
self.keep_prob = keep_prob
with tf.variable_scope('answer_graph'):
-
- self.question = [None]*self.batch_size
- for j in xrange(self.batch_size):
- self.question[j] = tf.reshape(
- self.question_vert_concat[j],
- [1, -1])
-
-
- self.answers = self.answers_vert_concat
-
- self.qa_proj = [None]*self.batch_size
+ self.obj_atr_qa_elementwise_prod = [None]*self.batch_size
for j in xrange(self.batch_size):
- self.qa_proj[j] = self.project_question_answer(
- self.question[j],
- self.answers[j],
- space_dim,
- 'question_answer_projection',
- None if j==0 else True)
+ self.obj_atr_qa_elementwise_prod[j] = self.elementwise_product(
+ self.object_feat[j],
+ self.attribute_feat[j],
+ self.question_vert_concat[j],
+ self.answers_vert_concat[j])
- self.obj_atr_proj = [None]*self.batch_size
- for j in xrange(self.batch_size):
- self.obj_atr_proj[j] = self.project_object_attribute(
- self.object_feat[j],
- self.attribute_feat[j],
- space_dim,
- 'object_attribute_projection',
- None if j==0 else True)
-
self.per_region_answer_scores = [None]*self.batch_size
for j in xrange(self.batch_size):
- self.per_region_answer_scores[j] = tf.matmul(
- self.obj_atr_proj[j],
- tf.transpose(self.qa_proj[j]),
- name='per_region_answer_scores')
+ if j==0:
+ reuse_vars = False
+ else:
+ reuse_vars = True
+
+ self.per_region_answer_scores[j] = layers.conv2d(
+ tf.expand_dims(
+ self.obj_atr_qa_elementwise_prod[j],
+ 0),
+ 1,
+ 1,
+ 'per_region_ans_score_conv',
+ func = None,
+ reuse_vars = reuse_vars)
+
+ self.per_region_answer_scores[j] = tf.squeeze(
+ self.per_region_answer_scores[j],
+ [0,3])
self.per_region_answer_prob = [None]*self.batch_size
self.answer_score = [None]*self.batch_size
@@ -73,6 +67,25 @@ class AnswerInference():
answer_score_tmp,
0,
keep_dims=True)
+
+ def elementwise_product(self, obj_feat, atr_feat, ques_feat, ans_feat):
+ tiled_ques = tf.tile(tf.reshape(ques_feat,[1, -1]),[self.num_answers,1])
+ qa_feat = tf.concat(
+ 1,
+ [tiled_ques, ans_feat])
+ qa_feat = tf.tile(qa_feat, [1,2])
+
+ obj_atr_feat = tf.concat(
+ 1,
+ [obj_feat, atr_feat])
+ obj_atr_feat = tf.tile(obj_atr_feat, [1,5])
+ obj_atr_feat = tf.expand_dims(obj_atr_feat,1)
+
+ feat = obj_atr_feat*qa_feat
+
+ return feat
+
+
def concat_object_attribute(self, object_feat, attribute_feat):
object_attribute = tf.concat(
diff --git a/answer_classifier_cached_features/train.py b/answer_classifier_cached_features/train.py
index 96d1984..e6de431 100644
--- a/answer_classifier_cached_features/train.py
+++ b/answer_classifier_cached_features/train.py
@@ -555,7 +555,7 @@ class attach_optimizer():
self.optimizer.add_variables(
self.graph.object_attribute_vars + self.graph.word_vec_vars,
- learning_rate = 0.0*self.lr)
+ learning_rate = 1.0*self.lr)
self.optimizer.add_variables(
diff --git a/constants_crunchy.py b/constants_crunchy.py
index bf4a63b..75a80b8 100644
--- a/constants_crunchy.py
+++ b/constants_crunchy.py
@@ -5,7 +5,7 @@ def mkdir_if_not_exists(dir_name):
if not os.path.exists(dir_name):
os.mkdir(dir_name)
-experiment_name = 'object_attribute_classifier_wordvec_xform' #'QA_joint_pretrain_genome_split'
+experiment_name = 'QA_classifier_wordvec_xform' #'QA_joint_pretrain_genome_split'
# Global output directory (all subexperiments will be saved here)
global_output_dir = '/home/tanmay/Code/GenVQA/Exp_Results/VQA'
@@ -104,7 +104,7 @@ region_batch_size = 200
region_num_epochs = 6
region_queue_size = 400
region_regularization_coeff = 1e-5
-region_lr = 1e-4
+region_lr = 1e-3
region_log_every_n_iter = 500
region_output_dir = os.path.join(
global_experiment_dir,
@@ -204,8 +204,8 @@ answer_output_dir = os.path.join(
mkdir_if_not_exists(answer_output_dir)
pretrained_model = '/home/tanmay/Code/GenVQA/Exp_Results/VQA/' + \
- 'object_attribute_classifier_large_images_vqa_split/' + \
- 'object_attribute_classifiers/model-80000'
+ 'object_attribute_classifier_wordvec_xform/' + \
+ 'object_attribute_classifiers/model-102000'
answer_model = os.path.join(
answer_output_dir,
@@ -215,7 +215,7 @@ answer_model = os.path.join(
num_regions_with_labels = 100
# Answer fine tune params
-answer_fine_tune_from_iter = 18500
+answer_fine_tune_from_iter = 13000
answer_fine_tune_from = answer_model + '-' + str(answer_fine_tune_from_iter)
# Answer eval params
--
GitLab