diff --git a/answer_classifier_cached_features/inference.py b/answer_classifier_cached_features/inference.py index fc9ec26136abe3ec702e6290d2d73296cc81b3c5..ff11c428ca3e1c9b322f09b7d6be7c9e6410d9cf 100644 --- a/answer_classifier_cached_features/inference.py +++ b/answer_classifier_cached_features/inference.py @@ -23,40 +23,34 @@ class AnswerInference(): self.num_answers = num_answers self.keep_prob = keep_prob with tf.variable_scope('answer_graph'): - - self.question = [None]*self.batch_size - for j in xrange(self.batch_size): - self.question[j] = tf.reshape( - self.question_vert_concat[j], - [1, -1]) - - - self.answers = self.answers_vert_concat - - self.qa_proj = [None]*self.batch_size + self.obj_atr_qa_elementwise_prod = [None]*self.batch_size for j in xrange(self.batch_size): - self.qa_proj[j] = self.project_question_answer( - self.question[j], - self.answers[j], - space_dim, - 'question_answer_projection', - None if j==0 else True) + self.obj_atr_qa_elementwise_prod[j] = self.elementwise_product( + self.object_feat[j], + self.attribute_feat[j], + self.question_vert_concat[j], + self.answers_vert_concat[j]) - self.obj_atr_proj = [None]*self.batch_size - for j in xrange(self.batch_size): - self.obj_atr_proj[j] = self.project_object_attribute( - self.object_feat[j], - self.attribute_feat[j], - space_dim, - 'object_attribute_projection', - None if j==0 else True) - self.per_region_answer_scores = [None]*self.batch_size for j in xrange(self.batch_size): - self.per_region_answer_scores[j] = tf.matmul( - self.obj_atr_proj[j], - tf.transpose(self.qa_proj[j]), - name='per_region_answer_scores') + if j==0: + reuse_vars = False + else: + reuse_vars = True + + self.per_region_answer_scores[j] = layers.conv2d( + tf.expand_dims( + self.obj_atr_qa_elementwise_prod[j], + 0), + 1, + 1, + 'per_region_ans_score_conv', + func = None, + reuse_vars = reuse_vars) + + self.per_region_answer_scores[j] = tf.squeeze( + self.per_region_answer_scores[j], + [0,3]) self.per_region_answer_prob = [None]*self.batch_size self.answer_score = [None]*self.batch_size @@ -73,6 +67,25 @@ class AnswerInference(): answer_score_tmp, 0, keep_dims=True) + + def elementwise_product(self, obj_feat, atr_feat, ques_feat, ans_feat): + tiled_ques = tf.tile(tf.reshape(ques_feat,[1, -1]),[self.num_answers,1]) + qa_feat = tf.concat( + 1, + [tiled_ques, ans_feat]) + qa_feat = tf.tile(qa_feat, [1,2]) + + obj_atr_feat = tf.concat( + 1, + [obj_feat, atr_feat]) + obj_atr_feat = tf.tile(obj_atr_feat, [1,5]) + obj_atr_feat = tf.expand_dims(obj_atr_feat,1) + + feat = obj_atr_feat*qa_feat + + return feat + + def concat_object_attribute(self, object_feat, attribute_feat): object_attribute = tf.concat( diff --git a/answer_classifier_cached_features/train.py b/answer_classifier_cached_features/train.py index 96d198453cd1bf5908731d39eab0a7e3c8b3a790..e6de4310b452630fa2df61618cb5f7dfff3c5590 100644 --- a/answer_classifier_cached_features/train.py +++ b/answer_classifier_cached_features/train.py @@ -555,7 +555,7 @@ class attach_optimizer(): self.optimizer.add_variables( self.graph.object_attribute_vars + self.graph.word_vec_vars, - learning_rate = 0.0*self.lr) + learning_rate = 1.0*self.lr) self.optimizer.add_variables( diff --git a/constants_crunchy.py b/constants_crunchy.py index bf4a63b88d6613eae9a07360a30cb81092c409bd..75a80b86660f5c8e6dc83b1b0ce40218ba235f70 100644 --- a/constants_crunchy.py +++ b/constants_crunchy.py @@ -5,7 +5,7 @@ def mkdir_if_not_exists(dir_name): if not os.path.exists(dir_name): os.mkdir(dir_name) -experiment_name = 'object_attribute_classifier_wordvec_xform' #'QA_joint_pretrain_genome_split' +experiment_name = 'QA_classifier_wordvec_xform' #'QA_joint_pretrain_genome_split' # Global output directory (all subexperiments will be saved here) global_output_dir = '/home/tanmay/Code/GenVQA/Exp_Results/VQA' @@ -104,7 +104,7 @@ region_batch_size = 200 region_num_epochs = 6 region_queue_size = 400 region_regularization_coeff = 1e-5 -region_lr = 1e-4 +region_lr = 1e-3 region_log_every_n_iter = 500 region_output_dir = os.path.join( global_experiment_dir, @@ -204,8 +204,8 @@ answer_output_dir = os.path.join( mkdir_if_not_exists(answer_output_dir) pretrained_model = '/home/tanmay/Code/GenVQA/Exp_Results/VQA/' + \ - 'object_attribute_classifier_large_images_vqa_split/' + \ - 'object_attribute_classifiers/model-80000' + 'object_attribute_classifier_wordvec_xform/' + \ + 'object_attribute_classifiers/model-102000' answer_model = os.path.join( answer_output_dir, @@ -215,7 +215,7 @@ answer_model = os.path.join( num_regions_with_labels = 100 # Answer fine tune params -answer_fine_tune_from_iter = 18500 +answer_fine_tune_from_iter = 13000 answer_fine_tune_from = answer_model + '-' + str(answer_fine_tune_from_iter) # Answer eval params