Refactoring code to have glorot initialization in obj and atr classifiers and...

Refactoring code to have glorot initialization in obj and atr classifiers and save only relevant variables

classifiers/answer_classifier/ans_data_io_helper.py

@@ -117,7 +117,7 @@ def ans_mini_batch_loader(qa_dict, region_anno_dict, ans_dict, vocab,
     region_score = np.zeros(shape=[1,count])
     partition = np.zeros(shape=[count])
     question_encodings = np.zeros(shape=[count, len(vocab)])
-        
+
     for i in xrange(start_index, start_index + batch_size):
         
         image_id = qa_dict[i].image_id
@@ -132,14 +132,25 @@ def ans_mini_batch_loader(qa_dict, region_anno_dict, ans_dict, vocab,
 
         end1 = time.time()
 #        print('Ranking Region: ' + str(end1-start1))
-
+        question_encoding_tmp = np.zeros(shape=[1, len(vocab)])
+        
         for word in question[0:-1].split():
+            
                 if word not in vocab:
                     word = 'unk'
-                question_encodings[0, vocab[word]] += 1 
-
+                question_encoding_tmp[0, vocab[word]] += 1 
+        question_len = np.sum(question_encoding_tmp)
+        # print(question[0:-1].split())
+        # print(question_len)
+        # print(question_encoding_tmp)
+        # print(vocab)
+        assert (not question_len==0)
+
+        question_encoding_tmp /= question_len
+        
         for j in xrange(num_proposals):
             counter = j + (i-start_index)*num_proposals
+            
             proposal = regions[j]
 
             start2 = time.time()
@@ -153,7 +164,7 @@ def ans_mini_batch_loader(qa_dict, region_anno_dict, ans_dict, vocab,
             region_score[0,counter] = proposal.score
             partition[counter] = i-start_index
             
-            question_encodings[counter,:] = question_encodings[0,:]
+            question_encodings[counter,:] = question_encoding_tmp
 
         score_start_id = (i-start_index)*num_proposals
         region_score[0, score_start_id:score_start_id+num_proposals] /= \

classifiers/answer_classifier/train_ans_classifier.py

@@ -5,6 +5,7 @@ import matplotlib.pyplot as plt
 import matplotlib.image as mpimg
 import numpy as np
 import math
+import pdb
 import tensorflow as tf
 import object_classifiers.obj_data_io_helper as obj_data_loader
 import attribute_classifiers.atr_data_io_helper as atr_data_loader
@@ -81,11 +82,11 @@ def train(train_params):
     # Save region crops
     if crop_n_save_regions == True:
         qa_anno_dict_test = ans_io_helper.parse_qa_anno(test_anno_filename)
-        ans_io_helper.save_regions(image_dir, image_regions_dir, 
-                                   qa_anno_dict, region_anno_dict, 
+        ans_io_helper.save_regions(image_dir, image_regions_dir,
+                                   qa_anno_dict, region_anno_dict,
                                    1, 111351, 75, 75)
-        ans_io_helper.save_regions(image_dir, image_regions_dir, 
-                                   qa_anno_dict_test, region_anno_dict, 
+        ans_io_helper.save_regions(image_dir, image_regions_dir,
+                                   qa_anno_dict_test, region_anno_dict,
                                    111352, 160725-111352+1, 75, 75)
 
     
@@ -99,7 +100,7 @@ def train(train_params):
     atr_feat = tf.get_collection('atr_feat', scope='atr/conv2')
 
     # model restoration
-    obj_atr_saver = tf.train.Saver()
+#    obj_atr_saver = tf.train.Saver()
     model_to_restore = '/home/tanmay/Code/GenVQA/GenVQA/classifiers/' + \
                        'saved_models/obj_atr_classifier-1'
 #    obj_atr_saver.restore(sess, model_to_restore)
@@ -120,18 +121,26 @@ def train(train_params):
     
     train_step = tf.train.AdamOptimizer(train_params['adam_lr']) \
                          .minimize(cross_entropy, var_list=vars_to_opt)
-    print(train_step.name)
-    vars_to_restore = tf.get_collection(tf.GraphKeys.VARIABLES,scope='obj') + \
-                      tf.get_collection(tf.GraphKeys.VARIABLES, scope='atr') + \
-                      tf.get_collection(tf.GraphKeys.VARIABLES, scope='ans/word_embed')
+
+    word_embed = tf.get_collection(tf.GraphKeys.VARIABLES, scope='ans/word_embed')
+    vars_to_restore = \
+    tf.get_collection(tf.GraphKeys.VARIABLES,scope='obj') + \
+    tf.get_collection(tf.GraphKeys.VARIABLES, scope='atr') + \
+    [word_embed[0]]
+    
     all_vars = tf.get_collection(tf.GraphKeys.VARIABLES)
 
     vars_to_init = [var for var in all_vars if var not in vars_to_restore]
+    vars_to_save = tf.get_collection(tf.GraphKeys.VARIABLES,scope='obj') + \
+                   tf.get_collection(tf.GraphKeys.VARIABLES, scope='atr') + \
+                   tf.get_collection(tf.GraphKeys.VARIABLES, scope='ans')
     
+    print('vars_to_save: ')
+    print([var.name for var in vars_to_save])
     # Session saver
     
     saver = tf.train.Saver(vars_to_restore)
-
+    saver2 = tf.train.Saver(vars_to_save)
     if restore_intermediate_model==True:
         intermediate_model = '/home/tanmay/Code/GenVQA/Exp_Results/' + \
                              'Ans_Classifier/ans_classifier_question_only-9'
@@ -155,6 +164,10 @@ def train(train_params):
 
     placeholders = [image_regions, questions, keep_prob, y, region_score]
     
+    # Variables to observe
+    W_fc2 = tf.get_collection(tf.GraphKeys.VARIABLES, scope='ans/fc2/W')
+    q_feat = tf.get_collection('q_feat', scope='ans/q_embed')
+
     # Start Training
 #    batch_size = 1
     max_epoch = 10
@@ -180,25 +193,58 @@ def train(train_params):
                 y: train_ans_labels,        
                 region_score: train_region_score,
             }
+
+
+            try:
+                assert (not np.any(np.isnan(q_feat[0].eval(feed_dict_train))))
+            except AssertionError:
+                print('NaN in q_feat')
+                print(1+i*batch_size)
+                print(train_questions)
+                print(logits.eval(feed_dict_train))
+                print(cross_entropy.eval(feed_dict_train))
+                exit(1)
+
             start1 = time.time()
-            _, current_train_batch_acc, y_avg_eval, y_pred_eval, logits_eval = \
-                sess.run([train_step, accuracy, y_avg, y_pred, logits],
+            _, current_train_batch_acc, y_avg_eval, y_pred_eval, logits_eval, W_fc2_eval = \
+                sess.run([train_step, accuracy, y_avg, y_pred, logits, W_fc2[0]],
                          feed_dict=feed_dict_train)
             end1 = time.time()
             #            print('Training Pass: ' + str(end1-start1))
             train_acc_array_epoch[epoch] = train_acc_array_epoch[epoch] + \
                                            current_train_batch_acc
+          #  pdb.set_trace()
+        
+            
+
+            try:
+                assert (not np.any(np.isnan(W_fc2_eval)))
+            except AssertionError:
+                print('NaN in W_fc2')
+                print(1+i*batch_size)
+                print(W_fc2_eval)
+                exit(1)
+            try:
+                assert (not np.any(np.isnan(logits_eval)))
+            except AssertionError:
+                print('NaN in logits')
+                print(1+i*batch_size)
+                print(y_avg_eval)
+                exit(1)
+
             try:
                 assert (not np.any(np.isnan(y_avg_eval)))
             except AssertionError:
-                print('Run NaNs coming')
+                print('NaN in y_avg')
                 print(1+i*batch_size)
+                print(logits_eval)
                 print(y_avg_eval)
                 exit(1)
 
             if (i+1)%500==0:
                 print(logits_eval[0:22,:])
                 print(train_region_score[0,0:22])
+                print(train_ans_labels[0,:])
 #                print(train_ans_labels[0,:])
                 print(y_avg_eval[0,:])
 #                print(y_pred_eval)
@@ -233,15 +279,15 @@ def train(train_params):
                                 savePath=os.path.join(outdir, 
                                 'acc_vs_epoch_q_o_atr.pdf'))
 
-        save_path = saver.save(sess, 
-                               os.path.join(outdir,'ans_classifier_question_obj_atr_only'), 
-                               global_step=epoch)
+        save_path = saver2.save(sess, 
+                                os.path.join(outdir,'ans_classifier_question_obj_atr'), 
+                                global_step=epoch)
 
     sess.close()
     tf.reset_default_graph()
     
 if __name__=='__main__':
     train_params = {
-        'adam_lr' : 0.00001,
+        'adam_lr' : 0.0001,
     }
     train(train_params)

classifiers/attribute_classifiers/eval_atr_classifier.py

@@ -15,13 +15,20 @@ def eval(eval_params):
     
     x, y, keep_prob = graph_creator.placeholder_inputs()
     _ = graph_creator.obj_comp_graph(x, 1.0)
-    obj_feat = tf.get_collection('obj_feat', scope='obj/conv2')
-
-    y_pred = graph_creator.atr_comp_graph(x, keep_prob, obj_feat[0])
+    g = tf.get_default_graph()
+    obj_feat = g.get_operation_by_name('obj/conv2/obj_feat')
+    y_pred = graph_creator.atr_comp_graph(x, keep_prob, obj_feat.outputs[0])
     accuracy = graph_creator.evaluation(y, y_pred)
 
-    saver = tf.train.Saver()
-    saver.restore(sess, eval_params['model_name'] + '-' + str(eval_params['global_step']))
+    # Object model restorer
+    vars_to_restore = tf.get_collection(tf.GraphKeys.VARIABLES, scope='obj') + \
+                      tf.get_collection(tf.GraphKeys.VARIABLES, scope='atr')
+    print('Variables to restore:')
+    print([var.name for var in vars_to_restore])
+
+    saver = tf.train.Saver(vars_to_restore)
+    saver.restore(sess, eval_params['model_name'] + '-' + \
+                  str(eval_params['global_step']))
 
     mean_image = np.load(os.path.join(eval_params['out_dir'], 'mean_image.npy'))
     test_json_filename = eval_params['test_json']
@@ -36,12 +43,17 @@ def eval(eval_params):
     html_dir = eval_params['html_dir']
     if not os.path.exists(html_dir):
         os.mkdir(html_dir)
-    html_writer = atr_data_loader.html_atr_table_writer(os.path.join(html_dir, 'index.html'))
+
+    html_writer = atr_data_loader \
+        .html_atr_table_writer(os.path.join(html_dir, 'index.html'))
+
     col_dict = {
         0: 'Grount Truth',
         1: 'Prediction',
         2: 'Image'}
+
     html_writer.add_element(col_dict)
+
     color_dict = {
         0: 'red', # blanks are treated as red
         1: 'green',
@@ -51,7 +63,9 @@ def eval(eval_params):
     batch_size = 100
     correct = 0
     for i in range(50):  
-        test_batch = atr_data_loader.atr_mini_batch_loader(test_json_data, image_dir, mean_image, 10000+i*batch_size, batch_size, 75, 75)
+        test_batch = atr_data_loader\
+            .atr_mini_batch_loader(test_json_data, image_dir, mean_image, 
+                                   10000+i*batch_size, batch_size, 75, 75)
         feed_dict_test={x: test_batch[0], y: test_batch[1], keep_prob: 1.0}
         result = sess.run([accuracy, y_pred], feed_dict=feed_dict_test)
         correct = correct + result[0]*batch_size

classifiers/attribute_classifiers/train_atr_classifier.py

@@ -15,31 +15,47 @@ def train(train_params):
 
     x, y, keep_prob = graph_creator.placeholder_inputs()
     _ = graph_creator.obj_comp_graph(x, 1.0)
-    obj_feat = tf.get_collection('obj_feat', scope='obj/conv2')
+    g = tf.get_default_graph()
+    obj_feat = g.get_operation_by_name('obj/conv2/obj_feat')
 
-    # Session Saver
-    obj_saver = tf.train.Saver()
+    # Object model restorer
+    vars_to_restore = tf.get_collection(tf.GraphKeys.VARIABLES, scope='obj')
+    print('Variables to restore:')
+    print([var.name for var in vars_to_restore])
 
-    # Restore obj network parameters
-    obj_saver.restore(sess, train_params['obj_model_name'] + '-' + str(train_params['obj_global_step']))
+    obj_saver = tf.train.Saver(vars_to_restore)
+    obj_saver.restore(sess, train_params['obj_model_name'] + '-' + \
+                      str(train_params['obj_global_step']))
 
-    y_pred = graph_creator.atr_comp_graph(x, keep_prob, obj_feat[0])
+    y_pred = graph_creator.atr_comp_graph(x, keep_prob, obj_feat.outputs[0])
     cross_entropy = graph_creator.loss(y, y_pred)
+    accuracy = graph_creator.evaluation(y, y_pred)
+
+    
+    # Collect variables to save or optimize
     vars_to_opt = tf.get_collection(tf.GraphKeys.VARIABLES, scope='atr')
-    vars_to_restore = tf.get_collection(tf.GraphKeys.VARIABLES, scope='obj')
-    train_step = tf.train.AdamOptimizer(train_params['adam_lr']).minimize(cross_entropy, var_list=vars_to_opt)
+    vars_to_save = vars_to_opt + vars_to_restore
+
+    print('Variables to optimize:')
+    print([var.name for var in vars_to_opt])
+    print('Variables to save:')
+    print([var.name for var in vars_to_save])
     
+
+    # Object and Attribute model saver
+    obj_atr_saver = tf.train.Saver(vars_to_save)
     
+    # Add optimization op
+    train_step = tf.train.AdamOptimizer(train_params['adam_lr']) \
+                         .minimize(cross_entropy, var_list=vars_to_opt)
+
+    # Collect variables to initialize
     all_vars = tf.get_collection(tf.GraphKeys.VARIABLES)
     vars_to_init = [var for var in all_vars if var not in vars_to_restore]
-    print('Variables that are being optimized: ' + ' '.join([var.name for var in vars_to_opt]))
-    print('Variables that will be initialized: ' + ' '.join([var.name for var in vars_to_init]))
-    accuracy = graph_creator.evaluation(y, y_pred)
+    print('Variables to initialize:')
+    print([var.name for var in vars_to_init])
+    
     
-    # Session saver for atr variables
-    atr_saver = tf.train.Saver(vars_to_opt)
-    obj_atr_saver = tf.train.Saver(all_vars)
-
     outdir = train_params['out_dir']
     if not os.path.exists(outdir):
         os.mkdir(outdir)
@@ -59,7 +75,9 @@ def train(train_params):
     image_dir = train_params['image_dir']
     if train_params['mean_image']=='':
         print('Computing mean image')
-        mean_image = atr_data_loader.mean_image(train_json_data, image_dir, 1000, 100, img_height, img_width)
+        mean_image = atr_data_loader.mean_image(train_json_data, image_dir, 
+                                                1000, 100, 
+                                                img_height, img_width)
     else:
         print('Loading mean image')
         mean_image = np.load(train_params['mean_image'])
@@ -67,7 +85,11 @@ def train(train_params):
 
     # Val Data
     print('Loading validation data')
-    val_batch = atr_data_loader.atr_mini_batch_loader(train_json_data, image_dir, mean_image, 9501, 499, img_height, img_width)
+    val_batch = atr_data_loader.atr_mini_batch_loader(train_json_data, 
+                                                      image_dir, mean_image, 
+                                                      9501, 499, 
+                                                      img_height, img_width)
+
     feed_dict_val={x: val_batch[0], y: val_batch[1], keep_prob: 1.0}
     
   
@@ -85,16 +107,31 @@ def train(train_params):
         for i in range(max_iter):
             if i%100==0:
                 print('Iter: ' + str(i))
-            train_batch = atr_data_loader.atr_mini_batch_loader(train_json_data, image_dir, mean_image, 1+i*batch_size, batch_size, img_height, img_width)
+                print('Val Acc: ' + str(accuracy.eval(feed_dict_val)))
+
+            train_batch = atr_data_loader \
+                .atr_mini_batch_loader(train_json_data, image_dir, mean_image, 
+                                       1+i*batch_size, batch_size, 
+                                       img_height, img_width)
             feed_dict_train={x: train_batch[0], y: train_batch[1], keep_prob: 0.5}
-            _, current_train_batch_acc = sess.run([train_step, accuracy], feed_dict=feed_dict_train)
-            train_acc_array_epoch[epoch] =  train_acc_array_epoch[epoch] + current_train_batch_acc
+
+            _, current_train_batch_acc = sess.run([train_step, accuracy], 
+                                                  feed_dict=feed_dict_train)
+            train_acc_array_epoch[epoch] =  train_acc_array_epoch[epoch] \
+                                            + current_train_batch_acc
         
         train_acc_array_epoch[epoch] = train_acc_array_epoch[epoch] / max_iter 
         val_acc_array_epoch[epoch] = accuracy.eval(feed_dict_val)
-        plotter.plot_accuracies(xdata=np.arange(0,epoch+1)+1, ydata_train=train_acc_array_epoch[0:epoch+1], ydata_val=val_acc_array_epoch[0:epoch+1], xlim=[1, max_epoch], ylim=[0, 1.], savePath=os.path.join(outdir,'acc_vs_epoch.pdf'))
-        _ = atr_saver.save(sess, os.path.join(outdir,'atr_classifier'), global_step=epoch)
-        _ = obj_atr_saver.save(sess, os.path.join(outdir,'obj_atr_classifier'), global_step=epoch)
+
+        plotter.plot_accuracies(xdata=np.arange(0,epoch+1)+1, 
+                                ydata_train=train_acc_array_epoch[0:epoch+1], 
+                                ydata_val=val_acc_array_epoch[0:epoch+1], 
+                                xlim=[1, max_epoch], ylim=[0, 1.], 
+                                savePath=os.path.join(outdir,
+                                                      'acc_vs_epoch.pdf'))
+
+        _ = obj_atr_saver.save(sess, os.path.join(outdir,'obj_atr_classifier'), 
+                               global_step=epoch)
             
     
     sess.close()

classifiers/object_classifiers/#obj_data_io_helper.py#

-import json
-import sys
-import os
-import matplotlib.pyplot as plt
-import matplotlib.image as mpimg
-import numpy as np
-import tensorflow as tf
-from scipy import misc
-
-def obj_mini_batch_loader(json_data, image_dir, mean_image, start_index, batch_size, img_height = 100, img_width = 100, channels = 3):
-
-    obj_images = np.empty(shape=[9 * batch_size, img_height / 3, img_width / 3, channels])
-    obj_labels = np.zeros(shape=[9 * batch_size, 4])
-
-    for i in range(start_index, start_index + batch_size):
-        image_name = os.path.join(image_dir, str(i) + '.jpg')
-        image = misc.imresize(mpimg.imread(image_name), (img_height, img_width), interp='nearest')
-        crop_shape = np.array([image.shape[0], image.shape[1]]) / 3
-        grid_config = json_data[i]
-
-        counter = 0
-        for grid_row in range(0, 3):
-            for grid_col in range(0, 3):
-                start_row = grid_row * crop_shape[0]
-                start_col = grid_col * crop_shape[1]
-                cropped_image = image[start_row:start_row + crop_shape[0], start_col:start_col + crop_shape[1], :]
-
-                if np.ndim(mean_image) == 0:
-                    obj_images[9 * (i - start_index) + counter, :, :, :] = cropped_image / 254.0
-                else:
-                    obj_images[9 * (i - start_index) + counter, :, :, :] = (cropped_image / 254.0) - mean_image
-
-                obj_labels[9 * (i - start_index) + counter, grid_config[6 * grid_row + 2 * grid_col]] = 1
-                counter = counter + 1
-
-    return (obj_images, obj_labels)
-
-
-def mean_image_batch(json_data, image_dir, start_index, batch_size, img_height = 100, img_width = 100, channels = 3):
-    batch = obj_mini_batch_loader(json_data, image_dir, np.empty([]), start_index, batch_size, img_height, img_width, channels)
-    mean_image = np.mean(batch[0], 0)
-    return mean_image
-
-
-def mean_image(json_data, image_dir, num_images, batch_size, img_height = 100, img_width = 100, channels = 3):
-    max_iter = np.floor(num_images / batch_size)
-    mean_image = np.zeros([img_height / 3, img_width / 3, channels])
-    for i in range(max_iter.astype(np.int16)):
-        mean_image = mean_image + mean_image_batch(json_data, image_dir, 1 + i * batch_size, batch_size, img_height, img_width, channels)
-
-    mean_image = mean_image / max_iter
-    return mean_image
-
-
-class html_obj_table_writer:
-
-    def __init__(self, filename):
-        self.filename = filename
-        self.html_file = open(self.filename, 'w')
-        self.html_file.write('<!DOCTYPE html>\n<html>\n<body>\n<table border="1" style="width:100%"> \n')
-
-    def add_element(self, col_dict):
-        self.html_file.write('    <tr>\n')
-        for key in range(len(col_dict)):
-            self.html_file.write('    <td>{}</td>\n'.format(col_dict[key]))
-
-        self.html_file.write('    </tr>\n')
-
-    def image_tag(self, image_path, height, width):
-        return '<img src="{}" alt="IMAGE NOT FOUND!" height={} width={}>'.format(image_path, height, width)
-
-    def close_file(self):
-        self.html_file.write('</table>\n</body>\n</html>')
-        self.html_file.close()
-
-
-if __name__ == '__main__':
-    html_writer = html_obj_table_writer('/home/tanmay/Code/GenVQA/Exp_Results/Shape_Classifier_v_1/trial.html')
-    col_dict = {0: 'sam',
-     1: html_writer.image_tag('something.png', 25, 25)}
-    html_writer.add_element(col_dict)
-    html_writer.close_file()

classifiers/object_classifiers/.#obj_data_io_helper.py

-tanmay@crunchy.15752:1450461082
\ No newline at end of file

classifiers/object_classifiers/eval_obj_classifier.py

@@ -16,9 +16,17 @@ def eval(eval_params):
     x, y, keep_prob = graph_creator.placeholder_inputs()
     y_pred = graph_creator.obj_comp_graph(x, keep_prob)
     accuracy = graph_creator.evaluation(y, y_pred)
-    
-    saver = tf.train.Saver()
-    saver.restore(sess, eval_params['model_name'] + '-' + str(eval_params['global_step']))
+
+    # Collect variables
+    vars_to_restore = tf.get_collection(tf.GraphKeys.VARIABLES, scope='obj')
+    print('Variables to restore:')
+    print([var.name for var in vars_to_restore])
+
+    saver = tf.train.Saver(vars_to_restore)
+    saver.restore(sess, eval_params['model_name'] + '-' + \
+                  str(eval_params['global_step']))
+
+    print 'Loading mean image'
     mean_image = np.load(os.path.join(eval_params['out_dir'], 'mean_image.npy'))
     test_json_filename = eval_params['test_json']
     with open(test_json_filename, 'r') as json_file: 
@@ -32,12 +40,17 @@ def eval(eval_params):
     html_dir = eval_params['html_dir']
     if not os.path.exists(html_dir):
         os.mkdir(html_dir)
-    html_writer = shape_data_loader.html_obj_table_writer(os.path.join(html_dir, 'index.html'))
+
+    html_writer = shape_data_loader \
+        .html_obj_table_writer(os.path.join(html_dir, 'index.html'))
+
     col_dict = {
         0: 'Grount Truth',
         1: 'Prediction',
         2: 'Image'}
+
     html_writer.add_element(col_dict)
+
     shape_dict = {
         0: 'blank',
         1: 'rectangle',
@@ -47,7 +60,8 @@ def eval(eval_params):
     batch_size = 100
     correct = 0
     for i in range(50):
-        test_batch = shape_data_loader.obj_mini_batch_loader(test_json_data, image_dir, mean_image, 10000 + i * batch_size, batch_size, 75, 75)
+        test_batch = shape_data_loader.obj_mini_batch_loader(test_json_data, 
+            image_dir, mean_image, 10000 + i * batch_size, batch_size, 75, 75)
         feed_dict_test = {x: test_batch[0], y: test_batch[1], keep_prob: 1.0}
         result = sess.run([accuracy, y_pred], feed_dict=feed_dict_test)
         correct = correct + result[0] * batch_size
@@ -57,12 +71,15 @@ def eval(eval_params):
             gt_id = np.argmax(test_batch[1][row, :])
             pred_id = np.argmax(result[1][row, :])
             if not gt_id == pred_id:
-                img_filename = os.path.join(html_dir, '{}_{}.png'.format(i, row))
-                misc.imsave(img_filename, test_batch[0][row, :, :, :] + mean_image)
+                img_filename = os.path.join(html_dir, 
+                                            '{}_{}.png'.format(i, row))
+                misc.imsave(img_filename, 
+                            test_batch[0][row, :, :, :] + mean_image)
                 col_dict = {
                     0: shape_dict[gt_id],
                     1: shape_dict[pred_id],
-                    2: html_writer.image_tag('{}_{}.png'.format(i, row), 25, 25)}
+                    2: html_writer.image_tag('{}_{}.png' \
+                                             .format(i, row), 25, 25)}
                 html_writer.add_element(col_dict)
 
     html_writer.close_file()

classifiers/object_classifiers/train_obj_classifier.py

@@ -15,7 +15,6 @@ def train(train_params):
     x, y, keep_prob = graph_creator.placeholder_inputs()
     y_pred = graph_creator.obj_comp_graph(x, keep_prob)
     cross_entropy = graph_creator.loss(y, y_pred)
-    train_step = tf.train.AdamOptimizer(train_params['adam_lr']).minimize(cross_entropy)
     accuracy = graph_creator.evaluation(y, y_pred)
 
     outdir = train_params['out_dir']
@@ -37,19 +36,36 @@ def train(train_params):
     image_dir = train_params['image_dir']
     if train_params['mean_image']=='':
         print('Computing mean image')
-        mean_image = shape_data_loader.mean_image(train_json_data, image_dir, 1000, 100, img_height, img_width)
+        mean_image = shape_data_loader.mean_image(train_json_data, 
+                                                  image_dir, 1000, 100, 
+                                                  img_height, img_width)
     else:
         print('Loading mean image')
         mean_image = np.load(train_params['mean_image'])
     np.save(os.path.join(outdir, 'mean_image.npy'), mean_image)
 
     # Val Data
-    val_batch = shape_data_loader.obj_mini_batch_loader(train_json_data, image_dir, mean_image, 9501, 499, img_height, img_width)
+    val_batch = shape_data_loader.obj_mini_batch_loader(train_json_data, 
+                                                        image_dir, mean_image, 
+                                                        9501, 499, 
+                                                        img_height, img_width)
     feed_dict_val = {x: val_batch[0], y: val_batch[1], keep_prob: 1.0}
     
+    # Collect variables
+    all_vars = tf.get_collection(tf.GraphKeys.VARIABLES)
+    vars_to_save = tf.get_collection(tf.GraphKeys.VARIABLES, scope='obj')
+
+    print('All variables:')
+    print([var.name for var in all_vars])
+    print('Variables to save:')
+    print([var.name for var in vars_to_save])
+
     # Session Saver
-    saver = tf.train.Saver()
+    saver = tf.train.Saver(vars_to_save)
     
+    # Add optimization op
+    train_step = tf.train.AdamOptimizer(train_params['adam_lr']) \
+                         .minimize(cross_entropy)
     # Start Training
     sess.run(tf.initialize_all_variables())
     batch_size = 10
@@ -64,15 +80,32 @@ def train(train_params):
                 print('Iter: ' + str(i))
                 print('Val Acc: ' + str(accuracy.eval(feed_dict_val)))
 
-            train_batch = shape_data_loader.obj_mini_batch_loader(train_json_data, image_dir, mean_image, 1 + i * batch_size, batch_size, img_height, img_width)
-            feed_dict_train = {x: train_batch[0], y: train_batch[1], keep_prob: 0.5}
-            _, current_train_batch_acc = sess.run([train_step, accuracy], feed_dict=feed_dict_train)
-            train_acc_array_epoch[epoch] = train_acc_array_epoch[epoch] + current_train_batch_acc
+            train_batch = shape_data_loader \
+                .obj_mini_batch_loader(train_json_data, image_dir, mean_image, 
+                                       1 + i * batch_size, batch_size, 
+                                       img_height, img_width)
+            feed_dict_train = {
+                x: train_batch[0], 
+                y: train_batch[1], 
+                keep_prob: 0.5
+            }
+
+            _, current_train_batch_acc = sess.run([train_step, accuracy], 
+                                                  feed_dict=feed_dict_train)
+            train_acc_array_epoch[epoch] = train_acc_array_epoch[epoch] \
+                                           + current_train_batch_acc
 
         train_acc_array_epoch[epoch] = train_acc_array_epoch[epoch] / max_iter
         val_acc_array_epoch[epoch] = accuracy.eval(feed_dict_val)
-        plotter.plot_accuracies(xdata=np.arange(0, epoch + 1) + 1, ydata_train=train_acc_array_epoch[0:epoch + 1], ydata_val=val_acc_array_epoch[0:epoch + 1], xlim=[1, max_epoch], ylim=[0, 1.0], savePath=os.path.join(outdir, 'acc_vs_epoch.pdf'))
-        save_path = saver.save(sess, os.path.join(outdir, 'obj_classifier'), global_step=epoch)
+        
+        plotter.plot_accuracies(xdata=np.arange(0, epoch + 1) + 1, 
+                                ydata_train=train_acc_array_epoch[0:epoch + 1], 
+                                ydata_val=val_acc_array_epoch[0:epoch + 1],
+                                xlim=[1, max_epoch], ylim=[0, 1.0], 
+                                savePath=os.path.join(outdir, 
+                                                      'acc_vs_epoch.pdf'))
+        save_path = saver.save(sess, os.path.join(outdir, 'obj_classifier'), 
+                               global_step=epoch)
 
     sess.close()
     tf.reset_default_graph()

classifiers/tf_graph_creation_helper.py

@@ -3,17 +3,28 @@ import math
 import tensorflow as tf
 import answer_classifier.ans_data_io_helper as ans_io_helper
 
-def weight_variable(shape, var_name = 'W', std=0.1):
-    initial = tf.truncated_normal(shape, stddev=std)
-    return tf.Variable(initial, name=var_name)
+graph_config = {
+    'num_objects': 4,
+    'num_attributes': 4,
+    'obj_feat_dim': 392,
+    'atr_feat_dim': 392,
+}
 
+def weight_variable(tensor_shape, fan_in=None, var_name='W'):
+    if fan_in==None:
+        fan_in = reduce(lambda x, y: x*y, tensor_shape[0:-1])
+    stddev = math.sqrt(2.0/fan_in)
+    print(stddev)
+    initial = tf.truncated_normal(shape=tensor_shape, mean=0.0, stddev=stddev)
+    return tf.Variable(initial_value=initial, name=var_name)
 
-def bias_variable(shape, var_name = 'b'):
-    initial = tf.constant(0.001, shape=shape)
-    return tf.Variable(initial, name=var_name)
 
+def bias_variable(tensor_shape, var_name='b'):
+    initial = tf.constant(value=0.0, shape=tensor_shape)
+    return tf.Variable(initial_value=initial, name=var_name)
 
-def conv2d(x, W, var_name = 'W'):
+
+def conv2d(x, W, var_name = 'conv'):
     return tf.nn.conv2d(x, W, strides=[1,1,1,1], padding='SAME', name=var_name)
 
 
@@ -31,6 +42,7 @@ def placeholder_inputs(mode = 'gt'):
     if mode == 'no_gt':
         print 'No placeholder for ground truth'
         return (x, keep_prob)
+    
 
 def placeholder_inputs_ans(total_vocab_size, ans_vocab_size, mode='gt'):
     image_regions = tf.placeholder(tf.float32, shape=[None,25,25,3])
@@ -49,50 +61,89 @@ def placeholder_inputs_ans(total_vocab_size, ans_vocab_size, mode='gt'):
 
 def obj_comp_graph(x, keep_prob):
     with tf.name_scope('obj') as obj_graph:
+
         with tf.name_scope('conv1') as conv1:
+
             W_conv1 = weight_variable([5,5,3,4])
             b_conv1 = bias_variable([4])
-            h_conv1 = tf.nn.relu(conv2d(x, W_conv1) + b_conv1, name='h')
+            a_conv1 = tf.add(conv2d(x, W_conv1), b_conv1, name='a')
+            h_conv1 = tf.nn.relu(a_conv1, name='h')
             h_pool1 = max_pool_2x2(h_conv1)
             h_conv1_drop = tf.nn.dropout(h_pool1, keep_prob, name='h_pool_drop')
+
         with tf.name_scope('conv2') as conv2:
+
             W_conv2 = weight_variable([3,3,4,8])
             b_conv2 = bias_variable([8])
-            h_conv2 = tf.nn.relu(conv2d(h_pool1, W_conv2) + b_conv2, name='h')
+            a_conv2 = tf.add(conv2d(h_pool1, W_conv2), b_conv2, name='a')
+            h_conv2 = tf.nn.relu(a_conv2, name='h')
             h_pool2 = max_pool_2x2(h_conv2)
             h_pool2_drop = tf.nn.dropout(h_pool2, keep_prob, name='h_pool_drop')
-            h_pool2_drop_flat = tf.reshape(h_pool2_drop, [-1, 392], name='h_pool_drop_flat')
+            h_pool2_drop_shape = h_pool2_drop.get_shape()
+            obj_feat_dim = reduce(lambda f, g: f*g, 
+                                  [dim.value for dim in h_pool2_drop_shape[1:]])
+            obj_feat = tf.reshape(h_pool2_drop, [-1, obj_feat_dim], 
+                                           name='obj_feat')
+
+            print('Object feature dimension: ' + str(obj_feat_dim)) #392
+
         with tf.name_scope('fc1') as fc1:
-            W_fc1 = weight_variable([392, 4])
+
+            W_fc1 = weight_variable([obj_feat_dim, graph_config['num_objects']])
             b_fc1 = bias_variable([4])
-        y_pred = tf.nn.softmax(tf.matmul(h_pool2_drop_flat, W_fc1) + b_fc1)
-        tf.add_to_collection('obj_feat', h_pool2_drop_flat)
+            logits = tf.add(tf.matmul(obj_feat, W_fc1), b_fc1, name='logits')
+        
+        y_pred = tf.nn.softmax(logits, name='softmax')
+#        tf.add_to_collection('obj_feat', h_pool2_drop_flat)
     return y_pred
 
 
 def atr_comp_graph(x, keep_prob, obj_feat):
     with tf.name_scope('atr') as obj_graph:
+
         with tf.name_scope('conv1') as conv1:
+            
             W_conv1 = weight_variable([5,5,3,4])
             b_conv1 = bias_variable([4])
-            h_conv1 = tf.nn.relu(conv2d(x, W_conv1) + b_conv1, name='h')
+            a_conv1 = tf.add(conv2d(x, W_conv1), b_conv1, name='a')
+            h_conv1 = tf.nn.relu(a_conv1, name='h')
             h_pool1 = max_pool_2x2(h_conv1)
             h_conv1_drop = tf.nn.dropout(h_pool1, keep_prob, name='h_pool_drop')
+
         with tf.name_scope('conv2') as conv2:
+
             W_conv2 = weight_variable([3,3,4,8])
             b_conv2 = bias_variable([8])
-            h_conv2 = tf.nn.relu(conv2d(h_pool1, W_conv2) + b_conv2, name='h')
+            a_conv2 = tf.add(conv2d(h_pool1, W_conv2), b_conv2, name='a')
+            h_conv2 = tf.nn.relu(a_conv2, name='h')
             h_pool2 = max_pool_2x2(h_conv2)
             h_pool2_drop = tf.nn.dropout(h_pool2, keep_prob, name='h_pool_drop')
-            h_pool2_drop_flat = tf.reshape(h_pool2_drop, [-1, 392], name='h_pool_drop_flat')
+            h_pool2_drop_shape = h_pool2_drop.get_shape()
+            atr_feat_dim = reduce(lambda f, g: f*g, 
+                                  [dim.value for dim in h_pool2_drop_shape[1:]])
+            atr_feat = tf.reshape(h_pool2_drop, [-1, atr_feat_dim], 
+                                  name='atr_feat')
+
+            print('Attribute feature dimension: ' + str(atr_feat_dim)) #392
+
         with tf.name_scope('fc1') as fc1:
-            W_obj_fc1 = weight_variable([392, 4], var_name='W_obj')
-            W_atr_fc1 = weight_variable([392, 4], var_name='W_atr')
+
+            W_obj_fc1 = weight_variable([graph_config['obj_feat_dim'], 
+                                         graph_config['num_attributes']], 
+                                        var_name='W_obj')
+            W_atr_fc1 = weight_variable([atr_feat_dim, 
+                                         graph_config['num_attributes']], 
+                                        var_name='W_atr')
             b_fc1 = bias_variable([4])
-        y_pred = tf.nn.softmax(tf.matmul(h_pool2_drop_flat, W_atr_fc1) + tf.matmul(obj_feat, W_obj_fc1) + b_fc1)
-        tf.add_to_collection('atr_feat', h_pool2_drop_flat)
+            logits_atr = tf.matmul(atr_feat, W_atr_fc1, name='logits_atr')
+            logits_obj = tf.matmul(obj_feat, W_obj_fc1, name='logits_obj')
+            logits = 0.5*logits_atr + 0.5*logits_obj + b_fc1
+
+        y_pred = tf.nn.softmax(logits, name='softmax')
+#        tf.add_to_collection('atr_feat', h_pool2_drop_flat)
     return y_pred
 
+
 def ans_comp_graph(image_regions, questions, keep_prob, \
                    obj_feat, atr_feat, vocab, inv_vocab, ans_vocab_size):
     with tf.name_scope('ans') as ans_graph:
@@ -104,7 +155,7 @@ def ans_comp_graph(image_regions, questions, keep_prob, \
         with tf.name_scope('q_embed') as q_embed:
             q_feat = tf.matmul(questions, word_vecs)
             # q_feat = tf.truediv(q_feat, tf.cast(len(vocab),tf.float32))
-            q_feat = tf.truediv(q_feat, tf.reduce_sum(questions,1,keep_dims=True))
+            # q_feat = tf.truediv(q_feat, tf.reduce_sum(questions,1,keep_dims=True))
 
         with tf.name_scope('conv1') as conv1:
             W_conv1 = weight_variable([5,5,3,4])
@@ -125,9 +176,9 @@ def ans_comp_graph(image_regions, questions, keep_prob, \
             fc1_dim = 300
             W_region_fc1 = weight_variable([392, fc1_dim], var_name='W_region')
             W_obj_fc1 = weight_variable([392, fc1_dim], var_name='W_obj',
-                                        std=math.sqrt(3/(392+ans_vocab_size)))
+                                        std=math.sqrt(3.0/(2.0*392.0+50.0+ans_vocab_size)))
             W_atr_fc1 = weight_variable([392, fc1_dim], var_name='W_atr',
-                                        std=math.sqrt(3/(392+ans_vocab_size)))
+                                        std=math.sqrt(3.0/(2.0*392.0+50.0+ans_vocab_size)))
             W_q_fc1 = weight_variable([50, fc1_dim], var_name='W_q', 
                                       std=math.sqrt(3.0/(50.0+ans_vocab_size)))
             b_fc1 = bias_variable([fc1_dim])
@@ -135,7 +186,6 @@ def ans_comp_graph(image_regions, questions, keep_prob, \
             h_tmp = tf.matmul(q_feat, W_q_fc1) + b_fc1 + \
                     tf.matmul(obj_feat, W_obj_fc1) + \
                     tf.matmul(atr_feat, W_atr_fc1)
-
                     #tf.matmul(h_pool2_drop_flat, W_region_fc1) + \
                     
             h_fc1 = tf.nn.relu(h_tmp, name='h')
@@ -144,6 +194,7 @@ def ans_comp_graph(image_regions, questions, keep_prob, \
         with tf.name_scope('fc2') as fc2:
             W_fc2 = weight_variable([fc1_dim, ans_vocab_size], 
                                     std=math.sqrt(3.0/(fc1_dim)))
+
             b_fc2 = bias_variable([ans_vocab_size])
             
             logits = tf.matmul(h_fc1_drop, W_fc2) + b_fc2

classifiers/train_classifiers.py

@@ -14,16 +14,16 @@ workflow = {
     'train_obj': False,
     'eval_obj': False,
     'train_atr': False,
-    'eval_atr': False,
+    'eval_atr': True,
 }
 
 obj_classifier_train_params = {
     'out_dir': '/home/tanmay/Code/GenVQA/Exp_Results/Obj_Classifier',
-    'adam_lr': 0.001,
+    'adam_lr': 0.0001,
     'train_json': '/home/tanmay/Code/GenVQA/GenVQA/shapes_dataset/train_anno.json',
     'image_dir': '/home/tanmay/Code/GenVQA/GenVQA/shapes_dataset/images',
-#    'mean_image': '/home/tanmay/Code/GenVQA/Exp_Results/Obj_Classifier/mean_image.npy',
-    'mean_image': '',
+    'mean_image': '/home/tanmay/Code/GenVQA/Exp_Results/Obj_Classifier/mean_image.npy',
+#    'mean_image': '',
 }
 
 obj_classifier_eval_params = {