select best model

answer_classifier_cached_features/select_best_model.py

@@ -13,6 +13,7 @@ import answer_classifier_cached_features.train as train
 import numpy as np
 import pdb
 import ujson
+import os
 import tensorflow as tf
 
 
@@ -59,7 +60,7 @@ def create_batch_generator(mode):
         constants.num_negative_answers,
         resnet_feat_dim=constants.resnet_feat_dim)
 
-    index_generator = tftools.data.random(
+    index_generator = tftools.data.sequential(
         constants.answer_batch_size, 
         num_questions, 
         1, 
@@ -129,10 +130,8 @@ def create_feed_dict_creator(plh, num_neg_answers):
 
 
 class eval_mgr():
-    def __init__(self, eval_data_json, results_json):
-        self.eval_data_json= eval_data_json
+    def __init__(self, results_json):
         self.results_json = results_json
-        self.eval_data = dict()
         self.correct = 0
         self.total = 0
         self.results = []
@@ -141,18 +140,6 @@ class eval_mgr():
         batch_size = len(batch['question_unencoded'])
         
         for j in xrange(batch_size):
-            dict_entry = dict()
-            dict_entry['question'] = batch['question_unencoded'][j]
-            dict_entry['positive_answer'] = {
-                batch['positive_answer_unencoded'][j]: 
-                str(eval_vars_dict['answer_score_' + str(j)][0,0])}
-
-            dict_entry['negative_answers'] = dict()
-            for i in xrange(len(batch['negative_answers_unencoded'][j])):
-                answer = batch['negative_answers_unencoded'][j][i]
-                dict_entry['negative_answers'][answer] = \
-                    str(eval_vars_dict['answer_score_' + str(j)][0,i+1])
-            
             question_id = batch['question_id'][j]
             pred_answer, pred_score = self.get_pred_answer(
                 [batch['positive_answer_unencoded'][j]] + \
@@ -166,18 +153,13 @@ class eval_mgr():
             }
             self.results.append(result_entry)
             
-            self.eval_data[str(question_id)] = dict_entry
-
-            print dict_entry
 
         self.total += batch_size
         
-        self.correct += eval_vars_dict['accuracy']*batch_size
-
-        self.print_accuracy()
-
+        self.correct += eval_vars_dict['accuracy'][0]*batch_size
 
-        if iter%100==0:
+        if iter%10==0:
+            self.print_accuracy()
             self.write_data()
 
     def get_pred_answer(self, answers, scores):
@@ -199,12 +181,12 @@ class eval_mgr():
         print 'Total: {}  Correct: {}  Accuracy: {}'.format(
             self.total,
             self.correct,
-            self.correct/float(self.total))
+            self.get_accuracy())
 
-    def write_data(self):
-        with open(self.eval_data_json, 'w') as file:
-            ujson.dump(self.eval_data, file, indent=4, sort_keys=True)
+    def get_accuracy(self):
+        return self.correct/float(self.total)
 
+    def write_data(self):
         with open(self.results_json, 'w') as file:
             ujson.dump(self.results, file, indent=4, sort_keys=True)
         
@@ -228,8 +210,9 @@ def eval(
 
         iter = 0
         for batch in batch_generator:
-            print '---'
-            print 'Iter: {}'.format(iter)
+            if iter%10 == 0:
+                print '---'
+                print 'Iter: {}'.format(iter)
             feed_dict = feed_dict_creator(batch)
             eval_vars = sess.run(
                 vars_to_eval,
@@ -240,10 +223,13 @@ def eval(
             evaluator.eval(iter, eval_vars_dict, batch)
             iter+=1
         
+        print '---'
+        print 'Iter: {}'.format(iter)
+        evaluator.print_accuracy()
         evaluator.write_data()
 
 
-if __name__=='__main__':
+def eval_model(model_to_eval, results_json):
     print 'Creating batch generator...'
     batch_generator = create_batch_generator(constants.answer_eval_on)
 
@@ -269,7 +255,7 @@ if __name__=='__main__':
     initializer = create_initializer(
         graph, 
         sess, 
-        constants.answer_model_to_eval)
+        model_to_eval)
 
     print 'Creating feed dict creator...'
     feed_dict_creator = create_feed_dict_creator(
@@ -285,9 +271,7 @@ if __name__=='__main__':
             graph.answer_inference.answer_score[j]
 
     print 'Creating evaluation manager...'
-    evaluator = eval_mgr(
-        constants.answer_eval_data_json,
-        constants.answer_results_json)
+    evaluator = eval_mgr(results_json)
 
     print 'Start training...'
     eval(
@@ -298,8 +282,42 @@ if __name__=='__main__':
         feed_dict_creator,
         evaluator)
 
+    return evaluator.get_accuracy()
 
-    
 
+def model_path_generator(models_dir, start_model, step_size):
+    model_number = start_model
+    filename = os.path.join(models_dir,'model-' + str(model_number))
+    while os.path.exists(filename):
+        yield filename, model_number
+        model_number += step_size
+        filename = os.path.join(models_dir,'model-' + str(model_number))
+    
+    
+if __name__=='__main__':
+    np.random.seed(seed=0)
+    model_paths = model_path_generator(
+        constants.models_dir,
+        constants.start_model,
+        constants.step_size)
+
+    model_accuracies_txt = open(constants.model_accuracies_txt, 'w')
+    best_model = (None, 0.0)
+    for model_path, model_number in model_paths:
+        results_json = os.path.join(
+            constants.answer_output_dir,
+            'eval_' + constants.answer_eval_on + '_results_' + str(model_number) + '.json')
+        accuracy = eval_model(model_path, results_json)
+        line = model_path + '\t' + str(accuracy)
+        print line
+        model_accuracies_txt.write(line + '\n')
+        if accuracy > best_model[1]:
+            best_model = (model_path, accuracy)
             
+        print 'best_model:' + '\t' + best_model[0] + '\t' + str(best_model[1])
+
+    line = 'best_model:' + '\t' + best_model[0] + '\t' + str(best_model[1])
+    model_accuracies_txt.write(line)
+    model_accuracies_txt.close()
+
     

constants_crunchy.py

@@ -141,6 +141,9 @@ vqa_train_anno = os.path.join(
 vqa_val_image_dir = os.path.join(
     vqa_basedir,
     'val2014_cropped_large')
+vqa_val_resnet_feat_dir = os.path.join(
+    vqa_basedir,
+    'val2014_cropped_large_resnet_features')
 vqa_val_anno = os.path.join(
     vqa_basedir,
     'mscoco_val2014_annotations_with_parsed_questions.json')
@@ -192,4 +195,14 @@ answer_eval_data_json = os.path.join(
     answer_output_dir,
     'eval_' + answer_eval_on + '_data.json')
 
+answer_eval_results_json = os.path.join(
+    answer_output_dir,
+    'eval_' + answer_eval_on + '_results.json')
 
+# Select best model
+models_dir = answer_output_dir
+start_model = 1000
+step_size = 2000
+model_accuracies_txt = os.path.join(
+    answer_output_dir,
+    'model_accuracies.txt')

constants_vision_gpu_2.py

@@ -140,6 +140,9 @@ vqa_train_anno = os.path.join(
 vqa_val_image_dir = os.path.join(
     vqa_basedir,
     'val2014_cropped_large')
+vqa_val_resnet_feat_dir = os.path.join(
+    vqa_basedir,
+    'val2014_cropped_large_resnet_features')
 vqa_val_anno = os.path.join(
     vqa_basedir,
     'mscoco_val2014_annotations_with_parsed_questions.json')
@@ -191,4 +194,8 @@ answer_eval_data_json = os.path.join(
     answer_output_dir,
     'eval_' + answer_eval_on + '_data.json')
 
+answer_eval_results_json = os.path.join(
+    answer_output_dir,
+    'eval_' + answer_eval_on + '_results.json')
+
 

tftools/data.py

@@ -7,7 +7,8 @@ def sequential(batch_size, num_samples, num_epochs=1, offset=0):
     """Generate sequence indices.
     """
     for epoch in range(num_epochs):
-        indices = range(num_samples)
+        indices = np.arange(num_samples) + offset
+        indices = indices.tolist()
         for i in range(0, num_samples - batch_size + 1, batch_size):
             yield indices[i:i+batch_size]
 

word2vec/word_vector_management.py

@@ -81,8 +81,6 @@ class word_vector_manager():
 
         object_label_vector_list = [None]*num_object_labels
         for i in xrange(num_object_labels):
-            if i==471:
-                print inv_object_labels[i]
             object_label_vector_list[i] = self.create_phrase_word_vectors(
                 inv_object_labels[i],
                 'object_label_' + str(i))