Visual Genome Data Manager

constants.py

+import os
+
+#height and width to which images are resized before feeding into networks
+image_size = (224, 224) 
+
+# Token to be used if object or attribute variable is unknown
+unknown_token = 'UNK'
+
+# Data paths
+data_absolute_path = '/home/tanmay/Data/VisualGenome'
+image_dir = os.path.join(data_absolute_path, 'images')
+object_labels_json = os.path.join(
+    data_absolute_path,
+    'restructured/object_labels.json')
+attribute_labels_json = os.path.join(
+    data_absolute_path,
+    'restructured/attribute_labels.json')
+regions_json = os.path.join(
+    data_absolute_path,
+    'restructured/region_with_labels.json')
+mean_image_filename = os.path.join(
+    data_absolute_path,
+    'restructured/mean_image.jpg')
+# Regions data partition
+# First 70% meant to be used for training
+# Next 10% is set aside for validation
+# Last 20% is to be used for testing
+num_total_regions = 1951768
+num_train_regions = 1366238 # First 70%
+num_val_regions = 195176 # Next 10%
+num_test_regions = num_total_regions \
+                   - num_train_regions \
+                   - num_val_regions 
+
+
+
+
+
+

data/compute_mean_region_image.py

+import numpy as np
+from data.regions import data
+import constants
+import image_io
+import pdb
+
+
+_image_size = constants.image_size
+_num_train_regions = constants.num_train_regions
+_mean_image_filename = constants.mean_image_filename
+
+
+if __name__=='__main__':
+    data_mgr = data(constants.image_dir,
+                    constants.object_labels_json,
+                    constants.attribute_labels_json,
+                    constants.regions_json,
+                    constants.image_size,
+                    channels=3,
+                    mean_image_filename=None)
+
+    h, w = _image_size
+    mean_image = np.zeros([h, w, 3], dtype=np.float32)
+    num_images_for_mean = min(10000, _num_train_regions)
+    for i in xrange(num_images_for_mean):
+        mean_image += data_mgr.get_region_image(i)
+    mean_image = mean_image/num_images_for_mean
+    image_io.imwrite(np.uint8(mean_image*255),
+                     _mean_image_filename)
+
+    

data/cropped_regions.py

+import numpy as np
+import json
+import os
+import pdb
+import time
+
+from multiprocessing import Pool
+
+import tftools.data 
+import image_io
+import constants
+
+import tensorflow as tf
+
+_unknown_token = constants.unknown_token
+
+class data():
+    def __init__(self,
+                 image_dir,
+                 object_labels_json,
+                 attribute_labels_json,
+                 regions_json,
+                 image_size,
+                 channels=3,
+                 mean_image_filename=None):
+        self.image_dir = image_dir
+        self.h = image_size[0]
+        self.w = image_size[1]
+        self.c = channels
+        self.mean_image = self.get_mean_image(mean_image_filename)
+        self.object_labels_dict = self.read_json_file(object_labels_json)
+        self.attribute_labels_dict = self.read_json_file(attribute_labels_json)
+        self.inv_object_labels_dict = self.invert_label_dict(
+            self.object_labels_dict)
+        self.inv_attribute_labels_dict = self.invert_label_dict(
+            self.attribute_labels_dict)
+        self.num_object_labels = len(self.object_labels_dict)
+        self.num_attribute_labels = len(self.attribute_labels_dict)
+        self.regions = self.read_json_file(regions_json)
+        self.num_regions = len(self.regions)
+        self.create_sample_to_region_dict()
+
+    def create_sample_to_region_dict(self):
+        self.sample_to_region_dict = \
+            {k: v for k, v in zip(xrange(self.num_regions),
+                                  self.regions.keys())}
+
+    def invert_label_dict(self, label_dict):
+        return {v: k for k, v in label_dict.items()}
+
+    def read_json_file(self, filename):
+        print 'Reading {} ...'.format(filename)
+        with open(filename, 'r') as file:
+            return json.load(file)
+
+    def get(self, samples):
+        batch_size = len(samples)
+        batch = dict()
+        batch['region_ids'] = dict()
+        batch['images'] = np.zeros(
+            [batch_size, self.h, self.w, self.c], np.float32)
+        batch['object_labels'] = np.zeros(
+            [batch_size, len(self.object_labels_dict)], np.float32)
+        batch['attribute_labels'] = np.zeros(
+            [batch_size, len(self.attribute_labels_dict)], np.float32)
+
+        for index, sample in enumerate(samples):
+            batch['region_ids'][index] = self.sample_to_region_dict[sample]
+            batch['images'][index, :, :, :] = self.get_region_image(sample)
+            batch['object_labels'][index, :] = self.get_object_label(sample)
+            batch['attribute_labels'][index,:] = \
+                self.get_attribute_label(sample)
+
+        return batch
+    
+    def get_single(self, sample):
+        print sample
+        batch = dict()
+        batch['region_ids'] = dict()
+        batch['images'] = np.zeros(
+            [self.h, self.w, self.c], np.float32)
+        batch['object_labels'] = np.zeros(
+            [len(self.object_labels_dict)], np.float32)
+        batch['attribute_labels'] = np.zeros(
+            [len(self.attribute_labels_dict)], np.float32)
+
+        batch['region_ids'] = self.sample_to_region_dict[sample]
+        batch['images'] = self.get_region_image(sample)
+        batch['object_labels'] = self.get_object_label(sample)
+        batch['attribute_labels'] = self.get_attribute_label(sample)
+
+        return batch
+
+    def get_parallel(self, samples):
+        batch_list = self.pool.map(self.get_single, samples)
+        batch_size = len(samples)
+        batch = dict()
+        batch['region_ids'] = dict()
+        batch['images'] = np.zeros(
+            [batch_size, self.h, self.w, self.c], np.float32)
+        batch['object_labels'] = np.zeros(
+            [batch_size, len(self.object_labels_dict)], np.float32)
+        batch['attribute_labels'] = np.zeros(
+            [batch_size, len(self.attribute_labels_dict)], np.float32)
+
+        for index, single_batch in enumerate(batch_list):
+            batch['region_ids'][index] = single_batch['region_ids']
+            batch['images'][index, :, :, :] = single_batch['images']
+            batch['object_labels'][index, :] = single_batch['object_labels']
+            batch['attribute_labels'][index,:] = single_batch['attribute_labels']
+
+
+        return batch
+        
+    def get_region_image(self, sample):
+        region_id = self.sample_to_region_dict[sample]
+        region = self.regions[region_id]
+        filename = os.path.join(self.image_dir,
+                                str(region['image_id']) + '.jpg')
+        image = image_io.imread(filename)
+        image = self.single_to_three_channel(image)
+        x, y, w, h = self.get_clipped_region_coords(region, image.shape[0:2])
+        region_image = image[y:y + h, x:x + w, :]
+
+        region_image = image_io.imresize(
+            region_image,
+            output_size=(self.h, self.w)).astype(np.float32)
+
+        return region_image / 255 - self.mean_image
+
+    def single_to_three_channel(self, image):
+        if len(image.shape)==3:
+            return image
+        elif len(image.shape)==2:
+            im_h, im_w =image.shape
+            image_tmp = np.zeros([im_h, im_w, 3], dtype=image.dtype)
+            for c in xrange(3):
+                image_tmp[:,:,c] = image
+            return image_tmp
+
+    def get_clipped_region_coords(self, region, image_size):
+        im_h, im_w = image_size
+        x = min(im_w - 1, max(0, region["x"]))
+        y = min(im_h - 1, max(0, region["y"]))
+        h = min(im_h - y, max(region["h"], 1))
+        w = min(im_w - x, max(region["w"], 1))
+        return x, y, w, h
+
+    def get_mean_image(self, mean_image_filename):
+        if mean_image_filename:
+            return image_io.imread(mean_image_filename).astype(
+                np.float32) / 255
+        else:
+            return np.zeros([self.h, self.w, self.c], np.float32)
+
+    def get_object_label(self, sample):
+        # Returns a multihot vector encoding of object labels
+        # If an object label is not found in the labels list, 
+        # _unknown_token is produced in that case. 
+        region_id = self.sample_to_region_dict[sample]
+        region = self.regions[region_id]
+        object_labels = region['object_names']
+        object_label_encoding = np.zeros([1, self.num_object_labels], 
+                                         dtype = np.float32)
+        if object_labels:
+            for object in object_labels:
+                if object not in self.object_labels_dict:
+                    label_id = self.object_labels_dict[_unknown_token]
+                else:
+                    label_id = self.object_labels_dict[object]
+                object_label_encoding[0,label_id] = 1.0
+        else:
+            label_id = self.object_labels_dict[_unknown_token]
+            object_label_encoding[0,label_id] = 1.0
+
+        return object_label_encoding/np.sum(object_label_encoding)
+            
+    def get_attribute_label(self, sample):
+        # Attribute is turned on if it is present 
+        region_id = self.sample_to_region_dict[sample]
+        region = self.regions[region_id]
+        attribute_labels = region['attributes']
+        attribute_label_encoding = np.zeros([1, self.num_attribute_labels], 
+                                            dtype = np.float32)
+        for attribute in attribute_labels:
+            if attribute in self.attribute_labels_dict:
+                label_id = self.attribute_labels_dict[attribute]
+                attribute_label_encoding[0,label_id] = 1.0
+
+        return attribute_label_encoding
+
+if __name__=='__main__':
+    data_mgr = data(constants.image_dir,
+                    constants.object_labels_json,
+                    constants.attribute_labels_json,
+                    constants.regions_json,
+                    constants.image_size,
+                    channels=3,
+                    mean_image_filename=None)
+    print 'Number of object labels: {}'.format(data_mgr.num_object_labels)
+    print 'Number of attribute labels: {}'.format(data_mgr.num_attribute_labels)
+    print 'Number of regions: {}'.format(data_mgr.num_regions)
+
+    #Test sample
+    samples = [1, 2]
+    sample = samples[0]
+    region_id = data_mgr.sample_to_region_dict[sample]
+    region = data_mgr.regions[region_id]
+    attribute_encoding = data_mgr.get_attribute_label(sample)
+    object_encoding = data_mgr.get_object_label(sample)
+    region_image = data_mgr.get_region_image(sample)
+
+    attributes = []
+    for i in xrange(attribute_encoding.shape[1]):
+        if attribute_encoding[0,i] > 0 :
+            attributes.append(data_mgr.inv_attribute_labels_dict[i])
+
+    objects = []
+    for i in xrange(object_encoding.shape[1]):
+        if object_encoding[0,i] > 0 :
+            objects.append(data_mgr.inv_object_labels_dict[i])
+    
+    print "Region: {}".format(region)
+    print "Attributes: {}".format(", ".join(attributes))
+    print "Objects: {}".format(", ".join(objects))
+    
+#    image_io.imshow(region_image)
+
+    batch_size = 200
+    num_samples = 1000
+    num_epochs = 1
+    offset = 0
+
+    index_generator = tftools.data.random(
+        batch_size, 
+        num_samples, 
+        num_epochs, 
+        offset)
+    
+    # start = time.time()
+    # count = 0
+    # for samples in index_generator:
+    #     batch = data_mgr.get(samples)
+    #     print 'Batch Count: {}'.format(count)
+    #     count += 1
+    # stop = time.time()
+    # print 'Time per batch: {}'.format((stop-start)/5.0)
+
+    batch_generator = tftools.data.async_batch_generator(
+        data_mgr, 
+        index_generator, 
+        1000)
+
+    count = 0 
+    start = time.time()
+    for batch in batch_generator:
+        print 'Batch Number: {}'.format(count)
+#        print batch['region_ids']
+        count += 1
+    stop = time.time()
+    print 'Time per batch: {}'.format((stop-start)/50.0)
+    print "Count: {}".format(count)
+
+    pool.close()

data/regions.py

+import numpy as np
+import json
+import os
+import pdb
+import time
+
+from multiprocessing import Pool
+
+import tftools.data 
+import image_io
+import constants
+
+import tensorflow as tf
+
+_unknown_token = constants.unknown_token
+
+class data():
+    def __init__(self,
+                 image_dir,
+                 object_labels_json,
+                 attribute_labels_json,
+                 regions_json,
+                 image_size,
+                 channels=3,
+                 mean_image_filename=None):
+        self.image_dir = image_dir
+        self.h = image_size[0]
+        self.w = image_size[1]
+        self.c = channels
+        self.mean_image = self.get_mean_image(mean_image_filename)
+        self.object_labels_dict = self.read_json_file(object_labels_json)
+        self.attribute_labels_dict = self.read_json_file(attribute_labels_json)
+        self.inv_object_labels_dict = self.invert_label_dict(
+            self.object_labels_dict)
+        self.inv_attribute_labels_dict = self.invert_label_dict(
+            self.attribute_labels_dict)
+        self.num_object_labels = len(self.object_labels_dict)
+        self.num_attribute_labels = len(self.attribute_labels_dict)
+        self.regions = self.read_json_file(regions_json)
+        self.num_regions = len(self.regions)
+        self.create_sample_to_region_dict()
+
+    def create_sample_to_region_dict(self):
+        self.sample_to_region_dict = \
+            {k: v for k, v in zip(xrange(self.num_regions),
+                                  self.regions.keys())}
+
+    def invert_label_dict(self, label_dict):
+        return {v: k for k, v in label_dict.items()}
+
+    def read_json_file(self, filename):
+        print 'Reading {} ...'.format(filename)
+        with open(filename, 'r') as file:
+            return json.load(file)
+
+    def get(self, samples):
+        batch_size = len(samples)
+        batch = dict()
+        batch['region_ids'] = dict()
+        batch['images'] = np.zeros(
+            [batch_size, self.h, self.w, self.c], np.float32)
+        batch['object_labels'] = np.zeros(
+            [batch_size, len(self.object_labels_dict)], np.float32)
+        batch['attribute_labels'] = np.zeros(
+            [batch_size, len(self.attribute_labels_dict)], np.float32)
+
+        for index, sample in enumerate(samples):
+            batch['region_ids'][index] = self.sample_to_region_dict[sample]
+            batch['images'][index, :, :, :] = self.get_region_image(sample)
+            batch['object_labels'][index, :] = self.get_object_label(sample)
+            batch['attribute_labels'][index,:] = \
+                self.get_attribute_label(sample)
+
+        return batch
+    
+    def get_single(self, sample):
+        print sample
+        batch = dict()
+        batch['region_ids'] = dict()
+        batch['images'] = np.zeros(
+            [self.h, self.w, self.c], np.float32)
+        batch['object_labels'] = np.zeros(
+            [len(self.object_labels_dict)], np.float32)
+        batch['attribute_labels'] = np.zeros(
+            [len(self.attribute_labels_dict)], np.float32)
+
+        batch['region_ids'] = self.sample_to_region_dict[sample]
+        batch['images'] = self.get_region_image(sample)
+        batch['object_labels'] = self.get_object_label(sample)
+        batch['attribute_labels'] = self.get_attribute_label(sample)
+
+        return batch
+
+    def get_parallel(self, samples):
+        batch_list = self.pool.map(self.get_single, samples)
+        batch_size = len(samples)
+        batch = dict()
+        batch['region_ids'] = dict()
+        batch['images'] = np.zeros(
+            [batch_size, self.h, self.w, self.c], np.float32)
+        batch['object_labels'] = np.zeros(
+            [batch_size, len(self.object_labels_dict)], np.float32)
+        batch['attribute_labels'] = np.zeros(
+            [batch_size, len(self.attribute_labels_dict)], np.float32)
+
+        for index, single_batch in enumerate(batch_list):
+            batch['region_ids'][index] = single_batch['region_ids']
+            batch['images'][index, :, :, :] = single_batch['images']
+            batch['object_labels'][index, :] = single_batch['object_labels']
+            batch['attribute_labels'][index,:] = single_batch['attribute_labels']
+
+
+        return batch
+        
+    def get_region_image(self, sample):
+        region_id = self.sample_to_region_dict[sample]
+        region = self.regions[region_id]
+        filename = os.path.join(self.image_dir,
+                                str(region['image_id']) + '.jpg')
+        image = image_io.imread(filename)
+        image = self.single_to_three_channel(image)
+        x, y, w, h = self.get_clipped_region_coords(region, image.shape[0:2])
+        region_image = image[y:y + h, x:x + w, :]
+
+        region_image = image_io.imresize(
+            region_image,
+            output_size=(self.h, self.w)).astype(np.float32)
+
+        return region_image / 255 - self.mean_image
+
+    def single_to_three_channel(self, image):
+        if len(image.shape)==3:
+            return image
+        elif len(image.shape)==2:
+            im_h, im_w =image.shape
+            image_tmp = np.zeros([im_h, im_w, 3], dtype=image.dtype)
+            for c in xrange(3):
+                image_tmp[:,:,c] = image
+            return image_tmp
+
+    def get_clipped_region_coords(self, region, image_size):
+        im_h, im_w = image_size
+        x = min(im_w - 1, max(0, region["x"]))
+        y = min(im_h - 1, max(0, region["y"]))
+        h = min(im_h - y, max(region["h"], 1))
+        w = min(im_w - x, max(region["w"], 1))
+        return x, y, w, h
+
+    def get_mean_image(self, mean_image_filename):
+        if mean_image_filename:
+            return image_io.imread(mean_image_filename).astype(
+                np.float32) / 255
+        else:
+            return np.zeros([self.h, self.w, self.c], np.float32)
+
+    def get_object_label(self, sample):
+        # Returns a multihot vector encoding of object labels
+        # If an object label is not found in the labels list, 
+        # _unknown_token is produced in that case. 
+        region_id = self.sample_to_region_dict[sample]
+        region = self.regions[region_id]
+        object_labels = region['object_names']
+        object_label_encoding = np.zeros([1, self.num_object_labels], 
+                                         dtype = np.float32)
+        if object_labels:
+            for object in object_labels:
+                if object not in self.object_labels_dict:
+                    label_id = self.object_labels_dict[_unknown_token]
+                else:
+                    label_id = self.object_labels_dict[object]
+                object_label_encoding[0,label_id] = 1.0
+        else:
+            label_id = self.object_labels_dict[_unknown_token]
+            object_label_encoding[0,label_id] = 1.0
+
+        return object_label_encoding/np.sum(object_label_encoding)
+            
+    def get_attribute_label(self, sample):
+        # Attribute is turned on if it is present 
+        region_id = self.sample_to_region_dict[sample]
+        region = self.regions[region_id]
+        attribute_labels = region['attributes']
+        attribute_label_encoding = np.zeros([1, self.num_attribute_labels], 
+                                            dtype = np.float32)
+        for attribute in attribute_labels:
+            if attribute in self.attribute_labels_dict:
+                label_id = self.attribute_labels_dict[attribute]
+                attribute_label_encoding[0,label_id] = 1.0
+
+        return attribute_label_encoding
+
+if __name__=='__main__':
+    data_mgr = data(constants.image_dir,
+                    constants.object_labels_json,
+                    constants.attribute_labels_json,
+                    constants.regions_json,
+                    constants.image_size,
+                    channels=3,
+                    mean_image_filename=None)
+    print 'Number of object labels: {}'.format(data_mgr.num_object_labels)
+    print 'Number of attribute labels: {}'.format(data_mgr.num_attribute_labels)
+    print 'Number of regions: {}'.format(data_mgr.num_regions)
+
+    #Test sample
+    samples = [1, 2]
+    sample = samples[0]
+    region_id = data_mgr.sample_to_region_dict[sample]
+    region = data_mgr.regions[region_id]
+    attribute_encoding = data_mgr.get_attribute_label(sample)
+    object_encoding = data_mgr.get_object_label(sample)
+    region_image = data_mgr.get_region_image(sample)
+
+    attributes = []
+    for i in xrange(attribute_encoding.shape[1]):
+        if attribute_encoding[0,i] > 0 :
+            attributes.append(data_mgr.inv_attribute_labels_dict[i])
+
+    objects = []
+    for i in xrange(object_encoding.shape[1]):
+        if object_encoding[0,i] > 0 :
+            objects.append(data_mgr.inv_object_labels_dict[i])
+    
+    print "Region: {}".format(region)
+    print "Attributes: {}".format(", ".join(attributes))
+    print "Objects: {}".format(", ".join(objects))
+    
+#    image_io.imshow(region_image)
+
+    batch_size = 200
+    num_samples = 1000
+    num_epochs = 1
+    offset = 0
+
+    index_generator = tftools.data.random(
+        batch_size, 
+        num_samples, 
+        num_epochs, 
+        offset)
+    
+    # start = time.time()
+    # count = 0
+    # for samples in index_generator:
+    #     batch = data_mgr.get(samples)
+    #     print 'Batch Count: {}'.format(count)
+    #     count += 1
+    # stop = time.time()
+    # print 'Time per batch: {}'.format((stop-start)/5.0)
+
+    batch_generator = tftools.data.async_batch_generator(
+        data_mgr, 
+        index_generator, 
+        1000)
+
+    count = 0 
+    start = time.time()
+    for batch in batch_generator:
+        print 'Batch Number: {}'.format(count)
+#        print batch['region_ids']
+        count += 1
+    stop = time.time()
+    print 'Time per batch: {}'.format((stop-start)/50.0)
+    print "Count: {}".format(count)
+
+    pool.close()

image_io.py

+from PIL import Image
+import numpy as np
+import math
+import pdb
+
+
+def imread(filename):
+    """
+    Matlab like function for reading an image file.
+    Returns:
+    im_array (numpy.ndarray): h x w x 3 ndarray for color images and 
+        h x w for grayscale images
+    """
+    im = Image.open(filename)
+
+    err_str = \
+        "imread only supports 'RGB' and 'L' modes, found '{}'".format(im.mode)
+    assert (im.mode == 'RGB' or im.mode == 'L'), err_str
+
+    im_array = np.array(im)
+
+    return im_array
+
+
+def imshow(np_im):
+    """
+    Matlab like function for displaying a numpy ndarray as an image
+    Args:
+    np_im (numpy.ndarray): h x w x 3 ndarray for color images and 
+        h x w for grayscale images with pixels stored in uint8 format
+    """
+    err_str = 'imshow expects ndarray of dimension h x w x c (RGB) or h x w (L)'
+    assert (len(np_im.shape) == 3 or len(np_im.shape) == 2), err_str
+
+    if len(np_im.shape) == 3:
+        assert (np_im.shape[2] == 3), 'imshow expected 3 channels'
+        im = Image.fromarray(np_im, 'RGB')
+    else:
+        im = Image.fromarray(np_im, 'L')
+
+    im.show()
+
+
+def imwrite(np_im, filename):
+    """
+    Matlab like function to save numpy image.
+    Args:
+    np_im (numpy.ndarray): h x w x 3 ndarray for color images and 
+        h x w for grayscale images with pixels stored in uint8 format
+    """
+    err_str = 'imshow expects ndarray of dimension h x w x c (RGB) or h x w (L)'
+    assert (len(np_im.shape) == 3 or len(np_im.shape) == 2), err_str
+
+    im = Image.fromarray(np_im)
+    im.save(filename)
+
+
+def imresize(np_im, method=Image.BILINEAR, **kwargs):
+    assert_str = "Only 1 keyword argument expected with key either" + \
+                 "'output_size' or 'scale'"
+    assert (len(kwargs)==1), assert_str
+    im_h = np_im.shape[0]
+    im_w = np_im.shape[1]
+    if 'output_size' in kwargs:
+        h, w = kwargs['output_size']
+    elif 'scale' in kwargs:
+        h = scale*im_h
+        w = scale*im_w
+    else:
+        assert_str = "Variable argument must be one of {'output_size','scale'}"
+        assert (False), assert_str
+    h = int(math.ceil(h))
+    w = int(math.ceil(w))
+    im = Image.fromarray(np_im)
+    return np.array(im.resize((w,h)))
+
+    
+    

tftools/data.py

+"""Utility functions for making batch generation easier."""
+import numpy as np
+from multiprocessing import Process, Queue
+import time
+
+def sequential(batch_size, num_samples, num_epochs=1, offset=0):
+    """Generate sequence indices.
+    """
+    for epoch in range(num_epochs):
+        indices = range(num_samples)
+        for i in range(0, num_samples - batch_size + 1, batch_size):
+            yield indices[i:i+batch_size]
+
+
+def random(batch_size, num_samples, num_epochs, offset=0):
+    """Generate random indices.
+    """
+    for epoch in range(num_epochs):
+        indices = np.random.permutation(num_samples) + offset
+        indices = indices.tolist()
+        for i in range(0, num_samples - batch_size + 1, batch_size):
+            yield indices[i:i+batch_size]
+
+
+def batch_generator(data, index_generator, batch_function=None):
+    """Generate batches of data.
+    """
+    for samples in index_generator:
+        batch = data.get(samples)
+        if batch_function:
+            output = batch_function(batch)
+        else:
+            output = batch
+        yield output
+
+
+def async_batch_generator(data, index_generator, queue_maxsize, batch_function=None):
+    """Create an asynchronous batch generator.
+    """
+    batcher = batch_generator(data, index_generator, batch_function)
+
+    queue = Queue(maxsize=queue_maxsize)
+    fetcher = BatchFetcher(queue, batcher)
+    fetcher.start()
+
+    time.sleep(10)
+    
+    queue_batcher = queue_generator(queue)
+    return queue_batcher
+
+
+def queue_generator(queue, sentinel=None):
+    """Create a generator from a queue.
+    """
+    while True:
+        value = queue.get()
+        if value is not sentinel:
+            yield value
+        else:
+            return
+
+
+class BatchFetcher(Process):
+    def __init__(self, queue, batch_generator):
+        super(BatchFetcher, self).__init__()
+        self.queue = queue
+        self.batch_generator = batch_generator
+
+    def run(self):
+
+        for batch in self.batch_generator:
+            self.queue.put(batch)
+
+        # Signal that fetcher is done.
+        self.queue.put(None)
+
+
+class NumpyData(object):
+    def __init__(self, array):
+        self.array = array
+
+    def get(self, indices):
+        if not isinstance(indices, np.ndarray):
+            indices = np.array(indices)
+        return self.array[indices]
+
+
+if __name__=='__main__':
+    batch_size = 10
+    num_samples = 20
+    num_epochs = 3
+    offset = 0
+    index_generator = random(batch_size, num_samples, num_epochs, offset)
+    for samples in index_generator:
+        print samples
+        

visual_genome_parser.py

+import json
+import os
+import operator
+import nltk
+import numpy as np
+import pdb
+from multiprocessing import Pool
+import image_io
+
+
+# Filenames
+_datadir = '/home/tanmay/Data/VisualGenome/'
+_outdir =  os.path.join(_datadir,'restructured')
+_cropped_regions_dir = os.path.join(_datadir, 'cropped_regions')
+_objects = 'objects.json'
+_attributes = 'attributes.json'
+_objects_in_image = 'objects_in_image.json'
+_regions_in_image = 'regions_in_image.json'
+_regions_with_attributes = 'regions_with_attributes.json'
+_regions = 'regions.json'
+_raw_object_labels = 'raw_object_labels.json'
+_raw_attribute_labels = 'raw_attribute_labels.json'
+_object_labels = 'object_labels.json'
+_attribute_labels = 'attribute_labels.json'
+_regions_with_labels = 'region_with_labels.json'
+_unknown_token = 'UNK'
+_im_w = 224
+_im_h = 224
+
+if not os.path.exists(_outdir):
+    os.mkdir(_outdir)
+
+def parse_objects():
+    filename = os.path.join(_datadir, _objects)
+    with open(filename,'r') as file:
+        data = json.load(file)
+
+    objects_in_image = dict()
+    for item in data:
+        objects_in_image[item['id']] = item['objects']
+
+    regions_in_image = dict()
+    for item in data:
+        region_ids = [object['id'] for object in item['objects']]
+        regions_in_image[item['id']] = region_ids
+
+    objects_in_image_out_filename = os.path.join(_outdir, 
+                                                 _objects_in_image)
+    with open(objects_in_image_out_filename, 'w') as outfile:
+        json.dump(objects_in_image, outfile, sort_keys=True, indent=4)
+        
+    regions_in_image_out_filename = os.path.join(_outdir, 
+                                                 _regions_in_image)
+    with open(regions_in_image_out_filename, 'w') as outfile:
+        json.dump(regions_in_image, outfile, sort_keys=True, indent=4)
+
+    
+def parse_attributes():
+    filename = os.path.join(_datadir, _attributes)
+    with open(filename,'r') as file:
+        data = json.load(file)
+
+    regions = dict()
+    for image_data in data:
+        for region_data in image_data['attributes']:
+            region_data_without_id = dict()
+            region_data_without_id['image_id'] = image_data['id']
+            for key, value in region_data.items():
+                if key != 'id':
+                    region_data_without_id[key] = value
+            regions[region_data['id']] = region_data_without_id
+            
+    regions_out_filename = os.path.join(_outdir, 
+                                        _regions_with_attributes)
+    with open(regions_out_filename, 'w') as outfile:
+        json.dump(regions, outfile, sort_keys=True, indent=4)
+
+
+def add_regions_without_attributes():
+    regions_with_attributes_filename = os.path.join(_outdir,
+                                                    _regions_with_attributes)
+    with open(regions_with_attributes_filename) as file:
+        regions_with_attributes_data = json.load(file)
+    
+    objects_in_image_filename = os.path.join(_outdir,
+                                             _objects_in_image)
+    with open(objects_in_image_filename) as file:
+        objects_in_image_data = json.load(file)
+
+    regions = regions_with_attributes_data
+    for image_id, object_regions in objects_in_image_data.items():
+        for object_region in object_regions:
+            if str(object_region['id']) not in regions_with_attributes_data:
+                region_data_without_id = dict()
+                region_data_without_id['image_id'] = int(image_id)
+                region_data_without_id['attributes'] = []
+                for key, value in object_region.items():
+                    if key != 'id':
+                        region_data_without_id[key] = value
+                regions[object_region['id']] = region_data_without_id
+
+    regions_out_filename = os.path.join(_outdir, 
+                                        _regions)
+    with open(regions_out_filename, 'w') as outfile:
+        json.dump(regions, outfile, sort_keys=True, indent=4)
+
+
+def stats():
+    regions_filename = os.path.join(_outdir, _regions)
+    with open(regions_filename) as file:
+        regions = json.load(file)
+    
+    num_regions = len(regions)
+    num_regions_with_attributes = 0
+    for region in regions.values():
+        if region['attributes']:
+            num_regions_with_attributes += 1
+
+    print 'Number of regions: {}'.format(num_regions)
+    print 'Number of regions with attributes: {}'.format(
+        num_regions_with_attributes)
+
+
+def normalize_object_label(label, lemmatizer):
+    words = nltk.tokenize.word_tokenize(label)
+    nouns = []
+    for word, pos_tag in nltk.pos_tag(words):
+        if pos_tag=='NN' or pos_tag=='NNS':
+            nouns.append(lemmatizer.lemmatize(word))
+    return " ".join(nouns).lower()
+
+            
+def normalize_attribute_label(label, tokenizer):
+    words = tokenizer.tokenize(label)
+    attributes = []
+    for word in words:
+        attributes.append("".join(word))
+    return " ".join(attributes).lower()
+
+
+def normalized_labels():
+    regions_with_attributes_filename = os.path.join(_outdir,
+                                                    _regions_with_attributes)
+    with open(regions_with_attributes_filename) as file:
+        regions = json.load(file)
+
+    object_labels = dict()
+    attribute_labels = dict()
+    object_count = 0
+    attribute_count = 0
+    lemmatizer = nltk.stem.WordNetLemmatizer()
+    tokenizer = nltk.tokenize.RegexpTokenizer("([^\W\d]+'[^\W\d]+)|([^\W\d]+)")
+    for region_id, region_data in regions.items():
+        for object in region_data['object_names']:
+            object = normalize_object_label(object, lemmatizer)
+            if object not in object_labels:
+                object_labels[object] = 1
+                object_count += 1
+            else:
+                object_labels[object] += 1
+        
+        for attribute in region_data['attributes']:
+            attribute = normalize_attribute_label(attribute, tokenizer)
+            if attribute not in attribute_labels:
+                attribute_labels[attribute] = 1
+                attribute_count += 1
+            else:
+                attribute_labels[attribute] += 1
+        
+        print 'Objects: {}    Attributes: {}'.format(object_count, 
+                                                     attribute_count)
+    object_labels_out_filename = os.path.join(_outdir, 
+                                              _raw_object_labels)
+    with open(object_labels_out_filename, 'w') as outfile:
+        json.dump(object_labels, outfile, sort_keys=True, indent=4)
+
+    attribute_labels_out_filename = os.path.join(_outdir, 
+                                                 _raw_attribute_labels)
+    with open(attribute_labels_out_filename, 'w') as outfile:
+        json.dump(attribute_labels, outfile, sort_keys=True, indent=4)
+
+    print 'Number of object labels: {}'.format(object_count)
+    print 'Number of attribute labels: {}'.format(attribute_count)
+
+
+def normalize_region_object_attribute_labels():
+    regions_with_attributes_filename = os.path.join(_outdir,
+                                                    _regions_with_attributes)
+    with open(regions_with_attributes_filename) as file:
+        regions = json.load(file)
+
+    lemmatizer = nltk.stem.WordNetLemmatizer()
+    tokenizer = nltk.tokenize.RegexpTokenizer("([^\W\d]+'[^\W\d]+)|([^\W\d]+)")
+    count = 0
+    for region_id, region_data in regions.items():
+        object_names = []
+        for object in region_data['object_names']:
+            if object_names == "":
+                continue
+            object_names.append(normalize_object_label(object, lemmatizer))
+        region_data['object_names'] = object_names
+
+        attributes = []
+        for attribute in region_data['attributes']:
+            if attributes == "":
+                continue
+            attributes.append(normalize_attribute_label(attribute, tokenizer))
+        region_data['attributes'] = attributes
+        count += 1
+        print '{}/{}'.format(count, len(regions))
+    regions_with_labels_out_filename = os.path.join(_outdir,
+                                                    _regions_with_labels)
+    with open(regions_with_labels_out_filename, 'w') as outfile:
+        json.dump(regions, outfile, sort_keys=True, indent=4)
+    
+
+def top_k_object_labels(k):
+    raw_object_labels_filename = os.path.join(_outdir,
+                                              _raw_object_labels)
+    with open(raw_object_labels_filename, 'r') as file:
+        raw_object_labels = json.load(file)
+        
+    sorted_raw_object_labels = \
+        [key for key, value in sorted(raw_object_labels.items(), 
+                              key = operator.itemgetter(1),
+                              reverse = True)]
+
+    object_labels = dict()
+    for i in xrange(min(k,len(sorted_raw_object_labels))):
+        object_labels[sorted_raw_object_labels[i]] = i
+
+    if "" in object_labels:
+        object_labels[_unknown_token] = object_labels[""]
+        del object_labels[""]
+
+    object_labels_filename = os.path.join(_outdir,
+                                          _object_labels)
+    with open(object_labels_filename, 'w') as outfile:
+        json.dump(object_labels, outfile, sort_keys=True, indent=4)
+
+
+def top_k_attribute_labels(k):
+    raw_attribute_labels_filename = os.path.join(_outdir,
+                                                 _raw_attribute_labels)
+    with open(raw_attribute_labels_filename, 'r') as file:
+        raw_attribute_labels = json.load(file)
+        
+    sorted_raw_attribute_labels = \
+        [key for key, value in sorted(raw_attribute_labels.items(), 
+                              key = operator.itemgetter(1),
+                              reverse = True)]
+
+    attribute_labels = dict()
+    for i in xrange(min(k,len(sorted_raw_attribute_labels))):
+        attribute_labels[sorted_raw_attribute_labels[i]] = i
+
+    if "" in attribute_labels:
+        attribute_labels[_unknown_token] = attribute_labels[""]
+        del attribute_labels[""]
+
+    attribute_labels_filename = os.path.join(_outdir,
+                                             _attribute_labels)
+    with open(attribute_labels_filename, 'w') as outfile:
+        json.dump(attribute_labels, outfile, sort_keys=True, indent=4)
+ 
+
+def crop_region(region_info):
+    region_id, region_data = region_info
+    image_filename = os.path.join(_datadir, 
+                                  'images/' + 
+                                  str(region_data['image_id']) + '.jpg')
+    image = image_io.imread(image_filename)
+    
+    if len(image.shape)==3:
+        im_h, im_w, im_c =image.shape
+    elif len(image.shape)==2:
+        im_h, im_w =image.shape
+        image_tmp = np.zeros([im_h, im_w, 3], dtype=image.dtype)        
+        for c in xrange(3):
+            image_tmp[:,:,c] = image
+        image = image_tmp
+
+    x = min(im_w-1,max(0,region_data["x"]))
+    y = min(im_h-1,max(0,region_data["y"]))
+    h = min(im_h-y,max(region_data["h"],1))
+    w = min(im_w-x,max(region_data["w"],1))
+    
+    cropped_region = image_io.imresize(image[y:y+h,x:x+w,:],
+                                       output_size=(_im_h, _im_w))
+    image_subdir = os.path.join(_cropped_regions_dir, 
+                                str(region_data['image_id']))
+    if not os.path.exists(image_subdir):
+        os.mkdir(image_subdir)
+        
+    image_out_filename = os.path.join(image_subdir, 
+                                      str(region_id) + '.jpg')
+        
+    image_io.imwrite(cropped_region, image_out_filename)
+
+
+def crop_regions_parallel():
+    regions_filename = os.path.join(_outdir,
+                                    _regions_with_labels)
+    with open(regions_filename) as file:
+        regions = json.load(file)
+        
+    if not os.path.exists(_cropped_regions_dir):
+        os.mkdir(_cropped_regions_dir)
+
+    pool = Pool(24)
+    try:
+        pool.map(crop_region, regions.items())
+    except:
+        pool.close()
+    pool.close()
+
+
+def crop_regions():
+    regions_filename = os.path.join(_outdir,
+                                    _regions_with_labels)
+    with open(regions_filename) as file:
+        regions = json.load(file)
+        
+    if not os.path.exists(_cropped_regions_dir):
+        os.mkdir(_cropped_regions_dir)
+
+    count = 0
+    for region_id, region_data in regions.items():
+        try:
+            image_filename = os.path.join(_datadir, 
+                                          'images/' + 
+                                          str(region_data['image_id']) + '.jpg')
+            image = image_io.imread(image_filename)
+            
+            if len(image.shape)==3:
+                im_h, im_w, im_c =image.shape
+            elif len(image.shape)==2:
+                im_h, im_w =image.shape
+                image_tmp = np.zeros([im_h, im_w, 3], dtype=image.dtype)
+                for c in xrange(3):
+                    image_tmp[:,:,c] = image
+                image = image_tmp
+
+            x = min(im_w-1,max(0,region_data["x"]))
+            y = min(im_h-1,max(0,region_data["y"]))
+            h = min(im_h-y,max(region_data["h"],1))
+            w = min(im_w-x,max(region_data["w"],1))
+
+            cropped_region = image_io.imresize(image[y:y+h,x:x+w,:],
+                                               output_size=(_im_h, _im_w))
+            image_subdir = os.path.join(_cropped_regions_dir, 
+                                        str(region_data['image_id']))
+            if not os.path.exists(image_subdir):
+                os.mkdir(image_subdir)
+
+            image_out_filename = os.path.join(image_subdir, 
+                                              str(region_id) + '.jpg')
+        
+            image_io.imwrite(cropped_region, image_out_filename)
+        
+            count += 1
+            
+            print '{}/{}'.format(count, len(regions))
+        except:
+            print region_id, region_data
+            raise
+
+  
+if __name__=='__main__':
+    # parse_objects()
+    # parse_attributes()
+    # add_regions_without_attributes()
+    # stats()
+    # normalized_labels()
+    # normalize_region_object_attribute_labels()
+    # top_k_object_labels(1000)
+    # top_k_attribute_labels(1000)
+    crop_regions_parallel()