From fe505614d44f3f26c8752f38f4786d348878954c Mon Sep 17 00:00:00 2001
From: tgupta6 <tgupta6@illinois.edu>
Date: Sat, 4 Jun 2016 19:23:17 -0500
Subject: [PATCH] Visual Genome Data Manager
---
__init__.py | 0
constants.py | 39 +++
data/__init__.py | 0
data/compute_mean_region_image.py | 31 +++
data/cropped_regions.py | 264 +++++++++++++++++++++
data/regions.py | 264 +++++++++++++++++++++
image_io.py | 78 ++++++
tftools/__init__.py | 0
tftools/data.py | 96 ++++++++
visual_genome_parser.py | 378 ++++++++++++++++++++++++++++++
10 files changed, 1150 insertions(+)
create mode 100644 __init__.py
create mode 100644 constants.py
create mode 100644 data/__init__.py
create mode 100644 data/compute_mean_region_image.py
create mode 100644 data/cropped_regions.py
create mode 100644 data/regions.py
create mode 100644 image_io.py
create mode 100644 tftools/__init__.py
create mode 100644 tftools/data.py
create mode 100644 visual_genome_parser.py
diff --git a/__init__.py b/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/constants.py b/constants.py
new file mode 100644
index 0000000..b292f0f
--- /dev/null
+++ b/constants.py
@@ -0,0 +1,39 @@
+import os
+
+#height and width to which images are resized before feeding into networks
+image_size = (224, 224)
+
+# Token to be used if object or attribute variable is unknown
+unknown_token = 'UNK'
+
+# Data paths
+data_absolute_path = '/home/tanmay/Data/VisualGenome'
+image_dir = os.path.join(data_absolute_path, 'images')
+object_labels_json = os.path.join(
+ data_absolute_path,
+ 'restructured/object_labels.json')
+attribute_labels_json = os.path.join(
+ data_absolute_path,
+ 'restructured/attribute_labels.json')
+regions_json = os.path.join(
+ data_absolute_path,
+ 'restructured/region_with_labels.json')
+mean_image_filename = os.path.join(
+ data_absolute_path,
+ 'restructured/mean_image.jpg')
+# Regions data partition
+# First 70% meant to be used for training
+# Next 10% is set aside for validation
+# Last 20% is to be used for testing
+num_total_regions = 1951768
+num_train_regions = 1366238 # First 70%
+num_val_regions = 195176 # Next 10%
+num_test_regions = num_total_regions \
+ - num_train_regions \
+ - num_val_regions
+
+
+
+
+
+
diff --git a/data/__init__.py b/data/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/data/compute_mean_region_image.py b/data/compute_mean_region_image.py
new file mode 100644
index 0000000..75806ca
--- /dev/null
+++ b/data/compute_mean_region_image.py
@@ -0,0 +1,31 @@
+import numpy as np
+from data.regions import data
+import constants
+import image_io
+import pdb
+
+
+_image_size = constants.image_size
+_num_train_regions = constants.num_train_regions
+_mean_image_filename = constants.mean_image_filename
+
+
+if __name__=='__main__':
+ data_mgr = data(constants.image_dir,
+ constants.object_labels_json,
+ constants.attribute_labels_json,
+ constants.regions_json,
+ constants.image_size,
+ channels=3,
+ mean_image_filename=None)
+
+ h, w = _image_size
+ mean_image = np.zeros([h, w, 3], dtype=np.float32)
+ num_images_for_mean = min(10000, _num_train_regions)
+ for i in xrange(num_images_for_mean):
+ mean_image += data_mgr.get_region_image(i)
+ mean_image = mean_image/num_images_for_mean
+ image_io.imwrite(np.uint8(mean_image*255),
+ _mean_image_filename)
+
+
diff --git a/data/cropped_regions.py b/data/cropped_regions.py
new file mode 100644
index 0000000..b99e6c0
--- /dev/null
+++ b/data/cropped_regions.py
@@ -0,0 +1,264 @@
+import numpy as np
+import json
+import os
+import pdb
+import time
+
+from multiprocessing import Pool
+
+import tftools.data
+import image_io
+import constants
+
+import tensorflow as tf
+
+_unknown_token = constants.unknown_token
+
+class data():
+ def __init__(self,
+ image_dir,
+ object_labels_json,
+ attribute_labels_json,
+ regions_json,
+ image_size,
+ channels=3,
+ mean_image_filename=None):
+ self.image_dir = image_dir
+ self.h = image_size[0]
+ self.w = image_size[1]
+ self.c = channels
+ self.mean_image = self.get_mean_image(mean_image_filename)
+ self.object_labels_dict = self.read_json_file(object_labels_json)
+ self.attribute_labels_dict = self.read_json_file(attribute_labels_json)
+ self.inv_object_labels_dict = self.invert_label_dict(
+ self.object_labels_dict)
+ self.inv_attribute_labels_dict = self.invert_label_dict(
+ self.attribute_labels_dict)
+ self.num_object_labels = len(self.object_labels_dict)
+ self.num_attribute_labels = len(self.attribute_labels_dict)
+ self.regions = self.read_json_file(regions_json)
+ self.num_regions = len(self.regions)
+ self.create_sample_to_region_dict()
+
+ def create_sample_to_region_dict(self):
+ self.sample_to_region_dict = \
+ {k: v for k, v in zip(xrange(self.num_regions),
+ self.regions.keys())}
+
+ def invert_label_dict(self, label_dict):
+ return {v: k for k, v in label_dict.items()}
+
+ def read_json_file(self, filename):
+ print 'Reading {} ...'.format(filename)
+ with open(filename, 'r') as file:
+ return json.load(file)
+
+ def get(self, samples):
+ batch_size = len(samples)
+ batch = dict()
+ batch['region_ids'] = dict()
+ batch['images'] = np.zeros(
+ [batch_size, self.h, self.w, self.c], np.float32)
+ batch['object_labels'] = np.zeros(
+ [batch_size, len(self.object_labels_dict)], np.float32)
+ batch['attribute_labels'] = np.zeros(
+ [batch_size, len(self.attribute_labels_dict)], np.float32)
+
+ for index, sample in enumerate(samples):
+ batch['region_ids'][index] = self.sample_to_region_dict[sample]
+ batch['images'][index, :, :, :] = self.get_region_image(sample)
+ batch['object_labels'][index, :] = self.get_object_label(sample)
+ batch['attribute_labels'][index,:] = \
+ self.get_attribute_label(sample)
+
+ return batch
+
+ def get_single(self, sample):
+ print sample
+ batch = dict()
+ batch['region_ids'] = dict()
+ batch['images'] = np.zeros(
+ [self.h, self.w, self.c], np.float32)
+ batch['object_labels'] = np.zeros(
+ [len(self.object_labels_dict)], np.float32)
+ batch['attribute_labels'] = np.zeros(
+ [len(self.attribute_labels_dict)], np.float32)
+
+ batch['region_ids'] = self.sample_to_region_dict[sample]
+ batch['images'] = self.get_region_image(sample)
+ batch['object_labels'] = self.get_object_label(sample)
+ batch['attribute_labels'] = self.get_attribute_label(sample)
+
+ return batch
+
+ def get_parallel(self, samples):
+ batch_list = self.pool.map(self.get_single, samples)
+ batch_size = len(samples)
+ batch = dict()
+ batch['region_ids'] = dict()
+ batch['images'] = np.zeros(
+ [batch_size, self.h, self.w, self.c], np.float32)
+ batch['object_labels'] = np.zeros(
+ [batch_size, len(self.object_labels_dict)], np.float32)
+ batch['attribute_labels'] = np.zeros(
+ [batch_size, len(self.attribute_labels_dict)], np.float32)
+
+ for index, single_batch in enumerate(batch_list):
+ batch['region_ids'][index] = single_batch['region_ids']
+ batch['images'][index, :, :, :] = single_batch['images']
+ batch['object_labels'][index, :] = single_batch['object_labels']
+ batch['attribute_labels'][index,:] = single_batch['attribute_labels']
+
+
+ return batch
+
+ def get_region_image(self, sample):
+ region_id = self.sample_to_region_dict[sample]
+ region = self.regions[region_id]
+ filename = os.path.join(self.image_dir,
+ str(region['image_id']) + '.jpg')
+ image = image_io.imread(filename)
+ image = self.single_to_three_channel(image)
+ x, y, w, h = self.get_clipped_region_coords(region, image.shape[0:2])
+ region_image = image[y:y + h, x:x + w, :]
+
+ region_image = image_io.imresize(
+ region_image,
+ output_size=(self.h, self.w)).astype(np.float32)
+
+ return region_image / 255 - self.mean_image
+
+ def single_to_three_channel(self, image):
+ if len(image.shape)==3:
+ return image
+ elif len(image.shape)==2:
+ im_h, im_w =image.shape
+ image_tmp = np.zeros([im_h, im_w, 3], dtype=image.dtype)
+ for c in xrange(3):
+ image_tmp[:,:,c] = image
+ return image_tmp
+
+ def get_clipped_region_coords(self, region, image_size):
+ im_h, im_w = image_size
+ x = min(im_w - 1, max(0, region["x"]))
+ y = min(im_h - 1, max(0, region["y"]))
+ h = min(im_h - y, max(region["h"], 1))
+ w = min(im_w - x, max(region["w"], 1))
+ return x, y, w, h
+
+ def get_mean_image(self, mean_image_filename):
+ if mean_image_filename:
+ return image_io.imread(mean_image_filename).astype(
+ np.float32) / 255
+ else:
+ return np.zeros([self.h, self.w, self.c], np.float32)
+
+ def get_object_label(self, sample):
+ # Returns a multihot vector encoding of object labels
+ # If an object label is not found in the labels list,
+ # _unknown_token is produced in that case.
+ region_id = self.sample_to_region_dict[sample]
+ region = self.regions[region_id]
+ object_labels = region['object_names']
+ object_label_encoding = np.zeros([1, self.num_object_labels],
+ dtype = np.float32)
+ if object_labels:
+ for object in object_labels:
+ if object not in self.object_labels_dict:
+ label_id = self.object_labels_dict[_unknown_token]
+ else:
+ label_id = self.object_labels_dict[object]
+ object_label_encoding[0,label_id] = 1.0
+ else:
+ label_id = self.object_labels_dict[_unknown_token]
+ object_label_encoding[0,label_id] = 1.0
+
+ return object_label_encoding/np.sum(object_label_encoding)
+
+ def get_attribute_label(self, sample):
+ # Attribute is turned on if it is present
+ region_id = self.sample_to_region_dict[sample]
+ region = self.regions[region_id]
+ attribute_labels = region['attributes']
+ attribute_label_encoding = np.zeros([1, self.num_attribute_labels],
+ dtype = np.float32)
+ for attribute in attribute_labels:
+ if attribute in self.attribute_labels_dict:
+ label_id = self.attribute_labels_dict[attribute]
+ attribute_label_encoding[0,label_id] = 1.0
+
+ return attribute_label_encoding
+
+if __name__=='__main__':
+ data_mgr = data(constants.image_dir,
+ constants.object_labels_json,
+ constants.attribute_labels_json,
+ constants.regions_json,
+ constants.image_size,
+ channels=3,
+ mean_image_filename=None)
+ print 'Number of object labels: {}'.format(data_mgr.num_object_labels)
+ print 'Number of attribute labels: {}'.format(data_mgr.num_attribute_labels)
+ print 'Number of regions: {}'.format(data_mgr.num_regions)
+
+ #Test sample
+ samples = [1, 2]
+ sample = samples[0]
+ region_id = data_mgr.sample_to_region_dict[sample]
+ region = data_mgr.regions[region_id]
+ attribute_encoding = data_mgr.get_attribute_label(sample)
+ object_encoding = data_mgr.get_object_label(sample)
+ region_image = data_mgr.get_region_image(sample)
+
+ attributes = []
+ for i in xrange(attribute_encoding.shape[1]):
+ if attribute_encoding[0,i] > 0 :
+ attributes.append(data_mgr.inv_attribute_labels_dict[i])
+
+ objects = []
+ for i in xrange(object_encoding.shape[1]):
+ if object_encoding[0,i] > 0 :
+ objects.append(data_mgr.inv_object_labels_dict[i])
+
+ print "Region: {}".format(region)
+ print "Attributes: {}".format(", ".join(attributes))
+ print "Objects: {}".format(", ".join(objects))
+
+# image_io.imshow(region_image)
+
+ batch_size = 200
+ num_samples = 1000
+ num_epochs = 1
+ offset = 0
+
+ index_generator = tftools.data.random(
+ batch_size,
+ num_samples,
+ num_epochs,
+ offset)
+
+ # start = time.time()
+ # count = 0
+ # for samples in index_generator:
+ # batch = data_mgr.get(samples)
+ # print 'Batch Count: {}'.format(count)
+ # count += 1
+ # stop = time.time()
+ # print 'Time per batch: {}'.format((stop-start)/5.0)
+
+ batch_generator = tftools.data.async_batch_generator(
+ data_mgr,
+ index_generator,
+ 1000)
+
+ count = 0
+ start = time.time()
+ for batch in batch_generator:
+ print 'Batch Number: {}'.format(count)
+# print batch['region_ids']
+ count += 1
+ stop = time.time()
+ print 'Time per batch: {}'.format((stop-start)/50.0)
+ print "Count: {}".format(count)
+
+ pool.close()
diff --git a/data/regions.py b/data/regions.py
new file mode 100644
index 0000000..b99e6c0
--- /dev/null
+++ b/data/regions.py
@@ -0,0 +1,264 @@
+import numpy as np
+import json
+import os
+import pdb
+import time
+
+from multiprocessing import Pool
+
+import tftools.data
+import image_io
+import constants
+
+import tensorflow as tf
+
+_unknown_token = constants.unknown_token
+
+class data():
+ def __init__(self,
+ image_dir,
+ object_labels_json,
+ attribute_labels_json,
+ regions_json,
+ image_size,
+ channels=3,
+ mean_image_filename=None):
+ self.image_dir = image_dir
+ self.h = image_size[0]
+ self.w = image_size[1]
+ self.c = channels
+ self.mean_image = self.get_mean_image(mean_image_filename)
+ self.object_labels_dict = self.read_json_file(object_labels_json)
+ self.attribute_labels_dict = self.read_json_file(attribute_labels_json)
+ self.inv_object_labels_dict = self.invert_label_dict(
+ self.object_labels_dict)
+ self.inv_attribute_labels_dict = self.invert_label_dict(
+ self.attribute_labels_dict)
+ self.num_object_labels = len(self.object_labels_dict)
+ self.num_attribute_labels = len(self.attribute_labels_dict)
+ self.regions = self.read_json_file(regions_json)
+ self.num_regions = len(self.regions)
+ self.create_sample_to_region_dict()
+
+ def create_sample_to_region_dict(self):
+ self.sample_to_region_dict = \
+ {k: v for k, v in zip(xrange(self.num_regions),
+ self.regions.keys())}
+
+ def invert_label_dict(self, label_dict):
+ return {v: k for k, v in label_dict.items()}
+
+ def read_json_file(self, filename):
+ print 'Reading {} ...'.format(filename)
+ with open(filename, 'r') as file:
+ return json.load(file)
+
+ def get(self, samples):
+ batch_size = len(samples)
+ batch = dict()
+ batch['region_ids'] = dict()
+ batch['images'] = np.zeros(
+ [batch_size, self.h, self.w, self.c], np.float32)
+ batch['object_labels'] = np.zeros(
+ [batch_size, len(self.object_labels_dict)], np.float32)
+ batch['attribute_labels'] = np.zeros(
+ [batch_size, len(self.attribute_labels_dict)], np.float32)
+
+ for index, sample in enumerate(samples):
+ batch['region_ids'][index] = self.sample_to_region_dict[sample]
+ batch['images'][index, :, :, :] = self.get_region_image(sample)
+ batch['object_labels'][index, :] = self.get_object_label(sample)
+ batch['attribute_labels'][index,:] = \
+ self.get_attribute_label(sample)
+
+ return batch
+
+ def get_single(self, sample):
+ print sample
+ batch = dict()
+ batch['region_ids'] = dict()
+ batch['images'] = np.zeros(
+ [self.h, self.w, self.c], np.float32)
+ batch['object_labels'] = np.zeros(
+ [len(self.object_labels_dict)], np.float32)
+ batch['attribute_labels'] = np.zeros(
+ [len(self.attribute_labels_dict)], np.float32)
+
+ batch['region_ids'] = self.sample_to_region_dict[sample]
+ batch['images'] = self.get_region_image(sample)
+ batch['object_labels'] = self.get_object_label(sample)
+ batch['attribute_labels'] = self.get_attribute_label(sample)
+
+ return batch
+
+ def get_parallel(self, samples):
+ batch_list = self.pool.map(self.get_single, samples)
+ batch_size = len(samples)
+ batch = dict()
+ batch['region_ids'] = dict()
+ batch['images'] = np.zeros(
+ [batch_size, self.h, self.w, self.c], np.float32)
+ batch['object_labels'] = np.zeros(
+ [batch_size, len(self.object_labels_dict)], np.float32)
+ batch['attribute_labels'] = np.zeros(
+ [batch_size, len(self.attribute_labels_dict)], np.float32)
+
+ for index, single_batch in enumerate(batch_list):
+ batch['region_ids'][index] = single_batch['region_ids']
+ batch['images'][index, :, :, :] = single_batch['images']
+ batch['object_labels'][index, :] = single_batch['object_labels']
+ batch['attribute_labels'][index,:] = single_batch['attribute_labels']
+
+
+ return batch
+
+ def get_region_image(self, sample):
+ region_id = self.sample_to_region_dict[sample]
+ region = self.regions[region_id]
+ filename = os.path.join(self.image_dir,
+ str(region['image_id']) + '.jpg')
+ image = image_io.imread(filename)
+ image = self.single_to_three_channel(image)
+ x, y, w, h = self.get_clipped_region_coords(region, image.shape[0:2])
+ region_image = image[y:y + h, x:x + w, :]
+
+ region_image = image_io.imresize(
+ region_image,
+ output_size=(self.h, self.w)).astype(np.float32)
+
+ return region_image / 255 - self.mean_image
+
+ def single_to_three_channel(self, image):
+ if len(image.shape)==3:
+ return image
+ elif len(image.shape)==2:
+ im_h, im_w =image.shape
+ image_tmp = np.zeros([im_h, im_w, 3], dtype=image.dtype)
+ for c in xrange(3):
+ image_tmp[:,:,c] = image
+ return image_tmp
+
+ def get_clipped_region_coords(self, region, image_size):
+ im_h, im_w = image_size
+ x = min(im_w - 1, max(0, region["x"]))
+ y = min(im_h - 1, max(0, region["y"]))
+ h = min(im_h - y, max(region["h"], 1))
+ w = min(im_w - x, max(region["w"], 1))
+ return x, y, w, h
+
+ def get_mean_image(self, mean_image_filename):
+ if mean_image_filename:
+ return image_io.imread(mean_image_filename).astype(
+ np.float32) / 255
+ else:
+ return np.zeros([self.h, self.w, self.c], np.float32)
+
+ def get_object_label(self, sample):
+ # Returns a multihot vector encoding of object labels
+ # If an object label is not found in the labels list,
+ # _unknown_token is produced in that case.
+ region_id = self.sample_to_region_dict[sample]
+ region = self.regions[region_id]
+ object_labels = region['object_names']
+ object_label_encoding = np.zeros([1, self.num_object_labels],
+ dtype = np.float32)
+ if object_labels:
+ for object in object_labels:
+ if object not in self.object_labels_dict:
+ label_id = self.object_labels_dict[_unknown_token]
+ else:
+ label_id = self.object_labels_dict[object]
+ object_label_encoding[0,label_id] = 1.0
+ else:
+ label_id = self.object_labels_dict[_unknown_token]
+ object_label_encoding[0,label_id] = 1.0
+
+ return object_label_encoding/np.sum(object_label_encoding)
+
+ def get_attribute_label(self, sample):
+ # Attribute is turned on if it is present
+ region_id = self.sample_to_region_dict[sample]
+ region = self.regions[region_id]
+ attribute_labels = region['attributes']
+ attribute_label_encoding = np.zeros([1, self.num_attribute_labels],
+ dtype = np.float32)
+ for attribute in attribute_labels:
+ if attribute in self.attribute_labels_dict:
+ label_id = self.attribute_labels_dict[attribute]
+ attribute_label_encoding[0,label_id] = 1.0
+
+ return attribute_label_encoding
+
+if __name__=='__main__':
+ data_mgr = data(constants.image_dir,
+ constants.object_labels_json,
+ constants.attribute_labels_json,
+ constants.regions_json,
+ constants.image_size,
+ channels=3,
+ mean_image_filename=None)
+ print 'Number of object labels: {}'.format(data_mgr.num_object_labels)
+ print 'Number of attribute labels: {}'.format(data_mgr.num_attribute_labels)
+ print 'Number of regions: {}'.format(data_mgr.num_regions)
+
+ #Test sample
+ samples = [1, 2]
+ sample = samples[0]
+ region_id = data_mgr.sample_to_region_dict[sample]
+ region = data_mgr.regions[region_id]
+ attribute_encoding = data_mgr.get_attribute_label(sample)
+ object_encoding = data_mgr.get_object_label(sample)
+ region_image = data_mgr.get_region_image(sample)
+
+ attributes = []
+ for i in xrange(attribute_encoding.shape[1]):
+ if attribute_encoding[0,i] > 0 :
+ attributes.append(data_mgr.inv_attribute_labels_dict[i])
+
+ objects = []
+ for i in xrange(object_encoding.shape[1]):
+ if object_encoding[0,i] > 0 :
+ objects.append(data_mgr.inv_object_labels_dict[i])
+
+ print "Region: {}".format(region)
+ print "Attributes: {}".format(", ".join(attributes))
+ print "Objects: {}".format(", ".join(objects))
+
+# image_io.imshow(region_image)
+
+ batch_size = 200
+ num_samples = 1000
+ num_epochs = 1
+ offset = 0
+
+ index_generator = tftools.data.random(
+ batch_size,
+ num_samples,
+ num_epochs,
+ offset)
+
+ # start = time.time()
+ # count = 0
+ # for samples in index_generator:
+ # batch = data_mgr.get(samples)
+ # print 'Batch Count: {}'.format(count)
+ # count += 1
+ # stop = time.time()
+ # print 'Time per batch: {}'.format((stop-start)/5.0)
+
+ batch_generator = tftools.data.async_batch_generator(
+ data_mgr,
+ index_generator,
+ 1000)
+
+ count = 0
+ start = time.time()
+ for batch in batch_generator:
+ print 'Batch Number: {}'.format(count)
+# print batch['region_ids']
+ count += 1
+ stop = time.time()
+ print 'Time per batch: {}'.format((stop-start)/50.0)
+ print "Count: {}".format(count)
+
+ pool.close()
diff --git a/image_io.py b/image_io.py
new file mode 100644
index 0000000..578b8d9
--- /dev/null
+++ b/image_io.py
@@ -0,0 +1,78 @@
+from PIL import Image
+import numpy as np
+import math
+import pdb
+
+
+def imread(filename):
+ """
+ Matlab like function for reading an image file.
+ Returns:
+ im_array (numpy.ndarray): h x w x 3 ndarray for color images and
+ h x w for grayscale images
+ """
+ im = Image.open(filename)
+
+ err_str = \
+ "imread only supports 'RGB' and 'L' modes, found '{}'".format(im.mode)
+ assert (im.mode == 'RGB' or im.mode == 'L'), err_str
+
+ im_array = np.array(im)
+
+ return im_array
+
+
+def imshow(np_im):
+ """
+ Matlab like function for displaying a numpy ndarray as an image
+ Args:
+ np_im (numpy.ndarray): h x w x 3 ndarray for color images and
+ h x w for grayscale images with pixels stored in uint8 format
+ """
+ err_str = 'imshow expects ndarray of dimension h x w x c (RGB) or h x w (L)'
+ assert (len(np_im.shape) == 3 or len(np_im.shape) == 2), err_str
+
+ if len(np_im.shape) == 3:
+ assert (np_im.shape[2] == 3), 'imshow expected 3 channels'
+ im = Image.fromarray(np_im, 'RGB')
+ else:
+ im = Image.fromarray(np_im, 'L')
+
+ im.show()
+
+
+def imwrite(np_im, filename):
+ """
+ Matlab like function to save numpy image.
+ Args:
+ np_im (numpy.ndarray): h x w x 3 ndarray for color images and
+ h x w for grayscale images with pixels stored in uint8 format
+ """
+ err_str = 'imshow expects ndarray of dimension h x w x c (RGB) or h x w (L)'
+ assert (len(np_im.shape) == 3 or len(np_im.shape) == 2), err_str
+
+ im = Image.fromarray(np_im)
+ im.save(filename)
+
+
+def imresize(np_im, method=Image.BILINEAR, **kwargs):
+ assert_str = "Only 1 keyword argument expected with key either" + \
+ "'output_size' or 'scale'"
+ assert (len(kwargs)==1), assert_str
+ im_h = np_im.shape[0]
+ im_w = np_im.shape[1]
+ if 'output_size' in kwargs:
+ h, w = kwargs['output_size']
+ elif 'scale' in kwargs:
+ h = scale*im_h
+ w = scale*im_w
+ else:
+ assert_str = "Variable argument must be one of {'output_size','scale'}"
+ assert (False), assert_str
+ h = int(math.ceil(h))
+ w = int(math.ceil(w))
+ im = Image.fromarray(np_im)
+ return np.array(im.resize((w,h)))
+
+
+
diff --git a/tftools/__init__.py b/tftools/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tftools/data.py b/tftools/data.py
new file mode 100644
index 0000000..9fe60c1
--- /dev/null
+++ b/tftools/data.py
@@ -0,0 +1,96 @@
+"""Utility functions for making batch generation easier."""
+import numpy as np
+from multiprocessing import Process, Queue
+import time
+
+def sequential(batch_size, num_samples, num_epochs=1, offset=0):
+ """Generate sequence indices.
+ """
+ for epoch in range(num_epochs):
+ indices = range(num_samples)
+ for i in range(0, num_samples - batch_size + 1, batch_size):
+ yield indices[i:i+batch_size]
+
+
+def random(batch_size, num_samples, num_epochs, offset=0):
+ """Generate random indices.
+ """
+ for epoch in range(num_epochs):
+ indices = np.random.permutation(num_samples) + offset
+ indices = indices.tolist()
+ for i in range(0, num_samples - batch_size + 1, batch_size):
+ yield indices[i:i+batch_size]
+
+
+def batch_generator(data, index_generator, batch_function=None):
+ """Generate batches of data.
+ """
+ for samples in index_generator:
+ batch = data.get(samples)
+ if batch_function:
+ output = batch_function(batch)
+ else:
+ output = batch
+ yield output
+
+
+def async_batch_generator(data, index_generator, queue_maxsize, batch_function=None):
+ """Create an asynchronous batch generator.
+ """
+ batcher = batch_generator(data, index_generator, batch_function)
+
+ queue = Queue(maxsize=queue_maxsize)
+ fetcher = BatchFetcher(queue, batcher)
+ fetcher.start()
+
+ time.sleep(10)
+
+ queue_batcher = queue_generator(queue)
+ return queue_batcher
+
+
+def queue_generator(queue, sentinel=None):
+ """Create a generator from a queue.
+ """
+ while True:
+ value = queue.get()
+ if value is not sentinel:
+ yield value
+ else:
+ return
+
+
+class BatchFetcher(Process):
+ def __init__(self, queue, batch_generator):
+ super(BatchFetcher, self).__init__()
+ self.queue = queue
+ self.batch_generator = batch_generator
+
+ def run(self):
+
+ for batch in self.batch_generator:
+ self.queue.put(batch)
+
+ # Signal that fetcher is done.
+ self.queue.put(None)
+
+
+class NumpyData(object):
+ def __init__(self, array):
+ self.array = array
+
+ def get(self, indices):
+ if not isinstance(indices, np.ndarray):
+ indices = np.array(indices)
+ return self.array[indices]
+
+
+if __name__=='__main__':
+ batch_size = 10
+ num_samples = 20
+ num_epochs = 3
+ offset = 0
+ index_generator = random(batch_size, num_samples, num_epochs, offset)
+ for samples in index_generator:
+ print samples
+
diff --git a/visual_genome_parser.py b/visual_genome_parser.py
new file mode 100644
index 0000000..d104401
--- /dev/null
+++ b/visual_genome_parser.py
@@ -0,0 +1,378 @@
+import json
+import os
+import operator
+import nltk
+import numpy as np
+import pdb
+from multiprocessing import Pool
+import image_io
+
+
+# Filenames
+_datadir = '/home/tanmay/Data/VisualGenome/'
+_outdir = os.path.join(_datadir,'restructured')
+_cropped_regions_dir = os.path.join(_datadir, 'cropped_regions')
+_objects = 'objects.json'
+_attributes = 'attributes.json'
+_objects_in_image = 'objects_in_image.json'
+_regions_in_image = 'regions_in_image.json'
+_regions_with_attributes = 'regions_with_attributes.json'
+_regions = 'regions.json'
+_raw_object_labels = 'raw_object_labels.json'
+_raw_attribute_labels = 'raw_attribute_labels.json'
+_object_labels = 'object_labels.json'
+_attribute_labels = 'attribute_labels.json'
+_regions_with_labels = 'region_with_labels.json'
+_unknown_token = 'UNK'
+_im_w = 224
+_im_h = 224
+
+if not os.path.exists(_outdir):
+ os.mkdir(_outdir)
+
+def parse_objects():
+ filename = os.path.join(_datadir, _objects)
+ with open(filename,'r') as file:
+ data = json.load(file)
+
+ objects_in_image = dict()
+ for item in data:
+ objects_in_image[item['id']] = item['objects']
+
+ regions_in_image = dict()
+ for item in data:
+ region_ids = [object['id'] for object in item['objects']]
+ regions_in_image[item['id']] = region_ids
+
+ objects_in_image_out_filename = os.path.join(_outdir,
+ _objects_in_image)
+ with open(objects_in_image_out_filename, 'w') as outfile:
+ json.dump(objects_in_image, outfile, sort_keys=True, indent=4)
+
+ regions_in_image_out_filename = os.path.join(_outdir,
+ _regions_in_image)
+ with open(regions_in_image_out_filename, 'w') as outfile:
+ json.dump(regions_in_image, outfile, sort_keys=True, indent=4)
+
+
+def parse_attributes():
+ filename = os.path.join(_datadir, _attributes)
+ with open(filename,'r') as file:
+ data = json.load(file)
+
+ regions = dict()
+ for image_data in data:
+ for region_data in image_data['attributes']:
+ region_data_without_id = dict()
+ region_data_without_id['image_id'] = image_data['id']
+ for key, value in region_data.items():
+ if key != 'id':
+ region_data_without_id[key] = value
+ regions[region_data['id']] = region_data_without_id
+
+ regions_out_filename = os.path.join(_outdir,
+ _regions_with_attributes)
+ with open(regions_out_filename, 'w') as outfile:
+ json.dump(regions, outfile, sort_keys=True, indent=4)
+
+
+def add_regions_without_attributes():
+ regions_with_attributes_filename = os.path.join(_outdir,
+ _regions_with_attributes)
+ with open(regions_with_attributes_filename) as file:
+ regions_with_attributes_data = json.load(file)
+
+ objects_in_image_filename = os.path.join(_outdir,
+ _objects_in_image)
+ with open(objects_in_image_filename) as file:
+ objects_in_image_data = json.load(file)
+
+ regions = regions_with_attributes_data
+ for image_id, object_regions in objects_in_image_data.items():
+ for object_region in object_regions:
+ if str(object_region['id']) not in regions_with_attributes_data:
+ region_data_without_id = dict()
+ region_data_without_id['image_id'] = int(image_id)
+ region_data_without_id['attributes'] = []
+ for key, value in object_region.items():
+ if key != 'id':
+ region_data_without_id[key] = value
+ regions[object_region['id']] = region_data_without_id
+
+ regions_out_filename = os.path.join(_outdir,
+ _regions)
+ with open(regions_out_filename, 'w') as outfile:
+ json.dump(regions, outfile, sort_keys=True, indent=4)
+
+
+def stats():
+ regions_filename = os.path.join(_outdir, _regions)
+ with open(regions_filename) as file:
+ regions = json.load(file)
+
+ num_regions = len(regions)
+ num_regions_with_attributes = 0
+ for region in regions.values():
+ if region['attributes']:
+ num_regions_with_attributes += 1
+
+ print 'Number of regions: {}'.format(num_regions)
+ print 'Number of regions with attributes: {}'.format(
+ num_regions_with_attributes)
+
+
+def normalize_object_label(label, lemmatizer):
+ words = nltk.tokenize.word_tokenize(label)
+ nouns = []
+ for word, pos_tag in nltk.pos_tag(words):
+ if pos_tag=='NN' or pos_tag=='NNS':
+ nouns.append(lemmatizer.lemmatize(word))
+ return " ".join(nouns).lower()
+
+
+def normalize_attribute_label(label, tokenizer):
+ words = tokenizer.tokenize(label)
+ attributes = []
+ for word in words:
+ attributes.append("".join(word))
+ return " ".join(attributes).lower()
+
+
+def normalized_labels():
+ regions_with_attributes_filename = os.path.join(_outdir,
+ _regions_with_attributes)
+ with open(regions_with_attributes_filename) as file:
+ regions = json.load(file)
+
+ object_labels = dict()
+ attribute_labels = dict()
+ object_count = 0
+ attribute_count = 0
+ lemmatizer = nltk.stem.WordNetLemmatizer()
+ tokenizer = nltk.tokenize.RegexpTokenizer("([^\W\d]+'[^\W\d]+)|([^\W\d]+)")
+ for region_id, region_data in regions.items():
+ for object in region_data['object_names']:
+ object = normalize_object_label(object, lemmatizer)
+ if object not in object_labels:
+ object_labels[object] = 1
+ object_count += 1
+ else:
+ object_labels[object] += 1
+
+ for attribute in region_data['attributes']:
+ attribute = normalize_attribute_label(attribute, tokenizer)
+ if attribute not in attribute_labels:
+ attribute_labels[attribute] = 1
+ attribute_count += 1
+ else:
+ attribute_labels[attribute] += 1
+
+ print 'Objects: {} Attributes: {}'.format(object_count,
+ attribute_count)
+ object_labels_out_filename = os.path.join(_outdir,
+ _raw_object_labels)
+ with open(object_labels_out_filename, 'w') as outfile:
+ json.dump(object_labels, outfile, sort_keys=True, indent=4)
+
+ attribute_labels_out_filename = os.path.join(_outdir,
+ _raw_attribute_labels)
+ with open(attribute_labels_out_filename, 'w') as outfile:
+ json.dump(attribute_labels, outfile, sort_keys=True, indent=4)
+
+ print 'Number of object labels: {}'.format(object_count)
+ print 'Number of attribute labels: {}'.format(attribute_count)
+
+
+def normalize_region_object_attribute_labels():
+ regions_with_attributes_filename = os.path.join(_outdir,
+ _regions_with_attributes)
+ with open(regions_with_attributes_filename) as file:
+ regions = json.load(file)
+
+ lemmatizer = nltk.stem.WordNetLemmatizer()
+ tokenizer = nltk.tokenize.RegexpTokenizer("([^\W\d]+'[^\W\d]+)|([^\W\d]+)")
+ count = 0
+ for region_id, region_data in regions.items():
+ object_names = []
+ for object in region_data['object_names']:
+ if object_names == "":
+ continue
+ object_names.append(normalize_object_label(object, lemmatizer))
+ region_data['object_names'] = object_names
+
+ attributes = []
+ for attribute in region_data['attributes']:
+ if attributes == "":
+ continue
+ attributes.append(normalize_attribute_label(attribute, tokenizer))
+ region_data['attributes'] = attributes
+ count += 1
+ print '{}/{}'.format(count, len(regions))
+ regions_with_labels_out_filename = os.path.join(_outdir,
+ _regions_with_labels)
+ with open(regions_with_labels_out_filename, 'w') as outfile:
+ json.dump(regions, outfile, sort_keys=True, indent=4)
+
+
+def top_k_object_labels(k):
+ raw_object_labels_filename = os.path.join(_outdir,
+ _raw_object_labels)
+ with open(raw_object_labels_filename, 'r') as file:
+ raw_object_labels = json.load(file)
+
+ sorted_raw_object_labels = \
+ [key for key, value in sorted(raw_object_labels.items(),
+ key = operator.itemgetter(1),
+ reverse = True)]
+
+ object_labels = dict()
+ for i in xrange(min(k,len(sorted_raw_object_labels))):
+ object_labels[sorted_raw_object_labels[i]] = i
+
+ if "" in object_labels:
+ object_labels[_unknown_token] = object_labels[""]
+ del object_labels[""]
+
+ object_labels_filename = os.path.join(_outdir,
+ _object_labels)
+ with open(object_labels_filename, 'w') as outfile:
+ json.dump(object_labels, outfile, sort_keys=True, indent=4)
+
+
+def top_k_attribute_labels(k):
+ raw_attribute_labels_filename = os.path.join(_outdir,
+ _raw_attribute_labels)
+ with open(raw_attribute_labels_filename, 'r') as file:
+ raw_attribute_labels = json.load(file)
+
+ sorted_raw_attribute_labels = \
+ [key for key, value in sorted(raw_attribute_labels.items(),
+ key = operator.itemgetter(1),
+ reverse = True)]
+
+ attribute_labels = dict()
+ for i in xrange(min(k,len(sorted_raw_attribute_labels))):
+ attribute_labels[sorted_raw_attribute_labels[i]] = i
+
+ if "" in attribute_labels:
+ attribute_labels[_unknown_token] = attribute_labels[""]
+ del attribute_labels[""]
+
+ attribute_labels_filename = os.path.join(_outdir,
+ _attribute_labels)
+ with open(attribute_labels_filename, 'w') as outfile:
+ json.dump(attribute_labels, outfile, sort_keys=True, indent=4)
+
+
+def crop_region(region_info):
+ region_id, region_data = region_info
+ image_filename = os.path.join(_datadir,
+ 'images/' +
+ str(region_data['image_id']) + '.jpg')
+ image = image_io.imread(image_filename)
+
+ if len(image.shape)==3:
+ im_h, im_w, im_c =image.shape
+ elif len(image.shape)==2:
+ im_h, im_w =image.shape
+ image_tmp = np.zeros([im_h, im_w, 3], dtype=image.dtype)
+ for c in xrange(3):
+ image_tmp[:,:,c] = image
+ image = image_tmp
+
+ x = min(im_w-1,max(0,region_data["x"]))
+ y = min(im_h-1,max(0,region_data["y"]))
+ h = min(im_h-y,max(region_data["h"],1))
+ w = min(im_w-x,max(region_data["w"],1))
+
+ cropped_region = image_io.imresize(image[y:y+h,x:x+w,:],
+ output_size=(_im_h, _im_w))
+ image_subdir = os.path.join(_cropped_regions_dir,
+ str(region_data['image_id']))
+ if not os.path.exists(image_subdir):
+ os.mkdir(image_subdir)
+
+ image_out_filename = os.path.join(image_subdir,
+ str(region_id) + '.jpg')
+
+ image_io.imwrite(cropped_region, image_out_filename)
+
+
+def crop_regions_parallel():
+ regions_filename = os.path.join(_outdir,
+ _regions_with_labels)
+ with open(regions_filename) as file:
+ regions = json.load(file)
+
+ if not os.path.exists(_cropped_regions_dir):
+ os.mkdir(_cropped_regions_dir)
+
+ pool = Pool(24)
+ try:
+ pool.map(crop_region, regions.items())
+ except:
+ pool.close()
+ pool.close()
+
+
+def crop_regions():
+ regions_filename = os.path.join(_outdir,
+ _regions_with_labels)
+ with open(regions_filename) as file:
+ regions = json.load(file)
+
+ if not os.path.exists(_cropped_regions_dir):
+ os.mkdir(_cropped_regions_dir)
+
+ count = 0
+ for region_id, region_data in regions.items():
+ try:
+ image_filename = os.path.join(_datadir,
+ 'images/' +
+ str(region_data['image_id']) + '.jpg')
+ image = image_io.imread(image_filename)
+
+ if len(image.shape)==3:
+ im_h, im_w, im_c =image.shape
+ elif len(image.shape)==2:
+ im_h, im_w =image.shape
+ image_tmp = np.zeros([im_h, im_w, 3], dtype=image.dtype)
+ for c in xrange(3):
+ image_tmp[:,:,c] = image
+ image = image_tmp
+
+ x = min(im_w-1,max(0,region_data["x"]))
+ y = min(im_h-1,max(0,region_data["y"]))
+ h = min(im_h-y,max(region_data["h"],1))
+ w = min(im_w-x,max(region_data["w"],1))
+
+ cropped_region = image_io.imresize(image[y:y+h,x:x+w,:],
+ output_size=(_im_h, _im_w))
+ image_subdir = os.path.join(_cropped_regions_dir,
+ str(region_data['image_id']))
+ if not os.path.exists(image_subdir):
+ os.mkdir(image_subdir)
+
+ image_out_filename = os.path.join(image_subdir,
+ str(region_id) + '.jpg')
+
+ image_io.imwrite(cropped_region, image_out_filename)
+
+ count += 1
+
+ print '{}/{}'.format(count, len(regions))
+ except:
+ print region_id, region_data
+ raise
+
+
+if __name__=='__main__':
+ # parse_objects()
+ # parse_attributes()
+ # add_regions_without_attributes()
+ # stats()
+ # normalized_labels()
+ # normalize_region_object_attribute_labels()
+ # top_k_object_labels(1000)
+ # top_k_attribute_labels(1000)
+ crop_regions_parallel()
--
GitLab