From 2a3cabf73f7760d243ce9005b4968d919e2b3237 Mon Sep 17 00:00:00 2001
From: xdu12 <xdu12@illinois.edu>
Date: Thu, 23 Nov 2017 23:40:35 -0600
Subject: [PATCH] Second commit
---
hicopyLoader.py | 930 ++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 930 insertions(+)
create mode 100644 hicopyLoader.py
diff --git a/hicopyLoader.py b/hicopyLoader.py
new file mode 100644
index 0000000..c50d405
--- /dev/null
+++ b/hicopyLoader.py
@@ -0,0 +1,930 @@
+# -*- coding: utf-8 -*-
+"""
+Created on Sun Oct 15 02:00:25 2017
+
+@author: du_xi
+"""
+__author__ = 'Xiaodan Du'
+__version__ = '1.0'
+
+import numpy as np
+from hicopy import hicopy, test_dim
+import copy
+import matplotlib.pyplot as plt
+import matplotlib.patches as patches
+import matplotlib.image as mpimg
+from matplotlib import rcParams
+from matplotlib.lines import Line2D
+from matplotlib.collections import PatchCollection
+import pdb
+"""
+This is a class developed for Shuai Tang's neural network.
+It will load and parse data from a hicopy object and pass it to the neural network
+index/id of image and cat(hoi) starts from 1 instead of 0
+index/ids must be integer and must be within the total number
+Default location of the JSON file:'C:\\Users\\du_xi\\Dropbox\\Research Shuai Tang\\list.json'
+Default location of the images:'C:\\Users\\du_xi\\.spyder-py3\\hico_20160224_det'
+__getitem__ fucntion has been overrided. hicopyLoader[i] will return information of the ith image
+The information returned by hicoipyLoader[i]: a dictionary 'inputs' that contains 3 sub-dictionaries - 'data', 'im_info' and 'labels'
+'data' contains 'im', 'boxes' and 'box_cls':
+ 'im': image in np array format (please see readImgs() in hicopy)
+ 'boxes': an array of all bounding boxes in the given image
+ 'box_cls': one-hot of all objects in the given image
+'im_info' contains 'im_id', 'im_hw', 'ids' and 'person_ids':
+ 'im_id': integer, index of the image
+ 'im_hw': list with two elements, height and width, respectively
+ 'ids': a list with the length of the number of objects in an image. The list starts from 0, increments 1 and ends at len(box_cls)-1
+ E.g. for an image with 6 objects, 'ids' should be [0,1,2,3,4,5]
+ 'person_ids': a list of index in 'ids' that are 'human'
+ E.g. if object 0 and 4 are humen in the above example, 'person_ids' should be [0,4]
+'labels' contains 'labels_prop' and 'labels_cls'
+ 'labels_prop': one-hot format of the interaction between humen and objects. np array with dimension of (len(person_ids),len(ids))
+ labels_prop[i,j]=1 if person_ids[i] has interaction with ids[j]
+ =0 otherwise (human does not have any interactions with himself/herself)
+ 'labels_cls': one-hot format of what kind of interaction between humen and objects.
+ labels_cls[i,j,k]=1 if person_ids[i] has interaction k with ids[j]
+ =0 otherwise
+
+
+Functions:
+ non_max_suppression_fast: suppress given boxes for given threshhold
+ it will return a list of box index showing which box will be replaced.
+ e.g. if boxes 0,1,2,3,4 are the original boxes and box 0 can be represented by 2 and box 4 can be represented by box 1,
+ the function will return [2,1,2,3,1]
+ borrowed from https://www.pyimagesearch.com/2015/02/16/faster-non-maximum-suppression-python/
+ onehot: turns list into one-hot format
+ uniqueList: uniqify given list
+ borrowed from https://www.peterbe.com/plog/uniqifiers-benchmark
+
+
+ hicopyLoader Class:
+ -hicopyLoader: It will load and parse data from a hicopy object and pass it to the neural network
+ -non_maximum_suppression_filter: a filter for the program to do non maximum suppression with given threshhold
+ -get_nms_index_human: list, result of non-maximum suppression of human bounding boxes
+ -get_nms_index_all: list, result of non-maximum suppression of all bounding boxes
+ -__getitem__: overrides the default getitem function and returns dictionary 'inputs'
+ -getData: returns dictionary 'data'
+ -getBoxes: returns np array 'boxes'
+ -getBoxesHuman: returns a np array that contains all human bounding boxes
+ -getBoxesObject: returns a np array that contains all object bounding boxes
+ -getBoxcls: returns one-hot 'box_cls'
+ -getImInfo: returns dictionary 'im_info'
+ -getIm_hw: returns list 'im_hw'
+ -getIds: returns list 'ids'
+ -getPerson_ids: returns list 'person_ids'
+ -getLabels: returns dictionary 'labels'
+ -getLabels_prop: returns one-hot 'labels_prop'
+ -getLabels_cls: returns one-hot 'labels_cls'
+ -getNumOfNoInteraction: returns an integer of the total number of 'no_interactions' in the image
+ -getObjectsOfGivenPerson: given a certain person in a certain image, the function returns the indices (in 'ids') of which objects are related to the person
+ -getVerbsOfGivenPerson: given a certain person in a certain image, the function returns the indices (in 'ids')of which actions are related to the person
+ -getObjectTypeOfGivenPerson: given a certain person in a certain image, the function returns the ids of the objects that are related to the person
+ -visualize: visualizes a given image for each hoi
+ -visualize_box_conn_one: a helper function of visualize()
+
+"""
+def onehot(dataList,totalState):
+ """
+ inputs:
+ dataList: a list of data
+ totalState: a list of all possible states
+ outputs:
+ a np array of one-hot
+ """
+ onehotlist=list()
+ for i, objn in enumerate(dataList):
+ temp=np.zeros(totalState)
+ temp[objn-1]=1
+ onehotlist.append(temp)
+ return np.asarray(onehotlist)
+
+def non_max_suppression_fast(boxes, overlapThresh):
+ """
+ inputs:
+ boxes: a list of bounding boxes
+ overlapThresh: the threshold (between 0 and 1)
+ outputs:
+ replace: a list of which box should be replaced by which box
+ """
+ # if there are no boxes, return an empty list
+ if len(boxes) == 0:
+ return []
+
+ # if the bounding boxes integers, convert them to floats --
+ # this is important since we'll be doing a bunch of divisions
+ if boxes.dtype.kind == "i":
+ boxes = boxes.astype("float")
+
+ # initialize the list of picked indexes
+ pick = []
+
+ # grab the coordinates of the bounding boxes
+ x1 = boxes[:,0]
+ y1 = boxes[:,1]
+ x2 = boxes[:,2]
+ y2 = boxes[:,3]
+
+ # compute the area of the bounding boxes and sort the bounding
+ # boxes by the bottom-right y-coordinate of the bounding box
+ area = (x2 - x1 + 1) * (y2 - y1 + 1)
+ idxs = np.argsort(y2)
+ replace=idxs*0
+ # keep looping while some indexes still remain in the indexes
+ # list
+ while len(idxs) > 0:
+ # grab the last index in the indexes list and add the
+ # index value to the list of picked indexes
+ last = len(idxs) - 1
+ i = idxs[last]
+ pick.append(i)
+ # find the largest (x, y) coordinates for the start of
+ # the bounding box and the smallest (x, y) coordinates
+ # for the end of the bounding box
+ xx1 = np.maximum(x1[i], x1[idxs[:last]])
+ yy1 = np.maximum(y1[i], y1[idxs[:last]])
+ xx2 = np.minimum(x2[i], x2[idxs[:last]])
+ yy2 = np.minimum(y2[i], y2[idxs[:last]])
+
+ # compute the width and height of the bounding box
+ w = np.maximum(0, xx2 - xx1 + 1)
+ h = np.maximum(0, yy2 - yy1 + 1)
+
+ # compute the ratio of overlap
+# pdb.set_trace()
+ overlap = (w * h) / area[idxs[:last]]
+ replace[idxs[np.concatenate(([last], np.where(overlap > overlapThresh)[0]))]]=i
+ # delete all indexes from the index list that have
+ idxs = np.delete(idxs, np.concatenate(([last], np.where(overlap > overlapThresh)[0])))
+
+ # return only the bounding boxes that were picked using the
+ # integer data type
+ return replace
+
+def uniqueList(seq):
+ """
+ inputs:
+ seq: a list to be uniquified
+ outputs:
+ noDupes: a list with unique elements
+ """
+ # order preserving
+ noDupes = []
+ [noDupes.append(i) for i in seq if not noDupes.count(i)]
+ return noDupes
+
+class hicopyLoader:
+ def __init__(self, hicopyObj, useNMS=False, thresh=0.8):
+ """
+ inputs:
+ hicopyObj: an hicopy object
+ useNMS: 'True' or 'False' to use non-maximum suppression
+ thresh: threshold of non-maximum suppression
+ outputs:
+ """
+ print('Creating hicopyLoader object...')
+ self.hicopyObj=hicopyObj
+ if useNMS==True:
+ self.thresh=thresh
+ self.non_maximum_suppression_filter(thresh)
+ print('Done.')
+
+ def non_maximum_suppression_filter(self,thresh):
+ """
+ inputs:
+ thresh: threshhold for non-maximum suppression
+ outputs:
+ """
+ for i,objn in enumerate(self.hicopyObj.bbox_train):
+ if self.hicopyObj.checkMultiHoi('bbox_train',i)==0:
+ #bboxhuman
+ if isinstance(self.hicopyObj.bbox_train[i]['hoi']['bboxhuman'],list):
+ allHuman=self.getBoxesHuman(i+1)
+ selectedHuman=non_max_suppression_fast(allHuman,thresh)
+ curr=0
+ for k in range(len(self.hicopyObj.bbox_train[i]['hoi']['bboxhuman'])):
+ self.hicopyObj.bbox_train[i]['hoi']['bboxhuman'][k]['x1']=allHuman[selectedHuman[curr]][0]
+ self.hicopyObj.bbox_train[i]['hoi']['bboxhuman'][k]['y1']=allHuman[selectedHuman[curr]][1]
+ self.hicopyObj.bbox_train[i]['hoi']['bboxhuman'][k]['x2']=allHuman[selectedHuman[curr]][2]
+ self.hicopyObj.bbox_train[i]['hoi']['bboxhuman'][k]['y2']=allHuman[selectedHuman[curr]][3]
+ curr=curr+1
+ #bboxobject
+ if isinstance(self.hicopyObj.bbox_train[i]['hoi']['bboxobject'],list):
+ allObject=self.getBoxesObject(i+1)
+ selectedObject=non_max_suppression_fast(allObject,thresh)
+# if i==243:
+# print(self.hicopyObj.bbox_train[i]['filename'])
+# print(allObject,selectedObject)
+ curr2=0
+ for k in range(len(self.hicopyObj.bbox_train[i]['hoi']['bboxobject'])):
+ self.hicopyObj.bbox_train[i]['hoi']['bboxobject'][k]['x1']=allObject[selectedObject[curr2]][0]
+ self.hicopyObj.bbox_train[i]['hoi']['bboxobject'][k]['y1']=allObject[selectedObject[curr2]][1]
+ self.hicopyObj.bbox_train[i]['hoi']['bboxobject'][k]['x2']=allObject[selectedObject[curr2]][2]
+ self.hicopyObj.bbox_train[i]['hoi']['bboxobject'][k]['y2']=allObject[selectedObject[curr2]][3]
+ curr2=curr2+1
+ else:
+ hoiNum=self.hicopyObj.checkMultiHoi('bbox_train',i)
+ allHuman=self.getBoxesHuman(i+1)
+ selectedHuman=non_max_suppression_fast(allHuman,thresh)
+ allObject=self.getBoxesObject(i+1)
+ selectedObject=non_max_suppression_fast(allObject,thresh)
+ curr=0
+ curr2=0
+ for j in range(hoiNum):
+ #bboxhuman
+ if isinstance(self.hicopyObj.bbox_train[i]['hoi'][j]['bboxhuman'],list):
+ for k in range(len(self.hicopyObj.bbox_train[i]['hoi'][j]['bboxhuman'])):
+ self.hicopyObj.bbox_train[i]['hoi'][j]['bboxhuman'][k]['x1']=allHuman[selectedHuman[curr]][0]
+ self.hicopyObj.bbox_train[i]['hoi'][j]['bboxhuman'][k]['y1']=allHuman[selectedHuman[curr]][1]
+ self.hicopyObj.bbox_train[i]['hoi'][j]['bboxhuman'][k]['x2']=allHuman[selectedHuman[curr]][2]
+ self.hicopyObj.bbox_train[i]['hoi'][j]['bboxhuman'][k]['y2']=allHuman[selectedHuman[curr]][3]
+ curr=curr+1
+ elif isinstance(self.hicopyObj.bbox_train[i]['hoi'][j]['bboxhuman'],dict):
+ self.hicopyObj.bbox_train[i]['hoi'][j]['bboxhuman']['x1']=allHuman[selectedHuman[curr]][0]
+ self.hicopyObj.bbox_train[i]['hoi'][j]['bboxhuman']['y1']=allHuman[selectedHuman[curr]][1]
+ self.hicopyObj.bbox_train[i]['hoi'][j]['bboxhuman']['x2']=allHuman[selectedHuman[curr]][2]
+ self.hicopyObj.bbox_train[i]['hoi'][j]['bboxhuman']['y2']=allHuman[selectedHuman[curr]][3]
+ curr=curr+1
+ #bboxobject
+ if isinstance(self.hicopyObj.bbox_train[i]['hoi'][j]['bboxobject'],list):
+ for k in range(len(self.hicopyObj.bbox_train[i]['hoi'][j]['bboxobject'])):
+ self.hicopyObj.bbox_train[i]['hoi'][j]['bboxobject'][k]['x1']=allObject[selectedObject[curr2]][0]
+ self.hicopyObj.bbox_train[i]['hoi'][j]['bboxobject'][k]['y1']=allObject[selectedObject[curr2]][1]
+ self.hicopyObj.bbox_train[i]['hoi'][j]['bboxobject'][k]['x2']=allObject[selectedObject[curr2]][2]
+ self.hicopyObj.bbox_train[i]['hoi'][j]['bboxobject'][k]['y2']=allObject[selectedObject[curr2]][3]
+ curr2=curr2+1
+ elif isinstance(self.hicopyObj.bbox_train[i]['hoi'][j]['bboxobject'],dict):
+ self.hicopyObj.bbox_train[i]['hoi'][j]['bboxobject']['x1']=allObject[selectedObject[curr2]][0]
+ self.hicopyObj.bbox_train[i]['hoi'][j]['bboxobject']['y1']=allObject[selectedObject[curr2]][1]
+ self.hicopyObj.bbox_train[i]['hoi'][j]['bboxobject']['x2']=allObject[selectedObject[curr2]][2]
+ self.hicopyObj.bbox_train[i]['hoi'][j]['bboxobject']['y2']=allObject[selectedObject[curr2]][3]
+ curr2=curr2+1
+
+ def get_nms_index_human(self,index,thresh):
+ """
+ inputs:
+ index: id of the image
+ thresh: threshold for nms
+ outputs:
+ selectedHuman: a np array of all the human boxes
+ """
+ allHuman=self.getBoxesHuman(index)
+ selectedHuman=non_max_suppression_fast(allHuman,thresh)
+ return selectedHuman
+
+ def get_nms_index_all(self,index,thresh):
+ """
+ inputs:
+ indexx: id of the image
+ thresh: threshold for nms
+ outputs:
+ selectedHumanAndObject: a np array of all the bounding boxes
+ """
+ allHumanAndObject=self.getBoxes(index,False)
+ selectedHumanAndObject=non_max_suppression_fast(allHumanAndObject,thresh)
+ return selectedHumanAndObject
+
+ def __getitem__(self,index):
+ """
+ inputs:
+ index: image id
+ outputs:
+ inputs: an dictionary 'inputs'
+ """
+ inputs={}
+ inputs['data'] = self.getData(index)
+ inputs['im_info'] = self.getImInfo(index)
+ inputs['labels'] = self.getLabels(index)
+ return inputs
+
+ def getData(self,index):
+ """
+ inputs:
+ index: image id
+ outputs:
+ data: a dictionary 'data'
+ """
+ data={}
+ data['im']=self.hicopyObj.readImgs([index]) if isinstance(index,int) else self.hicopyObj.readImgs(index)
+ data['boxes']=self.getBoxes(index)
+ data['box_cls']=self.getBoxcls(index)
+ return data
+
+ def getBoxes(self,index,unique=True):
+ """
+ inputs:
+ index: image id
+ unique: boolean, if you want the output to only include a box once even if it appears in different hois
+ outputs:
+ a np array of all boxes
+ Note: boxes will be represented in the form [x1,y1,x2,y2]
+
+ """
+ #[x1,y1,x2,y2]
+ allBoxes=list()
+ if self.hicopyObj.checkMultiHoi('bbox_train',index-1)==0:
+ if isinstance(self.hicopyObj.bbox_train[index-1]['hoi']['bboxhuman'],dict):
+ currentBox=[]
+ currentBox.append(self.hicopyObj.bbox_train[index-1]['hoi']['bboxhuman']['x1'])
+ currentBox.append(self.hicopyObj.bbox_train[index-1]['hoi']['bboxhuman']['y1'])
+ currentBox.append(self.hicopyObj.bbox_train[index-1]['hoi']['bboxhuman']['x2'])
+ currentBox.append(self.hicopyObj.bbox_train[index-1]['hoi']['bboxhuman']['y2'])
+ allBoxes.append(currentBox)
+ else:
+ if self.hicopyObj.bbox_train[index-1]['hoi']['bboxhuman']!=None:
+ for boxi in self.hicopyObj.bbox_train[index-1]['hoi']['bboxhuman']:
+ currentBox=[]
+ currentBox.append(boxi['x1'])
+ currentBox.append(boxi['y1'])
+ currentBox.append(boxi['x2'])
+ currentBox.append(boxi['y2'])
+ allBoxes.append(currentBox)
+ if isinstance(self.hicopyObj.bbox_train[index-1]['hoi']['bboxobject'],dict):
+ currentBox=[]
+ currentBox.append(self.hicopyObj.bbox_train[index-1]['hoi']['bboxobject']['x1'])
+ currentBox.append(self.hicopyObj.bbox_train[index-1]['hoi']['bboxobject']['y1'])
+ currentBox.append(self.hicopyObj.bbox_train[index-1]['hoi']['bboxobject']['x2'])
+ currentBox.append(self.hicopyObj.bbox_train[index-1]['hoi']['bboxobject']['y2'])
+ allBoxes.append(currentBox)
+ else:
+ if self.hicopyObj.bbox_train[index-1]['hoi']['bboxobject']!=None:
+ for boxi in self.hicopyObj.bbox_train[index-1]['hoi']['bboxobject']:
+ currentBox=[]
+ currentBox.append(boxi['x1'])
+ currentBox.append(boxi['y1'])
+ currentBox.append(boxi['x2'])
+ currentBox.append(boxi['y2'])
+ allBoxes.append(currentBox)
+ else:
+ for hoii in self.hicopyObj.bbox_train[index-1]['hoi']:
+ if isinstance(hoii['bboxhuman'],dict):
+ currentBox=[]
+ currentBox.append(hoii['bboxhuman']['x1'])
+ currentBox.append(hoii['bboxhuman']['y1'])
+ currentBox.append(hoii['bboxhuman']['x2'])
+ currentBox.append(hoii['bboxhuman']['y2'])
+ allBoxes.append(currentBox)
+ else:
+ if hoii['bboxhuman']!=None:
+ for boxi in hoii['bboxhuman']:
+ currentBox=[]
+ currentBox.append(boxi['x1'])
+ currentBox.append(boxi['y1'])
+ currentBox.append(boxi['x2'])
+ currentBox.append(boxi['y2'])
+ allBoxes.append(currentBox)
+ if isinstance(hoii['bboxobject'],dict):
+ currentBox=[]
+ currentBox.append(hoii['bboxobject']['x1'])
+ currentBox.append(hoii['bboxobject']['y1'])
+ currentBox.append(hoii['bboxobject']['x2'])
+ currentBox.append(hoii['bboxobject']['y2'])
+ allBoxes.append(currentBox)
+ else:
+ if hoii['bboxobject']!=None:
+ for boxi in hoii['bboxobject']:
+ currentBox=[]
+ currentBox.append(boxi['x1'])
+ currentBox.append(boxi['y1'])
+ currentBox.append(boxi['x2'])
+ currentBox.append(boxi['y2'])
+ allBoxes.append(currentBox)
+ if unique==True:
+ return np.asarray(uniqueList(allBoxes))
+ else:
+ return np.asarray(allBoxes)
+
+ def getBoxesHuman(self,index,unique=False):
+ """
+ inputs:
+ index: image id
+ unique: boolean, if you want the output to only include a box once even if it appears in different hois
+ outputs:
+ a np array of all human boxes
+ Note: boxes will be represented in the form [x1,y1,x2,y2]
+
+ """
+ #[x1,y1,x2,y2]
+ allBoxes=list()
+ if self.hicopyObj.checkMultiHoi('bbox_train',index-1)==0:
+ if isinstance(self.hicopyObj.bbox_train[index-1]['hoi']['bboxhuman'],dict):
+ currentBox=[]
+ currentBox.append(self.hicopyObj.bbox_train[index-1]['hoi']['bboxhuman']['x1'])
+ currentBox.append(self.hicopyObj.bbox_train[index-1]['hoi']['bboxhuman']['y1'])
+ currentBox.append(self.hicopyObj.bbox_train[index-1]['hoi']['bboxhuman']['x2'])
+ currentBox.append(self.hicopyObj.bbox_train[index-1]['hoi']['bboxhuman']['y2'])
+ allBoxes.append(currentBox)
+ else:
+ if self.hicopyObj.bbox_train[index-1]['hoi']['bboxhuman']!=None:
+ for boxi in self.hicopyObj.bbox_train[index-1]['hoi']['bboxhuman']:
+ currentBox=[]
+ currentBox.append(boxi['x1'])
+ currentBox.append(boxi['y1'])
+ currentBox.append(boxi['x2'])
+ currentBox.append(boxi['y2'])
+ allBoxes.append(currentBox)
+ else:
+ for hoii in self.hicopyObj.bbox_train[index-1]['hoi']:
+ if isinstance(hoii['bboxhuman'],dict):
+ currentBox=[]
+ currentBox.append(hoii['bboxhuman']['x1'])
+ currentBox.append(hoii['bboxhuman']['y1'])
+ currentBox.append(hoii['bboxhuman']['x2'])
+ currentBox.append(hoii['bboxhuman']['y2'])
+ allBoxes.append(currentBox)
+ else:
+ if hoii['bboxhuman']!=None:
+ for boxi in hoii['bboxhuman']:
+ currentBox=[]
+ currentBox.append(boxi['x1'])
+ currentBox.append(boxi['y1'])
+ currentBox.append(boxi['x2'])
+ currentBox.append(boxi['y2'])
+ allBoxes.append(currentBox)
+ if unique==False:
+ return np.asarray(allBoxes)
+ else:
+ return np.asarray(uniqueList(allBoxes))
+
+ def getBoxesObject(self,index):
+ """
+ inputs:
+ index: image id
+ unique: boolean, if you want the output to only include a box once even if it appears in different hois
+ outputs:
+ a np array of all object boxes
+ Note: boxes will be represented in the form [x1,y1,x2,y2]
+
+ """
+ #[x1,y1,x2,y2]
+ allBoxes=list()
+ if self.hicopyObj.checkMultiHoi('bbox_train',index-1)==0:
+ if isinstance(self.hicopyObj.bbox_train[index-1]['hoi']['bboxobject'],dict):
+ currentBox=[]
+ currentBox.append(self.hicopyObj.bbox_train[index-1]['hoi']['bboxobject']['x1'])
+ currentBox.append(self.hicopyObj.bbox_train[index-1]['hoi']['bboxobject']['y1'])
+ currentBox.append(self.hicopyObj.bbox_train[index-1]['hoi']['bboxobject']['x2'])
+ currentBox.append(self.hicopyObj.bbox_train[index-1]['hoi']['bboxobject']['y2'])
+ allBoxes.append(currentBox)
+ else:
+ if self.hicopyObj.bbox_train[index-1]['hoi']['bboxobject']!=None:
+ for boxi in self.hicopyObj.bbox_train[index-1]['hoi']['bboxobject']:
+ currentBox=[]
+ currentBox.append(boxi['x1'])
+ currentBox.append(boxi['y1'])
+ currentBox.append(boxi['x2'])
+ currentBox.append(boxi['y2'])
+ allBoxes.append(currentBox)
+ else:
+ for hoii in self.hicopyObj.bbox_train[index-1]['hoi']:
+ if isinstance(hoii['bboxobject'],dict):
+ currentBox=[]
+ currentBox.append(hoii['bboxobject']['x1'])
+ currentBox.append(hoii['bboxobject']['y1'])
+ currentBox.append(hoii['bboxobject']['x2'])
+ currentBox.append(hoii['bboxobject']['y2'])
+ allBoxes.append(currentBox)
+ else:
+ if hoii['bboxobject']!=None:
+ for boxi in hoii['bboxobject']:
+ currentBox=[]
+ currentBox.append(boxi['x1'])
+ currentBox.append(boxi['y1'])
+ currentBox.append(boxi['x2'])
+ currentBox.append(boxi['y2'])
+ allBoxes.append(currentBox)
+ return np.asarray(allBoxes)
+
+ def getBoxcls(self,index,unique=True):
+ """
+ inputs:
+ index: image id
+ unique: boolean, if you want the output to only include a box once even if it appears in different hois
+ outputs:
+ one-hot 'box_cls'
+ """
+ allNname=list()
+ if unique==False:
+ if self.hicopyObj.checkMultiHoi('bbox_train',index-1)==0:
+ if isinstance(self.hicopyObj.bbox_train[index-1]['hoi']['bboxhuman'],dict):
+ allNname.append(self.hicopyObj.getNnameIds('person'))
+ else:
+ if self.hicopyObj.bbox_train[index-1]['hoi']['bboxhuman']!=None:
+ for boxi in self.hicopyObj.bbox_train[index-1]['hoi']['bboxhuman']:
+ allNname.append(self.hicopyObj.getNnameIds('person'))
+ if isinstance(self.hicopyObj.bbox_train[index-1]['hoi']['bboxobject'],dict):
+ allNname.append(self.hicopyObj.getNnameIdsBasedOnHoiId(self.hicopyObj.bbox_train[index-1]['hoi']['id']))
+ else:
+ if self.hicopyObj.bbox_train[index-1]['hoi']['bboxobject']!=None:
+ for boxi in self.hicopyObj.bbox_train[index-1]['hoi']['bboxobject']:
+ allNname.append(self.hicopyObj.getNnameIdsBasedOnHoiId(self.hicopyObj.bbox_train[index-1]['hoi']['id']))
+ else:
+ for hoii in self.hicopyObj.bbox_train[index-1]['hoi']:
+ if isinstance(hoii['bboxhuman'],dict):
+ allNname.append(self.hicopyObj.getNnameIds('person'))
+ else:
+ if hoii['bboxhuman']!=None:
+ for boxi in hoii['bboxhuman']:
+ allNname.append(self.hicopyObj.getNnameIds('person'))
+ if isinstance(hoii['bboxobject'],dict):
+ allNname.append(self.hicopyObj.getNnameIdsBasedOnHoiId(hoii['id']))
+ else:
+ if hoii['bboxobject']!=None:
+ for boxi in hoii['bboxobject']:
+ allNname.append(self.hicopyObj.getNnameIdsBasedOnHoiId(hoii['id']))
+ return onehot(allNname,self.hicopyObj.getObjectNum())
+ else:
+ included=list()
+# nms_index=self.get_nms_index_all(index,self.thresh)
+ if self.hicopyObj.checkMultiHoi('bbox_train',index-1)==0:
+ if isinstance(self.hicopyObj.bbox_train[index-1]['hoi']['bboxhuman'],dict):
+ if self.hicopyObj.bbox_train[index-1]['hoi']['bboxhuman'] not in included:
+ allNname.append(self.hicopyObj.getNnameIds('person'))
+ included.append(self.hicopyObj.bbox_train[index-1]['hoi']['bboxhuman'])
+ else:
+ if self.hicopyObj.bbox_train[index-1]['hoi']['bboxhuman']!=None:
+ for boxi in self.hicopyObj.bbox_train[index-1]['hoi']['bboxhuman']:
+ if boxi not in included:
+ allNname.append(self.hicopyObj.getNnameIds('person'))
+ included.append(boxi)
+ if isinstance(self.hicopyObj.bbox_train[index-1]['hoi']['bboxobject'],dict):
+ if self.hicopyObj.bbox_train[index-1]['hoi']['bboxobject'] not in included:
+ allNname.append(self.hicopyObj.getNnameIdsBasedOnHoiId(self.hicopyObj.bbox_train[index-1]['hoi']['id']))
+ included.append(self.hicopyObj.bbox_train[index-1]['hoi']['bboxobject'])
+ else:
+ if self.hicopyObj.bbox_train[index-1]['hoi']['bboxobject']!=None:
+ for boxi in self.hicopyObj.bbox_train[index-1]['hoi']['bboxobject']:
+ if boxi not in included:
+ allNname.append(self.hicopyObj.getNnameIdsBasedOnHoiId(self.hicopyObj.bbox_train[index-1]['hoi']['id']))
+ included.append(boxi)
+ else:
+ for hoii in self.hicopyObj.bbox_train[index-1]['hoi']:
+ if isinstance(hoii['bboxhuman'],dict):
+ if hoii['bboxhuman'] not in included:
+ allNname.append(self.hicopyObj.getNnameIds('person'))
+ included.append(hoii['bboxhuman'])
+ else:
+ if hoii['bboxhuman']!=None:
+ for boxi in hoii['bboxhuman']:
+ if boxi not in included:
+ allNname.append(self.hicopyObj.getNnameIds('person'))
+ included.append(boxi)
+ if isinstance(hoii['bboxobject'],dict):
+ if hoii['bboxobject'] not in included:
+ allNname.append(self.hicopyObj.getNnameIdsBasedOnHoiId(hoii['id']))
+ included.append(hoii['bboxobject'])
+ else:
+ if hoii['bboxobject']!=None:
+ for boxi in hoii['bboxobject']:
+ if boxi not in included:
+ allNname.append(self.hicopyObj.getNnameIdsBasedOnHoiId(hoii['id']))
+ included.append(boxi)
+ return onehot(allNname,self.hicopyObj.getObjectNum())
+
+ def getImInfo(self,index):
+ """
+ inputs:
+ index: image id
+ outputs:
+ im_info: a dictionary 'im_info'
+ """
+ im_info={}
+ im_info['im_id']=index
+ im_info['im_hw']=self.getIm_hw(index)
+ im_info['ids']=self.getIds(index)
+ im_info['person_ids']=self.getPerson_ids(index)
+ return im_info
+
+ def getIm_hw(self,index):
+ """
+ inputs:
+ index: image id
+ outputs:
+ a list 'im_hw'
+ """
+ return [self.hicopyObj.bbox_train[index-1]['size']['height'],self.hicopyObj.bbox_train[index-1]['size']['width']]
+
+ def getIds(self,index,unique=True):
+ """
+ inputs:
+ index: image id
+ unique:boolean, if you want the output to only include a box once even if it appears in different hois
+ outputs:
+ a list 'ids'
+ """
+ return list(range(len(self.getBoxcls(index,unique))))
+
+ def getPerson_ids(self,index,unique=True):
+ """
+ inputs:
+ index: image id
+ unique:boolean, if you want the output to only include a box once even if it appears in different hois
+ outputs:
+ a sorted list 'person_ids'
+ """
+ wholeList=self.getIds(index,unique)
+ box_cls=self.getBoxcls(index,unique)
+ person_ids=list()
+ for i in wholeList:
+ if box_cls[i,self.hicopyObj.getNnameIds('person')-1]==1:
+ person_ids.append(i)
+ return sorted(person_ids)
+
+ def getLabels(self,index):
+ """
+ inputs:
+ index: image id
+ outputs:
+ labels: a dictionary 'labels'
+ """
+ labels={}
+ labels['labels_prop']=self.getLabels_prop(index)
+ labels['labels_cls']=self.getLabels_cls(index)
+ return labels
+
+ def getLabels_prop(self,index):
+ """
+ inputs:
+ index: image id
+ outputs:
+ one-hot 'labels_prop'
+ """
+ ids=self.getIds(index)
+ ids_notUnique=self.getIds(index,False)
+ person_ids=self.getPerson_ids(index)
+ person_ids_notUnique=self.getPerson_ids(index,False)
+ person_ids_nms_index=self.get_nms_index_human(index,0.999)
+ ids_nms_index=self.get_nms_index_all(index,0.999)
+ allBoxes=self.getBoxes(index,False)
+ humanBoxes_unique=self.getBoxesHuman(index,True)
+ allBoxes_unique=self.getBoxes(index,True)
+ labels_prop=np.zeros([len(person_ids),len(ids)])
+ hoiNum=len(self.hicopyObj.bbox_train[index-1]['hoi']) if self.hicopyObj.checkMultiHoi('bbox_train',index-1)>0 else 1
+ conn=list()
+ hum=np.zeros(hoiNum)
+ obj=np.zeros(hoiNum)
+ if hoiNum==1:
+ conn.append(self.hicopyObj.bbox_train[index-1]['hoi']['connection'])
+ if self.hicopyObj.bbox_train[index-1]['hoi']['bboxhuman']==None:
+ hum[0]=0
+ else:
+ hum[0]=len(self.hicopyObj.bbox_train[index-1]['hoi']['bboxhuman']) if isinstance(self.hicopyObj.bbox_train[index-1]['hoi']['bboxhuman'],list) else 1
+ if self.hicopyObj.bbox_train[index-1]['hoi']['bboxobject']==None:
+ obj[0]=0
+ else:
+ obj[0]=len(self.hicopyObj.bbox_train[index-1]['hoi']['bboxobject']) if isinstance(self.hicopyObj.bbox_train[index-1]['hoi']['bboxobject'],list) else 1
+ else:
+ for i in range(hoiNum):
+ conn.append(self.hicopyObj.bbox_train[index-1]['hoi'][i]['connection'])
+ if self.hicopyObj.bbox_train[index-1]['hoi'][i]['bboxhuman']==None:
+ hum[i]=0
+ else:
+ hum[i]=len(self.hicopyObj.bbox_train[index-1]['hoi'][i]['bboxhuman']) if isinstance(self.hicopyObj.bbox_train[index-1]['hoi'][i]['bboxhuman'],list) else 1
+ if self.hicopyObj.bbox_train[index-1]['hoi'][i]['bboxobject']==None:
+ obj[i]=0
+ else:
+ obj[i]=len(self.hicopyObj.bbox_train[index-1]['hoi'][i]['bboxobject']) if isinstance(self.hicopyObj.bbox_train[index-1]['hoi'][i]['bboxobject'],list) else 1
+
+ #print(hum,obj,conn)
+ currId=0
+ for i in range(hoiNum):
+ if test_dim(conn[i])>1:
+ for j in range(len(conn[i])):
+ currp=currId+conn[i][j][0]-1
+ curro=currId+hum[i]+conn[i][j][1]-1
+# labels_prop[person_ids.index(currp),ids.index(curro)]=1
+ currpBox=allBoxes[int(currp)]
+ curroBox=allBoxes[int(curro)]
+ for jj in range(len(person_ids)):
+ if (allBoxes_unique[person_ids[jj]]==currpBox).all():
+ for k in range(len(ids)):
+ if (allBoxes_unique[ids[k]]==curroBox).all():
+# if jj==3:
+# print('up','i=',i,'j=',j,'jj=',jj,'k=',k,currpBox,curroBox)
+ labels_prop[jj,k]=1
+ elif test_dim(conn[i])==1:
+ currp=currId+conn[i][0]-1
+ curro=currId+hum[i]+conn[i][1]-1
+# labels_prop[person_ids.index(currp),ids.index(curro)]=1
+ currpBox=allBoxes[int(currp)]
+ curroBox=allBoxes[int(curro)]
+ for jj in range(len(person_ids)):
+ if (allBoxes_unique[person_ids[jj]]==currpBox).all():
+ for k in range(len(ids)):
+ if (allBoxes_unique[ids[k]]==curroBox).all():
+# if jj==3:
+# print('down',k,currpBox,curroBox)
+ labels_prop[jj,k]=1
+ currId=currId+hum[i]+obj[i]
+ return labels_prop
+
+ def getLabels_cls(self,index):
+ """
+ inputs:
+ index: image id
+ outputs:
+ one-hot: 'labels_cls'
+ """
+ ids=self.getIds(index)
+ ids_notUnique=self.getIds(index,False)
+ person_ids=self.getPerson_ids(index)
+ person_ids_notUnique=self.getPerson_ids(index,False)
+ person_ids_nms_index=self.get_nms_index_human(index,0.999)
+ ids_nms_index=self.get_nms_index_all(index,0.999)
+ allBoxes=self.getBoxes(index,False)
+ humanBoxes_unique=self.getBoxesHuman(index,True)
+ allBoxes_unique=self.getBoxes(index,True)
+ labels_cls=np.zeros([len(person_ids),len(ids),self.hicopyObj.getVerbNum()])
+ hoiNum=len(self.hicopyObj.bbox_train[index-1]['hoi']) if self.hicopyObj.checkMultiHoi('bbox_train',index-1)>0 else 1
+ conn=list()
+ hum=np.zeros(hoiNum)
+ obj=np.zeros(hoiNum)
+ if hoiNum==1:
+ conn.append(self.hicopyObj.bbox_train[index-1]['hoi']['connection'])
+ if self.hicopyObj.bbox_train[index-1]['hoi']['bboxhuman']==None:
+ hum[0]=0
+ else:
+ hum[0]=len(self.hicopyObj.bbox_train[index-1]['hoi']['bboxhuman']) if isinstance(self.hicopyObj.bbox_train[index-1]['hoi']['bboxhuman'],list) else 1
+ if self.hicopyObj.bbox_train[index-1]['hoi']['bboxobject']==None:
+ obj[0]=0
+ else:
+ obj[0]=len(self.hicopyObj.bbox_train[index-1]['hoi']['bboxobject']) if isinstance(self.hicopyObj.bbox_train[index-1]['hoi']['bboxobject'],list) else 1
+ else:
+ for i in range(hoiNum):
+ conn.append(self.hicopyObj.bbox_train[index-1]['hoi'][i]['connection'])
+ if self.hicopyObj.bbox_train[index-1]['hoi'][i]['bboxhuman']==None:
+ hum[i]=0
+ else:
+ hum[i]=len(self.hicopyObj.bbox_train[index-1]['hoi'][i]['bboxhuman']) if isinstance(self.hicopyObj.bbox_train[index-1]['hoi'][i]['bboxhuman'],list) else 1
+ if self.hicopyObj.bbox_train[index-1]['hoi'][i]['bboxobject']==None:
+ obj[i]=0
+ else:
+ obj[i]=len(self.hicopyObj.bbox_train[index-1]['hoi'][i]['bboxobject']) if isinstance(self.hicopyObj.bbox_train[index-1]['hoi'][i]['bboxobject'],list) else 1
+
+ currId=0
+ for i in range(hoiNum):
+ if hoiNum==1:
+ verb=self.hicopyObj.getVnameIdsBasedOnHoiId(self.hicopyObj.bbox_train[index-1]['hoi']['id'])
+ else:
+ verb=self.hicopyObj.getVnameIdsBasedOnHoiId(self.hicopyObj.bbox_train[index-1]['hoi'][i]['id'])
+ if test_dim(conn[i])>1:
+ for j in range(len(conn[i])):
+ currp=currId+conn[i][j][0]-1
+ curro=currId+hum[i]+conn[i][j][1]-1
+# labels_cls[person_ids.index(currp),ids.index(curro),verb-1]=1
+ currpBox=allBoxes[int(currp)]
+ curroBox=allBoxes[int(curro)]
+ for jj in range(len(person_ids)):
+ if (allBoxes_unique[person_ids[jj]]==currpBox).all():
+ for k in range(len(ids)):
+ if (allBoxes_unique[ids[k]]==curroBox).all():
+# if jj==3:
+# print('up','i=',i,'j=',j,'jj=',jj,'k=',k,currpBox,curroBox)
+ labels_cls[jj,k,verb-1]=1
+ elif test_dim(conn[i])==1:
+ currp=currId+conn[i][0]-1
+ curro=currId+hum[i]+conn[i][1]-1
+# labels_cls[person_ids.index(currp),ids.index(curro),verb-1]=1
+ currpBox=allBoxes[int(currp)]
+ curroBox=allBoxes[int(curro)]
+ for jj in range(len(person_ids)):
+ if (allBoxes_unique[person_ids[jj]]==currpBox).all():
+ for k in range(len(ids)):
+ if (allBoxes_unique[ids[k]]==curroBox).all():
+# if jj==3:
+# print('down',k,currpBox,curroBox)
+ labels_cls[jj,k,verb-1]=1
+ currId=currId+hum[i]+obj[i]
+ return labels_cls
+
+ def getNumOfNoInteraction(self,index):
+ """
+ inputs:
+ index: image id
+ outputs:
+ an integer of the number of 'no_interaction'
+ """
+ return self.hicopyObj.getNumOfNoInteraction(index)
+
+ def getObjectsOfGivenPerson(self,index,i):
+ """
+ inputs:
+ index: image id
+ i: integer, person id in person_ids
+ outputs:
+ objects: a list of indices of objects that related to person i
+ """
+ objects=[]
+ labels_prop=self.getLabels_prop(index)
+ for j in range(len(labels_prop[0])):
+ if labels_prop[i,j]==1:
+ objects.append(j)
+ return objects
+
+ def getVerbsOfGivenPerson(self,index,i):
+ """
+ inputs:
+ index: image id
+ i: integer, person id in person_ids
+ outputs:
+ verbs: a list of indices of actions that related to person i
+ """
+ verbs=[]
+ labels_cls=self.getLabels_cls(index)
+ objects=self.getObjectsOfGivenPerson(index,i)
+ for j in objects:
+ currVerb=[]
+ for k in range(len(labels_cls[0,0])):
+ if labels_cls[i,j,k]==1:
+ currVerb.append(k+1)
+ verbs.append(currVerb)
+ return verbs
+
+ def getObjectTypeOfGivenPerson(self,index,i):
+ """
+ inputs:
+ index: image id
+ i: integer, person id in person_ids
+ outputs:
+ type: a list of ids of objects that related to person i
+ """
+ type=[]
+ objects=self.getObjectsOfGivenPerson(index,i)
+ boxcls=self.getBoxcls(index)
+ for j in objects:
+ for k in range(len(boxcls[0])):
+ if boxcls[j,k]==1:
+ type.append(k+1)
+ return type
+
+ def visualize(self,index):
+ """
+ inputs:
+ index: image id
+ outputs:
+ """
+ currItem=self[index]
+ img=currItem['data']['im'][0]
+ rcParams['figure.figsize'] = [20, 20]
+ personBoxes=self.getBoxesHuman(index,True)
+ allBoxes=self.getBoxes(index)
+ for i in range(len(self.getPerson_ids(index))):
+ print('person',i)
+ objects=self.getObjectsOfGivenPerson(index,i)
+ objectBoxes=[]
+ for j in objects:
+ objectBoxes.append(allBoxes[j])
+ fig,ax = plt.subplots(1)
+ imgHandle=plt.imshow(img)
+ plt.axis('off')
+ #print('imshow is above')
+ objectType=self.getObjectTypeOfGivenPerson(index,i)
+ verbs=self.getVerbsOfGivenPerson(index,i)
+ self.visualize_box_conn_one(ax,i,personBoxes[i],objectBoxes,objectType,self.getIm_hw(index)[0],verbs)
+ plt.show()
+ #print('show is above')
+
+ def visualize_box_conn_one(self, ax, i,personBox,objectBoxes,objectType,height,verbs):
+ """
+ helper function of visualize()
+ """
+ rt = [personBox[0], personBox[1], personBox[2]-personBox[0]+1, personBox[3]-personBox[1]+1]
+ rect = patches.Rectangle((rt[0],rt[1]),rt[2],rt[3],linewidth=7,edgecolor='blue',facecolor='none')
+ ax.add_patch(rect)
+ for jj,j in enumerate(objectBoxes):
+ rt2 = [j[0], j[1], j[2]-j[0]+1, j[3]-j[1]+1]
+ rect2 = patches.Rectangle((rt2[0],rt2[1]),rt2[2],rt2[3],linewidth=7,edgecolor='#00ff00',facecolor='none')
+ ax.add_patch(rect2)
+ if self.hicopyObj.getVnameIds('no_interaction') not in verbs[jj]:
+ ct1 = [(personBox[0]+personBox[2])/2, (personBox[1]+personBox[3])/2]
+ ct2 = [(j[0]+j[2])/2, (j[1]+j[3])/2]
+ markers_on = [0, -1]
+ line = Line2D([ct1[0],ct2[0]],[ct1[1],ct2[1]],linewidth=7,color='red',markevery=markers_on,marker='o',markersize=25,markerfacecolor='red')
+ ax.add_line(line)
+ ax.annotate(self.hicopyObj.getNnames(objectType[jj]), xy=(j[0], j[1]-10),fontsize=height/15,color='#00ff00')
+ verbxy=(j[0]+3, j[3]-5)
+ for k in range(len(verbs[jj])):
+ ax.annotate(self.hicopyObj.getVnames(verbs[jj][k]), xy=verbxy,fontsize=height/17,color='red')
+ verbxy=(verbxy[0],verbxy[1]-(height/40) - 4)
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
--
GitLab