Skip to content
GitLab
Explore
Sign in
Primary navigation
Search or go to…
Project
G
GenVQA
Manage
Activity
Members
Code
Repository
Branches
Commits
Tags
Repository graph
Compare revisions
Deploy
Releases
Model registry
Analyze
Contributor analytics
Repository analytics
Model experiments
Help
Help
Support
GitLab documentation
Compare GitLab plans
Community forum
Contribute to GitLab
Provide feedback
Keyboard shortcuts
?
Snippets
Groups
Projects
Show more breadcrumbs
Vision
GenVQA
Commits
3a05e7bc
Commit
3a05e7bc
authored
8 years ago
by
tgupta6
Browse files
Options
Downloads
Patches
Plain Diff
visualize relevance on same image but different questions
parent
977e2b2d
No related branches found
No related tags found
No related merge requests found
Changes
1
Hide whitespace changes
Inline
Side-by-side
Showing
1 changed file
visual_util/visualize_relevance_same_image_different_questions.py
+253
-0
253 additions, 0 deletions
...til/visualize_relevance_same_image_different_questions.py
with
253 additions
and
0 deletions
visual_util/visualize_relevance_same_image_different_questions.py
0 → 100644
+
253
−
0
View file @
3a05e7bc
import
ujson
import
os
import
csv
import
numpy
as
np
import
random
from
matplotlib
import
cm
import
image_io
from
html_writer
import
HtmlWriter
import
pdb
class
RelevanceVisualizer
():
def
__init__
(
self
,
eval_data_json
,
anno_data_json
,
image_dir
,
region_dir
,
output_dir
,
data_type
):
self
.
image_dir
=
image_dir
self
.
region_dir
=
region_dir
self
.
output_dir
=
output_dir
self
.
data_type
=
data_type
self
.
eval_data
=
self
.
read_json_file
(
eval_data_json
)
self
.
anno_data
=
self
.
read_json_file
(
anno_data_json
)
self
.
html_filename
=
os
.
path
.
join
(
output_dir
,
'
index.html
'
)
self
.
html_writer
=
HtmlWriter
(
self
.
html_filename
)
def
read_json_file
(
self
,
filename
):
print
'
Reading {} ...
'
.
format
(
filename
)
with
open
(
filename
,
'
r
'
)
as
file
:
return
ujson
.
load
(
file
)
def
get_image_name
(
self
,
qid
):
image_id
=
self
.
anno_data
[
qid
][
'
image_id
'
]
image_name
=
'
COCO_
'
+
self
.
data_type
+
'
_
'
+
str
(
image_id
).
zfill
(
12
)
image_path
=
os
.
path
.
join
(
image_dir
,
image_name
+
'
.jpg
'
)
return
image_name
,
image_path
def
get_image
(
self
,
key
):
image_name
,
image_path
=
self
.
get_image_name
(
key
)
im
=
image_io
.
imread
(
image_path
)
return
im
def
get_bboxes
(
self
,
key
):
image_name
,
image_path
=
self
.
get_image_name
(
key
)
bbox_dir
=
os
.
path
.
join
(
self
.
region_dir
,
image_name
)
bbox_csv
=
os
.
path
.
join
(
bbox_dir
,
'
edge_boxes.csv
'
)
bboxes
=
[]
with
open
(
bbox_csv
,
'
r
'
)
as
csvfile
:
bbox_reader
=
csv
.
DictReader
(
csvfile
,
delimiter
=
'
,
'
,
fieldnames
=
[
'
x
'
,
'
y
'
,
'
w
'
,
'
h
'
,
'
score
'
])
for
bbox
in
bbox_reader
:
bboxes
.
append
(
bbox
)
return
bboxes
def
get_pos_ans_rel_scores
(
self
,
key
):
ans
,
score
=
self
.
eval_data
[
key
][
'
positive_answer
'
].
items
()[
0
]
return
ans
,
float
(
score
),
self
.
eval_data
[
key
][
'
relevance_scores
'
][
0
]
def
get_pred_ans_rel_scores
(
self
,
key
):
ans
,
score
=
self
.
eval_data
[
key
][
'
positive_answer
'
].
items
()[
0
]
score
=
float
(
score
)
pred_ans_score
=
(
ans
,
score
)
pred_ans_id
=
0
count
=
1
for
ans
,
score
in
self
.
eval_data
[
key
][
'
negative_answers
'
].
items
():
score
=
float
(
score
)
if
score
>
pred_ans_score
[
1
]:
pred_ans_score
=
(
ans
,
score
)
pred_ans_id
=
count
count
+=
1
return
pred_ans_score
[
0
],
pred_ans_score
[
1
],
\
self
.
eval_data
[
key
][
'
relevance_scores
'
][
pred_ans_id
]
def
peakify_relevance
(
self
,
relevance
,
temp
=
10
):
relevance
=
np
.
asarray
(
relevance
)
relevance
=
np
.
log
(
relevance
+
1e-5
)
logits
=
(
relevance
-
np
.
min
(
relevance
))
*
temp
exp_logits
=
np
.
exp
(
logits
)
Z
=
np
.
sum
(
exp_logits
)
relevance
=
exp_logits
/
Z
return
relevance
.
tolist
()
def
get_box_score_pairs
(
self
,
bboxes
,
scores
):
pairs
=
[]
for
i
,
bbox
in
enumerate
(
bboxes
):
pairs
.
append
((
bbox
,
scores
[
i
]))
return
pairs
def
create_relevance_map
(
self
,
key
,
mode
=
'
pos
'
):
image_name
,
image_path
=
self
.
get_image_name
(
key
)
im
=
image_io
.
imread
(
image_path
)
if
len
(
im
.
shape
)
==
2
:
im_h
,
im_w
=
im
.
shape
im_
=
im
im
=
np
.
zeros
([
im_h
,
im_w
,
3
])
for
i
in
xrange
(
3
):
im
[:,:,
i
]
=
im_
bboxes
=
self
.
get_bboxes
(
key
)
if
mode
==
'
pos
'
:
ans
,
ans_score
,
rel_scores
=
self
.
get_pos_ans_rel_scores
(
key
)
elif
mode
==
'
pred
'
:
ans
,
ans_score
,
rel_scores
=
self
.
get_pred_ans_rel_scores
(
key
)
else
:
print
'
mode can only take values {
\'
pred
\'
or
\'
pos
\'
}
'
raise
# rel_scores = self.peakify_relevance(rel_scores)
box_score_pairs
=
self
.
get_box_score_pairs
(
bboxes
,
rel_scores
)
rel_map
=
np
.
zeros
(
im
.
shape
[
0
:
2
])
for
box
,
score
in
box_score_pairs
:
box_map
=
self
.
make_boxmap
(
box
,
im
.
shape
[
0
:
2
])
rel_map
=
rel_map
+
score
*
box_map
# gauss_map = self.make_gaussian(box, im.shape[0:2])
# rel_map = np.maximum(rel_map, score*box_map)
rel_map_
=
cm
.
jet
(
np
.
uint8
(
rel_map
*
255
))[:,:,
0
:
3
]
*
255
# im_rel_map = np.uint8(0.5*im+0.5*rel_map_)
im_rel_map
=
np
.
uint8
(
0.2
*
im
+
0.8
*
np
.
tile
(
rel_map
[:,:,
None
],
[
1
,
1
,
3
])
*
im
)
return
rel_map
,
im_rel_map
,
ans
,
ans_score
def
make_gaussian
(
self
,
box
,
im_size
):
im_h
,
im_w
=
im_size
x
=
np
.
arange
(
im_w
)
y
=
np
.
arange
(
im_h
)
xx
,
yy
=
np
.
meshgrid
(
x
,
y
)
sigma_x
=
float
(
box
[
'
w
'
])
/
4.0
sigma_y
=
float
(
box
[
'
h
'
])
/
4.0
cx
=
float
(
box
[
'
x
'
])
+
float
(
box
[
'
w
'
])
/
2.0
cy
=
float
(
box
[
'
y
'
])
+
float
(
box
[
'
w
'
])
/
2.0
g
=
np
.
exp
(
-
((
xx
-
cx
)
**
2
/
(
2
*
sigma_x
**
2
))
-
((
yy
-
cy
)
**
2
/
(
2
*
sigma_y
**
2
)))
return
g
def
make_boxmap
(
self
,
box
,
im_size
):
im_h
,
im_w
=
im_size
x
=
int
(
box
[
'
x
'
])
y
=
int
(
box
[
'
y
'
])
w
=
int
(
box
[
'
w
'
])
h
=
int
(
box
[
'
h
'
])
map
=
np
.
zeros
(
im_size
)
map
[
y
-
1
:
y
+
h
-
1
,
x
-
1
:
x
+
w
-
1
]
=
1.0
return
map
def
club_same_images_together
(
self
,
qids
):
image_id_to_qid
=
dict
()
for
qid
in
qids
:
image_id
=
rel_vis
.
anno_data
[
'
qid
'
][
'
image_id
'
]
if
image_id
not
in
image_id_to_qid
:
image_id_to_qid
[
image_id
]
=
[]
image_id_to_qid
[
image_id
].
append
(
qid
)
qids_new
=
[]
for
image_id
,
qid_list
in
image_id_to_qid
.
items
():
for
qid
in
qid_list
:
qids_new
.
append
(
qid
)
return
qids_new
def
write_html
(
self
):
col_dict
=
{
0
:
'
Question
'
,
1
:
'
Pos. Answer
'
,
2
:
'
Pos. Relevance
'
,
3
:
'
Pred. Answer
'
,
4
:
'
Pred. Relevance
'
,
5
:
'
Question Id
'
}
self
.
html_writer
.
add_element
(
col_dict
)
random
.
seed
(
0
)
qids
=
sorted
(
rel_vis
.
eval_data
.
keys
())
#qids = self.club_same_images_together(qids)
#random.shuffle(qids)
for
qid
in
qids
[
0
:
5000
]:
print
qid
question
=
rel_vis
.
anno_data
[
qid
][
'
question
'
]
pred_rel
,
pred_im_rel
,
pred_ans
,
pred_score
=
rel_vis
.
create_relevance_map
(
qid
,
mode
=
'
pred
'
)
pos_rel
,
pos_im_rel
,
pos_ans
,
pos_score
=
rel_vis
.
create_relevance_map
(
qid
,
mode
=
'
pos
'
)
# if np.max(pred_rel) < 0.5:
# continue
pred_im_name
=
'
pred_rel_
'
+
qid
+
'
.jpg
'
pos_im_name
=
'
pos_rel_
'
+
qid
+
'
.jpg
'
pred_rel_filename
=
os
.
path
.
join
(
self
.
output_dir
,
pred_im_name
)
pos_rel_filename
=
os
.
path
.
join
(
self
.
output_dir
,
pos_im_name
)
image_io
.
imwrite
(
pred_im_rel
,
pred_rel_filename
)
image_io
.
imwrite
(
pos_im_rel
,
pos_rel_filename
)
im_h
,
im_w
=
pred_rel
.
shape
[
0
:
2
]
col_dict
=
{
0
:
question
,
1
:
pos_ans
+
'
:
'
+
str
(
pos_score
),
2
:
self
.
html_writer
.
image_tag
(
pos_im_name
,
im_h
,
im_w
),
3
:
pred_ans
+
'
:
'
+
str
(
pred_score
),
4
:
self
.
html_writer
.
image_tag
(
pred_im_name
,
im_h
,
im_w
),
5
:
qid
,
}
self
.
html_writer
.
add_element
(
col_dict
)
self
.
html_writer
.
close_file
()
if
__name__
==
'
__main__
'
:
data_type
=
'
val2014
'
data_dir
=
'
/home/ssd/VQA
'
image_dir
=
os
.
path
.
join
(
data_dir
,
data_type
)
region_dir
=
os
.
path
.
join
(
data_dir
,
data_type
+
'
_cropped_large
'
)
anno_data_json
=
os
.
path
.
join
(
data_dir
,
'
mscoco_val2014_annotations_with_parsed_questions.json
'
)
exp_dir
=
'
/home/tanmay/Code/GenVQA/Exp_Results/VQA/ans_through_obj_atr_det_debug_rel/
'
# exp_dir = '/home/tanmay/Desktop/vision-gpu-2/data/tanmay/' + \
# 'GenVQA_Exp_Results/ans_through_obj_atr_det/'
eval_data_json
=
os
.
path
.
join
(
exp_dir
,
'
answer_classifiers/Results/eval_val_data.json
'
)
output_dir
=
os
.
path
.
join
(
exp_dir
,
'
qual_results_val_same_image
'
)
if
not
os
.
path
.
exists
(
output_dir
):
os
.
mkdir
(
output_dir
)
rel_vis
=
RelevanceVisualizer
(
eval_data_json
,
anno_data_json
,
image_dir
,
region_dir
,
output_dir
,
data_type
)
rel_vis
.
write_html
()
This diff is collapsed.
Click to expand it.
Preview
0%
Loading
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Save comment
Cancel
Please
register
or
sign in
to comment