added fast R-CNN

detector_wiggling/detector_wiggle_from_iou_score.m

@@ -0,0 +1,9 @@
+function [overlaps, score, class_ids, avg_cube, avg_cube_per_class] = detector_wiggle_from_iou_score(filename)
+  ld = load(filename, 'ious', 'scores', 'classes');
+
+  avg_cube = [];
+  avg_cube_per_class = [];
+  score = ld.scores;
+  overlaps = ld.ious;
+  class_ids = ld.classes;
+end

plot_correlation_edge_boxes.m

@@ -24,27 +24,65 @@ function plot_correlation_edge_boxes()
   % Recall matrix used for experiments nThresholds x nAlgorithms
   ld = load('data/pascal_voc07_test_recall.mat');
   R = ld.recalls(method_selection,:)';
+  ARs = ld.ARs(method_selection)';
 
   T = ld.iou_thresholds;
 
   ld = load('data/pascal_voc07_test_rcnn_aps.mat');
   rcnn_AP = ld.aps(method_selection);
+  ld = load('data/pascal_voc07_test_frcn_aps.mat');
+  frcn_AP = ld.aps(method_selection);
+  ld = load('data/pascal_voc07_test_frcn_noregr_aps.mat');
+  frcn_noregr_AP = ld.aps(method_selection);
 
   % assign distinct colors to the methods
-  for i = 1:numel(methods)
+  n = numel(methods);
+  for i = 1:n
     methods(i).line_style = '-';
   end
 
   % average recall
-  const_weights = ones(numel(T),1) ./ numel(T);
-  integral_area = sum(R .* repmat(const_weights, [1, n_methods]), 1);
-  methods = plot_weighted_area_color_coded(integral_area, ...
-    rcnn_AP, methods, custom_names, [0.34 0.55 43 55.5], true);
+  mirror_offset = 0.060;
+  offsets = zeros(n, 2);
+  offsets(:,1) = 0.010;
+  offsets(1,1) = offsets(1,1)-mirror_offset;
+  offsets(2,1) = offsets(2,1)-mirror_offset;
+  offsets(6,1) = offsets(6,1)-mirror_offset;
+  offsets(7,1) = offsets(7,1)-mirror_offset;
+  methods = plot_weighted_area_color_coded(ARs, ...
+    rcnn_AP, methods, custom_names, [0.3 0.6 43 55.5], true, offsets);
   hei = 7; wid = 7;
   set(gcf, 'Units','centimeters', 'Position',[0 0 wid hei]);
   set(gcf, 'PaperPositionMode','auto');
   printpdf('figures/RCNN_mAP_recall_area_voc07_edge_boxes.pdf');
 
+  mirror_offset = 0.060;
+  offsets = zeros(n, 2);
+  offsets(:,1) = 0.010;
+  offsets(1:4,1) = offsets(1:4,1)-mirror_offset;
+  offsets(5,2) = offsets(5,2)+0.5;
+  offsets(6:9,2) = offsets(6:9,2)-0.5;
+  plot_weighted_area_color_coded(ARs, ...
+    frcn_AP, methods, custom_names, [0.3 0.6 43 65], false, offsets);
+  hei = 7; wid = 7;
+  set(gcf, 'Units','centimeters', 'Position',[0 0 wid hei]);
+  set(gcf, 'PaperPositionMode','auto');
+  printpdf('figures/FRCN_mAP_recall_area_voc07_edge_boxes.pdf');
+
+  mirror_offset = 0.060;
+  offsets = zeros(n, 2);
+  offsets(:,1) = 0.010;
+  offsets(6:9,1) = offsets(6:9,1)-mirror_offset;
+  offsets(5:6,2) = offsets(5:6,2)+0.5;
+  offsets(1:4,2) = offsets(1:4,2)-0.6;
+  offsets(1:4,1) = offsets(1:4,1)-0.006;
+  plot_weighted_area_color_coded(ARs, ...
+    frcn_noregr_AP, methods, custom_names, [0.3 0.6 43 60], false, offsets);
+  hei = 7; wid = 7;
+  set(gcf, 'Units','centimeters', 'Position',[0 0 wid hei]);
+  set(gcf, 'PaperPositionMode','auto');
+  printpdf('figures/FRCN_noregr_mAP_recall_area_voc07_edge_boxes.pdf');
+
   % plot recall curves for each parameter setting
   fh = figure;
   plot_overlap_recall_curve({methods.best_voc07_candidates_file}, ...
@@ -64,7 +102,8 @@ function plot_correlation_edge_boxes()
   printpdf('figures/recall_1000_voc07_edge_boxes.pdf')
 end
 
-function [methods] = plot_weighted_area_color_coded(areas, AP, methods, custom_names, axis_lim, use_rainbow)
+function [methods] = plot_weighted_area_color_coded(areas, AP, methods, ...
+  custom_names, axis_lim, reassign_colors, label_offsets)
   additional_methods = methods(end);
   additional_areas = areas(end);
   additional_AP = AP(end);
@@ -92,10 +131,12 @@ function plot_correlation_edge_boxes()
     if i == 2; hold on; end
   end
 
-  for i = 1:numel(methods)
-    [~,c_idx] = min((yy(1,:) - areas(i)) .^ 2 + (yy(2,:) - AP(i)) .^ 2);
-    c_idx = max(c_idx - 1, 1);
-    methods(i).color = colors(c_idx,:);
+  if reassign_colors
+    for i = 1:numel(methods)
+      [~,c_idx] = min((yy(1,:) - areas(i)) .^ 2 + (yy(2,:) - AP(i)) .^ 2);
+      c_idx = max(c_idx - 1, 1);
+      methods(i).color = colors(c_idx,:);
+    end
   end
   hold on;
 
@@ -109,14 +150,11 @@ function plot_correlation_edge_boxes()
     additional_methods(i).color = [0 0 0];
   end
   grid on;
-  % move the labels aroundm so it looks nice
-  xpos = [areas additional_areas]+.010;
+  % move the labels around, so it looks nice
+  xpos = [areas additional_areas];
   ypos = [AP additional_AP];
-  mirror_offset = 0.060;
-  xpos(1) = xpos(1)-mirror_offset;
-  xpos(2) = xpos(2)-mirror_offset;
-  xpos(6) = xpos(6)-mirror_offset;
-  xpos(7) = xpos(7)-mirror_offset;
+  xpos = xpos + label_offsets(:,1)';
+  ypos = ypos + label_offsets(:,2)';
 
   text(xpos,ypos,custom_names);
   xlabel(sprintf('average recall')); axis(axis_lim)

plot_correlation_voc07.m

@@ -15,47 +15,106 @@ function plot_correlation_voc07()
   % Recall matrix used for experiments nThresholds x nAlgorithms
   ld = load('data/pascal_voc07_test_recall.mat');
   R = ld.recalls';
-
   T = ld.iou_thresholds;
+  ARs = ld.ARs(:);
+  ARs_per_class = ld.ARs_per_class';
 
   % define algorithm names, IoU thresholds, recall values and AP scores
   nms={methods.name};
 
   ld = load('data/pascal_voc07_test_llda_dpm_aps.mat');
-  axis_lim = [0 0.975 20 34];
+  axis_lim = [0.5 0.975 0.6 1];
   % the following three lines reproduce Piotr's plots
 %   whitelist = [8 7 13 9 15 1 2 4 10 3 6 12];
 %   AP = ld.aps(:,1)'; AP = AP(whitelist);
 %   ploteroo(T, R(:,whitelist), AP, nms(whitelist), axis_lim);
   llda_dpm_AP = ld.aps(:,2)';
+  llda_dpm_AP_per_class = ld.aps_per_class(:,:,2)';
 
   plot_correlation_over_recall(T, R(:,method_selection), ...
     llda_dpm_AP(method_selection), nms(method_selection), axis_lim);
   scale_and_save('figures/LLDA_DPM_mAP_recall_voc07.pdf');
 
   ld = load('data/pascal_voc07_test_rcnn_aps.mat');
-  axis_lim = [0 0.975 10 60];
   rcnn_AP = ld.aps;
+  rcnn_AP_per_class = ld.aps_per_class';
 
   plot_correlation_over_recall(T, R(:,method_selection), ...
     rcnn_AP(method_selection), nms(method_selection), axis_lim);
   scale_and_save('figures/RCNN_mAP_recall_voc07.pdf');
 
+  ld = load('data/pascal_voc07_test_frcn_aps.mat');
+  frcn_AP = ld.aps;
+  frcn_AP_per_class = ld.aps_per_class';
+
+  plot_correlation_over_recall(T, R(:,method_selection), ...
+    frcn_AP(method_selection), nms(method_selection), axis_lim);
+  scale_and_save('figures/FRCN_mAP_recall_voc07.pdf');
+
+  ld = load('data/pascal_voc07_test_frcn_noregr_aps.mat');
+  frcn_noregr_AP = ld.aps;
+  frcn_noregr_AP_per_class = ld.aps_per_class';
+
+  plot_correlation_over_recall(T, R(:,method_selection), ...
+    frcn_noregr_AP(method_selection), nms(method_selection), axis_lim);
+  scale_and_save('figures/FRCN_noregr_mAP_recall_voc07.pdf');
+
   % Average recall plots
-  const_weights = ones(numel(T),1) ./ numel(T);
-  integral_area = sum(R .* repmat(const_weights, [1, n_methods]), 1);
-  plot_weighted_area_color_coded(integral_area(method_selection), ...
+  plot_weighted_area_color_coded(ARs(method_selection), ...
     llda_dpm_AP(method_selection), methods(method_selection), [0 0.6 10 34]);
   scale_and_save('figures/LLDA_DPM_mAP_recall_area_voc07.pdf');
 
-  plot_weighted_area_color_coded(integral_area(method_selection), ...
+  plot_weighted_area_color_coded(ARs(method_selection), ...
     rcnn_AP(method_selection), methods(method_selection), [0 0.6 10 60]);
   scale_and_save('figures/RCNN_mAP_recall_area_voc07.pdf');
 
+  plot_weighted_area_color_coded(ARs(method_selection), ...
+    frcn_AP(method_selection), methods(method_selection), [0 0.6 15 65]);
+  scale_and_save('figures/FRCN_mAP_recall_area_voc07.pdf');
+
+  plot_weighted_area_color_coded(ARs(method_selection), ...
+    frcn_noregr_AP(method_selection), methods(method_selection), [0 0.6 15 65]);
+  scale_and_save('figures/FRCN_noregr_mAP_recall_area_voc07.pdf');
+
+  % Average recall plots per class
+  per_class_method_selection = method_selection;
+  plot_AR_per_class(ARs_per_class(:,per_class_method_selection), ...
+    llda_dpm_AP_per_class(:,per_class_method_selection), methods(per_class_method_selection), [0 .85 0 60]);
+  scale_and_save('figures/LLDA_DPM_correlation_per_class_lines_voc07.pdf', 6, 7);
+  plot_AR_per_class(ARs_per_class(:,per_class_method_selection), ...
+    rcnn_AP_per_class(:,per_class_method_selection), methods(per_class_method_selection), [0 .85 0 80]);
+  scale_and_save('figures/RCNN_correlation_per_class_lines_voc07.pdf', 6, 7);
+  plot_AR_per_class(ARs_per_class(:,per_class_method_selection), ...
+    frcn_AP_per_class(:,per_class_method_selection), methods(per_class_method_selection), [0 .85 0 85]);
+  scale_and_save('figures/FRCN_correlation_per_class_lines_voc07.pdf', 6, 7);
+  plot_AR_per_class(ARs_per_class(:,per_class_method_selection), ...
+    frcn_noregr_AP_per_class(:,per_class_method_selection), methods(per_class_method_selection), [0 .85 0 85]);
+  scale_and_save('figures/FRCN_noregr_correlation_per_class_lines_voc07.pdf', 6, 7);
+
+  plot_correlation_per_class(ARs_per_class(:,per_class_method_selection), ...
+    llda_dpm_AP_per_class(:,per_class_method_selection), true);
+  scale_and_save('figures/LLDA_DPM_correlation_per_class_colorbars_voc07.pdf', 6, 12);
+  plot_correlation_per_class(ARs_per_class(:,per_class_method_selection), ...
+    rcnn_AP_per_class(:,per_class_method_selection), true);
+  scale_and_save('figures/RCNN_correlation_per_class_colorbars_voc07.pdf', 6, 12);
+  plot_correlation_per_class(ARs_per_class(:,per_class_method_selection), ...
+    frcn_AP_per_class(:,per_class_method_selection), true);
+  scale_and_save('figures/FRCN_correlation_per_class_colorbars_voc07.pdf', 6, 12);
+  plot_correlation_per_class(ARs_per_class(:,per_class_method_selection), ...
+    frcn_noregr_AP_per_class(:,per_class_method_selection), true);
+  scale_and_save('figures/FRCN_noregr_correlation_per_class_colorbars_voc07.pdf', 6, 12);
+
+%   plot_correlation_per_class(ARs_per_class(:,per_class_method_selection), ...
+%     llda_dpm_AP_per_class(:,per_class_method_selection), false);
+%   scale_and_save('figures/LLDA_DPM_correlation_per_class_bars_voc07.pdf', 6, 12);
+%   plot_correlation_per_class(ARs_per_class(:,per_class_method_selection), ...
+%     rcnn_AP_per_class(:,per_class_method_selection), false);
+%   scale_and_save('figures/RCNN_correlation_per_class_bars_voc07.pdf', 6, 12);
 end
 
-function scale_and_save(output_filename)
-  hei = 6; wid = 6;
+function scale_and_save(output_filename, hei, wid)
+  if nargin < 2, hei = 6; end
+  if nargin < 3, wid = 6; end
   set(gcf, 'Units','centimeters', 'Position',[0 0 wid hei]);
   set(gcf, 'PaperPositionMode','auto');
   printpdf(output_filename);
@@ -67,10 +126,76 @@ function plot_weighted_area(areas, AP, nms, axis_lim)
   xlabel(sprintf('weighted area under recall')); axis(axis_lim)
   title(sprintf('correlation=%.3f',s)); ylabel('mAP'); hold on;
   p=polyfit(areas,AP,1); line([0 1],[p(2),sum(p)],'Color',[1 1 1]/3); hold off
+end
+
+function colors = class_colors()
+  colors = [[74,136,55],
+[127,137,221],
+[209,79,44],
+[95,40,42],
+[209,68,117],
+[218,145,59],
+[117,171,189],
+[205,132,115],
+[54,70,41],
+[198,147,179],
+[204,104,203],
+[114,198,70],
+[64,74,86],
+[121,60,105],
+[86,117,53],
+[121,113,53],
+[111,182,140],
+[138,75,38],
+[189,175,70],
+[82,92,146]]/255;
+end
+
+function plot_correlation_per_class(AR_per_class, AP_per_class, colorful)
+  colors = class_colors();
+  load('data/short_classes.mat');
+
+  n = numel(classes);
+  S=corrcoef([AR_per_class' AP_per_class']); S=diag(S(n+1:end,1:n));
+  figure; hold on;
+  if colorful
+    for i = 1:n
+      bar(i, S(i), 'FaceColor', colors(i,:), 'EdgeColor', colors(i,:));
+      hold on;
+    end
+  else
+    bar(S, 'FaceColor', [31,120,180]/256, 'EdgeColor', [31,120,180]/256);
   end
+%   bar(S);
+  set(gca,'XTick',1:numel(classes),'XTickLabel',classes);
+  set(gca,'XTickLabelRotation',60);
+  xlim([0,21]);
+  ylim([0.7 1]);
+  ylabel('correlation with mAP');
+end
 
-  function plot_weighted_area_color_coded(areas, AP, methods, axis_lim)
-  S=corrcoef([areas' AP']); s = S(1,end);
+function plot_AR_per_class(AR_per_class, AP_per_class, methods, axis_lim)
+  colors = class_colors();
+  load('data/short_classes.mat');
+
+  S=corrcoef([AR_per_class(:) AP_per_class(:)]); s = S(1,end);
+  figure;
+  for c = 1:numel(classes), cls = classes{c};
+    for i = 1:numel(methods)
+      plot(AR_per_class(c,i), AP_per_class(c,i), '.', 'MarkerSize', 10, 'Color', colors(c,:), 'LineWidth', 1.5);
+      hold on;
+    end
+    p=polyfit(AR_per_class(c,:),AP_per_class(c,:),1); line([0 1],[p(2),sum(p)],'Color',colors(c,:));
+  end
+  grid on;
+  xlabel(sprintf('average recall')); 
+  title(sprintf('correlation=%.3f',s)); ylabel('mAP'); hold off;
+  axis(axis_lim); 
+end
+
+function plot_weighted_area_color_coded(areas, AP, methods, axis_lim)
+  AP = AP(:);
+  S=corrcoef([areas AP]); s = S(1,end);
   figure;
   for i = 1:numel(methods)
     plot(areas(i), AP(i), '.', 'MarkerSize', 20, 'Color', methods(i).color, 'LineWidth', 1.5);
@@ -112,16 +237,8 @@ function ploteroo(T, R, AP, nms, axis_lim)
 
 function plot_correlation_over_recall(T, R, AP, nms, axis_lim)
   S=corrcoef([R' AP']); S=S(1:end-1,end);
-  figure(); plot(T,S,'.-', 'Color', [31,120,180]/256, 'MarkerSize', 20);
+  figure(); xlim([T(1), T(end-1)]);
+  plot(T,S,'.-', 'Color', [31,120,180]/256, 'MarkerSize', 20);
   xlabel('IoU overlap threshold'); ylabel('correlation with mAP'); grid on;
-  xlim([T(1), T(end-1)]);
-
-% figure;
-% for t = 1:numel(S)
-%   subplot(5,5,t);
-%   R1=R(t,:); plot(R1,AP,'dr'); grid on; %text(R1+.015,AP,nms);
-% xlabel(sprintf('recall at IoU=%.3f',T(t))); axis(axis_lim)
-% title(sprintf('correlation=%.3f',S(t))); ylabel('AP'); hold on;
-% p=polyfit(R1,AP,1); line([0 1],[p(2),sum(p)],'Color',[1 1 1]/3); hold off
-% end
+  axis(axis_lim);
 end

plot_detector_wiggle_voc07.m

@@ -4,9 +4,7 @@ function plot_detector_wiggle_voc07()
     83, 136, 173 ; ...
     219, 135, 45 ; ...
     218, 71, 56 ; ...
-    163, 163, 163 ; ...
     135, 130, 174 ; ...
-    225, 119, 174 ; ...
     142, 195, 129 ; ...
     138, 180, 66 ; ...
     223, 200, 51 ; ...
@@ -23,6 +21,12 @@ function plot_detector_wiggle_voc07()
   config(3).name = 'LM-LLDA bboxpred';
   config(3).func = @detector_wiggle_rcnn;
   config(3).filename = '/BS/candidate_detections/work/v1/pascal_voc07_test_dpm_bboxpred_scoremaps-3d.mat';
+  config(4).name = 'Fast R-CNN';
+  config(4).func = @detector_wiggle_from_iou_score;
+  config(4).filename = '/BS/candidate_detections/work/v1/pascal_voc07_test_frcn_iou_score.mat';
+  config(5).name = 'Fast R-CNN bboxpred';
+  config(5).func = @detector_wiggle_from_iou_score;
+  config(5).filename = '/BS/candidate_detections/work/v1/pascal_voc07_test_frcn_regr_iou_score.mat';
 
   autoscale = true;
 
@@ -41,7 +45,7 @@ function plot_detector_wiggle_voc07()
 %   bar(bin_centers', hists.bin_counts');
 %   legend({config.name}, 'Location', 'SouthEast');
 
-  if true
+  if false
     scale_factors = [0.8182    0.8485    0.8864    0.9242    0.9621    1.0000    1.0455    1.0833    1.1288  1.1742    1.2273];
     x = (-5:5) .* 4*227/128;
     y = x;
@@ -110,10 +114,10 @@ function plot_histograms(config, bin_centers, bin_edges, avg_scores, class_avg_s
   legend({config.name}, 'Location', 'SouthEast');
   xlabel('IoU with GT'); ylabel('detector score');
   xlim([min(bin_edges), max(bin_edges)]);
-%   hei = 8; wid = 8.5;
-%   set(gcf, 'Units','centimeters', 'Position',[0 0 wid hei]);
-%   set(gcf, 'PaperPositionMode','auto');
-%   printpdf('figures/detector_scoremaps_voc07.pdf');
+  hei = 8; wid = 12;
+  set(gcf, 'Units','centimeters', 'Position',[0 0 wid hei]);
+  set(gcf, 'PaperPositionMode','auto');
+  printpdf('figures/detector_scoremaps_voc07.pdf');
 
   fh_per_class = figure;
   for c = 1:20