4 * Copyright (C) 1997-2012 Adam Williams <broadcast at earthling dot net>
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation; either version 2 of the License, or
9 * (at your option) any later version.
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
16 * You should have received a copy of the GNU General Public License
17 * along with this program; if not, write to the Free Software
18 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
23 // This is mainly a test for object tracking
31 #include "findobject.h"
32 #include "findobjectwindow.h"
34 #include "overlayframe.h"
36 #include "transportque.h"
38 // Needed with OpenCV version 2.4.8
39 #include "opencv2/legacy/legacy.hpp"
40 #include "opencv2/legacy/compat.hpp"
42 #include "opencv2/video/tracking.hpp"
43 #include "opencv2/video/background_segm.hpp"
49 REGISTER_PLUGIN(FindObjectMain)
53 FindObjectConfig::FindObjectConfig()
60 draw_object_border = 1;
61 global_block_w = MIN_BLOCK;
62 global_block_h = MIN_BLOCK;
68 algorithm = NO_ALGORITHM;
75 void FindObjectConfig::boundaries()
77 CLAMP(global_range_w, MIN_RADIUS, MAX_RADIUS);
78 CLAMP(global_range_h, MIN_RADIUS, MAX_RADIUS);
79 CLAMP(global_block_w, MIN_BLOCK, MAX_BLOCK);
80 CLAMP(global_block_h, MIN_BLOCK, MAX_BLOCK);
81 CLAMP(block_x, 0, 100);
82 CLAMP(block_y, 0, 100);
83 CLAMP(object_layer, MIN_LAYER, MAX_LAYER);
84 CLAMP(replace_layer, MIN_LAYER, MAX_LAYER);
85 CLAMP(scene_layer, MIN_LAYER, MAX_LAYER);
86 CLAMP(vmin, MIN_CAMSHIFT, MAX_CAMSHIFT);
87 CLAMP(vmax, MIN_CAMSHIFT, MAX_CAMSHIFT);
88 CLAMP(smin, MIN_CAMSHIFT, MAX_CAMSHIFT);
89 CLAMP(blend, MIN_BLEND, MAX_BLEND);
92 int FindObjectConfig::equivalent(FindObjectConfig &that)
95 global_range_w == that.global_range_w &&
96 global_range_h == that.global_range_h &&
97 draw_keypoints == that.draw_keypoints &&
98 draw_border == that.draw_border &&
99 replace_object == that.replace_object &&
100 draw_object_border == that.draw_object_border &&
101 global_block_w == that.global_block_w &&
102 global_block_h == that.global_block_h &&
103 block_x == that.block_x &&
104 block_y == that.block_y &&
105 object_layer == that.object_layer &&
106 replace_layer == that.replace_layer &&
107 scene_layer == that.scene_layer &&
108 algorithm == that.algorithm &&
116 void FindObjectConfig::copy_from(FindObjectConfig &that)
118 global_range_w = that.global_range_w;
119 global_range_h = that.global_range_h;
120 draw_keypoints = that.draw_keypoints;
121 draw_border = that.draw_border;
122 replace_object = that.replace_object;
123 draw_object_border = that.draw_object_border;
124 global_block_w = that.global_block_w;
125 global_block_h = that.global_block_h;
126 block_x = that.block_x;
127 block_y = that.block_y;
128 object_layer = that.object_layer;
129 replace_layer = that.replace_layer;
130 scene_layer = that.scene_layer;
131 algorithm = that.algorithm;
138 void FindObjectConfig::interpolate(FindObjectConfig &prev,
139 FindObjectConfig &next,
142 int64_t current_frame)
154 FindObjectMain::FindObjectMain(PluginServer *server)
155 : PluginVClient(server)
157 bzero(&blob_param, sizeof(CvBlobTrackerAutoParam1));
169 object_keypoints = 0;
170 object_descriptors = 0;
172 scene_descriptors = 0;
179 FindObjectMain::~FindObjectMain()
181 // This releases all the arrays
182 if(storage) cvReleaseMemStorage(&storage);
183 if(object_image) cvReleaseImage(&object_image);
184 if(scene_image) cvReleaseImage(&scene_image);
185 if(prev_object) delete [] prev_object;
190 if(blob_param.pBT) cvReleaseBlobTracker(&blob_param.pBT);
191 if(blob_param.pBD) cvReleaseBlobDetector(&blob_param.pBD);
192 if(blob_param.pBTGen) cvReleaseBlobTrackGen(&blob_param.pBTGen);
193 if(blob_param.pBTA) cvReleaseBlobTrackAnalysis(&blob_param.pBTA);
194 if(blob_param.pFG) cvReleaseFGDetector(&blob_param.pFG);
195 if(blob_pTracker) cvReleaseBlobTrackerAuto(&blob_pTracker);
199 const char* FindObjectMain::plugin_title() { return _("Find Object"); }
200 int FindObjectMain::is_realtime() { return 1; }
201 int FindObjectMain::is_multichannel() { return 1; }
204 NEW_WINDOW_MACRO(FindObjectMain, FindObjectWindow)
206 LOAD_CONFIGURATION_MACRO(FindObjectMain, FindObjectConfig)
210 void FindObjectMain::update_gui()
214 if(load_configuration())
216 thread->window->lock_window("FindObjectMain::update_gui");
218 char string[BCTEXTLEN];
220 ((FindObjectWindow*)thread->window)->global_range_w->update(config.global_range_w);
221 ((FindObjectWindow*)thread->window)->global_range_h->update(config.global_range_h);
222 ((FindObjectWindow*)thread->window)->global_block_w->update(config.global_block_w);
223 ((FindObjectWindow*)thread->window)->global_block_h->update(config.global_block_h);
224 ((FindObjectWindow*)thread->window)->block_x->update(config.block_x);
225 ((FindObjectWindow*)thread->window)->block_y->update(config.block_y);
226 ((FindObjectWindow*)thread->window)->block_x_text->update((float)config.block_x);
227 ((FindObjectWindow*)thread->window)->block_y_text->update((float)config.block_y);
229 ((FindObjectWindow*)thread->window)->draw_keypoints->update(config.draw_keypoints);
230 ((FindObjectWindow*)thread->window)->draw_border->update(config.draw_border);
231 ((FindObjectWindow*)thread->window)->replace_object->update(config.replace_object);
232 ((FindObjectWindow*)thread->window)->draw_object_border->update(config.draw_object_border);
235 ((FindObjectWindow*)thread->window)->object_layer->update(
236 (int64_t)config.object_layer);
237 ((FindObjectWindow*)thread->window)->replace_layer->update(
238 (int64_t)config.replace_layer);
239 ((FindObjectWindow*)thread->window)->scene_layer->update(
240 (int64_t)config.scene_layer);
241 ((FindObjectWindow*)thread->window)->algorithm->set_text(
242 FindObjectAlgorithm::to_text(config.algorithm));
244 ((FindObjectWindow*)thread->window)->vmin->update(
245 (int64_t)config.vmin);
246 ((FindObjectWindow*)thread->window)->vmax->update(
247 (int64_t)config.vmax);
248 ((FindObjectWindow*)thread->window)->smin->update(
249 (int64_t)config.smin);
250 ((FindObjectWindow*)thread->window)->blend->update(
251 (int64_t)config.blend);
253 ((FindObjectWindow*)thread->window)->flush();
254 thread->window->unlock_window();
256 // printf("FindObjectMain::update_gui %d %d %d %d\n",
267 void FindObjectMain::save_data(KeyFrame *keyframe)
271 // cause data to be stored directly in text
272 output.set_shared_output(keyframe->get_data(), MESSAGESIZE);
273 output.tag.set_title("FINDOBJECT");
275 output.tag.set_property("GLOBAL_BLOCK_W", config.global_block_w);
276 output.tag.set_property("GLOBAL_BLOCK_H", config.global_block_h);
277 output.tag.set_property("BLOCK_X", config.block_x);
278 output.tag.set_property("BLOCK_Y", config.block_y);
279 output.tag.set_property("GLOBAL_RANGE_W", config.global_range_w);
280 output.tag.set_property("GLOBAL_RANGE_H", config.global_range_h);
281 output.tag.set_property("DRAW_KEYPOINTS", config.draw_keypoints);
282 output.tag.set_property("DRAW_BORDER", config.draw_border);
283 output.tag.set_property("REPLACE_OBJECT", config.replace_object);
284 output.tag.set_property("DRAW_OBJECT_BORDER", config.draw_object_border);
285 output.tag.set_property("OBJECT_LAYER", config.object_layer);
286 output.tag.set_property("REPLACE_LAYER", config.replace_layer);
287 output.tag.set_property("SCENE_LAYER", config.scene_layer);
288 output.tag.set_property("ALGORITHM", config.algorithm);
289 output.tag.set_property("VMIN", config.vmin);
290 output.tag.set_property("VMAX", config.vmax);
291 output.tag.set_property("SMIN", config.smin);
292 output.tag.set_property("BLEND", config.blend);
294 output.tag.set_title("/FINDOBJECT");
296 output.append_newline();
297 output.terminate_string();
300 void FindObjectMain::read_data(KeyFrame *keyframe)
304 input.set_shared_input(keyframe->get_data(), strlen(keyframe->get_data()));
310 result = input.read_tag();
314 if(input.tag.title_is("FINDOBJECT"))
316 config.global_block_w = input.tag.get_property("GLOBAL_BLOCK_W", config.global_block_w);
317 config.global_block_h = input.tag.get_property("GLOBAL_BLOCK_H", config.global_block_h);
318 config.block_x = input.tag.get_property("BLOCK_X", config.block_x);
319 config.block_y = input.tag.get_property("BLOCK_Y", config.block_y);
320 config.global_range_w = input.tag.get_property("GLOBAL_RANGE_W", config.global_range_w);
321 config.global_range_h = input.tag.get_property("GLOBAL_RANGE_H", config.global_range_h);
322 config.draw_keypoints = input.tag.get_property("DRAW_KEYPOINTS", config.draw_keypoints);
323 config.draw_border = input.tag.get_property("DRAW_BORDER", config.draw_border);
324 config.replace_object = input.tag.get_property("REPLACE_OBJECT", config.replace_object);
325 config.draw_object_border = input.tag.get_property("DRAW_OBJECT_BORDER", config.draw_object_border);
326 config.object_layer = input.tag.get_property("OBJECT_LAYER", config.object_layer);
327 config.replace_layer = input.tag.get_property("REPLACE_LAYER", config.replace_layer);
328 config.scene_layer = input.tag.get_property("SCENE_LAYER", config.scene_layer);
329 config.algorithm = input.tag.get_property("ALGORITHM", config.algorithm);
330 config.vmin = input.tag.get_property("VMIN", config.vmin);
331 config.vmax = input.tag.get_property("VMAX", config.vmax);
332 config.smin = input.tag.get_property("SMIN", config.smin);
333 config.blend = input.tag.get_property("BLEND", config.blend);
344 void FindObjectMain::draw_pixel(VFrame *frame, int x, int y)
346 if(!(x >= 0 && y >= 0 && x < frame->get_w() && y < frame->get_h())) return;
348 #define DRAW_PIXEL(x, y, components, do_yuv, max, type) \
350 type **rows = (type**)frame->get_rows(); \
351 rows[y][x * components] = max - rows[y][x * components]; \
354 rows[y][x * components + 1] = max - rows[y][x * components + 1]; \
355 rows[y][x * components + 2] = max - rows[y][x * components + 2]; \
359 rows[y][x * components + 1] = (max / 2 + 1) - rows[y][x * components + 1]; \
360 rows[y][x * components + 2] = (max / 2 + 1) - rows[y][x * components + 2]; \
362 if(components == 4) \
363 rows[y][x * components + 3] = max; \
367 switch(frame->get_color_model())
370 DRAW_PIXEL(x, y, 3, 0, 0xff, unsigned char);
373 DRAW_PIXEL(x, y, 4, 0, 0xff, unsigned char);
376 DRAW_PIXEL(x, y, 3, 0, 1.0, float);
379 DRAW_PIXEL(x, y, 4, 0, 1.0, float);
382 DRAW_PIXEL(x, y, 3, 1, 0xff, unsigned char);
385 DRAW_PIXEL(x, y, 4, 1, 0xff, unsigned char);
388 DRAW_PIXEL(x, y, 3, 0, 0xffff, uint16_t);
391 DRAW_PIXEL(x, y, 3, 1, 0xffff, uint16_t);
393 case BC_RGBA16161616:
394 DRAW_PIXEL(x, y, 4, 0, 0xffff, uint16_t);
396 case BC_YUVA16161616:
397 DRAW_PIXEL(x, y, 4, 1, 0xffff, uint16_t);
403 void FindObjectMain::draw_line(VFrame *frame, int x1, int y1, int x2, int y2)
405 int w = labs(x2 - x1);
406 int h = labs(y2 - y1);
407 //printf("FindObjectMain::draw_line 1 %d %d %d %d\n", x1, y1, x2, y2);
411 draw_pixel(frame, x1, y1);
416 // Flip coordinates so x1 < x2
426 int numerator = y2 - y1;
427 int denominator = x2 - x1;
428 for(int i = x1; i <= x2; i++)
430 int y = y1 + (int64_t)(i - x1) * (int64_t)numerator / (int64_t)denominator;
431 draw_pixel(frame, i, y);
436 // Flip coordinates so y1 < y2
446 int numerator = x2 - x1;
447 int denominator = y2 - y1;
448 for(int i = y1; i <= y2; i++)
450 int x = x1 + (int64_t)(i - y1) * (int64_t)numerator / (int64_t)denominator;
451 draw_pixel(frame, x, i);
454 //printf("FindObjectMain::draw_line 2\n");
457 void FindObjectMain::draw_rect(VFrame *frame, int x1, int y1, int x2, int y2)
459 draw_line(frame, x1, y1, x2, y1);
460 draw_line(frame, x2, y1 + 1, x2, y2);
461 draw_line(frame, x2 - 1, y2, x1, y2);
462 draw_line(frame, x1, y2 - 1, x1, y1 + 1);
467 // Convert to greyscale & crop
468 void FindObjectMain::grey_crop(unsigned char *dst,
477 // Dimensions of dst frame
481 bzero(dst, dst_w * dst_h);
483 //printf("FindObjectMain::grey_crop %d %d %d\n", __LINE__, w, h);
484 for(int i = 0; i < h; i++)
487 #define RGB_TO_VALUE(r, g, b) \
488 ((r) * R_TO_Y + (g) * G_TO_Y + (b) * B_TO_Y)
491 #define CONVERT(in_type, max, components, is_yuv) \
493 in_type *input = ((in_type*)src->get_rows()[i + y1]) + x1 * components; \
494 unsigned char *output = dst + i * dst_w; \
496 for(int j = 0; j < w; j++) \
498 /* Y channel only */ \
506 float r = (float)input[0] / max; \
507 float g = (float)input[1] / max; \
508 float b = (float)input[2] / max; \
509 *output = RGB_TO_VALUE(r, g, b); \
512 input += components; \
516 switch(src->get_color_model())
520 CONVERT(unsigned char, 0xff, 3, 0)
526 CONVERT(unsigned char, 0xff, 4, 0)
532 CONVERT(float, 1.0, 3, 0)
538 CONVERT(float, 1.0, 4, 0)
544 CONVERT(unsigned char, 0xff, 3, 1)
550 CONVERT(unsigned char, 0xff, 4, 1)
558 void FindObjectMain::process_surf()
563 // Only does greyscale
564 object_image = cvCreateImage(
565 cvSize(object_image_w, object_image_h),
572 // Only does greyscale
573 scene_image = cvCreateImage(
574 cvSize(scene_image_w, scene_image_h),
579 // Select only region with image size
580 // Does this do anything?
581 cvSetImageROI( object_image, cvRect( 0, 0, object_w, object_h ) );
582 cvSetImageROI( scene_image, cvRect( 0, 0, scene_w, scene_h ) );
584 if(!prev_object) prev_object = new unsigned char[object_image_w * object_image_h];
585 memcpy(prev_object, object_image->imageData, object_image_w * object_image_h);
586 grey_crop((unsigned char*)scene_image->imageData,
587 get_input(scene_layer),
596 grey_crop((unsigned char*)object_image->imageData,
597 get_input(object_layer),
606 if(!storage) storage = cvCreateMemStorage(0);
607 CvSURFParams params = cvSURFParams(500, 1);
610 //printf("FindObjectMain::process_surf %d\n", __LINE__);
612 // Only compute keypoints if the image changed
613 if(memcmp(prev_object, object_image->imageData, object_image_w * object_image_h))
615 if(object_keypoints) cvClearSeq(object_keypoints);
616 if(object_descriptors) cvClearSeq(object_descriptors);
617 cvExtractSURF(object_image,
626 //printf("FindObjectMain::process_surf %d object keypoints=%d\n", __LINE__, object_keypoints->total);
627 // Draw the keypoints
628 // for(int i = 0; i < object_keypoints->total; i++)
630 // CvSURFPoint* r1 = (CvSURFPoint*)cvGetSeqElem( object_keypoints, i );
631 // int size = r1->size / 4;
632 // draw_rect(frame[object_layer],
633 // r1->pt.x + object_x1 - size,
634 // r1->pt.y + object_y1 - size,
635 // r1->pt.x + object_x1 + size,
636 // r1->pt.y + object_y1 + size);
640 //printf("FindObjectMain::process_surf %d\n", __LINE__);
642 // TODO: make the surf data persistent & check for image changes instead
643 if(scene_keypoints) cvClearSeq(scene_keypoints);
644 if(scene_descriptors) cvClearSeq(scene_descriptors);
645 cvExtractSURF(scene_image,
653 // Draw the keypoints
654 // for(int i = 0; i < scene_keypoints->total; i++)
656 // CvSURFPoint* r1 = (CvSURFPoint*)cvGetSeqElem( scene_keypoints, i );
657 // int size = r1->size / 4;
658 // draw_rect(frame[scene_layer],
659 // r1->pt.x + scene_x1 - size,
660 // r1->pt.y + scene_y1 - size,
661 // r1->pt.x + scene_x1 + size,
662 // r1->pt.y + scene_y1 + size);
665 // printf("FindObjectMain::process_surf %d %d %d scene keypoints=%d\n",
669 // scene_keypoints->total);
671 int *point_pairs = 0;
673 CvPoint src_corners[4] =
677 { object_w, object_h },
681 CvPoint dst_corners[4] =
689 //printf("FindObjectMain::process_surf %d\n", __LINE__);
690 if(scene_keypoints->total &&
691 object_keypoints->total &&
692 locatePlanarObject(object_keypoints,
706 // Draw keypoints in the scene & object layer
707 if(config.draw_keypoints)
709 //printf("FindObjectMain::process_surf %d total pairs=%d\n", __LINE__, total_pairs);
710 for(int i = 0; i < total_pairs; i++)
712 CvSURFPoint* r1 = (CvSURFPoint*)cvGetSeqElem( object_keypoints, point_pairs[i * 2] );
713 CvSURFPoint* r2 = (CvSURFPoint*)cvGetSeqElem( scene_keypoints, point_pairs[i * 2 + 1] );
716 int size = r2->size * 1.2 / 9 * 2;
717 draw_rect(get_input(scene_layer),
718 r2->pt.x + scene_x1 - size,
719 r2->pt.y + scene_y1 - size,
720 r2->pt.x + scene_x1 + size,
721 r2->pt.y + scene_y1 + size);
722 draw_rect(get_input(object_layer),
723 r1->pt.x + object_x1 - size,
724 r1->pt.y + object_y1 - size,
725 r1->pt.x + object_x1 + size,
726 r1->pt.y + object_y1 + size);
731 //printf("FindObjectMain::process_surf %d\n", __LINE__);
732 // Get object outline in the scene layer
733 border_x1 = dst_corners[0].x + scene_x1;
734 border_y1 = dst_corners[0].y + scene_y1;
735 border_x2 = dst_corners[1].x + scene_x1;
736 border_y2 = dst_corners[1].y + scene_y1;
737 border_x3 = dst_corners[2].x + scene_x1;
738 border_y3 = dst_corners[2].y + scene_y1;
739 border_x4 = dst_corners[3].x + scene_x1;
740 border_y4 = dst_corners[3].y + scene_y1;
741 //printf("FindObjectMain::process_surf %d\n", __LINE__);
746 //printf("FindObjectMain::process_surf %d\n", __LINE__);
750 // for(int i = 0; i < object_y2 - object_y1; i++)
752 // unsigned char *dst = get_input(object_layer)->get_rows()[i];
753 // unsigned char *src = (unsigned char*)object_image->imageData + i * (object_x2 - object_x1);
754 // for(int j = 0; j < object_x2 - object_x1; j++)
764 // Frees the image structures
765 if(point_pairs) free(point_pairs);
770 void FindObjectMain::process_camshift()
772 // Some user defined parameters
773 int vmin = config.vmin;
774 int vmax = config.vmax;
775 int smin = config.smin;
776 float hranges[] = { 0, 180 };
777 const float* phranges = hranges;
780 // Create aligned, RGB images
783 object_image = cvCreateImage(
784 cvSize(object_image_w, object_image_h),
791 scene_image = cvCreateImage(
792 cvSize(scene_image_w, scene_image_h),
797 // Temporary row pointers
798 unsigned char **object_rows = new unsigned char*[object_image_h];
799 unsigned char **scene_rows = new unsigned char*[scene_image_h];
800 for(int i = 0; i < object_image_h; i++)
802 object_rows[i] = (unsigned char*)(object_image->imageData + i * object_image_w * 3);
804 for(int i = 0; i < scene_image_h; i++)
806 scene_rows[i] = (unsigned char*)(scene_image->imageData + i * scene_image_w * 3);
809 // Transfer object & scene to RGB images for OpenCV
810 if(!prev_object) prev_object = new unsigned char[object_image_w * object_image_h * 3];
811 // Back up old object image
812 memcpy(prev_object, object_image->imageData, object_image_w * object_image_h * 3);
814 BC_CModels::transfer(object_rows,
815 get_input(object_layer)->get_rows(),
830 get_input(object_layer)->get_color_model(),
835 BC_CModels::transfer(scene_rows,
836 get_input(scene_layer)->get_rows(),
851 get_input(scene_layer)->get_color_model(),
857 delete [] object_rows;
858 delete [] scene_rows;
860 // from camshiftdemo.cpp
861 // Compute new object
862 if(memcmp(prev_object,
863 object_image->imageData,
864 object_image_w * object_image_h * 3) ||
867 Mat image(object_image);
869 cvtColor(image, hsv, CV_RGB2HSV);
870 int _vmin = vmin, _vmax = vmax;
871 //printf("FindObjectMain::process_camshift %d\n", __LINE__);
874 Scalar(0, smin, MIN(_vmin,_vmax)),
875 Scalar(180, 256, MAX(_vmin, _vmax)),
878 hue.create(hsv.size(), hsv.depth());
879 mixChannels(&hsv, 1, &hue, 1, ch, 1);
881 Rect selection = Rect(0, 0, object_w, object_h);
882 trackWindow = selection;
884 Mat roi(hue, selection), maskroi(mask, selection);
885 calcHist(&roi, 1, 0, maskroi, hist, 1, &hsize, &phranges);
886 normalize(hist, hist, 0, 255, CV_MINMAX);
891 Mat image(scene_image);
892 Mat hsv, hue, mask, backproj;
893 cvtColor(image, hsv, CV_RGB2HSV);
894 int _vmin = vmin, _vmax = vmax;
897 Scalar(0, smin, MIN(_vmin,_vmax)),
898 Scalar(180, 256, MAX(_vmin, _vmax)),
901 hue.create(hsv.size(), hsv.depth());
902 mixChannels(&hsv, 1, &hue, 1, ch, 1);
904 //printf("FindObjectMain::process_camshift %d %d %d\n", __LINE__, hist.dims, hist.size[1]);
905 RotatedRect trackBox = RotatedRect(
906 Point2f((object_x1 + object_x2) / 2, (object_y1 + object_y2) / 2),
907 Size2f(object_w, object_h),
909 trackWindow = Rect(0,
917 calcBackProject(&hue, 1, 0, hist, backproj, &phranges);
919 //printf("FindObjectMain::process_camshift %d\n", __LINE__);
920 // if(trackWindow.width <= 0 ||
921 // trackWindow.height <= 0)
923 // trackWindow.width = object_w;
924 // trackWindow.height = object_h;
927 trackBox = CamShift(backproj,
929 TermCriteria( CV_TERMCRIT_EPS | CV_TERMCRIT_ITER, 10, 1 ));
930 //printf("FindObjectMain::process_camshift %d\n", __LINE__);
933 // if( trackWindow.area() <= 1 )
935 // int cols = backproj.cols;
936 // int rows = backproj.rows;
937 // int r = (MIN(cols, rows) + 5) / 6;
938 // trackWindow = Rect(trackWindow.x - r, trackWindow.y - r,
939 // trackWindow.x + r, trackWindow.y + r) &
940 // Rect(0, 0, cols, rows);
943 // printf("FindObjectMain::process_camshift %d %d %d %d %d\n",
947 // trackWindow.width,
948 // trackWindow.height);
951 // Draw mask over scene
952 if(config.draw_keypoints)
954 for(int i = 0; i < scene_h; i++)
956 switch(get_input(scene_layer)->get_color_model())
960 unsigned char *input = backproj.data + i * scene_image_w;
961 unsigned char *output = get_input(scene_layer)->get_rows()[i + scene_y1] + scene_x1 * 3;
962 for(int j = 0; j < scene_w; j++)
976 // Get object outline in the scene layer
977 // printf("FindObjectMain::process_camshift %d %d %d %d %d %d\n",
979 // (int)trackBox.center.x,
980 // (int)trackBox.center.y,
981 // (int)trackBox.size.width,
982 // (int)trackBox.size.height,
983 // (int)trackBox.angle);
984 double angle = trackBox.angle * 2 * M_PI / 360;
985 double angle1 = atan2(-(double)trackBox.size.height / 2, -(double)trackBox.size.width / 2) + angle;
986 double angle2 = atan2(-(double)trackBox.size.height / 2, (double)trackBox.size.width / 2) + angle;
987 double angle3 = atan2((double)trackBox.size.height / 2, (double)trackBox.size.width / 2) + angle;
988 double angle4 = atan2((double)trackBox.size.height / 2, -(double)trackBox.size.width / 2) + angle;
989 double radius = sqrt(SQR(trackBox.size.height / 2) + SQR(trackBox.size.width / 2));
990 border_x1 = (int)(trackBox.center.x + cos(angle1) * radius) + scene_x1;
991 border_y1 = (int)(trackBox.center.y + sin(angle1) * radius) + scene_y1;
992 border_x2 = (int)(trackBox.center.x + cos(angle2) * radius) + scene_x1;
993 border_y2 = (int)(trackBox.center.y + sin(angle2) * radius) + scene_y1;
994 border_x3 = (int)(trackBox.center.x + cos(angle3) * radius) + scene_x1;
995 border_y3 = (int)(trackBox.center.y + sin(angle3) * radius) + scene_y1;
996 border_x4 = (int)(trackBox.center.x + cos(angle4) * radius) + scene_x1;
997 border_y4 = (int)(trackBox.center.y + sin(angle4) * radius) + scene_y1;
1002 #define APPLY_MASK(type, max, components, do_yuv) \
1004 type *output_row = (type*)get_input(scene_layer)->get_rows()[i]; \
1005 unsigned char *mask_row = mask_rows[i]; \
1006 int chroma_offset = (int)(max + 1) / 2; \
1008 for(int j = 0; j < scene_w; j++) \
1010 if(components == 4) \
1012 output_row[j * 4 + 3] = output_row[j * 4 + 3] * mask_row[j] / 255; \
1016 output_row[j * 3] = output_row[j * 3] * mask_row[j] / 255; \
1017 output_row[j * 3 + 1] = output_row[j * 3 + 1] * mask_row[j] / 255; \
1018 output_row[j * 3 + 2] = output_row[j * 3 + 2] * mask_row[j] / 255; \
1022 output_row[j * 3 + 1] += chroma_offset * (255 - mask_row[j]) / 255; \
1023 output_row[j * 3 + 2] += chroma_offset * (255 - mask_row[j]) / 255; \
1031 // blobtrack_sample.cpp
1032 void FindObjectMain::process_blob()
1034 if(!blob_initialized)
1036 blob_initialized = 1;
1038 blob_param.FGTrainFrames = 5;
1041 /* Create FG Detection module: */
1042 blob_param.pFG = cvCreateFGDetectorBase(CV_BG_MODEL_FGD, NULL);
1043 /* Create Blob Entrance Detection module: */
1044 blob_param.pBD = cvCreateBlobDetectorCC();
1045 /* Create blob tracker module: */
1046 blob_param.pBT = cvCreateBlobTrackerCCMSPF();
1047 /* Create whole pipline: */
1048 blob_pTracker = cvCreateBlobTrackerAuto1(&blob_param);
1054 IplImage* pMask = NULL;
1056 // Create aligned, RGB images
1059 scene_image = cvCreateImage(
1060 cvSize(scene_image_w, scene_image_h),
1065 // Temporary row pointers
1066 unsigned char **scene_rows = new unsigned char*[scene_image_h];
1067 for(int i = 0; i < scene_image_h; i++)
1069 scene_rows[i] = (unsigned char*)(scene_image->imageData + i * scene_image_w * 3);
1072 BC_CModels::transfer(scene_rows,
1073 get_input(scene_layer)->get_rows(),
1088 get_input(scene_layer)->get_color_model(),
1094 delete [] scene_rows;
1096 blob_pTracker->Process(scene_image, pMask);
1097 printf("FindObjectMain::process_blob %d %jd %d\n", __LINE__, get_source_position(), blob_pTracker->GetBlobNum());
1101 if(blob_pTracker->GetFGMask())
1103 IplImage* pFG = blob_pTracker->GetFGMask();
1104 printf("FindObjectMain::process_blob %d %ld\n", __LINE__, get_source_position());
1106 // Temporary row pointers
1107 unsigned char **mask_rows = new unsigned char*[scene_image_h];
1108 for(int i = 0; i < scene_image_h; i++)
1110 mask_rows[i] = (unsigned char*)(pFG->imageData + i * scene_image_w);
1113 for(int i = 0; i < scene_image_h; i++)
1115 switch(get_input(scene_layer)->get_color_model())
1118 APPLY_MASK(unsigned char, 0xff, 3, 0)
1121 APPLY_MASK(float, 1.0, 3, 0)
1124 APPLY_MASK(unsigned char, 0xff, 3, 1)
1127 APPLY_MASK(unsigned char, 0xff, 4, 0)
1130 APPLY_MASK(float, 1.0, 4, 0)
1133 APPLY_MASK(unsigned char, 0xff, 4, 1)
1138 delete [] mask_rows;
1153 int FindObjectMain::process_buffer(VFrame **frame,
1154 int64_t start_position,
1157 int prev_algorithm = config.algorithm;
1158 if(load_configuration())
1163 w = frame[0]->get_w();
1164 h = frame[0]->get_h();
1165 //printf("FindObjectMain::process_buffer %d\n", __LINE__);
1168 // Get the layer containing the object.
1169 object_layer = config.object_layer;
1170 // Get the layer to search in.
1171 scene_layer = config.scene_layer;
1172 // Get the layer with the replacement object
1173 replace_layer = config.replace_layer;
1175 object_layer = MIN(object_layer, PluginClient::get_total_buffers() - 1);
1176 scene_layer = MIN(scene_layer, PluginClient::get_total_buffers() - 1);
1177 replace_layer = MIN(replace_layer, PluginClient::get_total_buffers() - 1);
1179 // printf("FindObjectMain::process_buffer %d %d %d %d %d %d\n",
1181 // PluginClient::get_total_buffers(),
1182 // config.object_layer,
1183 // config.scene_layer,
1187 // Create cropped images
1188 // TODO: use oblique corners & affine transform
1189 object_w = (int)(config.global_block_w * w / 100);
1190 object_h = (int)(config.global_block_h * h / 100);
1192 object_x1 = (int)(config.block_x * w / 100 - object_w / 2);
1193 object_y1 = (int)(config.block_y * h / 100 - object_h / 2);
1194 object_x2 = object_x1 + object_w;
1195 object_y2 = object_y1 + object_h;
1196 CLAMP(object_x1, 0, frame[0]->get_w() - 1);
1197 CLAMP(object_x2, 0, frame[0]->get_w() - 1);
1198 CLAMP(object_y1, 0, frame[0]->get_h() - 1);
1199 CLAMP(object_y2, 0, frame[0]->get_h() - 1);
1200 object_w = object_x2 - object_x1;
1201 object_h = object_y2 - object_y1;
1204 scene_w = (int)(config.global_range_w * w / 100);
1205 scene_h = (int)(config.global_range_h * h / 100);
1206 scene_x1 = (int)(config.block_x * w / 100 - scene_w / 2);
1207 scene_y1 = (int)(config.block_y * h / 100 - scene_h / 2);
1208 scene_x2 = scene_x1 + scene_w;
1209 scene_y2 = scene_y1 + scene_h;
1210 CLAMP(scene_x1, 0, frame[0]->get_w() - 1);
1211 CLAMP(scene_x2, 0, frame[0]->get_w() - 1);
1212 CLAMP(scene_y1, 0, frame[0]->get_h() - 1);
1213 CLAMP(scene_y2, 0, frame[0]->get_h() - 1);
1214 scene_w = scene_x2 - scene_x1;
1215 scene_h = scene_y2 - scene_y1;
1217 // Get quantized sizes
1218 int object_image_w = object_w;
1219 int object_image_h = object_h;
1220 int scene_image_w = scene_w;
1221 int scene_image_h = scene_h;
1222 if(object_w % QUANTIZE) object_image_w += QUANTIZE - (object_w % QUANTIZE);
1223 if(object_h % QUANTIZE) object_image_h += QUANTIZE - (object_h % QUANTIZE);
1224 if(scene_w % QUANTIZE) scene_image_w += QUANTIZE - (scene_w % QUANTIZE);
1225 if(scene_h % QUANTIZE) scene_image_h += QUANTIZE - (scene_h % QUANTIZE);
1228 (object_image_w != this->object_image_w ||
1229 object_image_h != this->object_image_h ||
1230 prev_algorithm != config.algorithm))
1232 cvReleaseImage(&object_image);
1234 delete [] prev_object;
1237 this->object_image_w = object_image_w;
1238 this->object_image_h = object_image_h;
1241 (scene_image_w != this->scene_image_w ||
1242 scene_image_h != this->scene_image_h ||
1243 prev_algorithm != config.algorithm))
1245 cvReleaseImage(&scene_image);
1248 this->scene_image_w = scene_image_w;
1249 this->scene_image_h = scene_image_h;
1254 //printf("FindObjectMain::process_buffer %d object_w=%d object_h=%d object_image_w=%d object_image_h=%d\n", __LINE__, object_w, object_h, object_image_w, object_image_h);
1255 //printf("FindObjectMain::process_buffer %d scene_w=%d scene_h=%d scene_image_w=%d scene_image_h=%d\n", __LINE__, scene_w, scene_h, scene_image_w, scene_image_h);
1256 //printf("FindObjectMain::process_buffer %d total_layers=%d\n", __LINE__, get_total_buffers());
1258 // Read in the input frames
1259 for(int i = 0; i < PluginClient::get_total_buffers(); i++)
1261 read_frame(frame[i],
1268 // Search for object
1269 if(config.algorithm != NO_ALGORITHM &&
1270 (config.replace_object ||
1271 config.draw_border ||
1272 config.draw_keypoints))
1276 switch(config.algorithm)
1278 #if HAVE_OPENCV_SURF
1279 case ALGORITHM_SURF:
1284 case ALGORITHM_CAMSHIFT:
1288 case ALGORITHM_BLOB:
1296 border_x1_accum = border_x1;
1297 border_y1_accum = border_y1;
1298 border_x2_accum = border_x2;
1299 border_y2_accum = border_y2;
1300 border_x3_accum = border_x3;
1301 border_y3_accum = border_y3;
1302 border_x4_accum = border_x4;
1303 border_y4_accum = border_y4;
1308 border_x1_accum = (float)border_x1 * config.blend / 100 +
1309 border_x1_accum * (100 - config.blend) / 100;
1310 border_y1_accum = (float)border_y1 * config.blend / 100 +
1311 border_y1_accum * (100 - config.blend) / 100;
1312 border_x2_accum = (float)border_x2 * config.blend / 100 +
1313 border_x2_accum * (100 - config.blend) / 100;
1314 border_y2_accum = (float)border_y2 * config.blend / 100 +
1315 border_y2_accum * (100 - config.blend) / 100;
1316 border_x3_accum = (float)border_x3 * config.blend / 100 +
1317 border_x3_accum * (100 - config.blend) / 100;
1318 border_y3_accum = (float)border_y3 * config.blend / 100 +
1319 border_y3_accum * (100 - config.blend) / 100;
1320 border_x4_accum = (float)border_x4 * config.blend / 100 +
1321 border_x4_accum * (100 - config.blend) / 100;
1322 border_y4_accum = (float)border_y4 * config.blend / 100 +
1323 border_y4_accum * (100 - config.blend) / 100;
1326 // Replace object in the scene layer
1327 if(config.replace_object)
1330 // Some trickery to get the affine transform to alpha blend into the output
1331 if(!affine) affine = new AffineEngine(get_project_smp() + 1,
1332 get_project_smp() + 1);
1334 //printf("FindObjectMain::process_surf %d replace_layer=%d\n", __LINE__, replace_layer);
1336 temp = new VFrame(w,
1338 get_input(scene_layer)->get_color_model());
1340 overlayer = new OverlayFrame(get_project_smp() + 1);
1342 temp->clear_frame();
1343 affine->process(temp,
1344 get_input(replace_layer),
1346 AffineEngine::PERSPECTIVE,
1347 border_x1_accum * 100 / w,
1348 border_y1_accum * 100 / h,
1349 border_x2_accum * 100 / w,
1350 border_y2_accum * 100 / h,
1351 border_x3_accum * 100 / w,
1352 border_y3_accum * 100 / h,
1353 border_x4_accum * 100 / w,
1354 border_y4_accum * 100 / h,
1357 overlayer->overlay(get_input(scene_layer),
1373 if(config.draw_border)
1375 draw_line(get_input(scene_layer),
1380 draw_line(get_input(scene_layer),
1385 draw_line(get_input(scene_layer),
1390 draw_line(get_input(scene_layer),
1399 // Draw object outline in the object layer
1400 if(config.draw_object_border)
1402 draw_line(frame[object_layer], object_x1, object_y1, object_x2, object_y1);
1403 draw_line(frame[object_layer], object_x2, object_y1, object_x2, object_y2);
1404 draw_line(frame[object_layer], object_x2, object_y2, object_x1, object_y2);
1405 draw_line(frame[object_layer], object_x1, object_y2, object_x1, object_y1);
1407 draw_line(frame[object_layer], scene_x1, scene_y1, scene_x2, scene_y1);
1408 draw_line(frame[object_layer], scene_x2, scene_y1, scene_x2, scene_y2);
1409 draw_line(frame[object_layer], scene_x2, scene_y2, scene_x1, scene_y2);
1410 draw_line(frame[object_layer], scene_x1, scene_y2, scene_x1, scene_y1);