4 * Copyright (C) 1997-2012 Adam Williams <broadcast at earthling dot net>
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation; either version 2 of the License, or
9 * (at your option) any later version.
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
16 * You should have received a copy of the GNU General Public License
17 * along with this program; if not, write to the Free Software
18 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
27 #include "findobjwindow.h"
29 #include "overlayframe.h"
31 #include "pluginserver.h"
38 REGISTER_PLUGIN(FindObjMain)
40 FindObjConfig::FindObjConfig()
42 algorithm = NO_ALGORITHM;
45 draw_scene_border = 0;
47 draw_object_border = 0;
48 object_x = 50; object_y = 50;
49 object_w = 100; object_h = 100;
51 scene_x = 50; scene_y = 50;
52 scene_w = 100; scene_h = 100;
60 void FindObjConfig::boundaries()
62 bclamp(drag_object, 0, 1); bclamp(drag_scene, 0, 1);
63 bclamp(object_x, 0, 100); bclamp(object_y, 0, 100);
64 bclamp(object_w, 0, 100); bclamp(object_h, 0, 100);
65 bclamp(scene_x, 0, 100); bclamp(scene_y, 0, 100);
66 bclamp(scene_w, 0, 100); bclamp(scene_h, 0, 100);
67 bclamp(object_layer, MIN_LAYER, MAX_LAYER);
68 bclamp(replace_layer, MIN_LAYER, MAX_LAYER);
69 bclamp(scene_layer, MIN_LAYER, MAX_LAYER);
70 bclamp(blend, MIN_BLEND, MAX_BLEND);
73 int FindObjConfig::equivalent(FindObjConfig &that)
76 algorithm == that.algorithm &&
77 use_flann == that.use_flann &&
78 draw_keypoints == that.draw_keypoints &&
79 draw_scene_border == that.draw_scene_border &&
80 replace_object == that.replace_object &&
81 draw_object_border == that.draw_object_border &&
82 object_x == that.object_x && object_y == that.object_y &&
83 object_w == that.object_w && object_h == that.object_h &&
84 drag_object == that.drag_object &&
85 scene_x == that.scene_x && scene_y == that.scene_y &&
86 scene_w == that.scene_w && scene_h == that.scene_h &&
87 drag_scene == that.drag_scene &&
88 object_layer == that.object_layer &&
89 replace_layer == that.replace_layer &&
90 scene_layer == that.scene_layer &&
95 void FindObjConfig::copy_from(FindObjConfig &that)
97 algorithm = that.algorithm;
98 use_flann = that.use_flann;
99 draw_keypoints = that.draw_keypoints;
100 draw_scene_border = that.draw_scene_border;
101 replace_object = that.replace_object;
102 draw_object_border = that.draw_object_border;
103 object_x = that.object_x; object_y = that.object_y;
104 object_w = that.object_w; object_h = that.object_h;
105 drag_object = that.drag_object;
106 scene_x = that.scene_x; scene_y = that.scene_y;
107 scene_w = that.scene_w; scene_h = that.scene_h;
108 drag_scene = that.drag_scene;
109 object_layer = that.object_layer;
110 replace_layer = that.replace_layer;
111 scene_layer = that.scene_layer;
115 void FindObjConfig::interpolate(FindObjConfig &prev, FindObjConfig &next,
116 int64_t prev_frame, int64_t next_frame, int64_t current_frame)
122 FindObjMain::FindObjMain(PluginServer *server)
123 : PluginVClient(server)
130 object = scene = replace = 0;
131 object_x = object_y = 0;
132 object_w = object_h = 0;
133 scene_x = scene_y = 0;
134 scene_w = scene_h = 0;
139 border_x1 = 0; border_y1 = 0;
140 border_x2 = 0; border_y2 = 0;
141 border_x3 = 0; border_y3 = 0;
142 border_x4 = 0; border_y4 = 0;
144 obj_x1 = 0; obj_y1 = 0;
145 obj_x2 = 0; obj_y2 = 0;
146 obj_x3 = 0; obj_y3 = 0;
147 obj_x4 = 0; obj_y4 = 0;
152 FindObjMain::~FindObjMain()
158 const char* FindObjMain::plugin_title() { return N_("FindObj"); }
159 int FindObjMain::is_realtime() { return 1; }
160 int FindObjMain::is_multichannel() { return 1; }
162 NEW_WINDOW_MACRO(FindObjMain, FindObjWindow)
163 LOAD_CONFIGURATION_MACRO(FindObjMain, FindObjConfig)
165 void FindObjMain::update_gui()
167 if( !thread ) return;
168 if( !load_configuration() ) return;
169 FindObjWindow *window = (FindObjWindow*)thread->window;
170 window->lock_window("FindObjMain::update_gui");
171 window->algorithm->set_text(FindObjAlgorithm::to_text(config.algorithm));
172 window->use_flann->update(config.use_flann);
173 window->drag_object->update(config.drag_object);
174 window->object_x->update(config.object_x);
175 window->object_x_text->update((float)config.object_x);
176 window->object_y->update(config.object_y);
177 window->object_y_text->update((float)config.object_y);
178 window->object_w->update(config.object_w);
179 window->object_w_text->update((float)config.object_w);
180 window->object_h->update(config.object_h);
181 window->object_h_text->update((float)config.object_h);
182 window->drag_scene->update(config.drag_scene);
183 window->scene_x->update(config.scene_x);
184 window->scene_x_text->update((float)config.scene_x);
185 window->scene_y->update(config.scene_y);
186 window->scene_y_text->update((float)config.scene_y);
187 window->scene_w->update(config.scene_w);
188 window->scene_w_text->update((float)config.scene_w);
189 window->scene_h->update(config.scene_h);
190 window->scene_h_text->update((float)config.scene_h);
191 window->draw_keypoints->update(config.draw_keypoints);
192 window->draw_scene_border->update(config.draw_scene_border);
193 window->replace_object->update(config.replace_object);
194 window->draw_object_border->update(config.draw_object_border);
195 window->object_layer->update( (int64_t)config.object_layer);
196 window->replace_layer->update( (int64_t)config.replace_layer);
197 window->scene_layer->update( (int64_t)config.scene_layer);
198 window->blend->update( (int64_t)config.blend);
200 window->unlock_window();
203 void FindObjMain::save_data(KeyFrame *keyframe)
207 // cause data to be stored directly in text
208 output.set_shared_output(keyframe->get_data(), MESSAGESIZE);
209 output.tag.set_title("FINDOBJ");
210 output.tag.set_property("ALGORITHM", config.algorithm);
211 output.tag.set_property("USE_FLANN", config.use_flann);
212 output.tag.set_property("DRAG_OBJECT", config.drag_object);
213 output.tag.set_property("OBJECT_X", config.object_x);
214 output.tag.set_property("OBJECT_Y", config.object_y);
215 output.tag.set_property("OBJECT_W", config.object_w);
216 output.tag.set_property("OBJECT_H", config.object_h);
217 output.tag.set_property("DRAG_SCENE", config.drag_scene);
218 output.tag.set_property("SCENE_X", config.scene_x);
219 output.tag.set_property("SCENE_Y", config.scene_y);
220 output.tag.set_property("SCENE_W", config.scene_w);
221 output.tag.set_property("SCENE_H", config.scene_h);
222 output.tag.set_property("DRAW_KEYPOINTS", config.draw_keypoints);
223 output.tag.set_property("DRAW_SCENE_BORDER", config.draw_scene_border);
224 output.tag.set_property("REPLACE_OBJECT", config.replace_object);
225 output.tag.set_property("DRAW_OBJECT_BORDER", config.draw_object_border);
226 output.tag.set_property("OBJECT_LAYER", config.object_layer);
227 output.tag.set_property("REPLACE_LAYER", config.replace_layer);
228 output.tag.set_property("SCENE_LAYER", config.scene_layer);
229 output.tag.set_property("BLEND", config.blend);
231 output.tag.set_title("/FINDOBJ");
233 output.append_newline();
234 output.terminate_string();
237 void FindObjMain::read_data(KeyFrame *keyframe)
241 input.set_shared_input(keyframe->get_data(), strlen(keyframe->get_data()));
245 while( !(result = input.read_tag()) ) {
246 if( input.tag.title_is("FINDOBJ") ) {
247 config.algorithm = input.tag.get_property("ALGORITHM", config.algorithm);
248 config.use_flann = input.tag.get_property("USE_FLANN", config.use_flann);
249 config.object_x = input.tag.get_property("OBJECT_X", config.object_x);
250 config.object_y = input.tag.get_property("OBJECT_Y", config.object_y);
251 config.object_w = input.tag.get_property("OBJECT_W", config.object_w);
252 config.object_h = input.tag.get_property("OBJECT_H", config.object_h);
253 config.drag_object = input.tag.get_property("DRAG_OBJECT", config.drag_object);
254 config.scene_x = input.tag.get_property("SCENE_X", config.scene_x);
255 config.scene_y = input.tag.get_property("SCENE_Y", config.scene_y);
256 config.scene_w = input.tag.get_property("SCENE_W", config.scene_w);
257 config.scene_h = input.tag.get_property("SCENE_H", config.scene_h);
258 config.drag_scene = input.tag.get_property("DRAG_SCENE", config.drag_scene);
259 config.draw_keypoints = input.tag.get_property("DRAW_KEYPOINTS", config.draw_keypoints);
260 config.draw_scene_border = input.tag.get_property("DRAW_SCENE_BORDER", config.draw_scene_border);
261 config.replace_object = input.tag.get_property("REPLACE_OBJECT", config.replace_object);
262 config.draw_object_border = input.tag.get_property("DRAW_OBJECT_BORDER", config.draw_object_border);
263 config.object_layer = input.tag.get_property("OBJECT_LAYER", config.object_layer);
264 config.replace_layer = input.tag.get_property("REPLACE_LAYER", config.replace_layer);
265 config.scene_layer = input.tag.get_property("SCENE_LAYER", config.scene_layer);
266 config.blend = input.tag.get_property("BLEND", config.blend);
273 void FindObjMain::draw_line(VFrame *vframe, int x1, int y1, int x2, int y2)
275 vframe->draw_line(x1, y1, x2, y2);
278 void FindObjMain::draw_rect(VFrame *vframe, int x1, int y1, int x2, int y2)
281 draw_line(vframe, x1, y1, x2, y1);
282 draw_line(vframe, x2, y1, x2, y2);
283 draw_line(vframe, x2, y2, x1, y2);
284 draw_line(vframe, x1, y2, x1, y1);
287 void FindObjMain::draw_circle(VFrame *vframe, int x, int y, int r)
289 int x1 = x-r, x2 = x+r;
290 int y1 = y-r, y2 = y+r;
291 vframe->draw_smooth(x1,y, x1,y1, x,y1);
292 vframe->draw_smooth(x,y1, x2,y1, x2,y);
293 vframe->draw_smooth(x2,y, x2,y2, x,y2);
294 vframe->draw_smooth(x,y2, x1,y2, x1,y);
297 void FindObjMain::filter_matches(ptV &p1, ptV &p2, double ratio)
299 DMatches::iterator it;
300 for( it=pairs.begin(); it!=pairs.end(); ++it ) {
302 if( m.size() == 2 && m[0].distance < m[1].distance*ratio ) {
303 p1.push_back(obj_keypts[m[0].queryIdx].pt);
304 p2.push_back(scn_keypts[m[0].trainIdx].pt);
309 void FindObjMain::to_mat(Mat &mat, int mcols, int mrows,
310 VFrame *inp, int ix,int iy, BC_CModel mcolor_model)
312 int mcomp = BC_CModels::components(mcolor_model);
313 int mbpp = BC_CModels::calculate_pixelsize(mcolor_model);
314 int psz = mbpp / mcomp;
315 int mdepth = psz < 2 ? CV_8U : CV_16U;
316 if( mat.dims != 2 || mat.depth() != mdepth || mat.channels() != mcomp ||
317 mat.cols != mcols || mat.rows != mrows ) {
321 int type = CV_MAKETYPE(mdepth, mcomp);
322 mat.create(mrows, mcols, type);
324 uint8_t *mat_rows[mrows];
325 for( int y=0; y<mrows; ++y ) mat_rows[y] = mat.ptr(y);
326 uint8_t **inp_rows = inp->get_rows();
327 int ibpl = inp->get_bytes_per_line(), obpl = mcols * mbpp;
328 int icolor_model = inp->get_color_model();
329 BC_CModels::transfer(mat_rows, mcolor_model, 0,0, mcols,mrows, obpl,
330 inp_rows, icolor_model, ix,iy, mcols,mrows, ibpl, 0);
331 // VFrame vfrm(mat_rows[0], -1, mcols,mrows, mcolor_model, mat_rows[1]-mat_rows[0]);
332 // static int vfrm_no = 0; char vfn[64]; sprintf(vfn,"/tmp/dat/%06d.png", vfrm_no++);
333 // vfrm.write_png(vfn);
336 void FindObjMain::detect(Mat &mat, KeyPointV &keypts,Mat &descrs)
341 detector->detectAndCompute(mat, noArray(), keypts, descrs);
342 } catch(std::exception e) { printf(_("detector exception: %s\n"), e.what()); }
345 void FindObjMain::match()
349 matcher->knnMatch(obj_descrs, scn_descrs, pairs, 2);
350 } catch(std::exception e) { printf(_("match execption: %s\n"), e.what()); }
353 Ptr<DescriptorMatcher> FindObjMain::flann_kdtree_matcher()
355 const Ptr<flann::IndexParams>& indexParams =
356 makePtr<flann::KDTreeIndexParams>(5);
357 const Ptr<flann::SearchParams>& searchParams =
358 makePtr<flann::SearchParams>();
359 return makePtr<FlannBasedMatcher>(indexParams, searchParams);
361 Ptr<DescriptorMatcher> FindObjMain::flann_lshidx_matcher()
362 { // table_number = 6#12, key_size = 12#20, multi_probe_level = 1#2
363 const Ptr<flann::IndexParams>& indexParams =
364 makePtr<flann::LshIndexParams>(6, 12, 1);
365 const Ptr<flann::SearchParams>& searchParams =
366 makePtr<flann::SearchParams>();
367 return makePtr<FlannBasedMatcher>(indexParams, searchParams);
369 Ptr<DescriptorMatcher> FindObjMain::bf_matcher_norm_l2()
371 return BFMatcher::create(NORM_L2);
373 Ptr<DescriptorMatcher> FindObjMain::bf_matcher_norm_hamming()
375 return BFMatcher::create(NORM_HAMMING);
379 void FindObjMain::set_sift()
382 detector = SIFT::create();
383 matcher = config.use_flann ?
384 flann_kdtree_matcher() : bf_matcher_norm_l2();
388 void FindObjMain::set_surf()
391 detector = SURF::create(800);
392 matcher = config.use_flann ?
393 flann_kdtree_matcher() : bf_matcher_norm_l2();
397 void FindObjMain::set_orb()
400 detector = ORB::create();
401 matcher = config.use_flann ?
402 flann_lshidx_matcher() : bf_matcher_norm_hamming();
406 void FindObjMain::set_akaze()
409 detector = AKAZE::create();
410 matcher = config.use_flann ?
411 flann_lshidx_matcher() : bf_matcher_norm_hamming();
415 void FindObjMain::set_brisk()
418 detector = BRISK::create();
419 matcher = config.use_flann ?
420 flann_lshidx_matcher() : bf_matcher_norm_hamming();
424 void FindObjMain::process_match()
426 if( config.algorithm == NO_ALGORITHM ) return;
427 if( !config.replace_object &&
428 !config.draw_scene_border &&
429 !config.draw_keypoints ) return;
431 if( detector.empty() ) {
432 switch( config.algorithm ) {
434 case ALGORITHM_SIFT: set_sift(); break;
437 case ALGORITHM_SURF: set_surf(); break;
440 case ALGORITHM_ORB: set_orb(); break;
443 case ALGORITHM_AKAZE: set_akaze(); break;
446 case ALGORITHM_BRISK: set_brisk(); break;
449 obj_keypts.clear(); obj_descrs.release();
450 to_mat(object_mat, object_w,object_h, object, object_x,object_y, cvmodel);
451 detect(object_mat, obj_keypts, obj_descrs);
452 //printf("detect obj %d features\n", (int)obj_keypts.size());
455 to_mat(scene_mat, scene_w,scene_h, scene, scene_x,scene_y, cvmodel);
456 detect(scene_mat, scn_keypts, scn_descrs);
457 //printf("detect scn %d features\n", (int)scn_keypts.size());
460 filter_matches(p1, p2);
461 if( p1.size() < 4 ) return;
462 Mat H = findHomography(p1, p2, RANSAC, 5.0);
463 if( !H.dims || !H.rows || !H.cols ) {
464 //printf("fail, size p1=%d,p2=%d\n",(int)p1.size(),(int)p2.size());
469 float obj_x1 = 0, obj_x2 = object_w;
470 float obj_y1 = 0, obj_y2 = object_h;
471 src.push_back(Point2f(obj_x1,obj_y1));
472 src.push_back(Point2f(obj_x2,obj_y1));
473 src.push_back(Point2f(obj_x2,obj_y2));
474 src.push_back(Point2f(obj_x1,obj_y2));
475 perspectiveTransform(src, dst, H);
477 border_x1 = dst[0].x + scene_x; border_y1 = dst[0].y + scene_y;
478 border_x2 = dst[1].x + scene_x; border_y2 = dst[1].y + scene_y;
479 border_x3 = dst[2].x + scene_x; border_y3 = dst[2].y + scene_y;
480 border_x4 = dst[3].x + scene_x; border_y4 = dst[3].y + scene_y;
481 //printf("src %f,%f %f,%f %f,%f %f,%f\n",
482 // src[0].x,src[0].y, src[1].x,src[1].y, src[2].x,src[2].y, src[3].x,src[3].y);
483 //printf("dst %f,%f %f,%f %f,%f %f,%f\n",
484 // dst[0].x,dst[0].y, dst[1].x,dst[1].y, dst[2].x,dst[2].y, dst[3].x,dst[3].y);
487 int FindObjMain::process_buffer(VFrame **frame, int64_t start_position, double frame_rate)
489 int prev_algorithm = config.algorithm;
490 int prev_use_flann = config.use_flann;
492 if( load_configuration() )
495 if( prev_algorithm != config.algorithm ||
496 prev_use_flann != config.use_flann ) {
501 object_layer = config.object_layer;
502 scene_layer = config.scene_layer;
503 replace_layer = config.replace_layer;
504 Track *track = server->plugin->track;
508 int max_layer = PluginClient::get_total_buffers() - 1;
509 object_layer = bclip(config.object_layer, 0, max_layer);
510 scene_layer = bclip(config.scene_layer, 0, max_layer);
511 replace_layer = bclip(config.replace_layer, 0, max_layer);
513 int cfg_w = (int)(w * config.object_w / 100.);
514 int cfg_h = (int)(h * config.object_h / 100.);
515 int cfg_x1 = (int)(w * config.object_x / 100. - cfg_w / 2);
516 int cfg_y1 = (int)(h * config.object_y / 100. - cfg_h / 2);
517 int cfg_x2 = cfg_x1 + cfg_w;
518 int cfg_y2 = cfg_y1 + cfg_h;
519 bclamp(cfg_x1, 0, w); object_x = cfg_x1;
520 bclamp(cfg_y1, 0, h); object_y = cfg_y1;
521 bclamp(cfg_x2, 0, w); object_w = cfg_x2 - cfg_x1;
522 bclamp(cfg_y2, 0, h); object_h = cfg_y2 - cfg_y1;
524 cfg_w = (int)(w * config.scene_w / 100.);
525 cfg_h = (int)(h * config.scene_h / 100.);
526 cfg_x1 = (int)(w * config.scene_x / 100. - cfg_w / 2);
527 cfg_y1 = (int)(h * config.scene_y / 100. - cfg_h / 2);
528 cfg_x2 = cfg_x1 + cfg_w;
529 cfg_y2 = cfg_y1 + cfg_h;
530 bclamp(cfg_x1, 0, w); scene_x = cfg_x1;
531 bclamp(cfg_y1, 0, h); scene_y = cfg_y1;
532 bclamp(cfg_x2, 0, w); scene_w = cfg_x2 - cfg_x1;
533 bclamp(cfg_y2, 0, h); scene_h = cfg_y2 - cfg_y1;
535 // Read in the input frames
536 for( int i = 0; i < PluginClient::get_total_buffers(); i++ ) {
537 read_frame(frame[i], i, start_position, frame_rate, 0);
540 object = frame[object_layer];
541 scene = frame[scene_layer];
542 replace = frame[replace_layer];
544 border_x1 = obj_x1; border_y1 = obj_y1;
545 border_x2 = obj_x2; border_y2 = obj_y2;
546 border_x3 = obj_x3; border_y3 = obj_y3;
547 border_x4 = obj_x4; border_y4 = obj_y4;
549 if( scene_w > 0 && scene_h > 0 && object_w > 0 && object_h > 0 ) {
553 double w0 = init_border ? (init_border=0, 1.) : config.blend/100., w1 = 1. - w0;
554 obj_x1 = border_x1*w0 + obj_x1*w1; obj_y1 = border_y1*w0 + obj_y1*w1;
555 obj_x2 = border_x2*w0 + obj_x2*w1; obj_y2 = border_y2*w0 + obj_y2*w1;
556 obj_x3 = border_x3*w0 + obj_x3*w1; obj_y3 = border_y3*w0 + obj_y3*w1;
557 obj_x4 = border_x4*w0 + obj_x4*w1; obj_y4 = border_y4*w0 + obj_y4*w1;
559 // Replace object in the scene layer
560 if( config.replace_object ) {
561 int cpus1 = get_project_smp() + 1;
563 affine = new AffineEngine(cpus1, cpus1);
565 overlayer = new OverlayFrame(cpus1);
567 VFrame *temp = new_temp(scene->get_w(), scene->get_h(), scene->get_color_model());
569 float sx = 100./w, sy = 100./h;
570 float x1 = sx * obj_x1, y1 = sy * obj_y1;
571 float x2 = sx * obj_x2, y2 = sy * obj_y2;
572 float x3 = sx * obj_x3, y3 = sy * obj_y3;
573 float x4 = sx * obj_x4, y4 = sy * obj_y4;
574 affine->process(temp, replace, 0, AffineEngine::PERSPECTIVE,
575 x1,y1, x2,y2, x3,y3, x4,y4, 1);
576 overlayer->overlay(scene, temp, 0,0, w,h, 0,0, w,h,
577 1, TRANSFER_NORMAL, NEAREST_NEIGHBOR);
581 if( config.draw_scene_border ) {
582 int wh = (w+h)>>8, ss = 1; while( wh>>=1 ) ss<<=1;
583 scene->set_pixel_color(WHITE); scene->set_stiple(ss*2);
584 draw_rect(scene, scene_x, scene_y, scene_x+scene_w, scene_y+scene_h);
586 if( config.draw_object_border ) {
587 int wh = (w+h)>>8, ss = 1; while( wh>>=1 ) ss<<=1;
588 scene->set_pixel_color(YELLOW); scene->set_stiple(ss*3);
589 draw_rect(scene, object_x, object_y, object_x+object_w, object_y+object_h);
591 if( config.draw_keypoints ) {
592 scene->set_pixel_color(RED); scene->set_stiple(0);
593 for( int i=0,n=scn_keypts.size(); i<n; ++i ) {
594 Point2f &pt = scn_keypts[i].pt;
595 int r = scn_keypts[i].size * 1.2/9 * 2;
596 int x = pt.x + scene_x, y = pt.y + scene_y;
597 draw_circle(scene, x, y, r);
602 if( config.drag_scene ) {
603 scene->set_pixel_color(WHITE);
604 DragCheckBox::draw_boundary(scene, scene_x, scene_y, scene_w, scene_h);
606 if( config.drag_object ) {
607 scene->set_pixel_color(YELLOW);
608 DragCheckBox::draw_boundary(scene, object_x, object_y, object_w, object_h);
612 scene->set_pixel_color(BLACK);
613 scene->set_stiple(0);