cinelerra-5.1/plugins/findobj/findobj.C

   1
   2 /*
   3  * CINELERRA
   4  * Copyright (C) 1997-2012 Adam Williams <broadcast at earthling dot net>
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License as published by
   8  * the Free Software Foundation; either version 2 of the License, or
   9  * (at your option) any later version.
  10  *
  11  * This program is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14  * GNU General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU General Public License
  17  * along with this program; if not, write to the Free Software
  18  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  19  */
  20
  21 #include "affine.h"
  22 #include "bccolors.h"
  23 #include "clip.h"
  24 #include "edl.h"
  25 #include "edlsession.h"
  26 #include "filexml.h"
  27 #include "language.h"
  28 #include "findobj.h"
  29 #include "findobjwindow.h"
  30 #include "mutex.h"
  31 #include "overlayframe.h"
  32 #include "plugin.h"
  33 #include "pluginserver.h"
  34 #include "track.h"
  35 #include "tracks.h"
  36
  37 #include <errno.h>
  38 #include <exception>
  39 #include <unistd.h>
  40
  41 REGISTER_PLUGIN(FindObjMain)
  42
  43 FindObjConfig::FindObjConfig()
  44 {
  45         reset();
  46 }
  47
  48 void FindObjConfig::reset()
  49 {
  50         algorithm = NO_ALGORITHM;
  51         use_flann = 1;
  52         mode = MODE_QUADRILATERAL;
  53         draw_match = 0;
  54         aspect = 1;
  55         scale = 1;
  56         translate = 1;
  57         rotate = 1;
  58         draw_keypoints = 0;
  59         draw_scene_border = 0;
  60         replace_object = 0;
  61         draw_object_border = 0;
  62         draw_replace_border = 0;
  63         object_x = 50;  object_y = 50;
  64         object_w = 100; object_h = 100;
  65         drag_object = 0;
  66         scene_x = 50;   scene_y = 50;
  67         scene_w = 100;  scene_h = 100;
  68         drag_replace = 0;
  69         replace_x = 50;   replace_y = 50;
  70         replace_w = 100;  replace_h = 100;
  71         replace_dx = 0;   replace_dy = 0;
  72         drag_scene = 0;
  73         scene_layer = 0;
  74         object_layer = 1;
  75         replace_layer = 2;
  76         blend = 100;
  77 }
  78
  79 void FindObjConfig::boundaries()
  80 {
  81         bclamp(object_x, 0, 100);  bclamp(object_y, 0, 100);
  82         bclamp(object_w, 0, 100);  bclamp(object_h, 0, 100);
  83         bclamp(scene_x, 0, 100);   bclamp(scene_y, 0, 100);
  84         bclamp(scene_w, 0, 100);   bclamp(scene_h, 0, 100);
  85         bclamp(object_layer, MIN_LAYER, MAX_LAYER);
  86         bclamp(replace_layer, MIN_LAYER, MAX_LAYER);
  87         bclamp(scene_layer, MIN_LAYER, MAX_LAYER);
  88         bclamp(blend, MIN_BLEND, MAX_BLEND);
  89 }
  90
  91 int FindObjConfig::equivalent(FindObjConfig &that)
  92 {
  93         int result =
  94                 algorithm == that.algorithm &&
  95                 use_flann == that.use_flann &&
  96                 mode == that.mode &&
  97                 aspect == that.aspect &&
  98                 scale == that.scale &&
  99                 translate == that.translate &&
 100                 rotate == that.rotate &&
 101                 draw_keypoints == that.draw_keypoints &&
 102                 draw_match == that.draw_match &&
 103                 draw_scene_border == that.draw_scene_border &&
 104                 replace_object == that.replace_object &&
 105                 draw_object_border == that.draw_object_border &&
 106                 draw_replace_border == that.draw_replace_border &&
 107                 object_x == that.object_x && object_y == that.object_y &&
 108                 object_w == that.object_w && object_h == that.object_h &&
 109                 drag_object == that.drag_object &&
 110                 scene_x == that.scene_x && scene_y == that.scene_y &&
 111                 scene_w == that.scene_w && scene_h == that.scene_h &&
 112                 drag_scene == that.drag_scene &&
 113                 replace_x == that.replace_x && replace_y == that.replace_y &&
 114                 replace_w == that.replace_w && replace_h == that.replace_h &&
 115                 replace_dx == that.replace_dx && replace_dy == that.replace_dy &&
 116                 drag_replace == that.drag_replace &&
 117                 object_layer == that.object_layer &&
 118                 replace_layer == that.replace_layer &&
 119                 scene_layer == that.scene_layer &&
 120                 blend == that.blend;
 121         return result;
 122 }
 123
 124 void FindObjConfig::copy_from(FindObjConfig &that)
 125 {
 126         algorithm = that.algorithm;
 127         use_flann = that.use_flann;
 128         mode = that.mode;
 129         aspect = that.aspect;
 130         scale = that.scale;
 131         translate = that.translate;
 132         rotate = that.rotate;
 133         draw_keypoints = that.draw_keypoints;
 134         draw_match = that.draw_match;
 135         draw_scene_border = that.draw_scene_border;
 136         replace_object = that.replace_object;
 137         draw_object_border = that.draw_object_border;
 138         draw_replace_border = that.draw_replace_border;
 139         object_x = that.object_x;  object_y = that.object_y;
 140         object_w = that.object_w;  object_h = that.object_h;
 141         drag_object = that.drag_object;
 142         scene_x = that.scene_x;    scene_y = that.scene_y;
 143         scene_w = that.scene_w;    scene_h = that.scene_h;
 144         drag_scene = that.drag_scene;
 145         replace_x = that.replace_x;   replace_y = that.replace_y;
 146         replace_w = that.replace_w;   replace_h = that.replace_h;
 147         replace_dx = that.replace_dx; replace_dy = that.replace_dy;
 148         drag_replace = that.drag_replace;
 149         object_layer = that.object_layer;
 150         replace_layer = that.replace_layer;
 151         scene_layer = that.scene_layer;
 152         blend = that.blend;
 153 }
 154
 155 void FindObjConfig::interpolate(FindObjConfig &prev, FindObjConfig &next,
 156         int64_t prev_frame, int64_t next_frame, int64_t current_frame)
 157 {
 158         copy_from(prev);
 159 }
 160
 161
 162 FindObjMain::FindObjMain(PluginServer *server)
 163  : PluginVClient(server)
 164 {
 165         affine = 0;
 166         overlayer = 0;
 167
 168         cvmodel = BC_RGB888;
 169         w = h = 0;
 170         object = scene = replace = 0;
 171         object_x = object_y = 0;
 172         object_w = object_h = 0;
 173         scene_x = scene_y = 0;
 174         scene_w = scene_h = 0;
 175         object_layer = 0;
 176         scene_layer = 1;
 177         replace_layer = 2;
 178
 179         match_x1 = 0;  match_y1 = 0;
 180         match_x2 = 0;  match_y2 = 0;
 181         match_x3 = 0;  match_y3 = 0;
 182         match_x4 = 0;  match_y4 = 0;
 183         shape_x1 = 0;  shape_y1 = 0;
 184         shape_x2 = 0;  shape_y2 = 0;
 185         shape_x3 = 0;  shape_y3 = 0;
 186         shape_x4 = 0;  shape_y4 = 0;
 187         out_x1 = 0;    out_y1 = 0;
 188         out_x2 = 0;    out_y2 = 0;
 189         out_x3 = 0;    out_y3 = 0;
 190         out_x4 = 0;    out_y4 = 0;
 191
 192         init_border = 1;
 193 }
 194
 195 FindObjMain::~FindObjMain()
 196 {
 197         delete affine;
 198         delete overlayer;
 199 }
 200
 201 const char* FindObjMain::plugin_title() { return N_("FindObj"); }
 202 int FindObjMain::is_realtime() { return 1; }
 203 int FindObjMain::is_multichannel() { return 1; }
 204
 205 NEW_WINDOW_MACRO(FindObjMain, FindObjWindow)
 206 LOAD_CONFIGURATION_MACRO(FindObjMain, FindObjConfig)
 207
 208 void FindObjMain::update_gui()
 209 {
 210         if( !thread ) return;
 211         if( !load_configuration() ) return;
 212         FindObjWindow *window = (FindObjWindow*)thread->window;
 213         window->lock_window("FindObjMain::update_gui");
 214         window->update_gui();
 215         window->flush();
 216         window->unlock_window();
 217 }
 218
 219 void FindObjMain::save_data(KeyFrame *keyframe)
 220 {
 221         FileXML output;
 222
 223 // cause data to be stored directly in text
 224         output.set_shared_output(keyframe->xbuf);
 225         output.tag.set_title("FINDOBJ");
 226         output.tag.set_property("ALGORITHM", config.algorithm);
 227         output.tag.set_property("USE_FLANN", config.use_flann);
 228         output.tag.set_property("MODE", config.mode);
 229         output.tag.set_property("ASPECT", config.aspect);
 230         output.tag.set_property("SCALE", config.scale);
 231         output.tag.set_property("TRANSLATE", config.translate);
 232         output.tag.set_property("ROTATE", config.rotate);
 233         output.tag.set_property("DRAG_OBJECT", config.drag_object);
 234         output.tag.set_property("OBJECT_X", config.object_x);
 235         output.tag.set_property("OBJECT_Y", config.object_y);
 236         output.tag.set_property("OBJECT_W", config.object_w);
 237         output.tag.set_property("OBJECT_H", config.object_h);
 238         output.tag.set_property("DRAG_SCENE", config.drag_scene);
 239         output.tag.set_property("SCENE_X", config.scene_x);
 240         output.tag.set_property("SCENE_Y", config.scene_y);
 241         output.tag.set_property("SCENE_W", config.scene_w);
 242         output.tag.set_property("SCENE_H", config.scene_h);
 243         output.tag.set_property("DRAG_REPLACE", config.drag_replace);
 244         output.tag.set_property("REPLACE_X", config.replace_x);
 245         output.tag.set_property("REPLACE_Y", config.replace_y);
 246         output.tag.set_property("REPLACE_W", config.replace_w);
 247         output.tag.set_property("REPLACE_H", config.replace_h);
 248         output.tag.set_property("REPLACE_DX", config.replace_dx);
 249         output.tag.set_property("REPLACE_DY", config.replace_dy);
 250         output.tag.set_property("DRAW_KEYPOINTS", config.draw_keypoints);
 251         output.tag.set_property("DRAW_MATCH", config.draw_match);
 252         output.tag.set_property("DRAW_SCENE_BORDER", config.draw_scene_border);
 253         output.tag.set_property("REPLACE_OBJECT", config.replace_object);
 254         output.tag.set_property("DRAW_OBJECT_BORDER", config.draw_object_border);
 255         output.tag.set_property("DRAW_REPLACE_BORDER", config.draw_replace_border);
 256         output.tag.set_property("OBJECT_LAYER", config.object_layer);
 257         output.tag.set_property("REPLACE_LAYER", config.replace_layer);
 258         output.tag.set_property("SCENE_LAYER", config.scene_layer);
 259         output.tag.set_property("BLEND", config.blend);
 260         output.append_tag();
 261         output.tag.set_title("/FINDOBJ");
 262         output.append_tag();
 263         output.append_newline();
 264         output.terminate_string();
 265 }
 266
 267 void FindObjMain::read_data(KeyFrame *keyframe)
 268 {
 269         FileXML input;
 270
 271         input.set_shared_input(keyframe->xbuf);
 272
 273         int result = 0;
 274
 275         while( !(result = input.read_tag()) ) {
 276                 if( input.tag.title_is("FINDOBJ") ) {
 277                         config.algorithm = input.tag.get_property("ALGORITHM", config.algorithm);
 278                         config.use_flann = input.tag.get_property("USE_FLANN", config.use_flann);
 279                         config.mode = input.tag.get_property("MODE", config.mode);
 280                         config.aspect = input.tag.get_property("ASPECT", config.aspect);
 281                         config.scale = input.tag.get_property("SCALE", config.scale);
 282                         config.translate = input.tag.get_property("TRANSLATE", config.translate);
 283                         config.rotate = input.tag.get_property("ROTATE", config.rotate);
 284                         config.object_x = input.tag.get_property("OBJECT_X", config.object_x);
 285                         config.object_y = input.tag.get_property("OBJECT_Y", config.object_y);
 286                         config.object_w = input.tag.get_property("OBJECT_W", config.object_w);
 287                         config.object_h = input.tag.get_property("OBJECT_H", config.object_h);
 288                         config.drag_object = input.tag.get_property("DRAG_OBJECT", config.drag_object);
 289                         config.scene_x = input.tag.get_property("SCENE_X", config.scene_x);
 290                         config.scene_y = input.tag.get_property("SCENE_Y", config.scene_y);
 291                         config.scene_w = input.tag.get_property("SCENE_W", config.scene_w);
 292                         config.scene_h = input.tag.get_property("SCENE_H", config.scene_h);
 293                         config.drag_scene = input.tag.get_property("DRAG_SCENE", config.drag_scene);
 294                         config.replace_x = input.tag.get_property("REPLACE_X", config.replace_x);
 295                         config.replace_y = input.tag.get_property("REPLACE_Y", config.replace_y);
 296                         config.replace_w = input.tag.get_property("REPLACE_W", config.replace_w);
 297                         config.replace_h = input.tag.get_property("REPLACE_H", config.replace_h);
 298                         config.replace_dx = input.tag.get_property("REPLACE_DX", config.replace_dx);
 299                         config.replace_dy = input.tag.get_property("REPLACE_DY", config.replace_dy);
 300                         config.drag_replace = input.tag.get_property("DRAG_REPLACE", config.drag_replace);
 301                         config.draw_keypoints = input.tag.get_property("DRAW_KEYPOINTS", config.draw_keypoints);
 302                         config.draw_match = input.tag.get_property("DRAW_MATCH", config.draw_match);
 303                         config.draw_scene_border = input.tag.get_property("DRAW_SCENE_BORDER", config.draw_scene_border);
 304                         config.replace_object = input.tag.get_property("REPLACE_OBJECT", config.replace_object);
 305                         config.draw_object_border = input.tag.get_property("DRAW_OBJECT_BORDER", config.draw_object_border);
 306                         config.draw_replace_border = input.tag.get_property("DRAW_REPLACE_BORDER", config.draw_replace_border);
 307                         config.object_layer = input.tag.get_property("OBJECT_LAYER", config.object_layer);
 308                         config.replace_layer = input.tag.get_property("REPLACE_LAYER", config.replace_layer);
 309                         config.scene_layer = input.tag.get_property("SCENE_LAYER", config.scene_layer);
 310                         config.blend = input.tag.get_property("BLEND", config.blend);
 311                 }
 312         }
 313
 314         config.boundaries();
 315 }
 316
 317 Track *FindObjMain::get_plugin_track()
 318 {
 319         int plugin_id = server->plugin_id;
 320         Plugin *plugin = server->edl->tracks->plugin_exists(plugin_id);
 321         return !plugin ? 0 : plugin->track;
 322 }
 323
 324 void FindObjMain::draw_line(VFrame *vframe, int x1, int y1, int x2, int y2)
 325 {
 326         vframe->draw_line(x1, y1, x2, y2);
 327 }
 328
 329 void FindObjMain::draw_quad(VFrame *vframe,
 330                 int x1, int y1, int x2, int y2,
 331                 int x3, int y3, int x4, int y4)
 332 {
 333         int r = bmin(vframe->get_w(), vframe->get_h()) / 200 + 1;
 334         for( int i=r; --i>0; ) {
 335                 draw_line(vframe, x1+i, y1+i, x2, y2);
 336                 draw_line(vframe, x1-i, y1-i, x2, y2);
 337                 draw_line(vframe, x2+i, y2+i, x3, y3);
 338                 draw_line(vframe, x2-i, y2-i, x3, y3);
 339                 draw_line(vframe, x3+i, y3+i, x4, y4);
 340                 draw_line(vframe, x3-i, y3-i, x4, y4);
 341                 draw_line(vframe, x4+i, y4+i, x1, y1);
 342                 draw_line(vframe, x4-i, y4-i, x1, y1);
 343         }
 344         draw_line(vframe, x1, y1, x2, y2);
 345         draw_line(vframe, x2, y2, x3, y3);
 346         draw_line(vframe, x3, y3, x4, y4);
 347         draw_line(vframe, x4, y4, x1, y1);
 348 }
 349
 350 void FindObjMain::draw_point(VFrame *vframe, int x1, int y1)
 351 {
 352         int r = bmin(vframe->get_w(), vframe->get_h()) / 200 + 1;
 353         for( int i=r; --i>0; )
 354                 draw_circle(vframe, x1, y1, i);
 355 }
 356
 357 void FindObjMain::draw_rect(VFrame *vframe, int x1, int y1, int x2, int y2)
 358 {
 359         int r = bmin(vframe->get_w(), vframe->get_h()) / 200 + 1;
 360         for( int i=r; --i>0; ) {
 361                 --x2;  --y2;
 362                 draw_line(vframe, x1, y1, x2, y1);
 363                 draw_line(vframe, x2, y1, x2, y2);
 364                 draw_line(vframe, x2, y2, x1, y2);
 365                 draw_line(vframe, x1, y2, x1, y1);
 366                 ++x1;  ++y1;
 367         }
 368 }
 369
 370 void FindObjMain::draw_circle(VFrame *vframe, int x, int y, int r)
 371 {
 372         int x1 = x-r, x2 = x+r;
 373         int y1 = y-r, y2 = y+r;
 374         vframe->draw_smooth(x1,y, x1,y1, x,y1);
 375         vframe->draw_smooth(x,y1, x2,y1, x2,y);
 376         vframe->draw_smooth(x2,y, x2,y2, x,y2);
 377         vframe->draw_smooth(x,y2, x1,y2, x1,y);
 378 }
 379
 380 void FindObjMain::filter_matches(ptV &p1, ptV &p2, double ratio)
 381 {
 382         DMatches::iterator it;
 383         for( it=pairs.begin(); it!=pairs.end(); ++it ) {
 384                 DMatchV &m = *it;
 385                 if( m.size() == 2 && m[0].distance < m[1].distance*ratio ) {
 386                         p1.push_back(obj_keypts[m[0].queryIdx].pt);
 387                         p2.push_back(scn_keypts[m[0].trainIdx].pt);
 388                 }
 389         }
 390 }
 391
 392 void FindObjMain::to_mat(Mat &mat, int mcols, int mrows,
 393         VFrame *inp, int ix,int iy, BC_CModel mcolor_model)
 394 {
 395         int mcomp = BC_CModels::components(mcolor_model);
 396         int mbpp = BC_CModels::calculate_pixelsize(mcolor_model);
 397         int psz = mbpp / mcomp;
 398         int mdepth = psz < 2 ? CV_8U : CV_16U;
 399         if( mat.dims != 2 || mat.depth() != mdepth || mat.channels() != mcomp ||
 400             mat.cols != mcols || mat.rows != mrows ) {
 401                 mat.release();
 402         }
 403         if( mat.empty() ) {
 404                 int type = CV_MAKETYPE(mdepth, mcomp);
 405                 mat.create(mrows, mcols, type);
 406         }
 407         uint8_t *mat_rows[mrows];
 408         for( int y=0; y<mrows; ++y ) mat_rows[y] = mat.ptr(y);
 409         uint8_t **inp_rows = inp->get_rows();
 410         int ibpl = inp->get_bytes_per_line(), obpl = mcols * mbpp;
 411         int icolor_model = inp->get_color_model();
 412         BC_CModels::transfer(mat_rows, mcolor_model, 0,0, mcols,mrows, obpl,
 413                 inp_rows, icolor_model, ix,iy, mcols,mrows, ibpl, 0);
 414 //      VFrame vfrm(mat_rows[0], -1, mcols,mrows, mcolor_model, mat_rows[1]-mat_rows[0]);
 415 //      static int vfrm_no = 0; char vfn[64]; sprintf(vfn,"/tmp/dat/%06d.png", vfrm_no++);
 416 //      vfrm.write_png(vfn);
 417 }
 418
 419 void FindObjMain::detect(Mat &mat, KeyPointV &keypts,Mat &descrs)
 420 {
 421         keypts.clear();
 422         descrs.release();
 423         try {
 424                 detector->detectAndCompute(mat, noArray(), keypts, descrs);
 425         } catch(std::exception &e) { printf(_("detector exception: %s\n"), e.what()); }
 426 }
 427
 428 void FindObjMain::match()
 429 {
 430         pairs.clear();
 431         try {
 432                 matcher->knnMatch(obj_descrs, scn_descrs, pairs, 2);
 433         } catch(std::exception &e) { printf(_("match execption: %s\n"), e.what()); }
 434 }
 435
 436 Ptr<DescriptorMatcher> FindObjMain::flann_kdtree_matcher()
 437 { // trees=5
 438         const Ptr<flann::IndexParams>& indexParams =
 439                 makePtr<flann::KDTreeIndexParams>(5);
 440         const Ptr<flann::SearchParams>& searchParams =
 441                 makePtr<flann::SearchParams>();
 442         return makePtr<FlannBasedMatcher>(indexParams, searchParams);
 443 }
 444 Ptr<DescriptorMatcher> FindObjMain::flann_lshidx_matcher()
 445 { // table_number = 6#12, key_size = 12#20, multi_probe_level = 1#2
 446         const Ptr<flann::IndexParams>& indexParams =
 447                 makePtr<flann::LshIndexParams>(6, 12, 1);
 448         const Ptr<flann::SearchParams>& searchParams =
 449                 makePtr<flann::SearchParams>();
 450         return makePtr<FlannBasedMatcher>(indexParams, searchParams);
 451 }
 452 Ptr<DescriptorMatcher> FindObjMain::bf_matcher_norm_l2()
 453 {
 454         return BFMatcher::create(NORM_L2);
 455 }
 456 Ptr<DescriptorMatcher> FindObjMain::bf_matcher_norm_hamming()
 457 {
 458         return BFMatcher::create(NORM_HAMMING);
 459 }
 460
 461 #ifdef _SIFT
 462 void FindObjMain::set_sift()
 463 {
 464         cvmodel = BC_GREY8;
 465         detector = SIFT::create();
 466         matcher = config.use_flann ?
 467                 flann_kdtree_matcher() : bf_matcher_norm_l2();
 468 }
 469 #endif
 470 #ifdef _SURF
 471 void FindObjMain::set_surf()
 472 {
 473         cvmodel = BC_GREY8;
 474         detector = SURF::create(800);
 475         matcher = config.use_flann ?
 476                 flann_kdtree_matcher() : bf_matcher_norm_l2();
 477 }
 478 #endif
 479 #ifdef _ORB
 480 void FindObjMain::set_orb()
 481 {
 482         cvmodel = BC_GREY8;
 483         detector = ORB::create();
 484         matcher = config.use_flann ?
 485                 flann_lshidx_matcher() : bf_matcher_norm_hamming();
 486 }
 487 #endif
 488 #ifdef _AKAZE
 489 void FindObjMain::set_akaze()
 490 {
 491         cvmodel = BC_GREY8;
 492         detector = AKAZE::create();
 493         matcher = config.use_flann ?
 494                 flann_lshidx_matcher() : bf_matcher_norm_hamming();
 495 }
 496 #endif
 497 #ifdef _BRISK
 498 void FindObjMain::set_brisk()
 499 {
 500         cvmodel = BC_GREY8;
 501         detector = BRISK::create();
 502         matcher = config.use_flann ?
 503                 flann_lshidx_matcher() : bf_matcher_norm_hamming();
 504 }
 505 #endif
 506
 507 void FindObjMain::process_match()
 508 {
 509         if( config.algorithm == NO_ALGORITHM ) return;
 510
 511         if( detector.empty() ) {
 512                 switch( config.algorithm ) {
 513 #ifdef _SIFT
 514                 case ALGORITHM_SIFT:   set_sift();   break;
 515 #endif
 516 #ifdef _SURF
 517                 case ALGORITHM_SURF:   set_surf();   break;
 518 #endif
 519 #ifdef _ORB
 520                 case ALGORITHM_ORB:    set_orb();    break;
 521 #endif
 522 #ifdef _AKAZE
 523                 case ALGORITHM_AKAZE:  set_akaze();  break;
 524 #endif
 525 #ifdef _BRISK
 526                 case ALGORITHM_BRISK:  set_brisk();  break;
 527 #endif
 528                 }
 529                 obj_keypts.clear();  obj_descrs.release();
 530                 to_mat(object_mat, object_w,object_h, object, object_x,object_y, cvmodel);
 531                 detect(object_mat, obj_keypts, obj_descrs);
 532 //printf("detect obj %d features\n", (int)obj_keypts.size());
 533         }
 534
 535         to_mat(scene_mat, scene_w,scene_h, scene, scene_x,scene_y, cvmodel);
 536         detect(scene_mat, scn_keypts, scn_descrs);
 537 //printf("detect scn %d features\n", (int)scn_keypts.size());
 538         match();
 539         ptV p1, p2;
 540         filter_matches(p1, p2);
 541         if( p1.size() < 4 ) return;
 542         Mat H = findHomography(p1, p2, RANSAC, 5.0);
 543         if( !H.dims || !H.rows || !H.cols ) {
 544 //printf("fail, size p1=%d,p2=%d\n",(int)p1.size(),(int)p2.size());
 545                 return;
 546         }
 547
 548         ptV src, dst;
 549         float out_x1 = 0, out_x2 = object_w;
 550         float out_y1 = 0, out_y2 = object_h;
 551         src.push_back(Point2f(out_x1,out_y1));
 552         src.push_back(Point2f(out_x2,out_y1));
 553         src.push_back(Point2f(out_x2,out_y2));
 554         src.push_back(Point2f(out_x1,out_y2));
 555         perspectiveTransform(src, dst, H);
 556
 557         match_x1 = dst[0].x + scene_x;  match_y1 = dst[0].y + scene_y;
 558         match_x2 = dst[1].x + scene_x;  match_y2 = dst[1].y + scene_y;
 559         match_x3 = dst[2].x + scene_x;  match_y3 = dst[2].y + scene_y;
 560         match_x4 = dst[3].x + scene_x;  match_y4 = dst[3].y + scene_y;
 561 }
 562
 563
 564 static double area(float x1, float y1, float x2, float y2,
 565                 float x3, float y3, float x4, float y4)
 566 { // quadrelateral area, sign is +ccw,-cw, use abs
 567         double dx1 = x3-x1, dy1 = y3-y1;
 568         double dx2 = x4-x2, dy2 = y4-y2;
 569         return 0.5 * (dx1 * dy2 - dx2 * dy1);
 570 }
 571 static double dist(float x1,float y1, float x2, float y2)
 572 {
 573         double dx = x2-x1, dy = y2-y1;
 574         return sqrt(dx*dx + dy*dy);
 575 }
 576 static int intersects(double x1, double y1, double x2, double y2,
 577                 double x3, double y3, double x4, double y4)
 578 {
 579         double dx12 = x2 - x1, dy12 = y2 - y1;
 580         double dx34 = x4 - x3, dy34 = y4 - y3;
 581         double d = dx12*dy34 - dx34*dy12;
 582         if( !d ) return 0; // parallel
 583         double dx13 = x3 - x1, dy13 = y3 - y1;
 584         double u = (dx13*dy34 - dx34*dy13) / d;
 585         if( u < 0 || u > 1 ) return 0;
 586         double v = (dx13*dy12 - dx12*dy13) / d;
 587         if( v < 0 || v > 1 ) return 0;
 588         return 1;
 589 }
 590
 591 /*
 592  * 4---------3    1---------2
 593  * |0,h   w,h|    |0,0   w,0|
 594  * |    +    |    |    +    |
 595  * |0,0   w,0|    |0,h   w,h|
 596  * 1---------2    1---------2
 597  * pt locations    screen pts
 598  */
 599 void FindObjMain::reshape()
 600 {
 601         if( config.mode == MODE_NONE ) return;
 602         const double pi = M_PI;
 603         double x1 = match_x1, y1 = match_y1;
 604         double x2 = match_x2, y2 = match_y2;
 605         double x3 = match_x3, y3 = match_y3;
 606         double x4 = match_x4, y4 = match_y4;
 607         double ia = area(x1,y1, x2,y2, x3,y3, x4,y4);
 608 // centroid
 609         double cx = (x1 + x2 + x3 + x4) / 4;
 610         double cy = (y1 + y2 + y3 + y4) / 4;
 611 // centered
 612         x1 -= cx;  x2 -= cx;  x3 -= cx;  x4 -= cx;
 613         y1 -= cy;  y2 -= cy;  y3 -= cy;  y4 -= cy;
 614 // bowtied
 615         if( intersects(x1,y1, x2,y2, x3,y3, x4,y4) ) {
 616                 double x = x2, y = y2;
 617                 x2 = x3;  y2 = y3;
 618                 x3 = x;   y3 = y;
 619         }
 620         else if( intersects(x1,y1, x4,y4, x3,y3, x2,y2) ) {
 621                 double x = x4, y = y4;
 622                 x4 = x3;  y4 = y3;
 623                 x3 = x;   y3 = y;
 624         }
 625
 626 // rotation, if mode is quad: reverse rotate
 627         double r = 0;
 628         if( (config.mode == MODE_QUADRILATERAL) ^ (config.rotate != 0) ) {
 629 // edge centers
 630                 double cx12 = (x1 + x2) / 2, cy12 = (y1 + y2) / 2;
 631                 double cx23 = (x2 + x3) / 2, cy23 = (y2 + y3) / 2;
 632                 double cx34 = (x3 + x4) / 2, cy34 = (y3 + y4) / 2;
 633                 double cx41 = (x4 + x1) / 2, cy41 = (y4 + y1) / 2;
 634                 double vx = cx34 - cx12, vy = cy34 - cy12;
 635                 double hx = cx23 - cx41, hy = cy23 - cy41;
 636                 double v = atan2(vy, vx);
 637                 double h = atan2(hy, hx);
 638                 r = (h + v - pi/2) / 2;
 639                 if( config.mode != MODE_QUADRILATERAL ) r = -r;
 640         }
 641 // diagonal length
 642         double a = dist(x1,y1, x3,y3) / 2;
 643         double b = dist(x2,y2, x4,y4) / 2;
 644         if( config.mode == MODE_SQUARE ||
 645             config.mode == MODE_RECTANGLE )
 646                 a = b = (a + b) / 2;
 647 // central angles
 648         double a1 = atan2(y1, x1);
 649         double a2 = atan2(y2, x2);
 650         double a3 = atan2(y3, x3);
 651         double a4 = atan2(y4, x4);
 652 // edge angles
 653         double a12 = a2 - a1, a23 = a3 - a2;
 654         double a34 = a4 - a3, a41 = a1 - a4;
 655         double dt = (a12 - a23 + a34 - a41)/4;
 656 // mirrored
 657         if( ia < 0 ) { ia = -ia;  dt = -dt;  r = -r; }
 658         switch( config.mode ) {
 659         case MODE_SQUARE:
 660         case MODE_RHOMBUS:
 661                 dt = pi/2;
 662         case MODE_RECTANGLE:
 663         case MODE_PARALLELOGRAM: {
 664                 double t = -(pi+dt)/2;
 665                 x1 = a*cos(t);  y1 = a*sin(t);  t += dt;
 666                 x2 = b*cos(t);  y2 = b*sin(t);  t += pi - dt;
 667                 x3 = a*cos(t);  y3 = a*sin(t);  t += dt;
 668                 x4 = b*cos(t);  y4 = b*sin(t); }
 669         case MODE_QUADRILATERAL:
 670                 break;
 671         }
 672 // aspect
 673         if( !config.aspect ) {
 674                 double cx12 = (x1 + x2) / 2, cy12 = (y1 + y2) / 2;
 675                 double cx23 = (x2 + x3) / 2, cy23 = (y2 + y3) / 2;
 676                 double cx34 = (x3 + x4) / 2, cy34 = (y3 + y4) / 2;
 677                 double cx41 = (x4 + x1) / 2, cy41 = (y4 + y1) / 2;
 678                 double iw = dist(cx41,cy41, cx23,cy23);
 679                 double ih = dist(cx12,cy12, cx34,cy34);
 680                 double ow = object_w, oh = object_h;
 681                 double sx = iw && ih ? sqrt((ih*ow)/(iw*oh)) : 1;
 682                 double sy = sx ? 1 / sx : 1;
 683                 x1 *= sx;  x2 *= sx;  x3 *= sx;  x4 *= sx;
 684                 y1 *= sy;  y2 *= sy;  y3 *= sy;  y4 *= sy;
 685         }
 686 // rotation
 687         if( r ) {
 688                 double ct = cos(r), st = sin(r), x, y;
 689                 x = x1*ct + y1*st;  y = y1*ct - x1*st;  x1 = x;  y1 = y;
 690                 x = x2*ct + y2*st;  y = y2*ct - x2*st;  x2 = x;  y2 = y;
 691                 x = x3*ct + y3*st;  y = y3*ct - x3*st;  x3 = x;  y3 = y;
 692                 x = x4*ct + y4*st;  y = y4*ct - x4*st;  x4 = x;  y4 = y;
 693         }
 694 // scaling
 695         ia = !config.scale ? object_w * object_h : ia;
 696         double oa = abs(area(x1,y1, x2,y2, x3,y3, x4,y4));
 697         double sf =  oa ? sqrt(ia / oa) : 0;
 698         x1 *= sf;  x2 *= sf;  x3 *= sf;  x4 *= sf;
 699         y1 *= sf;  y2 *= sf;  y3 *= sf;  y4 *= sf;
 700 // translation
 701         double ox = !config.translate ? object_x + object_w/2. : cx;
 702         double oy = !config.translate ? object_y + object_h/2. : cy;
 703         x1 += ox;  x2 += ox;  x3 += ox;  x4 += ox;
 704         y1 += oy;  y2 += oy;  y3 += oy;  y4 += oy;
 705
 706         shape_x1 = x1;  shape_y1 = y1;
 707         shape_x2 = x2;  shape_y2 = y2;
 708         shape_x3 = x3;  shape_y3 = y3;
 709         shape_x4 = x4;  shape_y4 = y4;
 710 }
 711
 712 int FindObjMain::process_buffer(VFrame **frame, int64_t start_position, double frame_rate)
 713 {
 714         int prev_algorithm = config.algorithm;
 715         int prev_use_flann = config.use_flann;
 716
 717         if( load_configuration() )
 718                 init_border = 1;
 719
 720         if( prev_algorithm != config.algorithm ||
 721             prev_use_flann != config.use_flann ) {
 722                 detector.release();
 723                 matcher.release();
 724         }
 725
 726         object_layer = config.object_layer;
 727         scene_layer = config.scene_layer;
 728         replace_layer = config.replace_layer;
 729         Track *track = get_plugin_track();
 730         w = track ? track->track_w : get_edl()->session->output_w;
 731         h = track ? track->track_h : get_edl()->session->output_h;
 732
 733         int max_layer = PluginClient::get_total_buffers() - 1;
 734         object_layer = bclip(config.object_layer, 0, max_layer);
 735         scene_layer = bclip(config.scene_layer, 0, max_layer);
 736         replace_layer = bclip(config.replace_layer, 0, max_layer);
 737
 738         int cfg_w = (int)(w * config.object_w / 100.);
 739         int cfg_h = (int)(h * config.object_h / 100.);
 740         int cfg_x1 = (int)(w * config.object_x / 100. - cfg_w / 2);
 741         int cfg_y1 = (int)(h * config.object_y / 100. - cfg_h / 2);
 742         int cfg_x2 = cfg_x1 + cfg_w;
 743         int cfg_y2 = cfg_y1 + cfg_h;
 744         bclamp(cfg_x1, 0, w);  object_x = cfg_x1;
 745         bclamp(cfg_y1, 0, h);  object_y = cfg_y1;
 746         bclamp(cfg_x2, 0, w);  object_w = cfg_x2 - cfg_x1;
 747         bclamp(cfg_y2, 0, h);  object_h = cfg_y2 - cfg_y1;
 748
 749         cfg_w = (int)(w * config.scene_w / 100.);
 750         cfg_h = (int)(h * config.scene_h / 100.);
 751         cfg_x1 = (int)(w * config.scene_x / 100. - cfg_w / 2);
 752         cfg_y1 = (int)(h * config.scene_y / 100. - cfg_h / 2);
 753         cfg_x2 = cfg_x1 + cfg_w;
 754         cfg_y2 = cfg_y1 + cfg_h;
 755         bclamp(cfg_x1, 0, w);  scene_x = cfg_x1;
 756         bclamp(cfg_y1, 0, h);  scene_y = cfg_y1;
 757         bclamp(cfg_x2, 0, w);  scene_w = cfg_x2 - cfg_x1;
 758         bclamp(cfg_y2, 0, h);  scene_h = cfg_y2 - cfg_y1;
 759
 760         cfg_w = (int)(w * config.replace_w / 100.);
 761         cfg_h = (int)(h * config.replace_h / 100.);
 762         cfg_x1 = (int)(w * config.replace_x / 100. - cfg_w / 2);
 763         cfg_y1 = (int)(h * config.replace_y / 100. - cfg_h / 2);
 764         cfg_x2 = cfg_x1 + cfg_w;
 765         cfg_y2 = cfg_y1 + cfg_h;
 766         bclamp(cfg_x1, 0, w);  replace_x = cfg_x1;
 767         bclamp(cfg_y1, 0, h);  replace_y = cfg_y1;
 768         bclamp(cfg_x2, 0, w);  replace_w = cfg_x2 - cfg_x1;
 769         bclamp(cfg_y2, 0, h);  replace_h = cfg_y2 - cfg_y1;
 770
 771         int cfg_dx = (int)(w * config.replace_dx / 100.);
 772         int cfg_dy = (int)(h * config.replace_dy / 100.);
 773         bclamp(cfg_dx, -h, h);  replace_dx = cfg_dx;
 774         bclamp(cfg_dy, -w, w);  replace_dy = cfg_dy;
 775
 776 // Read in the input frames
 777         for( int i = 0; i < PluginClient::get_total_buffers(); i++ ) {
 778                 read_frame(frame[i], i, start_position, frame_rate, 0);
 779         }
 780
 781         object = frame[object_layer];
 782         scene = frame[scene_layer];
 783         replace = frame[replace_layer];
 784
 785         shape_x1 = out_x1;  shape_y1 = out_y1;
 786         shape_x2 = out_x2;  shape_y2 = out_y2;
 787         shape_x3 = out_x3;  shape_y3 = out_y3;
 788         shape_x4 = out_x4;  shape_y4 = out_y4;
 789
 790         if( scene_w > 0 && scene_h > 0 && object_w > 0 && object_h > 0 ) {
 791                 process_match();
 792                 reshape();
 793         }
 794
 795         double w0 = init_border ? 1. : config.blend/100., w1 = 1. - w0;
 796         init_border = 0;
 797         out_x1 = shape_x1*w0 + out_x1*w1;  out_y1 = shape_y1*w0 + out_y1*w1;
 798         out_x2 = shape_x2*w0 + out_x2*w1;  out_y2 = shape_y2*w0 + out_y2*w1;
 799         out_x3 = shape_x3*w0 + out_x3*w1;  out_y3 = shape_y3*w0 + out_y3*w1;
 800         out_x4 = shape_x4*w0 + out_x4*w1;  out_y4 = shape_y4*w0 + out_y4*w1;
 801 // Replace object in the scene layer
 802         if( config.replace_object ) {
 803                 int cpus1 = get_project_smp() + 1;
 804                 if( !affine )
 805                         affine = new AffineEngine(cpus1, cpus1);
 806                 if( !overlayer )
 807                         overlayer = new OverlayFrame(cpus1);
 808                 VFrame *temp = new_temp(w, h, scene->get_color_model());
 809                 temp->clear_frame();
 810                 affine->set_in_viewport(replace_x, replace_y, replace_w, replace_h);
 811                 float ix1 = replace_x, ix2 = ix1 + replace_w;
 812                 float iy1 = replace_y, iy2 = iy1 + replace_h;
 813                 float dx = replace_dx, dy = replace_dy;
 814                 float ox1 = out_x1+dx, ox2 = out_x2+dx, ox3 = out_x3+dx, ox4 = out_x4+dx;
 815                 float oy1 = out_y1-dy, oy2 = out_y2-dy, oy3 = out_y3-dy, oy4 = out_y4-dy;
 816                 affine->set_matrix(ix1,iy1, ix2,iy2, ox1,oy1, ox2,oy2, ox4,oy4, ox3,oy3);
 817                 affine->process(temp, replace, 0,
 818                         AffineEngine::TRANSFORM, 0,0, 100,0, 100,100, 0,100, 1);
 819                 overlayer->overlay(scene, temp,  0,0, w,h,  0,0, w,h,
 820                         1, TRANSFER_NORMAL, NEAREST_NEIGHBOR);
 821
 822         }
 823
 824         int wh = (w+h)>>8, ss = 1; while( wh>>=1 ) ss<<=1;
 825         if( config.draw_scene_border ) {
 826                 scene->set_stiple(ss*2);
 827                 scene->set_pixel_color(WHITE);
 828                 draw_rect(scene, scene_x, scene_y, scene_x+scene_w, scene_y+scene_h);
 829         }
 830         if( config.draw_object_border ) {
 831                 scene->set_stiple(ss*3);
 832                 scene->set_pixel_color(YELLOW);
 833                 draw_rect(scene, object_x, object_y, object_x+object_w, object_y+object_h);
 834         }
 835         if( config.draw_replace_border ) {
 836                 scene->set_stiple(ss*3);
 837                 scene->set_pixel_color(GREEN);
 838                 draw_rect(scene, replace_x, replace_y, replace_x+replace_w, replace_y+replace_h);
 839         }
 840         scene->set_stiple(0);
 841         if( config.draw_keypoints ) {
 842                 scene->set_pixel_color(GREEN);
 843                 for( int i=0,n=obj_keypts.size(); i<n; ++i ) {
 844                         Point2f &pt = obj_keypts[i].pt;
 845                         int r = obj_keypts[i].size * 1.2/9 * 2;
 846                         int x = pt.x + object_x, y = pt.y + object_y;
 847                         draw_circle(scene, x, y, r);
 848                 }
 849                 scene->set_pixel_color(RED);
 850                 for( int i=0,n=scn_keypts.size(); i<n; ++i ) {
 851                         Point2f &pt = scn_keypts[i].pt;
 852                         int r = scn_keypts[i].size * 1.2/9 * 2;
 853                         int x = pt.x + scene_x, y = pt.y + scene_y;
 854                         draw_circle(scene, x, y, r);
 855                 }
 856         }
 857         if( config.draw_match ) {
 858                 scene->set_pixel_color(BLUE);
 859                 draw_quad(scene, match_x1, match_y1, match_x2, match_y2,
 860                                 match_x3, match_y3, match_x4, match_y4);
 861                 scene->set_pixel_color(LTGREEN);
 862                 draw_point(scene, match_x1, match_y1);
 863         }
 864
 865         if( gui_open() ) {
 866                 if( config.drag_scene ) {
 867                         scene->set_pixel_color(WHITE);
 868                         DragCheckBox::draw_boundary(scene, scene_x, scene_y, scene_w, scene_h);
 869                 }
 870                 if( config.drag_object ) {
 871                         scene->set_pixel_color(YELLOW);
 872                         DragCheckBox::draw_boundary(scene, object_x, object_y, object_w, object_h);
 873                 }
 874                 if( config.drag_replace ) {
 875                         scene->set_pixel_color(GREEN);
 876                         DragCheckBox::draw_boundary(scene, replace_x, replace_y, replace_w, replace_h);
 877                 }
 878         }
 879
 880         scene->set_pixel_color(BLACK);
 881         scene->set_stiple(0);
 882         return 0;
 883 }
 884