cinelerra-5.1/plugins/findobj/findobj.C

   1
   2 /*
   3  * CINELERRA
   4  * Copyright (C) 1997-2012 Adam Williams <broadcast at earthling dot net>
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License as published by
   8  * the Free Software Foundation; either version 2 of the License, or
   9  * (at your option) any later version.
  10  *
  11  * This program is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14  * GNU General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU General Public License
  17  * along with this program; if not, write to the Free Software
  18  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  19  */
  20
  21 #include "affine.h"
  22 #include "bccolors.h"
  23 #include "clip.h"
  24 #include "filexml.h"
  25 #include "language.h"
  26 #include "findobj.h"
  27 #include "findobjwindow.h"
  28 #include "mutex.h"
  29 #include "overlayframe.h"
  30 #include "plugin.h"
  31 #include "pluginserver.h"
  32 #include "track.h"
  33
  34 #include <errno.h>
  35 #include <exception>
  36 #include <unistd.h>
  37
  38 REGISTER_PLUGIN(FindObjMain)
  39
  40 FindObjConfig::FindObjConfig()
  41 {
  42         reset();
  43 }
  44
  45 void FindObjConfig::reset()
  46 {
  47         algorithm = NO_ALGORITHM;
  48         use_flann = 1;
  49         mode = MODE_QUADRILATERAL;
  50         scale = 1;
  51         translate = 1;
  52         rotate = 1;
  53         draw_keypoints = 0;
  54         draw_scene_border = 0;
  55         replace_object = 0;
  56         draw_object_border = 0;
  57         draw_replace_border = 0;
  58         object_x = 50;  object_y = 50;
  59         object_w = 100; object_h = 100;
  60         drag_object = 0;
  61         scene_x = 50;   scene_y = 50;
  62         scene_w = 100;  scene_h = 100;
  63         drag_replace = 0;
  64         replace_x = 50;   replace_y = 50;
  65         replace_w = 100;  replace_h = 100;
  66         replace_dx = 0;   replace_dy = 0;
  67         drag_scene = 0;
  68         scene_layer = 0;
  69         object_layer = 1;
  70         replace_layer = 2;
  71         blend = 100;
  72 }
  73
  74 void FindObjConfig::boundaries()
  75 {
  76         bclamp(object_x, 0, 100);  bclamp(object_y, 0, 100);
  77         bclamp(object_w, 0, 100);  bclamp(object_h, 0, 100);
  78         bclamp(scene_x, 0, 100);   bclamp(scene_y, 0, 100);
  79         bclamp(scene_w, 0, 100);   bclamp(scene_h, 0, 100);
  80         bclamp(object_layer, MIN_LAYER, MAX_LAYER);
  81         bclamp(replace_layer, MIN_LAYER, MAX_LAYER);
  82         bclamp(scene_layer, MIN_LAYER, MAX_LAYER);
  83         bclamp(blend, MIN_BLEND, MAX_BLEND);
  84 }
  85
  86 int FindObjConfig::equivalent(FindObjConfig &that)
  87 {
  88         int result =
  89                 algorithm == that.algorithm &&
  90                 use_flann == that.use_flann &&
  91                 mode == that.mode &&
  92                 scale == that.scale &&
  93                 translate == that.translate &&
  94                 rotate == that.rotate &&
  95                 draw_keypoints == that.draw_keypoints &&
  96                 draw_scene_border == that.draw_scene_border &&
  97                 replace_object == that.replace_object &&
  98                 draw_object_border == that.draw_object_border &&
  99                 draw_replace_border == that.draw_replace_border &&
 100                 object_x == that.object_x && object_y == that.object_y &&
 101                 object_w == that.object_w && object_h == that.object_h &&
 102                 drag_object == that.drag_object &&
 103                 scene_x == that.scene_x && scene_y == that.scene_y &&
 104                 scene_w == that.scene_w && scene_h == that.scene_h &&
 105                 drag_scene == that.drag_scene &&
 106                 replace_x == that.replace_x && replace_y == that.replace_y &&
 107                 replace_w == that.replace_w && replace_h == that.replace_h &&
 108                 replace_dx == that.replace_dx && replace_dy == that.replace_dy &&
 109                 drag_replace == that.drag_replace &&
 110                 object_layer == that.object_layer &&
 111                 replace_layer == that.replace_layer &&
 112                 scene_layer == that.scene_layer &&
 113                 blend == that.blend;
 114         return result;
 115 }
 116
 117 void FindObjConfig::copy_from(FindObjConfig &that)
 118 {
 119         algorithm = that.algorithm;
 120         use_flann = that.use_flann;
 121         mode = that.mode;
 122         scale = that.scale;
 123         translate = that.translate;
 124         rotate = that.rotate;
 125         draw_keypoints = that.draw_keypoints;
 126         draw_scene_border = that.draw_scene_border;
 127         replace_object = that.replace_object;
 128         draw_object_border = that.draw_object_border;
 129         draw_replace_border = that.draw_replace_border;
 130         object_x = that.object_x;  object_y = that.object_y;
 131         object_w = that.object_w;  object_h = that.object_h;
 132         drag_object = that.drag_object;
 133         scene_x = that.scene_x;    scene_y = that.scene_y;
 134         scene_w = that.scene_w;    scene_h = that.scene_h;
 135         drag_scene = that.drag_scene;
 136         replace_x = that.replace_x;   replace_y = that.replace_y;
 137         replace_w = that.replace_w;   replace_h = that.replace_h;
 138         replace_dx = that.replace_dx; replace_dy = that.replace_dy;
 139         drag_replace = that.drag_replace;
 140         object_layer = that.object_layer;
 141         replace_layer = that.replace_layer;
 142         scene_layer = that.scene_layer;
 143         blend = that.blend;
 144 }
 145
 146 void FindObjConfig::interpolate(FindObjConfig &prev, FindObjConfig &next,
 147         int64_t prev_frame, int64_t next_frame, int64_t current_frame)
 148 {
 149         copy_from(prev);
 150 }
 151
 152
 153 FindObjMain::FindObjMain(PluginServer *server)
 154  : PluginVClient(server)
 155 {
 156         affine = 0;
 157         overlayer = 0;
 158
 159         cvmodel = BC_RGB888;
 160         w = h = 0;
 161         object = scene = replace = 0;
 162         object_x = object_y = 0;
 163         object_w = object_h = 0;
 164         scene_x = scene_y = 0;
 165         scene_w = scene_h = 0;
 166         object_layer = 0;
 167         scene_layer = 1;
 168         replace_layer = 2;
 169
 170         border_x1 = 0;  border_y1 = 0;
 171         border_x2 = 0;  border_y2 = 0;
 172         border_x3 = 0;  border_y3 = 0;
 173         border_x4 = 0;  border_y4 = 0;
 174
 175         obj_x1 = 0;     obj_y1 = 0;
 176         obj_x2 = 0;     obj_y2 = 0;
 177         obj_x3 = 0;     obj_y3 = 0;
 178         obj_x4 = 0;     obj_y4 = 0;
 179
 180         init_border = 1;
 181 }
 182
 183 FindObjMain::~FindObjMain()
 184 {
 185         delete affine;
 186         delete overlayer;
 187 }
 188
 189 const char* FindObjMain::plugin_title() { return N_("FindObj"); }
 190 int FindObjMain::is_realtime() { return 1; }
 191 int FindObjMain::is_multichannel() { return 1; }
 192
 193 NEW_WINDOW_MACRO(FindObjMain, FindObjWindow)
 194 LOAD_CONFIGURATION_MACRO(FindObjMain, FindObjConfig)
 195
 196 void FindObjMain::update_gui()
 197 {
 198         if( !thread ) return;
 199         if( !load_configuration() ) return;
 200         FindObjWindow *window = (FindObjWindow*)thread->window;
 201         window->lock_window("FindObjMain::update_gui");
 202         window->update_gui();
 203         window->flush();
 204         window->unlock_window();
 205 }
 206
 207 void FindObjMain::save_data(KeyFrame *keyframe)
 208 {
 209         FileXML output;
 210
 211 // cause data to be stored directly in text
 212         output.set_shared_output(keyframe->get_data(), MESSAGESIZE);
 213         output.tag.set_title("FINDOBJ");
 214         output.tag.set_property("ALGORITHM", config.algorithm);
 215         output.tag.set_property("USE_FLANN", config.use_flann);
 216         output.tag.set_property("MODE", config.mode);
 217         output.tag.set_property("SCALE", config.scale);
 218         output.tag.set_property("TRANSLATE", config.translate);
 219         output.tag.set_property("ROTATE", config.rotate);
 220         output.tag.set_property("DRAG_OBJECT", config.drag_object);
 221         output.tag.set_property("OBJECT_X", config.object_x);
 222         output.tag.set_property("OBJECT_Y", config.object_y);
 223         output.tag.set_property("OBJECT_W", config.object_w);
 224         output.tag.set_property("OBJECT_H", config.object_h);
 225         output.tag.set_property("DRAG_SCENE", config.drag_scene);
 226         output.tag.set_property("SCENE_X", config.scene_x);
 227         output.tag.set_property("SCENE_Y", config.scene_y);
 228         output.tag.set_property("SCENE_W", config.scene_w);
 229         output.tag.set_property("SCENE_H", config.scene_h);
 230         output.tag.set_property("DRAG_REPLACE", config.drag_replace);
 231         output.tag.set_property("REPLACE_X", config.replace_x);
 232         output.tag.set_property("REPLACE_Y", config.replace_y);
 233         output.tag.set_property("REPLACE_W", config.replace_w);
 234         output.tag.set_property("REPLACE_H", config.replace_h);
 235         output.tag.set_property("REPLACE_DX", config.replace_dx);
 236         output.tag.set_property("REPLACE_DY", config.replace_dy);
 237         output.tag.set_property("DRAW_KEYPOINTS", config.draw_keypoints);
 238         output.tag.set_property("DRAW_SCENE_BORDER", config.draw_scene_border);
 239         output.tag.set_property("REPLACE_OBJECT", config.replace_object);
 240         output.tag.set_property("DRAW_OBJECT_BORDER", config.draw_object_border);
 241         output.tag.set_property("DRAW_REPLACE_BORDER", config.draw_replace_border);
 242         output.tag.set_property("OBJECT_LAYER", config.object_layer);
 243         output.tag.set_property("REPLACE_LAYER", config.replace_layer);
 244         output.tag.set_property("SCENE_LAYER", config.scene_layer);
 245         output.tag.set_property("BLEND", config.blend);
 246         output.append_tag();
 247         output.tag.set_title("/FINDOBJ");
 248         output.append_tag();
 249         output.append_newline();
 250         output.terminate_string();
 251 }
 252
 253 void FindObjMain::read_data(KeyFrame *keyframe)
 254 {
 255         FileXML input;
 256
 257         input.set_shared_input(keyframe->get_data(), strlen(keyframe->get_data()));
 258
 259         int result = 0;
 260
 261         while( !(result = input.read_tag()) ) {
 262                 if( input.tag.title_is("FINDOBJ") ) {
 263                         config.algorithm = input.tag.get_property("ALGORITHM", config.algorithm);
 264                         config.use_flann = input.tag.get_property("USE_FLANN", config.use_flann);
 265                         config.mode = input.tag.get_property("MODE", config.mode);
 266                         config.scale = input.tag.get_property("SCALE", config.scale);
 267                         config.translate = input.tag.get_property("TRANSLATE", config.translate);
 268                         config.rotate = input.tag.get_property("ROTATE", config.rotate);
 269                         config.object_x = input.tag.get_property("OBJECT_X", config.object_x);
 270                         config.object_y = input.tag.get_property("OBJECT_Y", config.object_y);
 271                         config.object_w = input.tag.get_property("OBJECT_W", config.object_w);
 272                         config.object_h = input.tag.get_property("OBJECT_H", config.object_h);
 273                         config.drag_object = input.tag.get_property("DRAG_OBJECT", config.drag_object);
 274                         config.scene_x = input.tag.get_property("SCENE_X", config.scene_x);
 275                         config.scene_y = input.tag.get_property("SCENE_Y", config.scene_y);
 276                         config.scene_w = input.tag.get_property("SCENE_W", config.scene_w);
 277                         config.scene_h = input.tag.get_property("SCENE_H", config.scene_h);
 278                         config.drag_scene = input.tag.get_property("DRAG_SCENE", config.drag_scene);
 279                         config.replace_x = input.tag.get_property("REPLACE_X", config.replace_x);
 280                         config.replace_y = input.tag.get_property("REPLACE_Y", config.replace_y);
 281                         config.replace_w = input.tag.get_property("REPLACE_W", config.replace_w);
 282                         config.replace_h = input.tag.get_property("REPLACE_H", config.replace_h);
 283                         config.replace_dx = input.tag.get_property("REPLACE_DX", config.replace_dx);
 284                         config.replace_dy = input.tag.get_property("REPLACE_DY", config.replace_dy);
 285                         config.drag_replace = input.tag.get_property("DRAG_REPLACE", config.drag_replace);
 286                         config.draw_keypoints = input.tag.get_property("DRAW_KEYPOINTS", config.draw_keypoints);
 287                         config.draw_scene_border = input.tag.get_property("DRAW_SCENE_BORDER", config.draw_scene_border);
 288                         config.replace_object = input.tag.get_property("REPLACE_OBJECT", config.replace_object);
 289                         config.draw_object_border = input.tag.get_property("DRAW_OBJECT_BORDER", config.draw_object_border);
 290                         config.draw_replace_border = input.tag.get_property("DRAW_REPLACE_BORDER", config.draw_replace_border);
 291                         config.object_layer = input.tag.get_property("OBJECT_LAYER", config.object_layer);
 292                         config.replace_layer = input.tag.get_property("REPLACE_LAYER", config.replace_layer);
 293                         config.scene_layer = input.tag.get_property("SCENE_LAYER", config.scene_layer);
 294                         config.blend = input.tag.get_property("BLEND", config.blend);
 295                 }
 296         }
 297
 298         config.boundaries();
 299 }
 300
 301 void FindObjMain::draw_line(VFrame *vframe, int x1, int y1, int x2, int y2)
 302 {
 303         vframe->draw_line(x1, y1, x2, y2);
 304 }
 305
 306 void FindObjMain::draw_rect(VFrame *vframe, int x1, int y1, int x2, int y2)
 307 {
 308         --x2;  --y2;
 309         draw_line(vframe, x1, y1, x2, y1);
 310         draw_line(vframe, x2, y1, x2, y2);
 311         draw_line(vframe, x2, y2, x1, y2);
 312         draw_line(vframe, x1, y2, x1, y1);
 313 }
 314
 315 void FindObjMain::draw_circle(VFrame *vframe, int x, int y, int r)
 316 {
 317         int x1 = x-r, x2 = x+r;
 318         int y1 = y-r, y2 = y+r;
 319         vframe->draw_smooth(x1,y, x1,y1, x,y1);
 320         vframe->draw_smooth(x,y1, x2,y1, x2,y);
 321         vframe->draw_smooth(x2,y, x2,y2, x,y2);
 322         vframe->draw_smooth(x,y2, x1,y2, x1,y);
 323 }
 324
 325 void FindObjMain::filter_matches(ptV &p1, ptV &p2, double ratio)
 326 {
 327         DMatches::iterator it;
 328         for( it=pairs.begin(); it!=pairs.end(); ++it ) {
 329                 DMatchV &m = *it;
 330                 if( m.size() == 2 && m[0].distance < m[1].distance*ratio ) {
 331                         p1.push_back(obj_keypts[m[0].queryIdx].pt);
 332                         p2.push_back(scn_keypts[m[0].trainIdx].pt);
 333                 }
 334         }
 335 }
 336
 337 void FindObjMain::to_mat(Mat &mat, int mcols, int mrows,
 338         VFrame *inp, int ix,int iy, BC_CModel mcolor_model)
 339 {
 340         int mcomp = BC_CModels::components(mcolor_model);
 341         int mbpp = BC_CModels::calculate_pixelsize(mcolor_model);
 342         int psz = mbpp / mcomp;
 343         int mdepth = psz < 2 ? CV_8U : CV_16U;
 344         if( mat.dims != 2 || mat.depth() != mdepth || mat.channels() != mcomp ||
 345             mat.cols != mcols || mat.rows != mrows ) {
 346                 mat.release();
 347         }
 348         if( mat.empty() ) {
 349                 int type = CV_MAKETYPE(mdepth, mcomp);
 350                 mat.create(mrows, mcols, type);
 351         }
 352         uint8_t *mat_rows[mrows];
 353         for( int y=0; y<mrows; ++y ) mat_rows[y] = mat.ptr(y);
 354         uint8_t **inp_rows = inp->get_rows();
 355         int ibpl = inp->get_bytes_per_line(), obpl = mcols * mbpp;
 356         int icolor_model = inp->get_color_model();
 357         BC_CModels::transfer(mat_rows, mcolor_model, 0,0, mcols,mrows, obpl,
 358                 inp_rows, icolor_model, ix,iy, mcols,mrows, ibpl, 0);
 359 //      VFrame vfrm(mat_rows[0], -1, mcols,mrows, mcolor_model, mat_rows[1]-mat_rows[0]);
 360 //      static int vfrm_no = 0; char vfn[64]; sprintf(vfn,"/tmp/dat/%06d.png", vfrm_no++);
 361 //      vfrm.write_png(vfn);
 362 }
 363
 364 void FindObjMain::detect(Mat &mat, KeyPointV &keypts,Mat &descrs)
 365 {
 366         keypts.clear();
 367         descrs.release();
 368         try {
 369                 detector->detectAndCompute(mat, noArray(), keypts, descrs);
 370         } catch(std::exception e) { printf(_("detector exception: %s\n"), e.what()); }
 371 }
 372
 373 void FindObjMain::match()
 374 {
 375         pairs.clear();
 376         try {
 377                 matcher->knnMatch(obj_descrs, scn_descrs, pairs, 2);
 378         } catch(std::exception e) { printf(_("match execption: %s\n"), e.what()); }
 379 }
 380
 381 Ptr<DescriptorMatcher> FindObjMain::flann_kdtree_matcher()
 382 { // trees=5
 383         const Ptr<flann::IndexParams>& indexParams =
 384                 makePtr<flann::KDTreeIndexParams>(5);
 385         const Ptr<flann::SearchParams>& searchParams =
 386                 makePtr<flann::SearchParams>();
 387         return makePtr<FlannBasedMatcher>(indexParams, searchParams);
 388 }
 389 Ptr<DescriptorMatcher> FindObjMain::flann_lshidx_matcher()
 390 { // table_number = 6#12, key_size = 12#20, multi_probe_level = 1#2
 391         const Ptr<flann::IndexParams>& indexParams =
 392                 makePtr<flann::LshIndexParams>(6, 12, 1);
 393         const Ptr<flann::SearchParams>& searchParams =
 394                 makePtr<flann::SearchParams>();
 395         return makePtr<FlannBasedMatcher>(indexParams, searchParams);
 396 }
 397 Ptr<DescriptorMatcher> FindObjMain::bf_matcher_norm_l2()
 398 {
 399         return BFMatcher::create(NORM_L2);
 400 }
 401 Ptr<DescriptorMatcher> FindObjMain::bf_matcher_norm_hamming()
 402 {
 403         return BFMatcher::create(NORM_HAMMING);
 404 }
 405
 406 #ifdef _SIFT
 407 void FindObjMain::set_sift()
 408 {
 409         cvmodel = BC_GREY8;
 410         detector = SIFT::create();
 411         matcher = config.use_flann ?
 412                 flann_kdtree_matcher() : bf_matcher_norm_l2();
 413 }
 414 #endif
 415 #ifdef _SURF
 416 void FindObjMain::set_surf()
 417 {
 418         cvmodel = BC_GREY8;
 419         detector = SURF::create(800);
 420         matcher = config.use_flann ?
 421                 flann_kdtree_matcher() : bf_matcher_norm_l2();
 422 }
 423 #endif
 424 #ifdef _ORB
 425 void FindObjMain::set_orb()
 426 {
 427         cvmodel = BC_GREY8;
 428         detector = ORB::create();
 429         matcher = config.use_flann ?
 430                 flann_lshidx_matcher() : bf_matcher_norm_hamming();
 431 }
 432 #endif
 433 #ifdef _AKAZE
 434 void FindObjMain::set_akaze()
 435 {
 436         cvmodel = BC_GREY8;
 437         detector = AKAZE::create();
 438         matcher = config.use_flann ?
 439                 flann_lshidx_matcher() : bf_matcher_norm_hamming();
 440 }
 441 #endif
 442 #ifdef _BRISK
 443 void FindObjMain::set_brisk()
 444 {
 445         cvmodel = BC_GREY8;
 446         detector = BRISK::create();
 447         matcher = config.use_flann ?
 448                 flann_lshidx_matcher() : bf_matcher_norm_hamming();
 449 }
 450 #endif
 451
 452
 453 void FindObjMain::process_match()
 454 {
 455         if( config.algorithm == NO_ALGORITHM ) return;
 456
 457         if( detector.empty() ) {
 458                 switch( config.algorithm ) {
 459 #ifdef _SIFT
 460                 case ALGORITHM_SIFT:   set_sift();   break;
 461 #endif
 462 #ifdef _SURF
 463                 case ALGORITHM_SURF:   set_surf();   break;
 464 #endif
 465 #ifdef _ORB
 466                 case ALGORITHM_ORB:    set_orb();    break;
 467 #endif
 468 #ifdef _AKAZE
 469                 case ALGORITHM_AKAZE:  set_akaze();  break;
 470 #endif
 471 #ifdef _BRISK
 472                 case ALGORITHM_BRISK:  set_brisk();  break;
 473 #endif
 474                 }
 475                 obj_keypts.clear();  obj_descrs.release();
 476                 to_mat(object_mat, object_w,object_h, object, object_x,object_y, cvmodel);
 477                 detect(object_mat, obj_keypts, obj_descrs);
 478 //printf("detect obj %d features\n", (int)obj_keypts.size());
 479         }
 480
 481         to_mat(scene_mat, scene_w,scene_h, scene, scene_x,scene_y, cvmodel);
 482         detect(scene_mat, scn_keypts, scn_descrs);
 483 //printf("detect scn %d features\n", (int)scn_keypts.size());
 484         match();
 485         ptV p1, p2;
 486         filter_matches(p1, p2);
 487         if( p1.size() < 4 ) return;
 488         Mat H = findHomography(p1, p2, RANSAC, 5.0);
 489         if( !H.dims || !H.rows || !H.cols ) {
 490 //printf("fail, size p1=%d,p2=%d\n",(int)p1.size(),(int)p2.size());
 491                 return;
 492         }
 493
 494         ptV src, dst;
 495         float obj_x1 = 0, obj_x2 = object_w;
 496         float obj_y1 = 0, obj_y2 = object_h;
 497         src.push_back(Point2f(obj_x1,obj_y1));
 498         src.push_back(Point2f(obj_x2,obj_y1));
 499         src.push_back(Point2f(obj_x2,obj_y2));
 500         src.push_back(Point2f(obj_x1,obj_y2));
 501         perspectiveTransform(src, dst, H);
 502
 503         border_x1 = dst[0].x;  border_y1 = dst[0].y;
 504         border_x2 = dst[1].x;  border_y2 = dst[1].y;
 505         border_x3 = dst[2].x;  border_y3 = dst[2].y;
 506         border_x4 = dst[3].x;  border_y4 = dst[3].y;
 507 }
 508
 509
 510 static double area(float x1, float y1, float x2, float y2, float x3, float y3, float x4, float y4)
 511 { // quadrelateral area
 512         double dx1 = x3-x1, dy1 = y3-y1;
 513         double dx2 = x4-x2, dy2 = y4-y2;
 514         return 0.5 * abs(dx1 * dy2 - dx2 * dy1);
 515 }
 516 static double dist(float x1,float y1, float x2, float y2)
 517 {
 518         double dx = x2-x1, dy = y2-y1;
 519         return sqrt(dx*dx + dy*dy);
 520 }
 521
 522 /*
 523  * 4---------3    1---------2
 524  * |0,h   w,h|    |0,0   w,0|
 525  * |    +    |    |    +    |
 526  * |0,0   w,0|    |0,h   w,h|
 527  * 1---------2    1---------2
 528  * pt locations    screen pts
 529  */
 530 void FindObjMain::reshape()
 531 {
 532         if( config.mode == MODE_NONE ) return;
 533         const double pi = M_PI;
 534         double x1 = border_x1, y1 = border_y1;
 535         double x2 = border_x2, y2 = border_y2;
 536         double x3 = border_x3, y3 = border_y3;
 537         double x4 = border_x4, y4 = border_y4;
 538 // centroid
 539         double cx = (x1 + x2 + x3 + x4) / 4;
 540         double cy = (y1 + y2 + y3 + y4) / 4;
 541 // centered
 542         x1 -= cx;  x2 -= cx;  x3 -= cx;  x4 -= cx;
 543         y1 -= cy;  y2 -= cy;  y3 -= cy;  y4 -= cy;
 544 // rotation, if mode is quad: reverse rotate
 545         double r = 0;
 546         if( (config.mode == MODE_QUADRILATERAL) ^ config.rotate ) {
 547 // edge centers
 548                 double cx12 = (x1 + x2) / 2, cy12 = (y1 + y2) / 2;
 549                 double cx23 = (x2 + x3) / 2, cy23 = (y2 + y3) / 2;
 550                 double cx34 = (x3 + x4) / 2, cy34 = (y3 + y4) / 2;
 551                 double cx41 = (x4 + x1) / 2, cy41 = (y4 + y1) / 2;
 552                 double vx = cx34 - cx12, vy = cy34 - cy12;
 553                 double hx = cx23 - cx41, hy = cy23 - cy41;
 554                 double v = atan2(vy, vx);
 555                 double h = atan2(hy, hx);
 556                 r = (h + v - pi/2) / 2;
 557         }
 558 // diagonal length
 559         double a = dist(x1,y1, x3,y3) / 2;
 560         double b = dist(x2,y2, x4,y4) / 2;
 561         if( config.mode == MODE_SQUARE ||
 562             config.mode == MODE_RECTANGLE )
 563                 a = b = (a + b) / 2;
 564 // central angles
 565         double a1 = atan2(y1, x1);
 566         double a2 = atan2(y2, x2);
 567         double a3 = atan2(y3, x3);
 568         double a4 = atan2(y4, x4);
 569 // edge angles
 570         double a12 = a2 - a1, a23 = a3 - a2;
 571         double a34 = a4 - a3, a41 = a1 - a4;
 572         double dt = (a12 - a23 + a34 - a41)/4;
 573         switch( config.mode ) {
 574         case MODE_SQUARE:
 575         case MODE_RHOMBUS:
 576                 dt = pi/2;
 577         case MODE_RECTANGLE:
 578         case MODE_PARALLELOGRAM: {
 579                 double t = -(pi+dt)/2;
 580                 x1 = a*cos(t);  y1 = a*sin(t);  t += dt;
 581                 x2 = b*cos(t);  y2 = b*sin(t);  t += pi - dt;
 582                 x3 = a*cos(t);  y3 = a*sin(t);  t += dt;
 583                 x4 = b*cos(t);  y4 = b*sin(t); }
 584         case MODE_QUADRILATERAL:
 585                 break;
 586         }
 587 // rotation
 588         if( r ) {
 589                 double ct = cos(r), st = sin(r), x, y;
 590                 x = x1*ct + y1*st;  y = y1*ct - x1*st;  x1 = x;  y1 = y;
 591                 x = x2*ct + y2*st;  y = y2*ct - x2*st;  x2 = x;  y2 = y;
 592                 x = x3*ct + y3*st;  y = y3*ct - x3*st;  x3 = x;  y3 = y;
 593                 x = x4*ct + y4*st;  y = y4*ct - x4*st;  x4 = x;  y4 = y;
 594         }
 595 // scaling
 596         double ia = !config.scale ? object_w * object_h :
 597                         area(border_x1,border_y1, border_x2,border_y2,
 598                              border_x3,border_y3, border_x4,border_y4);
 599         double oa = area(x1,y1, x2,y2, x3,y3, x4,y4);
 600         double sf =  oa ? sqrt(ia / oa) : 0;
 601         x1 *= sf;  x2 *= sf;  x3 *= sf;  x4 *= sf;
 602         y1 *= sf;  y2 *= sf;  y3 *= sf;  y4 *= sf;
 603 // translation
 604         double ox = !config.translate ? object_x + object_w/2. : cx;
 605         double oy = !config.translate ? object_y + object_h/2. : cy;
 606         x1 += ox;  x2 += ox;  x3 += ox;  x4 += ox;
 607         y1 += oy;  y2 += oy;  y3 += oy;  y4 += oy;
 608
 609         border_x1 = x1;  border_y1 = y1;
 610         border_x2 = x2;  border_y2 = y2;
 611         border_x3 = x3;  border_y3 = y3;
 612         border_x4 = x4;  border_y4 = y4;
 613 }
 614
 615 int FindObjMain::process_buffer(VFrame **frame, int64_t start_position, double frame_rate)
 616 {
 617         int prev_algorithm = config.algorithm;
 618         int prev_use_flann = config.use_flann;
 619
 620         if( load_configuration() )
 621                 init_border = 1;
 622
 623         if( prev_algorithm != config.algorithm ||
 624             prev_use_flann != config.use_flann ) {
 625                 detector.release();
 626                 matcher.release();
 627         }
 628
 629         object_layer = config.object_layer;
 630         scene_layer = config.scene_layer;
 631         replace_layer = config.replace_layer;
 632         Track *track = server->plugin->track;
 633         w = track->track_w;
 634         h = track->track_h;
 635
 636         int max_layer = PluginClient::get_total_buffers() - 1;
 637         object_layer = bclip(config.object_layer, 0, max_layer);
 638         scene_layer = bclip(config.scene_layer, 0, max_layer);
 639         replace_layer = bclip(config.replace_layer, 0, max_layer);
 640
 641         int cfg_w = (int)(w * config.object_w / 100.);
 642         int cfg_h = (int)(h * config.object_h / 100.);
 643         int cfg_x1 = (int)(w * config.object_x / 100. - cfg_w / 2);
 644         int cfg_y1 = (int)(h * config.object_y / 100. - cfg_h / 2);
 645         int cfg_x2 = cfg_x1 + cfg_w;
 646         int cfg_y2 = cfg_y1 + cfg_h;
 647         bclamp(cfg_x1, 0, w);  object_x = cfg_x1;
 648         bclamp(cfg_y1, 0, h);  object_y = cfg_y1;
 649         bclamp(cfg_x2, 0, w);  object_w = cfg_x2 - cfg_x1;
 650         bclamp(cfg_y2, 0, h);  object_h = cfg_y2 - cfg_y1;
 651
 652         cfg_w = (int)(w * config.scene_w / 100.);
 653         cfg_h = (int)(h * config.scene_h / 100.);
 654         cfg_x1 = (int)(w * config.scene_x / 100. - cfg_w / 2);
 655         cfg_y1 = (int)(h * config.scene_y / 100. - cfg_h / 2);
 656         cfg_x2 = cfg_x1 + cfg_w;
 657         cfg_y2 = cfg_y1 + cfg_h;
 658         bclamp(cfg_x1, 0, w);  scene_x = cfg_x1;
 659         bclamp(cfg_y1, 0, h);  scene_y = cfg_y1;
 660         bclamp(cfg_x2, 0, w);  scene_w = cfg_x2 - cfg_x1;
 661         bclamp(cfg_y2, 0, h);  scene_h = cfg_y2 - cfg_y1;
 662
 663         cfg_w = (int)(w * config.replace_w / 100.);
 664         cfg_h = (int)(h * config.replace_h / 100.);
 665         cfg_x1 = (int)(w * config.replace_x / 100. - cfg_w / 2);
 666         cfg_y1 = (int)(h * config.replace_y / 100. - cfg_h / 2);
 667         cfg_x2 = cfg_x1 + cfg_w;
 668         cfg_y2 = cfg_y1 + cfg_h;
 669         bclamp(cfg_x1, 0, w);  replace_x = cfg_x1;
 670         bclamp(cfg_y1, 0, h);  replace_y = cfg_y1;
 671         bclamp(cfg_x2, 0, w);  replace_w = cfg_x2 - cfg_x1;
 672         bclamp(cfg_y2, 0, h);  replace_h = cfg_y2 - cfg_y1;
 673
 674         int cfg_dx = (int)(w * config.replace_dx / 100.);
 675         int cfg_dy = (int)(h * config.replace_dy / 100.);
 676         bclamp(cfg_dx, -h, h);  replace_dx = cfg_dx;
 677         bclamp(cfg_dy, -w, w);  replace_dy = cfg_dy;
 678
 679 // Read in the input frames
 680         for( int i = 0; i < PluginClient::get_total_buffers(); i++ ) {
 681                 read_frame(frame[i], i, start_position, frame_rate, 0);
 682         }
 683
 684         object = frame[object_layer];
 685         scene = frame[scene_layer];
 686         replace = frame[replace_layer];
 687
 688         border_x1 = obj_x1;  border_y1 = obj_y1;
 689         border_x2 = obj_x2;  border_y2 = obj_y2;
 690         border_x3 = obj_x3;  border_y3 = obj_y3;
 691         border_x4 = obj_x4;  border_y4 = obj_y4;
 692
 693         if( scene_w > 0 && scene_h > 0 && object_w > 0 && object_h > 0 ) {
 694                 process_match();
 695                 reshape();
 696         }
 697
 698         double w0 = init_border ? 1. : config.blend/100., w1 = 1. - w0;
 699         init_border = 0;
 700         obj_x1 = border_x1*w0 + obj_x1*w1;  obj_y1 = border_y1*w0 + obj_y1*w1;
 701         obj_x2 = border_x2*w0 + obj_x2*w1;  obj_y2 = border_y2*w0 + obj_y2*w1;
 702         obj_x3 = border_x3*w0 + obj_x3*w1;  obj_y3 = border_y3*w0 + obj_y3*w1;
 703         obj_x4 = border_x4*w0 + obj_x4*w1;  obj_y4 = border_y4*w0 + obj_y4*w1;
 704 // Replace object in the scene layer
 705         if( config.replace_object ) {
 706                 int cpus1 = get_project_smp() + 1;
 707                 if( !affine )
 708                         affine = new AffineEngine(cpus1, cpus1);
 709                 if( !overlayer )
 710                         overlayer = new OverlayFrame(cpus1);
 711                 VFrame *temp = new_temp(w, h, scene->get_color_model());
 712                 temp->clear_frame();
 713                 affine->set_in_viewport(replace_x, replace_y, replace_w, replace_h);
 714                 float dx = replace_dx, dy = replace_dy;
 715                 float x1 = obj_x1+dx, x2 = obj_x2+dx, x3 = obj_x3+dx, x4 = obj_x4+dx;
 716                 float y1 = obj_y1-dy, y2 = obj_y2-dy, y3 = obj_y3-dy, y4 = obj_y4-dy;
 717                 affine->set_matrix(
 718                         replace_x, replace_y, replace_x+replace_w, replace_y+replace_h,
 719                         x1,y1, x2,y2, x4,y4, x3,y3);
 720                 affine->process(temp, replace, 0,
 721                         AffineEngine::TRANSFORM, 0,0, 100,0, 100,100, 0,100, 1);
 722                 overlayer->overlay(scene, temp,  0,0, w,h,  0,0, w,h,
 723                         1, TRANSFER_NORMAL, NEAREST_NEIGHBOR);
 724
 725         }
 726
 727         if( config.draw_scene_border ) {
 728                 int wh = (w+h)>>8, ss = 1; while( wh>>=1 ) ss<<=1;
 729                 scene->set_pixel_color(WHITE);  scene->set_stiple(ss*2);
 730                 draw_rect(scene, scene_x, scene_y, scene_x+scene_w, scene_y+scene_h);
 731         }
 732         if( config.draw_object_border ) {
 733                 int wh = (w+h)>>8, ss = 1; while( wh>>=1 ) ss<<=1;
 734                 scene->set_pixel_color(YELLOW);  scene->set_stiple(ss*3);
 735                 draw_rect(scene, object_x, object_y, object_x+object_w, object_y+object_h);
 736         }
 737         if( config.draw_replace_border ) {
 738                 int wh = (w+h)>>8, ss = 1; while( wh>>=1 ) ss<<=1;
 739                 scene->set_pixel_color(GREEN);  scene->set_stiple(ss*3);
 740                 draw_rect(scene, replace_x, replace_y, replace_x+replace_w, replace_y+replace_h);
 741         }
 742         if( config.draw_keypoints ) {
 743                 scene->set_pixel_color(GREEN);  scene->set_stiple(0);
 744                 for( int i=0,n=obj_keypts.size(); i<n; ++i ) {
 745                         Point2f &pt = obj_keypts[i].pt;
 746                         int r = obj_keypts[i].size * 1.2/9 * 2;
 747                         int x = pt.x + object_x, y = pt.y + object_y;
 748                         draw_circle(scene, x, y, r);
 749                 }
 750                 scene->set_pixel_color(RED);  scene->set_stiple(0);
 751                 for( int i=0,n=scn_keypts.size(); i<n; ++i ) {
 752                         Point2f &pt = scn_keypts[i].pt;
 753                         int r = scn_keypts[i].size * 1.2/9 * 2;
 754                         int x = pt.x + scene_x, y = pt.y + scene_y;
 755                         draw_circle(scene, x, y, r);
 756                 }
 757         }
 758
 759         if( gui_open() ) {
 760                 if( config.drag_scene ) {
 761                         scene->set_pixel_color(WHITE);
 762                         DragCheckBox::draw_boundary(scene, scene_x, scene_y, scene_w, scene_h);
 763                 }
 764                 if( config.drag_object ) {
 765                         scene->set_pixel_color(YELLOW);
 766                         DragCheckBox::draw_boundary(scene, object_x, object_y, object_w, object_h);
 767                 }
 768                 if( config.drag_replace ) {
 769                         scene->set_pixel_color(GREEN);
 770                         DragCheckBox::draw_boundary(scene, replace_x, replace_y, replace_w, replace_h);
 771                 }
 772         }
 773
 774         scene->set_pixel_color(BLACK);
 775         scene->set_stiple(0);
 776         return 0;
 777 }
 778