cinelerra-5.0/plugins/findobject/findobject.C

   1
   2 /*
   3  * CINELERRA
   4  * Copyright (C) 1997-2012 Adam Williams <broadcast at earthling dot net>
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License as published by
   8  * the Free Software Foundation; either version 2 of the License, or
   9  * (at your option) any later version.
  10  *
  11  * This program is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14  * GNU General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU General Public License
  17  * along with this program; if not, write to the Free Software
  18  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  19  *
  20  */
  21
  22
  23 // This is mainly a test for object tracking
  24
  25 #include "affine.h"
  26 #include "cicolors.h"
  27 #include "clip.h"
  28 #include "filexml.h"
  29 #include "keyframe.h"
  30 #include "language.h"
  31 #include "findobject.h"
  32 #include "findobjectwindow.h"
  33 #include "mutex.h"
  34 #include "overlayframe.h"
  35 #include "surfscan.h"
  36 #include "transportque.h"
  37
  38 #include "opencv2/video/tracking.hpp"
  39 #include "opencv2/video/background_segm.hpp"
  40
  41
  42 #include <errno.h>
  43 #include <unistd.h>
  44
  45 REGISTER_PLUGIN(FindObjectMain)
  46
  47
  48
  49 FindObjectConfig::FindObjectConfig()
  50 {
  51         global_range_w = 5;
  52         global_range_h = 5;
  53         draw_keypoints = 1;
  54         draw_border = 1;
  55         replace_object = 0;
  56         draw_object_border = 1;
  57         global_block_w = MIN_BLOCK;
  58         global_block_h = MIN_BLOCK;
  59         block_x = 50;
  60         block_y = 50;
  61         object_layer = 0;
  62         replace_layer = 1;
  63         scene_layer = 2;
  64         algorithm = NO_ALGORITHM;
  65         vmin = 10;
  66         vmax = 256;
  67         smin = 30;
  68         blend = 100;
  69 }
  70
  71 void FindObjectConfig::boundaries()
  72 {
  73         CLAMP(global_range_w, MIN_RADIUS, MAX_RADIUS);
  74         CLAMP(global_range_h, MIN_RADIUS, MAX_RADIUS);
  75         CLAMP(global_block_w, MIN_BLOCK, MAX_BLOCK);
  76         CLAMP(global_block_h, MIN_BLOCK, MAX_BLOCK);
  77         CLAMP(block_x, 0, 100);
  78         CLAMP(block_y, 0, 100);
  79         CLAMP(object_layer, MIN_LAYER, MAX_LAYER);
  80         CLAMP(replace_layer, MIN_LAYER, MAX_LAYER);
  81         CLAMP(scene_layer, MIN_LAYER, MAX_LAYER);
  82         CLAMP(vmin, MIN_CAMSHIFT, MAX_CAMSHIFT);
  83         CLAMP(vmax, MIN_CAMSHIFT, MAX_CAMSHIFT);
  84         CLAMP(smin, MIN_CAMSHIFT, MAX_CAMSHIFT);
  85         CLAMP(blend, MIN_BLEND, MAX_BLEND);
  86 }
  87
  88 int FindObjectConfig::equivalent(FindObjectConfig &that)
  89 {
  90         int result =
  91                 global_range_w == that.global_range_w &&
  92                 global_range_h == that.global_range_h &&
  93                 draw_keypoints == that.draw_keypoints &&
  94                 draw_border == that.draw_border &&
  95                 replace_object == that.replace_object &&
  96                 draw_object_border == that.draw_object_border &&
  97                 global_block_w == that.global_block_w &&
  98                 global_block_h == that.global_block_h &&
  99                 block_x == that.block_x &&
 100                 block_y == that.block_y &&
 101                 object_layer == that.object_layer &&
 102                 replace_layer == that.replace_layer &&
 103                 scene_layer == that.scene_layer &&
 104                 algorithm == that.algorithm &&
 105                 vmin == that.vmin &&
 106                 vmax == that.vmax &&
 107                 smin == that.smin &&
 108                 blend == that.blend;
 109         return result;
 110 }
 111
 112 void FindObjectConfig::copy_from(FindObjectConfig &that)
 113 {
 114         global_range_w = that.global_range_w;
 115         global_range_h = that.global_range_h;
 116         draw_keypoints = that.draw_keypoints;
 117         draw_border = that.draw_border;
 118         replace_object = that.replace_object;
 119         draw_object_border = that.draw_object_border;
 120         global_block_w = that.global_block_w;
 121         global_block_h = that.global_block_h;
 122         block_x = that.block_x;
 123         block_y = that.block_y;
 124         object_layer = that.object_layer;
 125         replace_layer = that.replace_layer;
 126         scene_layer = that.scene_layer;
 127         algorithm = that.algorithm;
 128         vmin = that.vmin;
 129         vmax = that.vmax;
 130         smin = that.smin;
 131         blend = that.blend;
 132 }
 133
 134 void FindObjectConfig::interpolate(FindObjectConfig &prev,
 135         FindObjectConfig &next,
 136         int64_t prev_frame,
 137         int64_t next_frame,
 138         int64_t current_frame)
 139 {
 140         copy_from(prev);
 141 }
 142
 143
 144
 145
 146
 147
 148
 149
 150 FindObjectMain::FindObjectMain(PluginServer *server)
 151  : PluginVClient(server)
 152 {
 153         bzero(&blob_param, sizeof(CvBlobTrackerAutoParam1));
 154         blob_pTracker = 0;
 155
 156
 157         object_image = 0;
 158         prev_object = 0;
 159         scene_image = 0;
 160         object_image_w = 0;
 161         object_image_h = 0;
 162         scene_image_w = 0;
 163         scene_image_h = 0;
 164         storage = 0;
 165         object_keypoints = 0;
 166         object_descriptors = 0;
 167         scene_keypoints = 0;
 168         scene_descriptors = 0;
 169         affine = 0;
 170         temp = 0;
 171         overlayer = 0;
 172         init_border = 1;
 173 }
 174
 175 FindObjectMain::~FindObjectMain()
 176 {
 177 // This releases all the arrays
 178         if(storage) cvReleaseMemStorage(&storage);
 179         if(object_image) cvReleaseImage(&object_image);
 180         if(scene_image) cvReleaseImage(&scene_image);
 181         if(prev_object) delete [] prev_object;
 182         delete affine;
 183         delete temp;
 184         delete overlayer;
 185
 186     if(blob_param.pBT) cvReleaseBlobTracker(&blob_param.pBT);
 187     if(blob_param.pBD) cvReleaseBlobDetector(&blob_param.pBD);
 188     if(blob_param.pBTGen) cvReleaseBlobTrackGen(&blob_param.pBTGen);
 189     if(blob_param.pBTA) cvReleaseBlobTrackAnalysis(&blob_param.pBTA);
 190     if(blob_param.pFG) cvReleaseFGDetector(&blob_param.pFG);
 191     if(blob_pTracker) cvReleaseBlobTrackerAuto(&blob_pTracker);
 192
 193 }
 194
 195 const char* FindObjectMain::plugin_title() { return _("Find Object"); }
 196 int FindObjectMain::is_realtime() { return 1; }
 197 int FindObjectMain::is_multichannel() { return 1; }
 198
 199
 200 NEW_WINDOW_MACRO(FindObjectMain, FindObjectWindow)
 201
 202 LOAD_CONFIGURATION_MACRO(FindObjectMain, FindObjectConfig)
 203
 204
 205
 206 void FindObjectMain::update_gui()
 207 {
 208         if(thread)
 209         {
 210                 if(load_configuration())
 211                 {
 212                         thread->window->lock_window("FindObjectMain::update_gui");
 213
 214                         char string[BCTEXTLEN];
 215
 216                         ((FindObjectWindow*)thread->window)->global_range_w->update(config.global_range_w);
 217                         ((FindObjectWindow*)thread->window)->global_range_h->update(config.global_range_h);
 218                         ((FindObjectWindow*)thread->window)->global_block_w->update(config.global_block_w);
 219                         ((FindObjectWindow*)thread->window)->global_block_h->update(config.global_block_h);
 220                         ((FindObjectWindow*)thread->window)->block_x->update(config.block_x);
 221                         ((FindObjectWindow*)thread->window)->block_y->update(config.block_y);
 222                         ((FindObjectWindow*)thread->window)->block_x_text->update((float)config.block_x);
 223                         ((FindObjectWindow*)thread->window)->block_y_text->update((float)config.block_y);
 224
 225                         ((FindObjectWindow*)thread->window)->draw_keypoints->update(config.draw_keypoints);
 226                         ((FindObjectWindow*)thread->window)->draw_border->update(config.draw_border);
 227                         ((FindObjectWindow*)thread->window)->replace_object->update(config.replace_object);
 228                         ((FindObjectWindow*)thread->window)->draw_object_border->update(config.draw_object_border);
 229
 230
 231                         ((FindObjectWindow*)thread->window)->object_layer->update(
 232                                 (int64_t)config.object_layer);
 233                         ((FindObjectWindow*)thread->window)->replace_layer->update(
 234                                 (int64_t)config.replace_layer);
 235                         ((FindObjectWindow*)thread->window)->scene_layer->update(
 236                                 (int64_t)config.scene_layer);
 237                         ((FindObjectWindow*)thread->window)->algorithm->set_text(
 238                                 FindObjectAlgorithm::to_text(config.algorithm));
 239
 240                         ((FindObjectWindow*)thread->window)->vmin->update(
 241                                 (int64_t)config.vmin);
 242                         ((FindObjectWindow*)thread->window)->vmax->update(
 243                                 (int64_t)config.vmax);
 244                         ((FindObjectWindow*)thread->window)->smin->update(
 245                                 (int64_t)config.smin);
 246                         ((FindObjectWindow*)thread->window)->blend->update(
 247                                 (int64_t)config.blend);
 248
 249                         ((FindObjectWindow*)thread->window)->flush();
 250                         thread->window->unlock_window();
 251                 }
 252 // printf("FindObjectMain::update_gui %d %d %d %d\n",
 253 // __LINE__,
 254 // config.mode1,
 255 // config.mode2,
 256 // config.mode3);
 257         }
 258 }
 259
 260
 261
 262
 263 void FindObjectMain::save_data(KeyFrame *keyframe)
 264 {
 265         FileXML output;
 266
 267 // cause data to be stored directly in text
 268         output.set_shared_output(keyframe->get_data(), MESSAGESIZE);
 269         output.tag.set_title("FINDOBJECT");
 270
 271         output.tag.set_property("GLOBAL_BLOCK_W", config.global_block_w);
 272         output.tag.set_property("GLOBAL_BLOCK_H", config.global_block_h);
 273         output.tag.set_property("BLOCK_X", config.block_x);
 274         output.tag.set_property("BLOCK_Y", config.block_y);
 275         output.tag.set_property("GLOBAL_RANGE_W", config.global_range_w);
 276         output.tag.set_property("GLOBAL_RANGE_H", config.global_range_h);
 277         output.tag.set_property("DRAW_KEYPOINTS", config.draw_keypoints);
 278         output.tag.set_property("DRAW_BORDER", config.draw_border);
 279         output.tag.set_property("REPLACE_OBJECT", config.replace_object);
 280         output.tag.set_property("DRAW_OBJECT_BORDER", config.draw_object_border);
 281         output.tag.set_property("OBJECT_LAYER", config.object_layer);
 282         output.tag.set_property("REPLACE_LAYER", config.replace_layer);
 283         output.tag.set_property("SCENE_LAYER", config.scene_layer);
 284         output.tag.set_property("ALGORITHM", config.algorithm);
 285         output.tag.set_property("VMIN", config.vmin);
 286         output.tag.set_property("VMAX", config.vmax);
 287         output.tag.set_property("SMIN", config.smin);
 288         output.tag.set_property("BLEND", config.blend);
 289         output.append_tag();
 290         output.terminate_string();
 291 }
 292
 293 void FindObjectMain::read_data(KeyFrame *keyframe)
 294 {
 295         FileXML input;
 296
 297         input.set_shared_input(keyframe->get_data(), strlen(keyframe->get_data()));
 298
 299         int result = 0;
 300
 301         while(!result)
 302         {
 303                 result = input.read_tag();
 304
 305                 if(!result)
 306                 {
 307                         if(input.tag.title_is("FINDOBJECT"))
 308                         {
 309                                 config.global_block_w = input.tag.get_property("GLOBAL_BLOCK_W", config.global_block_w);
 310                                 config.global_block_h = input.tag.get_property("GLOBAL_BLOCK_H", config.global_block_h);
 311                                 config.block_x = input.tag.get_property("BLOCK_X", config.block_x);
 312                                 config.block_y = input.tag.get_property("BLOCK_Y", config.block_y);
 313                                 config.global_range_w = input.tag.get_property("GLOBAL_RANGE_W", config.global_range_w);
 314                                 config.global_range_h = input.tag.get_property("GLOBAL_RANGE_H", config.global_range_h);
 315                                 config.draw_keypoints = input.tag.get_property("DRAW_KEYPOINTS", config.draw_keypoints);
 316                                 config.draw_border = input.tag.get_property("DRAW_BORDER", config.draw_border);
 317                                 config.replace_object = input.tag.get_property("REPLACE_OBJECT", config.replace_object);
 318                                 config.draw_object_border = input.tag.get_property("DRAW_OBJECT_BORDER", config.draw_object_border);
 319                                 config.object_layer = input.tag.get_property("OBJECT_LAYER", config.object_layer);
 320                                 config.replace_layer = input.tag.get_property("REPLACE_LAYER", config.replace_layer);
 321                                 config.scene_layer = input.tag.get_property("SCENE_LAYER", config.scene_layer);
 322                                 config.algorithm = input.tag.get_property("ALGORITHM", config.algorithm);
 323                                 config.vmin = input.tag.get_property("VMIN", config.vmin);
 324                                 config.vmax = input.tag.get_property("VMAX", config.vmax);
 325                                 config.smin = input.tag.get_property("SMIN", config.smin);
 326                                 config.blend = input.tag.get_property("BLEND", config.blend);
 327                         }
 328                 }
 329         }
 330         config.boundaries();
 331 }
 332
 333
 334
 335
 336
 337 void FindObjectMain::draw_pixel(VFrame *frame, int x, int y)
 338 {
 339         if(!(x >= 0 && y >= 0 && x < frame->get_w() && y < frame->get_h())) return;
 340
 341 #define DRAW_PIXEL(x, y, components, do_yuv, max, type) \
 342 { \
 343         type **rows = (type**)frame->get_rows(); \
 344         rows[y][x * components] = max - rows[y][x * components]; \
 345         if(!do_yuv) \
 346         { \
 347                 rows[y][x * components + 1] = max - rows[y][x * components + 1]; \
 348                 rows[y][x * components + 2] = max - rows[y][x * components + 2]; \
 349         } \
 350         else \
 351         { \
 352                 rows[y][x * components + 1] = (max / 2 + 1) - rows[y][x * components + 1]; \
 353                 rows[y][x * components + 2] = (max / 2 + 1) - rows[y][x * components + 2]; \
 354         } \
 355         if(components == 4) \
 356                 rows[y][x * components + 3] = max; \
 357 }
 358
 359
 360         switch(frame->get_color_model())
 361         {
 362                 case BC_RGB888:
 363                         DRAW_PIXEL(x, y, 3, 0, 0xff, unsigned char);
 364                         break;
 365                 case BC_RGBA8888:
 366                         DRAW_PIXEL(x, y, 4, 0, 0xff, unsigned char);
 367                         break;
 368                 case BC_RGB_FLOAT:
 369                         DRAW_PIXEL(x, y, 3, 0, 1.0, float);
 370                         break;
 371                 case BC_RGBA_FLOAT:
 372                         DRAW_PIXEL(x, y, 4, 0, 1.0, float);
 373                         break;
 374                 case BC_YUV888:
 375                         DRAW_PIXEL(x, y, 3, 1, 0xff, unsigned char);
 376                         break;
 377                 case BC_YUVA8888:
 378                         DRAW_PIXEL(x, y, 4, 1, 0xff, unsigned char);
 379                         break;
 380                 case BC_RGB161616:
 381                         DRAW_PIXEL(x, y, 3, 0, 0xffff, uint16_t);
 382                         break;
 383                 case BC_YUV161616:
 384                         DRAW_PIXEL(x, y, 3, 1, 0xffff, uint16_t);
 385                         break;
 386                 case BC_RGBA16161616:
 387                         DRAW_PIXEL(x, y, 4, 0, 0xffff, uint16_t);
 388                         break;
 389                 case BC_YUVA16161616:
 390                         DRAW_PIXEL(x, y, 4, 1, 0xffff, uint16_t);
 391                         break;
 392         }
 393 }
 394
 395
 396 void FindObjectMain::draw_line(VFrame *frame, int x1, int y1, int x2, int y2)
 397 {
 398         int w = labs(x2 - x1);
 399         int h = labs(y2 - y1);
 400 //printf("FindObjectMain::draw_line 1 %d %d %d %d\n", x1, y1, x2, y2);
 401
 402         if(!w && !h)
 403         {
 404                 draw_pixel(frame, x1, y1);
 405         }
 406         else
 407         if(w > h)
 408         {
 409 // Flip coordinates so x1 < x2
 410                 if(x2 < x1)
 411                 {
 412                         y2 ^= y1;
 413                         y1 ^= y2;
 414                         y2 ^= y1;
 415                         x1 ^= x2;
 416                         x2 ^= x1;
 417                         x1 ^= x2;
 418                 }
 419                 int numerator = y2 - y1;
 420                 int denominator = x2 - x1;
 421                 for(int i = x1; i <= x2; i++)
 422                 {
 423                         int y = y1 + (int64_t)(i - x1) * (int64_t)numerator / (int64_t)denominator;
 424                         draw_pixel(frame, i, y);
 425                 }
 426         }
 427         else
 428         {
 429 // Flip coordinates so y1 < y2
 430                 if(y2 < y1)
 431                 {
 432                         y2 ^= y1;
 433                         y1 ^= y2;
 434                         y2 ^= y1;
 435                         x1 ^= x2;
 436                         x2 ^= x1;
 437                         x1 ^= x2;
 438                 }
 439                 int numerator = x2 - x1;
 440                 int denominator = y2 - y1;
 441                 for(int i = y1; i <= y2; i++)
 442                 {
 443                         int x = x1 + (int64_t)(i - y1) * (int64_t)numerator / (int64_t)denominator;
 444                         draw_pixel(frame, x, i);
 445                 }
 446         }
 447 //printf("FindObjectMain::draw_line 2\n");
 448 }
 449
 450 void FindObjectMain::draw_rect(VFrame *frame, int x1, int y1, int x2, int y2)
 451 {
 452         draw_line(frame, x1, y1, x2, y1);
 453         draw_line(frame, x2, y1 + 1, x2, y2);
 454         draw_line(frame, x2 - 1, y2, x1, y2);
 455         draw_line(frame, x1, y2 - 1, x1, y1 + 1);
 456 }
 457
 458
 459
 460 // Convert to greyscale & crop
 461 void FindObjectMain::grey_crop(unsigned char *dst,
 462         VFrame *src,
 463         int x1,
 464         int y1,
 465         int x2,
 466         int y2,
 467         int dst_w,
 468         int dst_h)
 469 {
 470 // Dimensions of dst frame
 471         int w = x2 - x1;
 472         int h = y2 - y1;
 473
 474         bzero(dst, dst_w * dst_h);
 475
 476 //printf("FindObjectMain::grey_crop %d %d %d\n", __LINE__, w, h);
 477         for(int i = 0; i < h; i++)
 478         {
 479
 480 #define RGB_TO_VALUE(r, g, b) \
 481 ((r) * R_TO_Y + (g) * G_TO_Y + (b) * B_TO_Y)
 482
 483
 484 #define CONVERT(in_type, max, components, is_yuv) \
 485 { \
 486         in_type *input = ((in_type*)src->get_rows()[i + y1]) + x1 * components; \
 487         unsigned char *output = dst + i * dst_w; \
 488  \
 489         for(int j = 0; j < w; j++) \
 490         { \
 491 /* Y channel only */ \
 492                 if(is_yuv) \
 493                 { \
 494                         *output = *input; \
 495                 } \
 496 /* RGB */ \
 497                 else \
 498                 { \
 499                         float r = (float)input[0] / max; \
 500                         float g = (float)input[1] / max; \
 501                         float b = (float)input[2] / max; \
 502                         *output = RGB_TO_VALUE(r, g, b); \
 503                 } \
 504  \
 505                 input += components; \
 506                 output++; \
 507         } \
 508 }
 509                 switch(src->get_color_model())
 510                 {
 511                         case BC_RGB888:
 512                         {
 513                                 CONVERT(unsigned char, 0xff, 3, 0)
 514                                 break;
 515                         }
 516
 517                         case BC_RGBA8888:
 518                         {
 519                                 CONVERT(unsigned char, 0xff, 4, 0)
 520                                 break;
 521                         }
 522
 523                         case BC_RGB_FLOAT:
 524                         {
 525                                 CONVERT(float, 1.0, 3, 0)
 526                                 break;
 527                         }
 528
 529                         case BC_RGBA_FLOAT:
 530                         {
 531                                 CONVERT(float, 1.0, 4, 0)
 532                                 break;
 533                         }
 534
 535                         case BC_YUV888:
 536                         {
 537                                 CONVERT(unsigned char, 0xff, 3, 1)
 538                                 break;
 539                         }
 540
 541                         case BC_YUVA8888:
 542                         {
 543                                 CONVERT(unsigned char, 0xff, 4, 1)
 544                                 break;
 545                         }
 546                 }
 547         }
 548 }
 549
 550
 551 void FindObjectMain::process_surf()
 552 {
 553
 554         if(!object_image)
 555         {
 556 // Only does greyscale
 557                 object_image = cvCreateImage(
 558                         cvSize(object_image_w, object_image_h),
 559                         8,
 560                         1);
 561         }
 562
 563         if(!scene_image)
 564         {
 565 // Only does greyscale
 566                 scene_image = cvCreateImage(
 567                         cvSize(scene_image_w, scene_image_h),
 568                         8,
 569                         1);
 570         }
 571
 572 // Select only region with image size
 573 // Does this do anything?
 574         cvSetImageROI( object_image, cvRect( 0, 0, object_w, object_h ) );
 575         cvSetImageROI( scene_image, cvRect( 0, 0, scene_w, scene_h ) );
 576
 577         if(!prev_object) prev_object = new unsigned char[object_image_w * object_image_h];
 578         memcpy(prev_object, object_image->imageData, object_image_w * object_image_h);
 579         grey_crop((unsigned char*)scene_image->imageData,
 580                 get_input(scene_layer),
 581                 scene_x1,
 582                 scene_y1,
 583                 scene_x2,
 584                 scene_y2,
 585                 scene_image_w,
 586                 scene_image_h);
 587
 588
 589         grey_crop((unsigned char*)object_image->imageData,
 590                 get_input(object_layer),
 591                 object_x1,
 592                 object_y1,
 593                 object_x2,
 594                 object_y2,
 595                 object_image_w,
 596                 object_image_h);
 597
 598
 599         if(!storage) storage = cvCreateMemStorage(0);
 600         CvSURFParams params = cvSURFParams(500, 1);
 601
 602
 603 //printf("FindObjectMain::process_surf %d\n", __LINE__);
 604
 605 // Only compute keypoints if the image changed
 606         if(memcmp(prev_object, object_image->imageData, object_image_w * object_image_h))
 607         {
 608                 if(object_keypoints) cvClearSeq(object_keypoints);
 609                 if(object_descriptors) cvClearSeq(object_descriptors);
 610                 cvExtractSURF(object_image,
 611                         0,
 612                         &object_keypoints,
 613                         &object_descriptors,
 614                         storage,
 615                         params,
 616                         0);
 617         }
 618
 619 //printf("FindObjectMain::process_surf %d object keypoints=%d\n", __LINE__, object_keypoints->total);
 620 // Draw the keypoints
 621 //              for(int i = 0; i < object_keypoints->total; i++)
 622 //              {
 623 //              CvSURFPoint* r1 = (CvSURFPoint*)cvGetSeqElem( object_keypoints, i );
 624 //                      int size = r1->size / 4;
 625 //                      draw_rect(frame[object_layer],
 626 //                              r1->pt.x + object_x1 - size,
 627 //                              r1->pt.y + object_y1 - size,
 628 //                              r1->pt.x + object_x1 + size,
 629 //                              r1->pt.y + object_y1 + size);
 630 //              }
 631
 632
 633 //printf("FindObjectMain::process_surf %d\n", __LINE__);
 634
 635 // TODO: make the surf data persistent & check for image changes instead
 636         if(scene_keypoints) cvClearSeq(scene_keypoints);
 637         if(scene_descriptors) cvClearSeq(scene_descriptors);
 638         cvExtractSURF(scene_image,
 639                 0,
 640                 &scene_keypoints,
 641                 &scene_descriptors,
 642                 storage,
 643                 params,
 644                 0);
 645
 646 // Draw the keypoints
 647 //              for(int i = 0; i < scene_keypoints->total; i++)
 648 //              {
 649 //              CvSURFPoint* r1 = (CvSURFPoint*)cvGetSeqElem( scene_keypoints, i );
 650 //                      int size = r1->size / 4;
 651 //                      draw_rect(frame[scene_layer],
 652 //                              r1->pt.x + scene_x1 - size,
 653 //                              r1->pt.y + scene_y1 - size,
 654 //                              r1->pt.x + scene_x1 + size,
 655 //                              r1->pt.y + scene_y1 + size);
 656 //              }
 657
 658 // printf("FindObjectMain::process_surf %d %d %d scene keypoints=%d\n",
 659 // __LINE__,
 660 // scene_w,
 661 // scene_h,
 662 // scene_keypoints->total);
 663
 664         int *point_pairs = 0;
 665         int total_pairs = 0;
 666         CvPoint src_corners[4] =
 667         {
 668                 { 0, 0 },
 669                 { object_w, 0 },
 670                 { object_w, object_h },
 671                 { 0, object_h }
 672         };
 673
 674         CvPoint dst_corners[4] =
 675         {
 676                 { 0, 0 },
 677                 { 0, 0 },
 678                 { 0, 0 },
 679                 { 0, 0 }
 680         };
 681
 682 //printf("FindObjectMain::process_surf %d\n", __LINE__);
 683         if(scene_keypoints->total &&
 684                 object_keypoints->total &&
 685                         locatePlanarObject(object_keypoints,
 686                         object_descriptors,
 687                         scene_keypoints,
 688                         scene_descriptors,
 689                         src_corners,
 690                         dst_corners,
 691                         &point_pairs,
 692                         &total_pairs))
 693         {
 694
 695
 696
 697
 698
 699 // Draw keypoints in the scene & object layer
 700                 if(config.draw_keypoints)
 701                 {
 702 //printf("FindObjectMain::process_surf %d total pairs=%d\n", __LINE__, total_pairs);
 703                         for(int i = 0; i < total_pairs; i++)
 704                         {
 705                         CvSURFPoint* r1 = (CvSURFPoint*)cvGetSeqElem( object_keypoints, point_pairs[i * 2] );
 706                         CvSURFPoint* r2 = (CvSURFPoint*)cvGetSeqElem( scene_keypoints, point_pairs[i * 2 + 1] );
 707
 708
 709                                 int size = r2->size * 1.2 / 9 * 2;
 710                                 draw_rect(get_input(scene_layer),
 711                                         r2->pt.x + scene_x1 - size,
 712                                         r2->pt.y + scene_y1 - size,
 713                                         r2->pt.x + scene_x1 + size,
 714                                         r2->pt.y + scene_y1 + size);
 715                                 draw_rect(get_input(object_layer),
 716                                         r1->pt.x + object_x1 - size,
 717                                         r1->pt.y + object_y1 - size,
 718                                         r1->pt.x + object_x1 + size,
 719                                         r1->pt.y + object_y1 + size);
 720                         }
 721                 }
 722
 723
 724 //printf("FindObjectMain::process_surf %d\n", __LINE__);
 725 // Get object outline in the scene layer
 726                 border_x1 = dst_corners[0].x + scene_x1;
 727                 border_y1 = dst_corners[0].y + scene_y1;
 728                 border_x2 = dst_corners[1].x + scene_x1;
 729                 border_y2 = dst_corners[1].y + scene_y1;
 730                 border_x3 = dst_corners[2].x + scene_x1;
 731                 border_y3 = dst_corners[2].y + scene_y1;
 732                 border_x4 = dst_corners[3].x + scene_x1;
 733                 border_y4 = dst_corners[3].y + scene_y1;
 734 //printf("FindObjectMain::process_surf %d\n", __LINE__);
 735
 736
 737
 738         }
 739 //printf("FindObjectMain::process_surf %d\n", __LINE__);
 740
 741
 742
 743 // for(int i = 0; i < object_y2 - object_y1; i++)
 744 // {
 745 //      unsigned char *dst = get_input(object_layer)->get_rows()[i];
 746 //      unsigned char *src = (unsigned char*)object_image->imageData + i * (object_x2 - object_x1);
 747 //      for(int j = 0; j < object_x2 - object_x1; j++)
 748 //      {
 749 //              *dst++ = *src;
 750 //              *dst++ = 0x80;
 751 //              *dst++ = 0x80;
 752 //              src++;
 753 //      }
 754 // }
 755
 756
 757 // Frees the image structures
 758         if(point_pairs) free(point_pairs);
 759 }
 760
 761
 762
 763 void FindObjectMain::process_camshift()
 764 {
 765 // Some user defined parameters
 766         int vmin = config.vmin;
 767         int vmax = config.vmax;
 768         int smin = config.smin;
 769         float hranges[] = { 0, 180 };
 770         const float* phranges = hranges;
 771
 772
 773 // Create aligned, RGB images
 774         if(!object_image)
 775         {
 776                 object_image = cvCreateImage(
 777                         cvSize(object_image_w, object_image_h),
 778                         8,
 779                         3);
 780         }
 781
 782         if(!scene_image)
 783         {
 784                 scene_image = cvCreateImage(
 785                         cvSize(scene_image_w, scene_image_h),
 786                         8,
 787                         3);
 788         }
 789
 790 // Temporary row pointers
 791         unsigned char **object_rows = new unsigned char*[object_image_h];
 792         unsigned char **scene_rows = new unsigned char*[scene_image_h];
 793         for(int i = 0; i < object_image_h; i++)
 794         {
 795                 object_rows[i] = (unsigned char*)(object_image->imageData + i * object_image_w * 3);
 796         }
 797         for(int i = 0; i < scene_image_h; i++)
 798         {
 799                 scene_rows[i] = (unsigned char*)(scene_image->imageData + i * scene_image_w * 3);
 800         }
 801
 802 // Transfer object & scene to RGB images for OpenCV
 803         if(!prev_object) prev_object = new unsigned char[object_image_w * object_image_h * 3];
 804 // Back up old object image
 805         memcpy(prev_object, object_image->imageData, object_image_w * object_image_h * 3);
 806
 807         BC_CModels::transfer(object_rows,
 808                 get_input(object_layer)->get_rows(),
 809                 0,
 810                 0,
 811                 0,
 812                 0,
 813                 0,
 814                 0,
 815                 object_x1,
 816                 object_y1,
 817                 object_w,
 818                 object_h,
 819                 0,
 820                 0,
 821                 object_w,
 822                 object_h,
 823                 get_input(object_layer)->get_color_model(),
 824                 BC_RGB888,
 825                 0,
 826                 0,
 827                 0);
 828         BC_CModels::transfer(scene_rows,
 829                 get_input(scene_layer)->get_rows(),
 830                 0,
 831                 0,
 832                 0,
 833                 0,
 834                 0,
 835                 0,
 836                 scene_x1,
 837                 scene_y1,
 838                 scene_w,
 839                 scene_h,
 840                 0,
 841                 0,
 842                 scene_w,
 843                 scene_h,
 844                 get_input(scene_layer)->get_color_model(),
 845                 BC_RGB888,
 846                 0,
 847                 0,
 848                 0);
 849
 850         delete [] object_rows;
 851         delete [] scene_rows;
 852
 853 // from camshiftdemo.cpp
 854 // Compute new object
 855         if(memcmp(prev_object,
 856                 object_image->imageData,
 857                 object_image_w * object_image_h * 3) ||
 858                 !hist.dims)
 859         {
 860                 Mat image(object_image);
 861                 Mat hsv, hue, mask;
 862                 cvtColor(image, hsv, CV_RGB2HSV);
 863         int _vmin = vmin, _vmax = vmax;
 864 //printf("FindObjectMain::process_camshift %d\n", __LINE__);
 865
 866         inRange(hsv,
 867                         Scalar(0, smin, MIN(_vmin,_vmax)),
 868                 Scalar(180, 256, MAX(_vmin, _vmax)),
 869                         mask);
 870         int ch[] = { 0, 0 };
 871         hue.create(hsv.size(), hsv.depth());
 872         mixChannels(&hsv, 1, &hue, 1, ch, 1);
 873
 874                 Rect selection = Rect(0, 0, object_w, object_h);
 875                 trackWindow = selection;
 876                 int hsize = 16;
 877                 Mat roi(hue, selection), maskroi(mask, selection);
 878                 calcHist(&roi, 1, 0, maskroi, hist, 1, &hsize, &phranges);
 879                 normalize(hist, hist, 0, 255, CV_MINMAX);
 880         }
 881
 882
 883 // compute scene
 884         Mat image(scene_image);
 885         Mat hsv, hue, mask, backproj;
 886         cvtColor(image, hsv, CV_RGB2HSV);
 887     int _vmin = vmin, _vmax = vmax;
 888
 889     inRange(hsv,
 890                 Scalar(0, smin, MIN(_vmin,_vmax)),
 891         Scalar(180, 256, MAX(_vmin, _vmax)),
 892                 mask);
 893     int ch[] = {0, 0};
 894     hue.create(hsv.size(), hsv.depth());
 895     mixChannels(&hsv, 1, &hue, 1, ch, 1);
 896
 897 //printf("FindObjectMain::process_camshift %d %d %d\n", __LINE__, hist.dims, hist.size[1]);
 898         RotatedRect trackBox = RotatedRect(
 899                 Point2f((object_x1 + object_x2) / 2, (object_y1 + object_y2) / 2),
 900                 Size2f(object_w, object_h),
 901                 0);
 902         trackWindow = Rect(0,
 903                 0,
 904         scene_w,
 905                 scene_h);
 906         if(hist.dims > 0)
 907         {
 908
 909
 910                 calcBackProject(&hue, 1, 0, hist, backproj, &phranges);
 911                 backproj &= mask;
 912 //printf("FindObjectMain::process_camshift %d\n", __LINE__);
 913 //              if(trackWindow.width <= 0 ||
 914 //                      trackWindow.height <= 0)
 915 //              {
 916 //                      trackWindow.width = object_w;
 917 //                      trackWindow.height = object_h;
 918 //              }
 919
 920                 trackBox = CamShift(backproj,
 921                         trackWindow,
 922                 TermCriteria( CV_TERMCRIT_EPS | CV_TERMCRIT_ITER, 10, 1 ));
 923 //printf("FindObjectMain::process_camshift %d\n", __LINE__);
 924
 925
 926 //      if( trackWindow.area() <= 1 )
 927 //      {
 928 //              int cols = backproj.cols;
 929 //                      int rows = backproj.rows;
 930 //                      int r = (MIN(cols, rows) + 5) / 6;
 931 //              trackWindow = Rect(trackWindow.x - r, trackWindow.y - r,
 932 //                                 trackWindow.x + r, trackWindow.y + r) &
 933 //                        Rect(0, 0, cols, rows);
 934 //      }
 935         }
 936 // printf("FindObjectMain::process_camshift %d %d %d %d %d\n",
 937 // __LINE__,
 938 // trackWindow.x,
 939 // trackWindow.y,
 940 // trackWindow.width,
 941 // trackWindow.height);
 942
 943
 944 // Draw mask over scene
 945         if(config.draw_keypoints)
 946         {
 947                 for(int i = 0; i < scene_h; i++)
 948                 {
 949                         switch(get_input(scene_layer)->get_color_model())
 950                         {
 951                                 case BC_YUV888:
 952                                 {
 953                                         unsigned char *input = backproj.data + i * scene_image_w;
 954                                         unsigned char *output = get_input(scene_layer)->get_rows()[i + scene_y1] + scene_x1 * 3;
 955                                         for(int j = 0; j < scene_w; j++)
 956                                         {
 957                                                 output[0] = *input;
 958                                                 output[1] = 0x80;
 959                                                 output[2] = 0x80;
 960                                                 output += 3;
 961                                                 input++;
 962                                         }
 963                                         break;
 964                                 }
 965                         }
 966                 }
 967         }
 968
 969 // Get object outline in the scene layer
 970 // printf("FindObjectMain::process_camshift %d %d %d %d %d %d\n",
 971 // __LINE__,
 972 // (int)trackBox.center.x,
 973 // (int)trackBox.center.y,
 974 // (int)trackBox.size.width,
 975 // (int)trackBox.size.height,
 976 // (int)trackBox.angle);
 977         double angle = trackBox.angle * 2 * M_PI / 360;
 978         double angle1 = atan2(-(double)trackBox.size.height / 2, -(double)trackBox.size.width / 2) + angle;
 979         double angle2 = atan2(-(double)trackBox.size.height / 2, (double)trackBox.size.width / 2) + angle;
 980         double angle3 = atan2((double)trackBox.size.height / 2, (double)trackBox.size.width / 2) + angle;
 981         double angle4 = atan2((double)trackBox.size.height / 2, -(double)trackBox.size.width / 2) + angle;
 982         double radius = sqrt(SQR(trackBox.size.height / 2) + SQR(trackBox.size.width / 2));
 983         border_x1 = (int)(trackBox.center.x + cos(angle1) * radius) + scene_x1;
 984         border_y1 = (int)(trackBox.center.y + sin(angle1) * radius) + scene_y1;
 985         border_x2 = (int)(trackBox.center.x + cos(angle2) * radius) + scene_x1;
 986         border_y2 = (int)(trackBox.center.y + sin(angle2) * radius) + scene_y1;
 987         border_x3 = (int)(trackBox.center.x + cos(angle3) * radius) + scene_x1;
 988         border_y3 = (int)(trackBox.center.y + sin(angle3) * radius) + scene_y1;
 989         border_x4 = (int)(trackBox.center.x + cos(angle4) * radius) + scene_x1;
 990         border_y4 = (int)(trackBox.center.y + sin(angle4) * radius) + scene_y1;
 991
 992 }
 993
 994
 995 #define APPLY_MASK(type, max, components, do_yuv) \
 996 { \
 997         type *output_row = (type*)get_input(scene_layer)->get_rows()[i]; \
 998         unsigned char *mask_row = mask_rows[i]; \
 999         int chroma_offset = (int)(max + 1) / 2; \
1000  \
1001         for(int j  = 0; j < scene_w; j++) \
1002         { \
1003                 if(components == 4) \
1004                 { \
1005                         output_row[j * 4 + 3] = output_row[j * 4 + 3] * mask_row[j] / 255; \
1006                 } \
1007                 else \
1008                 { \
1009                         output_row[j * 3] = output_row[j * 3] * mask_row[j] / 255; \
1010                         output_row[j * 3 + 1] = output_row[j * 3 + 1] * mask_row[j] / 255; \
1011                         output_row[j * 3 + 2] = output_row[j * 3 + 2] * mask_row[j] / 255; \
1012  \
1013                         if(do_yuv) \
1014                         { \
1015                                 output_row[j * 3 + 1] += chroma_offset * (255 - mask_row[j]) / 255; \
1016                                 output_row[j * 3 + 2] += chroma_offset * (255 - mask_row[j]) / 255; \
1017                         } \
1018                 } \
1019         } \
1020 }
1021
1022
1023
1024 // blobtrack_sample.cpp
1025 void FindObjectMain::process_blob()
1026 {
1027         if(!blob_initialized)
1028         {
1029                 blob_initialized = 1;
1030
1031                 blob_param.FGTrainFrames = 5;
1032
1033
1034 /* Create FG Detection module: */
1035                 blob_param.pFG = cvCreateFGDetectorBase(CV_BG_MODEL_FGD, NULL);
1036 /* Create Blob Entrance Detection module: */
1037         blob_param.pBD = cvCreateBlobDetectorCC();
1038 /* Create blob tracker module: */
1039         blob_param.pBT = cvCreateBlobTrackerCCMSPF();
1040 /* Create whole pipline: */
1041         blob_pTracker = cvCreateBlobTrackerAuto1(&blob_param);
1042
1043         }
1044
1045
1046 /* Process: */
1047         IplImage*   pMask = NULL;
1048
1049 // Create aligned, RGB images
1050         if(!scene_image)
1051         {
1052                 scene_image = cvCreateImage(
1053                         cvSize(scene_image_w, scene_image_h),
1054                         8,
1055                         3);
1056         }
1057
1058 // Temporary row pointers
1059         unsigned char **scene_rows = new unsigned char*[scene_image_h];
1060         for(int i = 0; i < scene_image_h; i++)
1061         {
1062                 scene_rows[i] = (unsigned char*)(scene_image->imageData + i * scene_image_w * 3);
1063         }
1064
1065         BC_CModels::transfer(scene_rows,
1066                 get_input(scene_layer)->get_rows(),
1067                 0,
1068                 0,
1069                 0,
1070                 0,
1071                 0,
1072                 0,
1073                 scene_x1,
1074                 scene_y1,
1075                 scene_w,
1076                 scene_h,
1077                 0,
1078                 0,
1079                 scene_w,
1080                 scene_h,
1081                 get_input(scene_layer)->get_color_model(),
1082                 BC_RGB888,
1083                 0,
1084                 0,
1085                 0);
1086
1087         delete [] scene_rows;
1088
1089     blob_pTracker->Process(scene_image, pMask);
1090 printf("FindObjectMain::process_blob %d %ld %d\n", __LINE__, get_source_position(), blob_pTracker->GetBlobNum());
1091
1092
1093 #if 0
1094         if(blob_pTracker->GetFGMask())
1095         {
1096                 IplImage* pFG = blob_pTracker->GetFGMask();
1097 printf("FindObjectMain::process_blob %d %ld\n", __LINE__, get_source_position());
1098
1099 // Temporary row pointers
1100                 unsigned char **mask_rows = new unsigned char*[scene_image_h];
1101                 for(int i = 0; i < scene_image_h; i++)
1102                 {
1103                         mask_rows[i] = (unsigned char*)(pFG->imageData + i * scene_image_w);
1104                 }
1105
1106                 for(int i = 0; i < scene_image_h; i++)
1107                 {
1108                         switch(get_input(scene_layer)->get_color_model())
1109                         {
1110                                 case BC_RGB888:
1111                                         APPLY_MASK(unsigned char, 0xff, 3, 0)
1112                                         break;
1113                                 case BC_RGB_FLOAT:
1114                                         APPLY_MASK(float, 1.0, 3, 0)
1115                                         break;
1116                                 case BC_YUV888:
1117                                         APPLY_MASK(unsigned char, 0xff, 3, 1)
1118                                         break;
1119                                 case BC_RGBA8888:
1120                                         APPLY_MASK(unsigned char, 0xff, 4, 0)
1121                                         break;
1122                                 case BC_RGBA_FLOAT:
1123                                         APPLY_MASK(float, 1.0, 4, 0)
1124                                         break;
1125                                 case BC_YUVA8888:
1126                                         APPLY_MASK(unsigned char, 0xff, 4, 1)
1127                                         break;
1128                         }
1129                 }
1130
1131                 delete [] mask_rows;
1132
1133
1134         }
1135 #endif
1136
1137
1138 }
1139
1140
1141
1142
1143
1144
1145
1146 int FindObjectMain::process_buffer(VFrame **frame,
1147         int64_t start_position,
1148         double frame_rate)
1149 {
1150         int prev_algorithm = config.algorithm;
1151         if(load_configuration())
1152         {
1153                 init_border = 1;
1154         }
1155
1156         w = frame[0]->get_w();
1157         h = frame[0]->get_h();
1158 //printf("FindObjectMain::process_buffer %d\n", __LINE__);
1159
1160
1161 // Get the layer containing the object.
1162         object_layer = config.object_layer;
1163 // Get the layer to search in.
1164         scene_layer = config.scene_layer;
1165 // Get the layer with the replacement object
1166         replace_layer = config.replace_layer;
1167
1168         object_layer = MIN(object_layer, PluginClient::get_total_buffers() - 1);
1169         scene_layer = MIN(scene_layer, PluginClient::get_total_buffers() - 1);
1170         replace_layer = MIN(replace_layer, PluginClient::get_total_buffers() - 1);
1171
1172 // printf("FindObjectMain::process_buffer %d %d %d %d %d %d\n",
1173 // __LINE__,
1174 // PluginClient::get_total_buffers(),
1175 // config.object_layer,
1176 // config.scene_layer,
1177 // object_layer,
1178 // scene_layer);
1179 //
1180 // Create cropped images
1181 // TODO: use oblique corners & affine transform
1182         object_w = (int)(config.global_block_w * w / 100);
1183         object_h = (int)(config.global_block_h * h / 100);
1184         object_x1;
1185         object_y1;
1186         object_x2;
1187         object_y2;
1188
1189         object_x1 = (int)(config.block_x * w / 100 - object_w / 2);
1190         object_y1 = (int)(config.block_y * h / 100 - object_h / 2);
1191         object_x2 = object_x1 + object_w;
1192         object_y2 = object_y1 + object_h;
1193         CLAMP(object_x1, 0, frame[0]->get_w() - 1);
1194         CLAMP(object_x2, 0, frame[0]->get_w() - 1);
1195         CLAMP(object_y1, 0, frame[0]->get_h() - 1);
1196         CLAMP(object_y2, 0, frame[0]->get_h() - 1);
1197         object_w = object_x2 - object_x1;
1198         object_h = object_y2 - object_y1;
1199
1200
1201         scene_w = (int)(config.global_range_w * w / 100);
1202         scene_h = (int)(config.global_range_h * h / 100);
1203         scene_x1;
1204         scene_y1;
1205         scene_x2;
1206         scene_y2;
1207         scene_x1 = (int)(config.block_x * w / 100 - scene_w / 2);
1208         scene_y1 = (int)(config.block_y * h / 100 - scene_h / 2);
1209         scene_x2 = scene_x1 + scene_w;
1210         scene_y2 = scene_y1 + scene_h;
1211         CLAMP(scene_x1, 0, frame[0]->get_w() - 1);
1212         CLAMP(scene_x2, 0, frame[0]->get_w() - 1);
1213         CLAMP(scene_y1, 0, frame[0]->get_h() - 1);
1214         CLAMP(scene_y2, 0, frame[0]->get_h() - 1);
1215         scene_w = scene_x2 - scene_x1;
1216         scene_h = scene_y2 - scene_y1;
1217
1218 // Get quantized sizes
1219         int object_image_w = object_w;
1220         int object_image_h = object_h;
1221         int scene_image_w = scene_w;
1222         int scene_image_h = scene_h;
1223         if(object_w % QUANTIZE) object_image_w += QUANTIZE - (object_w % QUANTIZE);
1224         if(object_h % QUANTIZE) object_image_h += QUANTIZE - (object_h % QUANTIZE);
1225         if(scene_w % QUANTIZE) scene_image_w += QUANTIZE - (scene_w % QUANTIZE);
1226         if(scene_h % QUANTIZE) scene_image_h += QUANTIZE - (scene_h % QUANTIZE);
1227
1228         if(object_image &&
1229                 (object_image_w != this->object_image_w ||
1230                 object_image_h != this->object_image_h ||
1231                 prev_algorithm != config.algorithm))
1232         {
1233                 cvReleaseImage(&object_image);
1234                 object_image = 0;
1235                 delete [] prev_object;
1236                 prev_object = 0;
1237         }
1238         this->object_image_w = object_image_w;
1239         this->object_image_h = object_image_h;
1240
1241         if(scene_image &&
1242                 (scene_image_w != this->scene_image_w ||
1243                 scene_image_h != this->scene_image_h ||
1244                 prev_algorithm != config.algorithm))
1245         {
1246                 cvReleaseImage(&scene_image);
1247                 scene_image = 0;
1248         }
1249         this->scene_image_w = scene_image_w;
1250         this->scene_image_h = scene_image_h;
1251
1252
1253
1254
1255 //printf("FindObjectMain::process_buffer %d object_w=%d object_h=%d object_image_w=%d object_image_h=%d\n", __LINE__, object_w, object_h, object_image_w, object_image_h);
1256 //printf("FindObjectMain::process_buffer %d scene_w=%d scene_h=%d scene_image_w=%d scene_image_h=%d\n", __LINE__, scene_w, scene_h, scene_image_w, scene_image_h);
1257 //printf("FindObjectMain::process_buffer %d total_layers=%d\n", __LINE__, get_total_buffers());
1258
1259 // Read in the input frames
1260         for(int i = 0; i < PluginClient::get_total_buffers(); i++)
1261         {
1262                 read_frame(frame[i],
1263                         i,
1264                         start_position,
1265                         frame_rate);
1266         }
1267
1268
1269 // Search for object
1270         if(config.algorithm != NO_ALGORITHM &&
1271                 (config.replace_object ||
1272                 config.draw_border ||
1273                 config.draw_keypoints))
1274         {
1275
1276
1277                 switch(config.algorithm)
1278                 {
1279                         case ALGORITHM_SURF:
1280                                 process_surf();
1281                                 break;
1282
1283                         case ALGORITHM_CAMSHIFT:
1284                                 process_camshift();
1285                                 break;
1286
1287                         case ALGORITHM_BLOB:
1288                                 process_blob();
1289                                 break;
1290                 }
1291
1292
1293                 if(init_border)
1294                 {
1295                         border_x1_accum = border_x1;
1296                         border_y1_accum = border_y1;
1297                         border_x2_accum = border_x2;
1298                         border_y2_accum = border_y2;
1299                         border_x3_accum = border_x3;
1300                         border_y3_accum = border_y3;
1301                         border_x4_accum = border_x4;
1302                         border_y4_accum = border_y4;
1303                         init_border = 0;
1304                 }
1305                 else
1306                 {
1307                         border_x1_accum = (float)border_x1 * config.blend / 100 +
1308                                 border_x1_accum * (100 - config.blend) / 100;
1309                         border_y1_accum = (float)border_y1 * config.blend / 100 +
1310                                 border_y1_accum * (100 - config.blend) / 100;
1311                         border_x2_accum = (float)border_x2 * config.blend / 100 +
1312                                 border_x2_accum * (100 - config.blend) / 100;
1313                         border_y2_accum = (float)border_y2 * config.blend / 100 +
1314                                 border_y2_accum * (100 - config.blend) / 100;
1315                         border_x3_accum = (float)border_x3 * config.blend / 100 +
1316                                 border_x3_accum * (100 - config.blend) / 100;
1317                         border_y3_accum = (float)border_y3 * config.blend / 100 +
1318                                 border_y3_accum * (100 - config.blend) / 100;
1319                         border_x4_accum = (float)border_x4 * config.blend / 100 +
1320                                 border_x4_accum * (100 - config.blend) / 100;
1321                         border_y4_accum = (float)border_y4 * config.blend / 100 +
1322                                 border_y4_accum * (100 - config.blend) / 100;
1323                 }
1324
1325 // Replace object in the scene layer
1326                 if(config.replace_object)
1327                 {
1328
1329 // Some trickery to get the affine transform to alpha blend into the output
1330                         if(!affine) affine = new AffineEngine(get_project_smp() + 1,
1331                                 get_project_smp() + 1);
1332
1333 //printf("FindObjectMain::process_surf %d replace_layer=%d\n", __LINE__, replace_layer);
1334                         if(!temp)
1335                                 temp = new VFrame(w,
1336                                         h,
1337                                         get_input(scene_layer)->get_color_model());
1338                         if(!overlayer)
1339                                 overlayer = new OverlayFrame(get_project_smp() + 1);
1340
1341                         temp->clear_frame();
1342                         affine->process(temp,
1343                                 get_input(replace_layer),
1344                                 0,
1345                                 AffineEngine::PERSPECTIVE,
1346                                 border_x1_accum * 100 / w,
1347                                 border_y1_accum * 100 / h,
1348                                 border_x2_accum * 100 / w,
1349                                 border_y2_accum * 100 / h,
1350                                 border_x3_accum * 100 / w,
1351                                 border_y3_accum * 100 / h,
1352                                 border_x4_accum * 100 / w,
1353                                 border_y4_accum * 100 / h,
1354                                 1);
1355
1356                         overlayer->overlay(get_input(scene_layer),
1357                                 temp,
1358                                 0,
1359                                 0,
1360                                 w,
1361                                 h,
1362                                 0,
1363                                 0,
1364                                 w,
1365                                 h,
1366                                 1,        // 0 - 1
1367                                 TRANSFER_NORMAL,
1368                                 NEAREST_NEIGHBOR);
1369                 }
1370
1371
1372                 if(config.draw_border)
1373                 {
1374                         draw_line(get_input(scene_layer),
1375                                 border_x1_accum,
1376                                 border_y1_accum,
1377                                 border_x2_accum,
1378                                 border_y2_accum);
1379                         draw_line(get_input(scene_layer),
1380                                 border_x2_accum,
1381                                 border_y2_accum,
1382                                 border_x3_accum,
1383                                 border_y3_accum);
1384                         draw_line(get_input(scene_layer),
1385                                 border_x3_accum,
1386                                 border_y3_accum,
1387                                 border_x4_accum,
1388                                 border_y4_accum);
1389                         draw_line(get_input(scene_layer),
1390                                 border_x4_accum,
1391                                 border_y4_accum,
1392                                 border_x1_accum,
1393                                 border_y1_accum);
1394                 }
1395
1396         }
1397
1398 // Draw object outline in the object layer
1399         if(config.draw_object_border)
1400         {
1401                 draw_line(frame[object_layer], object_x1, object_y1, object_x2, object_y1);
1402                 draw_line(frame[object_layer], object_x2, object_y1, object_x2, object_y2);
1403                 draw_line(frame[object_layer], object_x2, object_y2, object_x1, object_y2);
1404                 draw_line(frame[object_layer], object_x1, object_y2, object_x1, object_y1);
1405
1406                 draw_line(frame[object_layer], scene_x1, scene_y1, scene_x2, scene_y1);
1407                 draw_line(frame[object_layer], scene_x2, scene_y1, scene_x2, scene_y2);
1408                 draw_line(frame[object_layer], scene_x2, scene_y2, scene_x1, scene_y2);
1409                 draw_line(frame[object_layer], scene_x1, scene_y2, scene_x1, scene_y1);
1410         }
1411
1412
1413
1414
1415         return 0;
1416 }
1417
1418