cinelerra-5.1/plugins/interpolatevideo/opticflow.C

   1 /*
   2  * CINELERRA
   3  * Copyright (C) 1997-2011 Adam Williams <broadcast at earthling dot net>
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation; either version 2 of the License, or
   8  * (at your option) any later version.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13  * GNU General Public License for more details.
  14  * *
  15  * You should have received a copy of the GNU General Public License
  16  * along with this program; if not, write to the Free Software
  17  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  18  *
  19  */
  20
  21
  22
  23
  24
  25 #include "clip.h"
  26 #include "interpolatevideo.h"
  27 #include "motionscan.h"
  28 #include "opticflow.h"
  29
  30 #include <sys/time.h>
  31 #include <unistd.h>
  32
  33
  34
  35
  36 OpticFlowMacroblock::OpticFlowMacroblock()
  37 {
  38         dx = 0;
  39         dy = 0;
  40         visible = 0;
  41 }
  42
  43 void OpticFlowMacroblock::copy_from(OpticFlowMacroblock *src)
  44 {
  45         this->x = src->x;
  46         this->y = src->y;
  47         this->dx = src->dx;
  48         this->dy = src->dy;
  49         this->is_valid = src->is_valid;
  50 // Temporaries for blending macroblocks
  51         this->angle1 = src->angle1;
  52         this->angle2 = src->angle2;
  53         this->dist = src->dist;
  54         this->visible = src->visible;
  55 }
  56
  57
  58
  59
  60
  61
  62
  63 OpticFlowPackage::OpticFlowPackage()
  64  : LoadPackage()
  65 {
  66 }
  67
  68
  69
  70
  71
  72
  73
  74 OpticFlowUnit::OpticFlowUnit(OpticFlow *server)
  75  : LoadClient(server)
  76 {
  77         this->server = server;
  78         motion = 0;
  79 }
  80
  81 OpticFlowUnit::~OpticFlowUnit()
  82 {
  83         delete motion;
  84 }
  85
  86 void OpticFlowUnit::process_package(LoadPackage *package)
  87 {
  88         OpticFlowPackage *pkg = (OpticFlowPackage*)package;
  89         InterpolateVideo *plugin = server->plugin;
  90         int w = plugin->frames[0]->get_w();
  91         int h = plugin->frames[0]->get_h();
  92         struct timeval start_time;
  93         gettimeofday(&start_time, 0);
  94
  95         if(!motion) motion = new MotionScan(1, 1);
  96
  97         motion->set_test_match(0);
  98 // printf("OpticFlowUnit::process_package %d %d %d\n",
  99 // __LINE__,
 100 // pkg->macroblock0,
 101 // pkg->macroblock1);
 102
 103         for(int i = pkg->macroblock0; i < pkg->macroblock1; i++)
 104         {
 105                 OpticFlowMacroblock *mb = plugin->macroblocks.get(i);
 106                 motion->scan_frame(plugin->frames[0],
 107 // Frame after motion
 108                         plugin->frames[1],
 109                         100 * plugin->config.search_radius / w,
 110                         100 * plugin->config.search_radius / h,
 111                         100 * plugin->config.macroblock_size / w,
 112                         100 * plugin->config.macroblock_size / h,
 113                         100 * mb->x / w,
 114                         100 * mb->y / h,
 115                         MotionScan::TRACK_PREVIOUS,
 116                         MotionScan::CALCULATE,
 117 // Get it to do the subpixel step
 118                         MotionScan::STABILIZE,
 119                         0,
 120                         0,
 121                         0,
 122                         MIN(MAX_SEARCH_STEPS, plugin->config.search_radius * plugin->config.search_radius),
 123                         0,
 124                         0,
 125                         0,
 126                         0);
 127 // Degrees from center to maximum angle
 128 //                      0,
 129 // Accumulated angle from previous frames
 130 //                      0,
 131 // Total number of angles to test in each pass
 132 //                      0,
 133 //                      0);
 134
 135
 136                 mb->dx = motion->dx_result;
 137                 mb->dy = motion->dy_result;
 138 //              mb->is_valid = motion->result_valid;
 139                 mb->is_valid = 1;
 140         }
 141
 142         struct timeval end_time;
 143         gettimeofday(&end_time, 0);
 144 //      printf("OpticFlowUnit::process_package %d %d\n",
 145 //              __LINE__,
 146 //              end_time.tv_sec * 1000 + end_time.tv_usec / 1000 -
 147 //              start_time.tv_sec * 1000 - start_time.tv_usec / 1000);
 148 }
 149
 150
 151
 152
 153
 154
 155 OpticFlow::OpticFlow(InterpolateVideo *plugin,
 156         int total_clients,
 157         int total_packages)
 158 // : LoadServer(1,
 159 //      total_packages)
 160  : LoadServer(total_clients,
 161         total_packages)
 162 {
 163         this->plugin = plugin;
 164 }
 165
 166
 167 OpticFlow::~OpticFlow()
 168 {
 169 }
 170
 171 void OpticFlow::init_packages()
 172 {
 173 //printf("OpticFlow::init_packages %d %d\n", __LINE__, get_total_packages());
 174         for(int i = 0; i < get_total_packages(); i++)
 175         {
 176                 OpticFlowPackage *pkg = (OpticFlowPackage*)get_package(i);
 177                 pkg->macroblock0 = plugin->total_macroblocks * i / get_total_packages();
 178                 pkg->macroblock1 = plugin->total_macroblocks * (i + 1) / get_total_packages();
 179         }
 180 }
 181
 182 LoadClient* OpticFlow::new_client()
 183 {
 184         return new OpticFlowUnit(this);
 185 }
 186
 187 LoadPackage* OpticFlow::new_package()
 188 {
 189         return new OpticFlowPackage;
 190 }
 191
 192
 193
 194
 195
 196
 197
 198 WarpPackage::WarpPackage()
 199  : LoadPackage()
 200 {
 201 }
 202
 203
 204
 205
 206
 207
 208
 209 WarpUnit::WarpUnit(Warp *server)
 210  : LoadClient(server)
 211 {
 212         this->server = server;
 213 }
 214
 215 WarpUnit::~WarpUnit()
 216 {
 217 }
 218
 219
 220 #define AVERAGE2(type, components, max) \
 221 { \
 222         type *prev0 = (type*)prev_rows[(int)prev_y0] + ((int)prev_x0) * components; \
 223         type *prev1 = (type*)prev_rows[(int)prev_y0] + ((int)prev_x1) * components; \
 224         type *prev2 = (type*)prev_rows[(int)prev_y1] + ((int)prev_x0) * components; \
 225         type *prev3 = (type*)prev_rows[(int)prev_y1] + ((int)prev_x1) * components; \
 226  \
 227  \
 228  \
 229         type *next0 = (type*)next_rows[(int)next_y0] + ((int)next_x0) * components; \
 230         type *next1 = (type*)next_rows[(int)next_y0] + ((int)next_x1) * components; \
 231         type *next2 = (type*)next_rows[(int)next_y1] + ((int)next_x0) * components; \
 232         type *next3 = (type*)next_rows[(int)next_y1] + ((int)next_x1) * components; \
 233  \
 234  \
 235  \
 236         type *out_row = (type*)out_rows[i] + j * components; \
 237  \
 238         for(int k = 0; k < components; k++) \
 239         { \
 240                 float value = \
 241                         prev_alpha * (*prev0 * prev_fraction_x0 * prev_fraction_y0 + \
 242                                 *prev1 * prev_fraction_x1 * prev_fraction_y0 + \
 243                                 *prev2 * prev_fraction_x0 * prev_fraction_y1 + \
 244                                 *prev3 * prev_fraction_x1 * prev_fraction_y1) + \
 245                         next_alpha * (*next0 * next_fraction_x0 * next_fraction_y0 + \
 246                                 *next1 * next_fraction_x1 * next_fraction_y0 + \
 247                                 *next2 * next_fraction_x0 * next_fraction_y1 + \
 248                                 *next3 * next_fraction_x1 * next_fraction_y1); \
 249                 CLAMP(value, 0, max); \
 250                 *out_row++ = (type)value; \
 251                 prev0++; \
 252                 prev1++; \
 253                 prev2++; \
 254                 prev3++; \
 255                 next0++; \
 256                 next1++; \
 257                 next2++; \
 258                 next3++; \
 259         } \
 260 }
 261
 262
 263 void WarpUnit::process_package(LoadPackage *package)
 264 {
 265         WarpPackage *pkg = (WarpPackage*)package;
 266         InterpolateVideo *plugin = server->plugin;
 267         int w = plugin->frames[0]->get_w();
 268         int h = plugin->frames[0]->get_h();
 269         unsigned char **prev_rows = plugin->frames[0]->get_rows();
 270         unsigned char **next_rows = plugin->frames[1]->get_rows();
 271         unsigned char **out_rows = plugin->get_output()->get_rows();
 272         int color_model = plugin->get_output()->get_color_model();
 273         int macroblock_size = plugin->config.macroblock_size;
 274
 275         float lowest_fraction = plugin->lowest_fraction;
 276         //float highest_fraction = 1.0 - lowest_fraction;
 277
 278         float prev_alpha = lowest_fraction;
 279         float next_alpha = 1.0 - prev_alpha;
 280 //printf("WarpUnit::process_package %d %p %d %d\n", __LINE__, this, pkg->y1, pkg->y2);
 281
 282 // Count all macroblocks as valid
 283         for(int i = pkg->y1; i < pkg->y2; i++)
 284         {
 285                 for(int j = 0; j < w; j++)
 286                 {
 287 // Get the motion vector for each pixel, based on the nearest motion vectors
 288                         int x_macroblock = j / macroblock_size;
 289                         int y_macroblock = i / macroblock_size;
 290
 291                         int x_macroblock2 = x_macroblock + 1;
 292                         int y_macroblock2 = y_macroblock + 1;
 293
 294                         x_macroblock2 = MIN(x_macroblock2, plugin->x_macroblocks - 1);
 295                         y_macroblock2 = MIN(y_macroblock2, plugin->y_macroblocks - 1);
 296
 297                         float x_fraction = (float)(j - x_macroblock * macroblock_size) / macroblock_size;
 298                         float y_fraction = (float)(i - y_macroblock * macroblock_size) / macroblock_size;
 299
 300                         OpticFlowMacroblock *mb;
 301                         mb = plugin->macroblocks.get(
 302                                 x_macroblock + y_macroblock * plugin->x_macroblocks);
 303
 304                         float dx = (float)mb->dx * (1.0 - x_fraction) * (1.0 - y_fraction);
 305                         float dy = (float)mb->dy * (1.0 - x_fraction) * (1.0 - y_fraction);
 306
 307                         mb = plugin->macroblocks.get(
 308                                 x_macroblock2 + y_macroblock * plugin->x_macroblocks);
 309                         dx += (float)mb->dx * (x_fraction) * (1.0 - y_fraction);
 310                         dy += (float)mb->dy * (x_fraction) * (1.0 - y_fraction);
 311
 312                         mb = plugin->macroblocks.get(
 313                                 x_macroblock + y_macroblock2 * plugin->x_macroblocks);
 314                         dx += (float)mb->dx * (1.0 - x_fraction) * (y_fraction);
 315                         dy += (float)mb->dy * (1.0 - x_fraction) * (y_fraction);
 316
 317                         mb = plugin->macroblocks.get(
 318                                 x_macroblock2 + y_macroblock2 * plugin->x_macroblocks);
 319                         dx += (float)mb->dx * (x_fraction) * (y_fraction);
 320                         dy += (float)mb->dy * (x_fraction) * (y_fraction);
 321
 322                         dx /= (float)OVERSAMPLE;
 323                         dy /= (float)OVERSAMPLE;
 324
 325
 326 // Input pixels
 327
 328 // 4 pixels from prev frame
 329                         float prev_x0 = (float)j + dx * (1.0 - lowest_fraction);
 330                         float prev_y0 = (float)i + dy * (1.0 - lowest_fraction);
 331                         float prev_x1 = prev_x0 + 1;
 332                         float prev_y1 = prev_y0 + 1;
 333                         float prev_fraction_x1 = prev_x0 - floor(prev_x0);
 334                         float prev_fraction_x0 = 1.0 - prev_fraction_x1;
 335                         float prev_fraction_y1 = prev_y0 - floor(prev_y0);
 336                         float prev_fraction_y0 = 1.0 - prev_fraction_y1;
 337
 338
 339 // 4 pixels from next frame
 340                         float next_x0 = (float)j - dx * plugin->lowest_fraction;
 341                         float next_y0 = (float)i - dy * plugin->lowest_fraction;
 342                         float next_x1 = next_x0 + 1;
 343                         float next_y1 = next_y0 + 1;
 344                         float next_fraction_x1 = next_x0 - floor(next_x0);
 345                         float next_fraction_x0 = 1.0 - next_fraction_x1;
 346                         float next_fraction_y1 = next_y0 - floor(next_y0);
 347                         float next_fraction_y0 = 1.0 - next_fraction_y1;
 348
 349
 350                         CLAMP(prev_x0, 0, w - 1);
 351                         CLAMP(prev_y0, 0, h - 1);
 352                         CLAMP(prev_x1, 0, w - 1);
 353                         CLAMP(prev_y1, 0, h - 1);
 354
 355                         CLAMP(next_x0, 0, w - 1);
 356                         CLAMP(next_y0, 0, h - 1);
 357                         CLAMP(next_x1, 0, w - 1);
 358                         CLAMP(next_y1, 0, h - 1);
 359
 360 //printf("WarpUnit::process_package %d\n", __LINE__);
 361
 362                         switch(color_model)
 363                         {
 364                                 case BC_RGB_FLOAT:
 365                                         AVERAGE2(float, 3, 1.0);
 366                                         break;
 367                                 case BC_RGB888:
 368                                 case BC_YUV888:
 369                                         AVERAGE2(unsigned char, 3, 0xff);
 370                                         break;
 371                                 case BC_RGBA_FLOAT:
 372                                         AVERAGE2(float, 4, 1.0);
 373                                         break;
 374                                 case BC_RGBA8888:
 375                                 case BC_YUVA8888:
 376                                         AVERAGE2(unsigned char, 4, 0xff);
 377                                         break;
 378                         }
 379 //printf("WarpUnit::process_package %d\n", __LINE__);
 380                 }
 381         }
 382 }
 383
 384
 385
 386
 387
 388
 389 Warp::Warp(InterpolateVideo *plugin,
 390         int total_clients,
 391         int total_packages)
 392  : LoadServer(total_clients,
 393         total_packages)
 394 {
 395         this->plugin = plugin;
 396 }
 397
 398
 399 Warp::~Warp()
 400 {
 401 }
 402
 403 void Warp::init_packages()
 404 {
 405         int out_h = plugin->frames[0]->get_h();
 406         for(int i = 0; i < get_total_packages(); i++)
 407         {
 408                 WarpPackage *pkg = (WarpPackage*)get_package(i);
 409                 pkg->y1 = out_h * i / get_total_packages();
 410                 pkg->y2 = out_h * (i + 1) / get_total_packages();
 411         }
 412 }
 413
 414 LoadClient* Warp::new_client()
 415 {
 416         return new WarpUnit(this);
 417 }
 418
 419 LoadPackage* Warp::new_package()
 420 {
 421         return new WarpPackage;
 422 }
 423
 424
 425
 426
 427
 428
 429
 430
 431 BlendPackage::BlendPackage()
 432  : LoadPackage()
 433 {
 434 }
 435
 436
 437
 438
 439
 440
 441
 442
 443 BlendMacroblockUnit::BlendMacroblockUnit(BlendMacroblock *server)
 444  : LoadClient(server)
 445 {
 446         this->server = server;
 447 }
 448
 449 BlendMacroblockUnit::~BlendMacroblockUnit()
 450 {
 451 }
 452
 453 void BlendMacroblockUnit::process_package(LoadPackage *package)
 454 {
 455         BlendPackage *pkg = (BlendPackage*)package;
 456         InterpolateVideo *plugin = server->plugin;
 457
 458         for(int i = pkg->number0; i < pkg->number1; i++)
 459         {
 460                 plugin->blend_macroblock(plugin->invalid_blocks.get(i));
 461         }
 462 }
 463
 464
 465
 466
 467
 468
 469
 470
 471
 472
 473 BlendMacroblock::BlendMacroblock(InterpolateVideo *plugin,
 474         int total_clients,
 475         int total_packages)
 476  : LoadServer(total_clients,
 477         total_packages)
 478 {
 479         this->plugin = plugin;
 480 }
 481
 482
 483 BlendMacroblock::~BlendMacroblock()
 484 {
 485 }
 486
 487 void BlendMacroblock::init_packages()
 488 {
 489         for(int i = 0; i < get_total_packages(); i++)
 490         {
 491                 BlendPackage *pkg = (BlendPackage*)get_package(i);
 492                 pkg->number0 = plugin->invalid_blocks.size() * i / get_total_packages();
 493                 pkg->number1 = plugin->invalid_blocks.size() * (i + 1) / get_total_packages();
 494         }
 495 }
 496
 497
 498
 499 LoadClient* BlendMacroblock::new_client()
 500 {
 501         return new BlendMacroblockUnit(this);
 502 }
 503
 504 LoadPackage* BlendMacroblock::new_package()
 505 {
 506         return new BlendPackage;
 507 }
 508
 509
 510
 511