cinelerra-5.1/plugins/interpolatevideo/opticflow.C

   1 /*
   2  * CINELERRA
   3  * Copyright (C) 1997-2011 Adam Williams <broadcast at earthling dot net>
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation; either version 2 of the License, or
   8  * (at your option) any later version.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13  * GNU General Public License for more details.
  14  * *
  15  * You should have received a copy of the GNU General Public License
  16  * along with this program; if not, write to the Free Software
  17  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  18  *
  19  */
  20
  21
  22
  23
  24
  25 #include "clip.h"
  26 #include "interpolatevideo.h"
  27 #include "motioncache-hv.h"
  28 #include "motionscan-hv.h"
  29 #include "opticflow.h"
  30
  31 #include <sys/time.h>
  32 #include <unistd.h>
  33
  34
  35
  36
  37 OpticFlowMacroblock::OpticFlowMacroblock()
  38 {
  39         dx = 0;
  40         dy = 0;
  41         visible = 0;
  42 }
  43
  44 void OpticFlowMacroblock::copy_from(OpticFlowMacroblock *src)
  45 {
  46         this->x = src->x;
  47         this->y = src->y;
  48         this->dx = src->dx;
  49         this->dy = src->dy;
  50         this->is_valid = src->is_valid;
  51 // Temporaries for blending macroblocks
  52         this->angle1 = src->angle1;
  53         this->angle2 = src->angle2;
  54         this->dist = src->dist;
  55         this->visible = src->visible;
  56 }
  57
  58
  59
  60
  61
  62
  63
  64 OpticFlowPackage::OpticFlowPackage()
  65  : LoadPackage()
  66 {
  67 }
  68
  69
  70
  71
  72
  73
  74
  75 OpticFlowUnit::OpticFlowUnit(OpticFlow *server)
  76  : LoadClient(server)
  77 {
  78         this->server = server;
  79         motion = 0;
  80 }
  81
  82 OpticFlowUnit::~OpticFlowUnit()
  83 {
  84         delete motion;
  85 }
  86
  87 void OpticFlowUnit::process_package(LoadPackage *package)
  88 {
  89         OpticFlowPackage *pkg = (OpticFlowPackage*)package;
  90         InterpolateVideo *plugin = server->plugin;
  91         //int w = plugin->frames[0]->get_w();
  92         //int h = plugin->frames[0]->get_h();
  93         struct timeval start_time;
  94         gettimeofday(&start_time, 0);
  95
  96         if(!motion) motion = new MotionHVScan(1, 1);
  97
  98         motion->set_test_match(0);
  99         motion->set_cache(server->downsample_cache);
 100
 101 // printf("OpticFlowUnit::process_package %d %d %d\n",
 102 // __LINE__,
 103 // pkg->macroblock0,
 104 // pkg->macroblock1);
 105
 106         for(int i = pkg->macroblock0; i < pkg->macroblock1; i++)
 107         {
 108                 OpticFlowMacroblock *mb = plugin->macroblocks.get(i);
 109 //printf("OpticFlowUnit::process_package %d i=%d x=%d y=%d\n", __LINE__, i, mb->x, mb->y);
 110                 motion->scan_frame(plugin->frames[0],
 111 // Frame after motion
 112                         plugin->frames[1],
 113                         plugin->config.search_radius,
 114                         plugin->config.search_radius,
 115                         plugin->config.macroblock_size,
 116                         plugin->config.macroblock_size,
 117                         mb->x,
 118                         mb->y,
 119                         MotionHVScan::TRACK_PREVIOUS,
 120                         MotionHVScan::CALCULATE,
 121 // Get it to do the subpixel step
 122                         MotionHVScan::STABILIZE,
 123                         0,
 124                         0,
 125                         0,
 126                         0,
 127                         0,
 128                         0,
 129                         0,
 130                         1,
 131                         0,
 132                         0,
 133                         0);
 134 //printf("OpticFlowUnit::process_package 2\n", __LINE__);
 135
 136
 137                 mb->dx = motion->dx_result;
 138                 mb->dy = motion->dy_result;
 139 //              mb->is_valid = motion->result_valid;
 140                 mb->is_valid = 1;
 141         }
 142
 143         struct timeval end_time;
 144         gettimeofday(&end_time, 0);
 145 //      printf("OpticFlowUnit::process_package %d %d\n",
 146 //              __LINE__,
 147 //              end_time.tv_sec * 1000 + end_time.tv_usec / 1000 -
 148 //              start_time.tv_sec * 1000 - start_time.tv_usec / 1000);
 149 }
 150
 151
 152
 153
 154
 155
 156 OpticFlow::OpticFlow(InterpolateVideo *plugin,
 157         int total_clients,
 158         int total_packages)
 159 // : LoadServer(1,
 160 //      total_packages)
 161  : LoadServer(total_clients,
 162         total_packages)
 163 {
 164         this->plugin = plugin;
 165         downsample_cache = 0;
 166 }
 167
 168
 169 OpticFlow::~OpticFlow()
 170 {
 171         if(downsample_cache)
 172         {
 173 //printf("OpticFlow::~OpticFlow %d %p\n", __LINE__, downsample_cache);
 174                 delete downsample_cache;
 175         }
 176 }
 177
 178 void OpticFlow::init_packages()
 179 {
 180         if(!downsample_cache)
 181         {
 182                 downsample_cache = new MotionHVCache();
 183         }
 184
 185         downsample_cache->clear();
 186
 187         for(int i = 0; i < get_total_packages(); i++)
 188         {
 189                 OpticFlowPackage *pkg = (OpticFlowPackage*)get_package(i);
 190                 pkg->macroblock0 = plugin->total_macroblocks * i / get_total_packages();
 191                 pkg->macroblock1 = plugin->total_macroblocks * (i + 1) / get_total_packages();
 192 // printf("OpticFlow::init_packages %d %d %d %d %d\n",
 193 // __LINE__,
 194 // plugin->total_macroblocks,
 195 // get_total_packages(),
 196 // pkg->macroblock0,
 197 // pkg->macroblock1);
 198         }
 199 }
 200
 201 LoadClient* OpticFlow::new_client()
 202 {
 203         return new OpticFlowUnit(this);
 204 }
 205
 206 LoadPackage* OpticFlow::new_package()
 207 {
 208         return new OpticFlowPackage;
 209 }
 210
 211
 212
 213
 214
 215
 216
 217 WarpPackage::WarpPackage()
 218  : LoadPackage()
 219 {
 220 }
 221
 222
 223
 224
 225
 226
 227
 228 WarpUnit::WarpUnit(Warp *server)
 229  : LoadClient(server)
 230 {
 231         this->server = server;
 232 }
 233
 234 WarpUnit::~WarpUnit()
 235 {
 236 }
 237
 238
 239 #define AVERAGE2(type, components, max) \
 240 { \
 241         type *prev0 = (type*)prev_rows[(int)prev_y0] + ((int)prev_x0) * components; \
 242         type *prev1 = (type*)prev_rows[(int)prev_y0] + ((int)prev_x1) * components; \
 243         type *prev2 = (type*)prev_rows[(int)prev_y1] + ((int)prev_x0) * components; \
 244         type *prev3 = (type*)prev_rows[(int)prev_y1] + ((int)prev_x1) * components; \
 245  \
 246  \
 247  \
 248         type *next0 = (type*)next_rows[(int)next_y0] + ((int)next_x0) * components; \
 249         type *next1 = (type*)next_rows[(int)next_y0] + ((int)next_x1) * components; \
 250         type *next2 = (type*)next_rows[(int)next_y1] + ((int)next_x0) * components; \
 251         type *next3 = (type*)next_rows[(int)next_y1] + ((int)next_x1) * components; \
 252  \
 253  \
 254  \
 255         type *out_row = (type*)out_rows[i] + j * components; \
 256  \
 257         for(int k = 0; k < components; k++) \
 258         { \
 259                 float value = \
 260                         prev_alpha * (*prev0 * prev_fraction_x0 * prev_fraction_y0 + \
 261                                 *prev1 * prev_fraction_x1 * prev_fraction_y0 + \
 262                                 *prev2 * prev_fraction_x0 * prev_fraction_y1 + \
 263                                 *prev3 * prev_fraction_x1 * prev_fraction_y1) + \
 264                         next_alpha * (*next0 * next_fraction_x0 * next_fraction_y0 + \
 265                                 *next1 * next_fraction_x1 * next_fraction_y0 + \
 266                                 *next2 * next_fraction_x0 * next_fraction_y1 + \
 267                                 *next3 * next_fraction_x1 * next_fraction_y1); \
 268                 CLAMP(value, 0, max); \
 269                 *out_row++ = (type)value; \
 270                 prev0++; \
 271                 prev1++; \
 272                 prev2++; \
 273                 prev3++; \
 274                 next0++; \
 275                 next1++; \
 276                 next2++; \
 277                 next3++; \
 278         } \
 279 }
 280
 281
 282 void WarpUnit::process_package(LoadPackage *package)
 283 {
 284         WarpPackage *pkg = (WarpPackage*)package;
 285         InterpolateVideo *plugin = server->plugin;
 286         int w = plugin->frames[0]->get_w();
 287         int h = plugin->frames[0]->get_h();
 288         unsigned char **prev_rows = plugin->frames[0]->get_rows();
 289         unsigned char **next_rows = plugin->frames[1]->get_rows();
 290         unsigned char **out_rows = plugin->get_output()->get_rows();
 291         int color_model = plugin->get_output()->get_color_model();
 292         int macroblock_size = plugin->config.macroblock_size;
 293
 294         float lowest_fraction = plugin->lowest_fraction;
 295         //float highest_fraction = 1.0 - lowest_fraction;
 296
 297         float prev_alpha = lowest_fraction;
 298         float next_alpha = 1.0 - prev_alpha;
 299 //printf("WarpUnit::process_package %d %p %d %d\n", __LINE__, this, pkg->y1, pkg->y2);
 300
 301 // Count all macroblocks as valid
 302         for(int i = pkg->y1; i < pkg->y2; i++)
 303         {
 304                 for(int j = 0; j < w; j++)
 305                 {
 306 // Get the motion vector for each pixel, based on the nearest motion vectors
 307                         int x_macroblock = j / macroblock_size;
 308                         int y_macroblock = i / macroblock_size;
 309
 310                         int x_macroblock2 = x_macroblock + 1;
 311                         int y_macroblock2 = y_macroblock + 1;
 312
 313                         x_macroblock2 = MIN(x_macroblock2, plugin->x_macroblocks - 1);
 314                         y_macroblock2 = MIN(y_macroblock2, plugin->y_macroblocks - 1);
 315
 316                         float x_fraction = (float)(j - x_macroblock * macroblock_size) / macroblock_size;
 317                         float y_fraction = (float)(i - y_macroblock * macroblock_size) / macroblock_size;
 318
 319                         OpticFlowMacroblock *mb;
 320                         mb = plugin->macroblocks.get(
 321                                 x_macroblock + y_macroblock * plugin->x_macroblocks);
 322
 323                         float dx = (float)mb->dx * (1.0 - x_fraction) * (1.0 - y_fraction);
 324                         float dy = (float)mb->dy * (1.0 - x_fraction) * (1.0 - y_fraction);
 325
 326                         mb = plugin->macroblocks.get(
 327                                 x_macroblock2 + y_macroblock * plugin->x_macroblocks);
 328                         dx += (float)mb->dx * (x_fraction) * (1.0 - y_fraction);
 329                         dy += (float)mb->dy * (x_fraction) * (1.0 - y_fraction);
 330
 331                         mb = plugin->macroblocks.get(
 332                                 x_macroblock + y_macroblock2 * plugin->x_macroblocks);
 333                         dx += (float)mb->dx * (1.0 - x_fraction) * (y_fraction);
 334                         dy += (float)mb->dy * (1.0 - x_fraction) * (y_fraction);
 335
 336                         mb = plugin->macroblocks.get(
 337                                 x_macroblock2 + y_macroblock2 * plugin->x_macroblocks);
 338                         dx += (float)mb->dx * (x_fraction) * (y_fraction);
 339                         dy += (float)mb->dy * (x_fraction) * (y_fraction);
 340
 341                         dx /= (float)OVERSAMPLE;
 342                         dy /= (float)OVERSAMPLE;
 343
 344
 345 // Input pixels
 346
 347 // 4 pixels from prev frame
 348                         float prev_x0 = (float)j + dx * (1.0 - lowest_fraction);
 349                         float prev_y0 = (float)i + dy * (1.0 - lowest_fraction);
 350                         float prev_x1 = prev_x0 + 1;
 351                         float prev_y1 = prev_y0 + 1;
 352                         float prev_fraction_x1 = prev_x0 - floor(prev_x0);
 353                         float prev_fraction_x0 = 1.0 - prev_fraction_x1;
 354                         float prev_fraction_y1 = prev_y0 - floor(prev_y0);
 355                         float prev_fraction_y0 = 1.0 - prev_fraction_y1;
 356
 357
 358 // 4 pixels from next frame
 359                         float next_x0 = (float)j - dx * plugin->lowest_fraction;
 360                         float next_y0 = (float)i - dy * plugin->lowest_fraction;
 361                         float next_x1 = next_x0 + 1;
 362                         float next_y1 = next_y0 + 1;
 363                         float next_fraction_x1 = next_x0 - floor(next_x0);
 364                         float next_fraction_x0 = 1.0 - next_fraction_x1;
 365                         float next_fraction_y1 = next_y0 - floor(next_y0);
 366                         float next_fraction_y0 = 1.0 - next_fraction_y1;
 367
 368
 369                         CLAMP(prev_x0, 0, w - 1);
 370                         CLAMP(prev_y0, 0, h - 1);
 371                         CLAMP(prev_x1, 0, w - 1);
 372                         CLAMP(prev_y1, 0, h - 1);
 373
 374                         CLAMP(next_x0, 0, w - 1);
 375                         CLAMP(next_y0, 0, h - 1);
 376                         CLAMP(next_x1, 0, w - 1);
 377                         CLAMP(next_y1, 0, h - 1);
 378
 379 //printf("WarpUnit::process_package %d\n", __LINE__);
 380
 381                         switch(color_model)
 382                         {
 383                                 case BC_RGB_FLOAT:
 384                                         AVERAGE2(float, 3, 1.0);
 385                                         break;
 386                                 case BC_RGB888:
 387                                 case BC_YUV888:
 388                                         AVERAGE2(unsigned char, 3, 0xff);
 389                                         break;
 390                                 case BC_RGBA_FLOAT:
 391                                         AVERAGE2(float, 4, 1.0);
 392                                         break;
 393                                 case BC_RGBA8888:
 394                                 case BC_YUVA8888:
 395                                         AVERAGE2(unsigned char, 4, 0xff);
 396                                         break;
 397                         }
 398 //printf("WarpUnit::process_package %d\n", __LINE__);
 399                 }
 400         }
 401 }
 402
 403
 404
 405
 406
 407
 408 Warp::Warp(InterpolateVideo *plugin,
 409         int total_clients,
 410         int total_packages)
 411  : LoadServer(total_clients,
 412         total_packages)
 413 {
 414         this->plugin = plugin;
 415 }
 416
 417
 418 Warp::~Warp()
 419 {
 420 }
 421
 422 void Warp::init_packages()
 423 {
 424         int out_h = plugin->frames[0]->get_h();
 425         for(int i = 0; i < get_total_packages(); i++)
 426         {
 427                 WarpPackage *pkg = (WarpPackage*)get_package(i);
 428                 pkg->y1 = out_h * i / get_total_packages();
 429                 pkg->y2 = out_h * (i + 1) / get_total_packages();
 430         }
 431 }
 432
 433 LoadClient* Warp::new_client()
 434 {
 435         return new WarpUnit(this);
 436 }
 437
 438 LoadPackage* Warp::new_package()
 439 {
 440         return new WarpPackage;
 441 }
 442
 443
 444
 445
 446
 447
 448
 449
 450 BlendPackage::BlendPackage()
 451  : LoadPackage()
 452 {
 453 }
 454
 455
 456
 457
 458
 459
 460
 461
 462 BlendMacroblockUnit::BlendMacroblockUnit(BlendMacroblock *server)
 463  : LoadClient(server)
 464 {
 465         this->server = server;
 466 }
 467
 468 BlendMacroblockUnit::~BlendMacroblockUnit()
 469 {
 470 }
 471
 472 void BlendMacroblockUnit::process_package(LoadPackage *package)
 473 {
 474         BlendPackage *pkg = (BlendPackage*)package;
 475         InterpolateVideo *plugin = server->plugin;
 476
 477         for(int i = pkg->number0; i < pkg->number1; i++)
 478         {
 479                 plugin->blend_macroblock(plugin->invalid_blocks.get(i));
 480         }
 481 }
 482
 483
 484
 485
 486
 487
 488
 489
 490
 491
 492 BlendMacroblock::BlendMacroblock(InterpolateVideo *plugin,
 493         int total_clients,
 494         int total_packages)
 495  : LoadServer(total_clients,
 496         total_packages)
 497 {
 498         this->plugin = plugin;
 499 }
 500
 501
 502 BlendMacroblock::~BlendMacroblock()
 503 {
 504 }
 505
 506 void BlendMacroblock::init_packages()
 507 {
 508         for(int i = 0; i < get_total_packages(); i++)
 509         {
 510                 BlendPackage *pkg = (BlendPackage*)get_package(i);
 511                 pkg->number0 = plugin->invalid_blocks.size() * i / get_total_packages();
 512                 pkg->number1 = plugin->invalid_blocks.size() * (i + 1) / get_total_packages();
 513         }
 514 }
 515
 516
 517
 518 LoadClient* BlendMacroblock::new_client()
 519 {
 520         return new BlendMacroblockUnit(this);
 521 }
 522
 523 LoadPackage* BlendMacroblock::new_package()
 524 {
 525         return new BlendPackage;
 526 }
 527
 528
 529
 530