cinelerra-5.1/plugins/interpolatevideo/opticflow.C

   1 /*
   2  * CINELERRA
   3  * Copyright (C) 1997-2011 Adam Williams <broadcast at earthling dot net>
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation; either version 2 of the License, or
   8  * (at your option) any later version.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13  * GNU General Public License for more details.
  14  * *
  15  * You should have received a copy of the GNU General Public License
  16  * along with this program; if not, write to the Free Software
  17  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  18  *
  19  */
  20
  21
  22
  23
  24
  25 #include "clip.h"
  26 #include "interpolatevideo.h"
  27 #include "motionscan.h"
  28 #include "opticflow.h"
  29
  30 #include <sys/time.h>
  31 #include <unistd.h>
  32
  33
  34
  35
  36 OpticFlowMacroblock::OpticFlowMacroblock()
  37 {
  38         dx = 0;
  39         dy = 0;
  40         visible = 0;
  41 }
  42
  43 void OpticFlowMacroblock::copy_from(OpticFlowMacroblock *src)
  44 {
  45         this->x = src->x;
  46         this->y = src->y;
  47         this->dx = src->dx;
  48         this->dy = src->dy;
  49         this->is_valid = src->is_valid;
  50 // Temporaries for blending macroblocks
  51         this->angle1 = src->angle1;
  52         this->angle2 = src->angle2;
  53         this->dist = src->dist;
  54         this->visible = src->visible;
  55 }
  56
  57
  58
  59
  60
  61
  62
  63 OpticFlowPackage::OpticFlowPackage()
  64  : LoadPackage()
  65 {
  66 }
  67
  68
  69
  70
  71
  72
  73
  74 OpticFlowUnit::OpticFlowUnit(OpticFlow *server)
  75  : LoadClient(server)
  76 {
  77         this->server = server;
  78         motion = 0;
  79 }
  80
  81 OpticFlowUnit::~OpticFlowUnit()
  82 {
  83         delete motion;
  84 }
  85
  86 void OpticFlowUnit::process_package(LoadPackage *package)
  87 {
  88         OpticFlowPackage *pkg = (OpticFlowPackage*)package;
  89         InterpolateVideo *plugin = server->plugin;
  90         int w = plugin->frames[0]->get_w();
  91         int h = plugin->frames[0]->get_h();
  92         struct timeval start_time;
  93         gettimeofday(&start_time, 0);
  94
  95         if(!motion) motion = new MotionScan(1, 1);
  96
  97         motion->set_test_match(0);
  98 // printf("OpticFlowUnit::process_package %d %d %d\n",
  99 // __LINE__,
 100 // pkg->macroblock0,
 101 // pkg->macroblock1);
 102
 103         for(int i = pkg->macroblock0; i < pkg->macroblock1; i++)
 104         {
 105                 OpticFlowMacroblock *mb = plugin->macroblocks.get(i);
 106                 motion->scan_frame(plugin->frames[0],
 107 // Frame after motion
 108                         plugin->frames[1],
 109                         plugin->config.search_radius,
 110                         plugin->config.search_radius,
 111                         plugin->config.macroblock_size,
 112                         plugin->config.macroblock_size,
 113                         mb->x,
 114                         mb->y,
 115                         MotionScan::TRACK_PREVIOUS,
 116                         MotionScan::CALCULATE,
 117 // Get it to do the subpixel step
 118                         MotionScan::STABILIZE,
 119                         0,
 120                         0,
 121                         0,
 122                         0,
 123                         0,
 124                         0,
 125                         0,
 126                         1,
 127                         0,
 128                         0,
 129                         0);
 130
 131
 132                 mb->dx = motion->dx_result;
 133                 mb->dy = motion->dy_result;
 134 //              mb->is_valid = motion->result_valid;
 135                 mb->is_valid = 1;
 136         }
 137
 138         struct timeval end_time;
 139         gettimeofday(&end_time, 0);
 140 //      printf("OpticFlowUnit::process_package %d %d\n",
 141 //              __LINE__,
 142 //              end_time.tv_sec * 1000 + end_time.tv_usec / 1000 -
 143 //              start_time.tv_sec * 1000 - start_time.tv_usec / 1000);
 144 }
 145
 146
 147
 148
 149
 150
 151 OpticFlow::OpticFlow(InterpolateVideo *plugin,
 152         int total_clients,
 153         int total_packages)
 154 // : LoadServer(1,
 155 //      total_packages)
 156  : LoadServer(total_clients,
 157         total_packages)
 158 {
 159         this->plugin = plugin;
 160 }
 161
 162
 163 OpticFlow::~OpticFlow()
 164 {
 165 }
 166
 167 void OpticFlow::init_packages()
 168 {
 169 //printf("OpticFlow::init_packages %d %d\n", __LINE__, get_total_packages());
 170         for(int i = 0; i < get_total_packages(); i++)
 171         {
 172                 OpticFlowPackage *pkg = (OpticFlowPackage*)get_package(i);
 173                 pkg->macroblock0 = plugin->total_macroblocks * i / get_total_packages();
 174                 pkg->macroblock1 = plugin->total_macroblocks * (i + 1) / get_total_packages();
 175         }
 176 }
 177
 178 LoadClient* OpticFlow::new_client()
 179 {
 180         return new OpticFlowUnit(this);
 181 }
 182
 183 LoadPackage* OpticFlow::new_package()
 184 {
 185         return new OpticFlowPackage;
 186 }
 187
 188
 189
 190
 191
 192
 193
 194 WarpPackage::WarpPackage()
 195  : LoadPackage()
 196 {
 197 }
 198
 199
 200
 201
 202
 203
 204
 205 WarpUnit::WarpUnit(Warp *server)
 206  : LoadClient(server)
 207 {
 208         this->server = server;
 209 }
 210
 211 WarpUnit::~WarpUnit()
 212 {
 213 }
 214
 215
 216 #define AVERAGE2(type, components, max) \
 217 { \
 218         type *prev0 = (type*)prev_rows[(int)prev_y0] + ((int)prev_x0) * components; \
 219         type *prev1 = (type*)prev_rows[(int)prev_y0] + ((int)prev_x1) * components; \
 220         type *prev2 = (type*)prev_rows[(int)prev_y1] + ((int)prev_x0) * components; \
 221         type *prev3 = (type*)prev_rows[(int)prev_y1] + ((int)prev_x1) * components; \
 222  \
 223  \
 224  \
 225         type *next0 = (type*)next_rows[(int)next_y0] + ((int)next_x0) * components; \
 226         type *next1 = (type*)next_rows[(int)next_y0] + ((int)next_x1) * components; \
 227         type *next2 = (type*)next_rows[(int)next_y1] + ((int)next_x0) * components; \
 228         type *next3 = (type*)next_rows[(int)next_y1] + ((int)next_x1) * components; \
 229  \
 230  \
 231  \
 232         type *out_row = (type*)out_rows[i] + j * components; \
 233  \
 234         for(int k = 0; k < components; k++) \
 235         { \
 236                 float value = \
 237                         prev_alpha * (*prev0 * prev_fraction_x0 * prev_fraction_y0 + \
 238                                 *prev1 * prev_fraction_x1 * prev_fraction_y0 + \
 239                                 *prev2 * prev_fraction_x0 * prev_fraction_y1 + \
 240                                 *prev3 * prev_fraction_x1 * prev_fraction_y1) + \
 241                         next_alpha * (*next0 * next_fraction_x0 * next_fraction_y0 + \
 242                                 *next1 * next_fraction_x1 * next_fraction_y0 + \
 243                                 *next2 * next_fraction_x0 * next_fraction_y1 + \
 244                                 *next3 * next_fraction_x1 * next_fraction_y1); \
 245                 CLAMP(value, 0, max); \
 246                 *out_row++ = (type)value; \
 247                 prev0++; \
 248                 prev1++; \
 249                 prev2++; \
 250                 prev3++; \
 251                 next0++; \
 252                 next1++; \
 253                 next2++; \
 254                 next3++; \
 255         } \
 256 }
 257
 258
 259 void WarpUnit::process_package(LoadPackage *package)
 260 {
 261         WarpPackage *pkg = (WarpPackage*)package;
 262         InterpolateVideo *plugin = server->plugin;
 263         int w = plugin->frames[0]->get_w();
 264         int h = plugin->frames[0]->get_h();
 265         unsigned char **prev_rows = plugin->frames[0]->get_rows();
 266         unsigned char **next_rows = plugin->frames[1]->get_rows();
 267         unsigned char **out_rows = plugin->get_output()->get_rows();
 268         int color_model = plugin->get_output()->get_color_model();
 269         int macroblock_size = plugin->config.macroblock_size;
 270
 271         float lowest_fraction = plugin->lowest_fraction;
 272         //float highest_fraction = 1.0 - lowest_fraction;
 273
 274         float prev_alpha = lowest_fraction;
 275         float next_alpha = 1.0 - prev_alpha;
 276 //printf("WarpUnit::process_package %d %p %d %d\n", __LINE__, this, pkg->y1, pkg->y2);
 277
 278 // Count all macroblocks as valid
 279         for(int i = pkg->y1; i < pkg->y2; i++)
 280         {
 281                 for(int j = 0; j < w; j++)
 282                 {
 283 // Get the motion vector for each pixel, based on the nearest motion vectors
 284                         int x_macroblock = j / macroblock_size;
 285                         int y_macroblock = i / macroblock_size;
 286
 287                         int x_macroblock2 = x_macroblock + 1;
 288                         int y_macroblock2 = y_macroblock + 1;
 289
 290                         x_macroblock2 = MIN(x_macroblock2, plugin->x_macroblocks - 1);
 291                         y_macroblock2 = MIN(y_macroblock2, plugin->y_macroblocks - 1);
 292
 293                         float x_fraction = (float)(j - x_macroblock * macroblock_size) / macroblock_size;
 294                         float y_fraction = (float)(i - y_macroblock * macroblock_size) / macroblock_size;
 295
 296                         OpticFlowMacroblock *mb;
 297                         mb = plugin->macroblocks.get(
 298                                 x_macroblock + y_macroblock * plugin->x_macroblocks);
 299
 300                         float dx = (float)mb->dx * (1.0 - x_fraction) * (1.0 - y_fraction);
 301                         float dy = (float)mb->dy * (1.0 - x_fraction) * (1.0 - y_fraction);
 302
 303                         mb = plugin->macroblocks.get(
 304                                 x_macroblock2 + y_macroblock * plugin->x_macroblocks);
 305                         dx += (float)mb->dx * (x_fraction) * (1.0 - y_fraction);
 306                         dy += (float)mb->dy * (x_fraction) * (1.0 - y_fraction);
 307
 308                         mb = plugin->macroblocks.get(
 309                                 x_macroblock + y_macroblock2 * plugin->x_macroblocks);
 310                         dx += (float)mb->dx * (1.0 - x_fraction) * (y_fraction);
 311                         dy += (float)mb->dy * (1.0 - x_fraction) * (y_fraction);
 312
 313                         mb = plugin->macroblocks.get(
 314                                 x_macroblock2 + y_macroblock2 * plugin->x_macroblocks);
 315                         dx += (float)mb->dx * (x_fraction) * (y_fraction);
 316                         dy += (float)mb->dy * (x_fraction) * (y_fraction);
 317
 318                         dx /= (float)OVERSAMPLE;
 319                         dy /= (float)OVERSAMPLE;
 320
 321
 322 // Input pixels
 323
 324 // 4 pixels from prev frame
 325                         float prev_x0 = (float)j + dx * (1.0 - lowest_fraction);
 326                         float prev_y0 = (float)i + dy * (1.0 - lowest_fraction);
 327                         float prev_x1 = prev_x0 + 1;
 328                         float prev_y1 = prev_y0 + 1;
 329                         float prev_fraction_x1 = prev_x0 - floor(prev_x0);
 330                         float prev_fraction_x0 = 1.0 - prev_fraction_x1;
 331                         float prev_fraction_y1 = prev_y0 - floor(prev_y0);
 332                         float prev_fraction_y0 = 1.0 - prev_fraction_y1;
 333
 334
 335 // 4 pixels from next frame
 336                         float next_x0 = (float)j - dx * plugin->lowest_fraction;
 337                         float next_y0 = (float)i - dy * plugin->lowest_fraction;
 338                         float next_x1 = next_x0 + 1;
 339                         float next_y1 = next_y0 + 1;
 340                         float next_fraction_x1 = next_x0 - floor(next_x0);
 341                         float next_fraction_x0 = 1.0 - next_fraction_x1;
 342                         float next_fraction_y1 = next_y0 - floor(next_y0);
 343                         float next_fraction_y0 = 1.0 - next_fraction_y1;
 344
 345
 346                         CLAMP(prev_x0, 0, w - 1);
 347                         CLAMP(prev_y0, 0, h - 1);
 348                         CLAMP(prev_x1, 0, w - 1);
 349                         CLAMP(prev_y1, 0, h - 1);
 350
 351                         CLAMP(next_x0, 0, w - 1);
 352                         CLAMP(next_y0, 0, h - 1);
 353                         CLAMP(next_x1, 0, w - 1);
 354                         CLAMP(next_y1, 0, h - 1);
 355
 356 //printf("WarpUnit::process_package %d\n", __LINE__);
 357
 358                         switch(color_model)
 359                         {
 360                                 case BC_RGB_FLOAT:
 361                                         AVERAGE2(float, 3, 1.0);
 362                                         break;
 363                                 case BC_RGB888:
 364                                 case BC_YUV888:
 365                                         AVERAGE2(unsigned char, 3, 0xff);
 366                                         break;
 367                                 case BC_RGBA_FLOAT:
 368                                         AVERAGE2(float, 4, 1.0);
 369                                         break;
 370                                 case BC_RGBA8888:
 371                                 case BC_YUVA8888:
 372                                         AVERAGE2(unsigned char, 4, 0xff);
 373                                         break;
 374                         }
 375 //printf("WarpUnit::process_package %d\n", __LINE__);
 376                 }
 377         }
 378 }
 379
 380
 381
 382
 383
 384
 385 Warp::Warp(InterpolateVideo *plugin,
 386         int total_clients,
 387         int total_packages)
 388  : LoadServer(total_clients,
 389         total_packages)
 390 {
 391         this->plugin = plugin;
 392 }
 393
 394
 395 Warp::~Warp()
 396 {
 397 }
 398
 399 void Warp::init_packages()
 400 {
 401         int out_h = plugin->frames[0]->get_h();
 402         for(int i = 0; i < get_total_packages(); i++)
 403         {
 404                 WarpPackage *pkg = (WarpPackage*)get_package(i);
 405                 pkg->y1 = out_h * i / get_total_packages();
 406                 pkg->y2 = out_h * (i + 1) / get_total_packages();
 407         }
 408 }
 409
 410 LoadClient* Warp::new_client()
 411 {
 412         return new WarpUnit(this);
 413 }
 414
 415 LoadPackage* Warp::new_package()
 416 {
 417         return new WarpPackage;
 418 }
 419
 420
 421
 422
 423
 424
 425
 426
 427 BlendPackage::BlendPackage()
 428  : LoadPackage()
 429 {
 430 }
 431
 432
 433
 434
 435
 436
 437
 438
 439 BlendMacroblockUnit::BlendMacroblockUnit(BlendMacroblock *server)
 440  : LoadClient(server)
 441 {
 442         this->server = server;
 443 }
 444
 445 BlendMacroblockUnit::~BlendMacroblockUnit()
 446 {
 447 }
 448
 449 void BlendMacroblockUnit::process_package(LoadPackage *package)
 450 {
 451         BlendPackage *pkg = (BlendPackage*)package;
 452         InterpolateVideo *plugin = server->plugin;
 453
 454         for(int i = pkg->number0; i < pkg->number1; i++)
 455         {
 456                 plugin->blend_macroblock(plugin->invalid_blocks.get(i));
 457         }
 458 }
 459
 460
 461
 462
 463
 464
 465
 466
 467
 468
 469 BlendMacroblock::BlendMacroblock(InterpolateVideo *plugin,
 470         int total_clients,
 471         int total_packages)
 472  : LoadServer(total_clients,
 473         total_packages)
 474 {
 475         this->plugin = plugin;
 476 }
 477
 478
 479 BlendMacroblock::~BlendMacroblock()
 480 {
 481 }
 482
 483 void BlendMacroblock::init_packages()
 484 {
 485         for(int i = 0; i < get_total_packages(); i++)
 486         {
 487                 BlendPackage *pkg = (BlendPackage*)get_package(i);
 488                 pkg->number0 = plugin->invalid_blocks.size() * i / get_total_packages();
 489                 pkg->number1 = plugin->invalid_blocks.size() * (i + 1) / get_total_packages();
 490         }
 491 }
 492
 493
 494
 495 LoadClient* BlendMacroblock::new_client()
 496 {
 497         return new BlendMacroblockUnit(this);
 498 }
 499
 500 LoadPackage* BlendMacroblock::new_package()
 501 {
 502         return new BlendPackage;
 503 }
 504
 505
 506
 507