bsd/cygwin patch sync, ffmpeg color_space/color_range fix
[goodguy/cinelerra.git] / cinelerra-5.1 / plugins / interpolatevideo / opticflow.C
1 /*
2  * CINELERRA
3  * Copyright (C) 1997-2011 Adam Williams <broadcast at earthling dot net>
4  *
5  * This program is free software; you can redistribute it and/or modify
6  * it under the terms of the GNU General Public License as published by
7  * the Free Software Foundation; either version 2 of the License, or
8  * (at your option) any later version.
9  *
10  * This program is distributed in the hope that it will be useful,
11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13  * GNU General Public License for more details.
14  * *
15  * You should have received a copy of the GNU General Public License
16  * along with this program; if not, write to the Free Software
17  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
18  *
19  */
20
21
22
23
24
25 #include "clip.h"
26 #include "interpolatevideo.h"
27 #include "motioncache-hv.h"
28 #include "motionscan-hv.h"
29 #include "opticflow.h"
30
31 #include <sys/time.h>
32 #include <unistd.h>
33
34
35
36
37 OpticFlowMacroblock::OpticFlowMacroblock()
38 {
39         dx = 0;
40         dy = 0;
41         visible = 0;
42 }
43
44 void OpticFlowMacroblock::copy_from(OpticFlowMacroblock *src)
45 {
46         this->x = src->x;
47         this->y = src->y;
48         this->dx = src->dx;
49         this->dy = src->dy;
50         this->is_valid = src->is_valid;
51 // Temporaries for blending macroblocks
52         this->angle1 = src->angle1;
53         this->angle2 = src->angle2;
54         this->dist = src->dist;
55         this->visible = src->visible;
56 }
57
58
59
60
61
62
63
64 OpticFlowPackage::OpticFlowPackage()
65  : LoadPackage()
66 {
67 }
68
69
70
71
72
73
74
75 OpticFlowUnit::OpticFlowUnit(OpticFlow *server)
76  : LoadClient(server)
77 {
78         this->server = server;
79         motion = 0;
80 }
81
82 OpticFlowUnit::~OpticFlowUnit()
83 {
84         delete motion;
85 }
86
87 void OpticFlowUnit::process_package(LoadPackage *package)
88 {
89         OpticFlowPackage *pkg = (OpticFlowPackage*)package;
90         InterpolateVideo *plugin = server->plugin;
91         //int w = plugin->frames[0]->get_w();
92         //int h = plugin->frames[0]->get_h();
93         struct timeval start_time;
94         gettimeofday(&start_time, 0);
95
96         if(!motion) motion = new MotionHVScan(1, 1);
97
98         motion->set_test_match(0);
99         motion->set_cache(server->downsample_cache);
100         
101 // printf("OpticFlowUnit::process_package %d %d %d\n",
102 // __LINE__,
103 // pkg->macroblock0,
104 // pkg->macroblock1);
105
106         for(int i = pkg->macroblock0; i < pkg->macroblock1; i++)
107         {
108                 OpticFlowMacroblock *mb = plugin->macroblocks.get(i);
109 //printf("OpticFlowUnit::process_package %d i=%d x=%d y=%d\n", __LINE__, i, mb->x, mb->y);
110                 motion->scan_frame(plugin->frames[0],
111 // Frame after motion
112                         plugin->frames[1],
113                         plugin->config.search_radius,
114                         plugin->config.search_radius,
115                         plugin->config.macroblock_size,
116                         plugin->config.macroblock_size,
117                         mb->x,
118                         mb->y,
119                         MotionHVScan::TRACK_PREVIOUS,
120                         MotionHVScan::CALCULATE,
121 // Get it to do the subpixel step
122                         MotionHVScan::STABILIZE,
123                         0,
124                         0,
125                         0,
126                         0,
127                         0,
128                         0,
129                         0,
130                         1,
131                         0,
132                         0,
133                         0);
134 //printf("OpticFlowUnit::process_package 2\n", __LINE__);
135
136
137                 mb->dx = motion->dx_result;
138                 mb->dy = motion->dy_result;
139 //              mb->is_valid = motion->result_valid;
140                 mb->is_valid = 1;
141         }
142
143         struct timeval end_time;
144         gettimeofday(&end_time, 0);
145 //      printf("OpticFlowUnit::process_package %d %d\n",
146 //              __LINE__,
147 //              end_time.tv_sec * 1000 + end_time.tv_usec / 1000 -
148 //              start_time.tv_sec * 1000 - start_time.tv_usec / 1000);
149 }
150
151
152
153
154
155
156 OpticFlow::OpticFlow(InterpolateVideo *plugin,
157         int total_clients,
158         int total_packages)
159 // : LoadServer(1,
160 //      total_packages)
161  : LoadServer(total_clients,
162         total_packages)
163 {
164         this->plugin = plugin;
165         downsample_cache = 0;
166 }
167
168
169 OpticFlow::~OpticFlow()
170 {
171         if(downsample_cache)
172         {
173 //printf("OpticFlow::~OpticFlow %d %p\n", __LINE__, downsample_cache);
174                 delete downsample_cache;
175         }
176 }
177
178 void OpticFlow::init_packages()
179 {
180         if(!downsample_cache)
181         {
182                 downsample_cache = new MotionHVCache();
183         }
184         
185         downsample_cache->clear();
186
187         for(int i = 0; i < get_total_packages(); i++)
188         {
189                 OpticFlowPackage *pkg = (OpticFlowPackage*)get_package(i);
190                 pkg->macroblock0 = plugin->total_macroblocks * i / get_total_packages();
191                 pkg->macroblock1 = plugin->total_macroblocks * (i + 1) / get_total_packages();
192 // printf("OpticFlow::init_packages %d %d %d %d %d\n", 
193 // __LINE__, 
194 // plugin->total_macroblocks,
195 // get_total_packages(),
196 // pkg->macroblock0,
197 // pkg->macroblock1);
198         }
199 }
200
201 LoadClient* OpticFlow::new_client()
202 {
203         return new OpticFlowUnit(this);
204 }
205
206 LoadPackage* OpticFlow::new_package()
207 {
208         return new OpticFlowPackage;
209 }
210
211
212
213
214
215
216
217 WarpPackage::WarpPackage()
218  : LoadPackage()
219 {
220 }
221
222
223
224
225
226
227
228 WarpUnit::WarpUnit(Warp *server)
229  : LoadClient(server)
230 {
231         this->server = server;
232 }
233
234 WarpUnit::~WarpUnit()
235 {
236 }
237
238
239 #define AVERAGE2(type, components, max) \
240 { \
241         type *prev0 = (type*)prev_rows[(int)prev_y0] + ((int)prev_x0) * components; \
242         type *prev1 = (type*)prev_rows[(int)prev_y0] + ((int)prev_x1) * components; \
243         type *prev2 = (type*)prev_rows[(int)prev_y1] + ((int)prev_x0) * components; \
244         type *prev3 = (type*)prev_rows[(int)prev_y1] + ((int)prev_x1) * components; \
245  \
246  \
247  \
248         type *next0 = (type*)next_rows[(int)next_y0] + ((int)next_x0) * components; \
249         type *next1 = (type*)next_rows[(int)next_y0] + ((int)next_x1) * components; \
250         type *next2 = (type*)next_rows[(int)next_y1] + ((int)next_x0) * components; \
251         type *next3 = (type*)next_rows[(int)next_y1] + ((int)next_x1) * components; \
252  \
253  \
254  \
255         type *out_row = (type*)out_rows[i] + j * components; \
256  \
257         for(int k = 0; k < components; k++) \
258         { \
259                 float value = \
260                         prev_alpha * (*prev0 * prev_fraction_x0 * prev_fraction_y0 + \
261                                 *prev1 * prev_fraction_x1 * prev_fraction_y0 + \
262                                 *prev2 * prev_fraction_x0 * prev_fraction_y1 + \
263                                 *prev3 * prev_fraction_x1 * prev_fraction_y1) + \
264                         next_alpha * (*next0 * next_fraction_x0 * next_fraction_y0 + \
265                                 *next1 * next_fraction_x1 * next_fraction_y0 + \
266                                 *next2 * next_fraction_x0 * next_fraction_y1 + \
267                                 *next3 * next_fraction_x1 * next_fraction_y1); \
268                 CLAMP(value, 0, max); \
269                 *out_row++ = (type)value; \
270                 prev0++; \
271                 prev1++; \
272                 prev2++; \
273                 prev3++; \
274                 next0++; \
275                 next1++; \
276                 next2++; \
277                 next3++; \
278         } \
279 }
280
281
282 void WarpUnit::process_package(LoadPackage *package)
283 {
284         WarpPackage *pkg = (WarpPackage*)package;
285         InterpolateVideo *plugin = server->plugin;
286         int w = plugin->frames[0]->get_w();
287         int h = plugin->frames[0]->get_h();
288         unsigned char **prev_rows = plugin->frames[0]->get_rows();
289         unsigned char **next_rows = plugin->frames[1]->get_rows();
290         unsigned char **out_rows = plugin->get_output()->get_rows();
291         int color_model = plugin->get_output()->get_color_model();
292         int macroblock_size = plugin->config.macroblock_size;
293
294         float lowest_fraction = plugin->lowest_fraction;
295         //float highest_fraction = 1.0 - lowest_fraction;
296
297         float prev_alpha = lowest_fraction;
298         float next_alpha = 1.0 - prev_alpha;
299 //printf("WarpUnit::process_package %d %p %d %d\n", __LINE__, this, pkg->y1, pkg->y2);
300
301 // Count all macroblocks as valid
302         for(int i = pkg->y1; i < pkg->y2; i++)
303         {
304                 for(int j = 0; j < w; j++)
305                 {
306 // Get the motion vector for each pixel, based on the nearest motion vectors
307                         int x_macroblock = j / macroblock_size;
308                         int y_macroblock = i / macroblock_size;
309
310                         int x_macroblock2 = x_macroblock + 1;
311                         int y_macroblock2 = y_macroblock + 1;
312
313                         x_macroblock2 = MIN(x_macroblock2, plugin->x_macroblocks - 1);
314                         y_macroblock2 = MIN(y_macroblock2, plugin->y_macroblocks - 1);
315
316                         float x_fraction = (float)(j - x_macroblock * macroblock_size) / macroblock_size;
317                         float y_fraction = (float)(i - y_macroblock * macroblock_size) / macroblock_size;
318
319                         OpticFlowMacroblock *mb;
320                         mb = plugin->macroblocks.get(
321                                 x_macroblock + y_macroblock * plugin->x_macroblocks);
322
323                         float dx = (float)mb->dx * (1.0 - x_fraction) * (1.0 - y_fraction);
324                         float dy = (float)mb->dy * (1.0 - x_fraction) * (1.0 - y_fraction);
325
326                         mb = plugin->macroblocks.get(
327                                 x_macroblock2 + y_macroblock * plugin->x_macroblocks);
328                         dx += (float)mb->dx * (x_fraction) * (1.0 - y_fraction);
329                         dy += (float)mb->dy * (x_fraction) * (1.0 - y_fraction);
330
331                         mb = plugin->macroblocks.get(
332                                 x_macroblock + y_macroblock2 * plugin->x_macroblocks);
333                         dx += (float)mb->dx * (1.0 - x_fraction) * (y_fraction);
334                         dy += (float)mb->dy * (1.0 - x_fraction) * (y_fraction);
335
336                         mb = plugin->macroblocks.get(
337                                 x_macroblock2 + y_macroblock2 * plugin->x_macroblocks);
338                         dx += (float)mb->dx * (x_fraction) * (y_fraction);
339                         dy += (float)mb->dy * (x_fraction) * (y_fraction);
340
341                         dx /= (float)OVERSAMPLE;
342                         dy /= (float)OVERSAMPLE;
343
344
345 // Input pixels
346
347 // 4 pixels from prev frame
348                         float prev_x0 = (float)j + dx * (1.0 - lowest_fraction);
349                         float prev_y0 = (float)i + dy * (1.0 - lowest_fraction);
350                         float prev_x1 = prev_x0 + 1;
351                         float prev_y1 = prev_y0 + 1;
352                         float prev_fraction_x1 = prev_x0 - floor(prev_x0);
353                         float prev_fraction_x0 = 1.0 - prev_fraction_x1;
354                         float prev_fraction_y1 = prev_y0 - floor(prev_y0);
355                         float prev_fraction_y0 = 1.0 - prev_fraction_y1;
356
357
358 // 4 pixels from next frame
359                         float next_x0 = (float)j - dx * plugin->lowest_fraction;
360                         float next_y0 = (float)i - dy * plugin->lowest_fraction;
361                         float next_x1 = next_x0 + 1;
362                         float next_y1 = next_y0 + 1;
363                         float next_fraction_x1 = next_x0 - floor(next_x0);
364                         float next_fraction_x0 = 1.0 - next_fraction_x1;
365                         float next_fraction_y1 = next_y0 - floor(next_y0);
366                         float next_fraction_y0 = 1.0 - next_fraction_y1;
367
368
369                         CLAMP(prev_x0, 0, w - 1);
370                         CLAMP(prev_y0, 0, h - 1);
371                         CLAMP(prev_x1, 0, w - 1);
372                         CLAMP(prev_y1, 0, h - 1);
373
374                         CLAMP(next_x0, 0, w - 1);
375                         CLAMP(next_y0, 0, h - 1);
376                         CLAMP(next_x1, 0, w - 1);
377                         CLAMP(next_y1, 0, h - 1);
378
379 //printf("WarpUnit::process_package %d\n", __LINE__);
380
381                         switch(color_model)
382                         {
383                                 case BC_RGB_FLOAT:
384                                         AVERAGE2(float, 3, 1.0);
385                                         break;
386                                 case BC_RGB888:
387                                 case BC_YUV888:
388                                         AVERAGE2(unsigned char, 3, 0xff);
389                                         break;
390                                 case BC_RGBA_FLOAT:
391                                         AVERAGE2(float, 4, 1.0);
392                                         break;
393                                 case BC_RGBA8888:
394                                 case BC_YUVA8888:
395                                         AVERAGE2(unsigned char, 4, 0xff);
396                                         break;
397                         }
398 //printf("WarpUnit::process_package %d\n", __LINE__);
399                 }
400         }
401 }
402
403
404
405
406
407
408 Warp::Warp(InterpolateVideo *plugin,
409         int total_clients,
410         int total_packages)
411  : LoadServer(total_clients,
412         total_packages)
413 {
414         this->plugin = plugin;
415 }
416
417
418 Warp::~Warp()
419 {
420 }
421
422 void Warp::init_packages()
423 {
424         int out_h = plugin->frames[0]->get_h();
425         for(int i = 0; i < get_total_packages(); i++)
426         {
427                 WarpPackage *pkg = (WarpPackage*)get_package(i);
428                 pkg->y1 = out_h * i / get_total_packages();
429                 pkg->y2 = out_h * (i + 1) / get_total_packages();
430         }
431 }
432
433 LoadClient* Warp::new_client()
434 {
435         return new WarpUnit(this);
436 }
437
438 LoadPackage* Warp::new_package()
439 {
440         return new WarpPackage;
441 }
442
443
444
445
446
447
448
449
450 BlendPackage::BlendPackage()
451  : LoadPackage()
452 {
453 }
454
455
456
457
458
459
460
461
462 BlendMacroblockUnit::BlendMacroblockUnit(BlendMacroblock *server)
463  : LoadClient(server)
464 {
465         this->server = server;
466 }
467
468 BlendMacroblockUnit::~BlendMacroblockUnit()
469 {
470 }
471
472 void BlendMacroblockUnit::process_package(LoadPackage *package)
473 {
474         BlendPackage *pkg = (BlendPackage*)package;
475         InterpolateVideo *plugin = server->plugin;
476
477         for(int i = pkg->number0; i < pkg->number1; i++)
478         {
479                 plugin->blend_macroblock(plugin->invalid_blocks.get(i));
480         }
481 }
482
483
484
485
486
487
488
489
490
491
492 BlendMacroblock::BlendMacroblock(InterpolateVideo *plugin,
493         int total_clients,
494         int total_packages)
495  : LoadServer(total_clients,
496         total_packages)
497 {
498         this->plugin = plugin;
499 }
500
501
502 BlendMacroblock::~BlendMacroblock()
503 {
504 }
505
506 void BlendMacroblock::init_packages()
507 {
508         for(int i = 0; i < get_total_packages(); i++)
509         {
510                 BlendPackage *pkg = (BlendPackage*)get_package(i);
511                 pkg->number0 = plugin->invalid_blocks.size() * i / get_total_packages();
512                 pkg->number1 = plugin->invalid_blocks.size() * (i + 1) / get_total_packages();
513         }
514 }
515
516
517
518 LoadClient* BlendMacroblock::new_client()
519 {
520         return new BlendMacroblockUnit(this);
521 }
522
523 LoadPackage* BlendMacroblock::new_package()
524 {
525         return new BlendPackage;
526 }
527
528
529
530