3 * Copyright (C) 1997-2011 Adam Williams <broadcast at earthling dot net>
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software
17 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
21 #include "bcsignals.h"
24 #include "interpolatevideo.h"
25 #include "interpolatewindow.h"
27 #include "motionscan.h"
28 #include "opticflow.h"
29 #include "transportque.inc"
40 InterpolateVideoConfig::InterpolateVideoConfig()
42 input_rate = (double)30000 / 1001;
50 void InterpolateVideoConfig::copy_from(InterpolateVideoConfig *config)
52 this->input_rate = config->input_rate;
53 this->use_keyframes = config->use_keyframes;
54 this->optic_flow = config->optic_flow;
55 this->draw_vectors = config->draw_vectors;
56 this->search_radius = config->search_radius;
57 this->macroblock_size = config->macroblock_size;
60 int InterpolateVideoConfig::equivalent(InterpolateVideoConfig *config)
62 return EQUIV(this->input_rate, config->input_rate) &&
63 (this->use_keyframes == config->use_keyframes) &&
64 this->optic_flow == config->optic_flow &&
65 this->draw_vectors == config->draw_vectors &&
66 this->search_radius == config->search_radius &&
67 this->macroblock_size == config->macroblock_size;
77 REGISTER_PLUGIN(InterpolateVideo)
85 InterpolateVideo::InterpolateVideo(PluginServer *server)
86 : PluginVClient(server)
88 optic_flow_engine = 0;
91 bzero(frames, sizeof(VFrame*) * 2);
92 for(int i = 0; i < 2; i++)
96 last_macroblock_size = 0;
97 last_search_radius = 0;
98 total_macroblocks = 0;
102 InterpolateVideo::~InterpolateVideo()
104 delete optic_flow_engine;
107 if(frames[0]) delete frames[0];
108 if(frames[1]) delete frames[1];
109 macroblocks.remove_all_objects();
113 void InterpolateVideo::fill_border(double frame_rate, int64_t start_position)
115 // A border frame changed or the start position is not identical to the last
118 int64_t frame_start = range_start + (get_direction() == PLAY_REVERSE ? 1 : 0);
119 int64_t frame_end = range_end + (get_direction() == PLAY_REVERSE ? 1 : 0);
121 if( last_position == start_position && EQUIV(last_rate, frame_rate) &&
122 frame_number[0] >= 0 && frame_number[0] == frame_start &&
123 frame_number[1] >= 0 && frame_number[1] == frame_end ) return;
125 if( frame_start == frame_number[1] || frame_end == frame_number[0] )
127 int64_t n = frame_number[0];
128 frame_number[0] = frame_number[1];
130 VFrame *f = frames[0];
131 frames[0] = frames[1];
135 if( frame_start != frame_number[0] )
137 //printf("InterpolateVideo::fill_border 1 %lld\n", range_start);
138 read_frame(frames[0], 0, frame_start, active_input_rate, 0);
139 frame_number[0] = frame_start;
142 if( frame_end != frame_number[1] )
144 //printf("InterpolateVideo::fill_border 2 %lld\n", range_start);
145 read_frame(frames[1], 0, frame_end, active_input_rate, 0);
146 frame_number[1] = frame_end;
149 last_position = start_position;
150 last_rate = frame_rate;
157 void InterpolateVideo::draw_pixel(VFrame *frame, int x, int y)
159 if(!(x >= 0 && y >= 0 && x < frame->get_w() && y < frame->get_h())) return;
161 #define DRAW_PIXEL(x, y, components, do_yuv, max, type) \
163 type **rows = (type**)frame->get_rows(); \
164 rows[y][x * components] = max - rows[y][x * components]; \
167 rows[y][x * components + 1] = max - rows[y][x * components + 1]; \
168 rows[y][x * components + 2] = max - rows[y][x * components + 2]; \
172 rows[y][x * components + 1] = (max / 2 + 1) - rows[y][x * components + 1]; \
173 rows[y][x * components + 2] = (max / 2 + 1) - rows[y][x * components + 2]; \
175 if(components == 4) \
176 rows[y][x * components + 3] = max; \
180 switch(frame->get_color_model())
183 DRAW_PIXEL(x, y, 3, 0, 0xff, unsigned char);
186 DRAW_PIXEL(x, y, 4, 0, 0xff, unsigned char);
189 DRAW_PIXEL(x, y, 3, 0, 1.0, float);
192 DRAW_PIXEL(x, y, 4, 0, 1.0, float);
195 DRAW_PIXEL(x, y, 3, 1, 0xff, unsigned char);
198 DRAW_PIXEL(x, y, 4, 1, 0xff, unsigned char);
201 DRAW_PIXEL(x, y, 3, 0, 0xffff, uint16_t);
204 DRAW_PIXEL(x, y, 3, 1, 0xffff, uint16_t);
206 case BC_RGBA16161616:
207 DRAW_PIXEL(x, y, 4, 0, 0xffff, uint16_t);
209 case BC_YUVA16161616:
210 DRAW_PIXEL(x, y, 4, 1, 0xffff, uint16_t);
216 void InterpolateVideo::draw_line(VFrame *frame, int x1, int y1, int x2, int y2)
218 int w = labs(x2 - x1);
219 int h = labs(y2 - y1);
220 //printf("InterpolateVideo::draw_line 1 %d %d %d %d\n", x1, y1, x2, y2);
224 draw_pixel(frame, x1, y1);
229 // Flip coordinates so x1 < x2
239 int numerator = y2 - y1;
240 int denominator = x2 - x1;
241 for(int i = x1; i < x2; i++)
243 int y = y1 + (int64_t)(i - x1) * (int64_t)numerator / (int64_t)denominator;
244 draw_pixel(frame, i, y);
249 // Flip coordinates so y1 < y2
259 int numerator = x2 - x1;
260 int denominator = y2 - y1;
261 for(int i = y1; i < y2; i++)
263 int x = x1 + (int64_t)(i - y1) * (int64_t)numerator / (int64_t)denominator;
264 draw_pixel(frame, x, i);
267 //printf("InterpolateVideo::draw_line 2\n");
270 #define ARROW_SIZE 10
271 void InterpolateVideo::draw_arrow(VFrame *frame,
277 double angle = atan((float)(y2 - y1) / (float)(x2 - x1));
278 double angle1 = angle + (float)145 / 360 * 2 * 3.14159265;
279 double angle2 = angle - (float)145 / 360 * 2 * 3.14159265;
286 x3 = x2 - (int)(ARROW_SIZE * cos(angle1));
287 y3 = y2 - (int)(ARROW_SIZE * sin(angle1));
288 x4 = x2 - (int)(ARROW_SIZE * cos(angle2));
289 y4 = y2 - (int)(ARROW_SIZE * sin(angle2));
293 x3 = x2 + (int)(ARROW_SIZE * cos(angle1));
294 y3 = y2 + (int)(ARROW_SIZE * sin(angle1));
295 x4 = x2 + (int)(ARROW_SIZE * cos(angle2));
296 y4 = y2 + (int)(ARROW_SIZE * sin(angle2));
300 draw_line(frame, x1, y1, x2, y2);
301 // draw_line(frame, x1, y1 + 1, x2, y2 + 1);
304 if(abs(y2 - y1) || abs(x2 - x1)) draw_line(frame, x2, y2, x3, y3);
305 // draw_line(frame, x2, y2 + 1, x3, y3 + 1);
307 if(abs(y2 - y1) || abs(x2 - x1)) draw_line(frame, x2, y2, x4, y4);
308 // draw_line(frame, x2, y2 + 1, x4, y4 + 1);
311 void InterpolateVideo::draw_rect(VFrame *frame,
317 draw_line(frame, x1, y1, x2, y1);
318 draw_line(frame, x2, y1, x2, y2);
319 draw_line(frame, x2, y2, x1, y2);
320 draw_line(frame, x1, y2, x1, y1);
324 void InterpolateVideo::create_macroblocks()
326 // Get macroblock size
327 x_macroblocks = frames[0]->get_w() / config.macroblock_size;
328 y_macroblocks = frames[0]->get_h() / config.macroblock_size;
330 if(config.macroblock_size * x_macroblocks < frames[0]->get_w())
335 if(config.macroblock_size * y_macroblocks < frames[0]->get_h())
340 total_macroblocks = x_macroblocks * y_macroblocks;
342 if(total_macroblocks != macroblocks.size())
344 macroblocks.remove_all_objects();
347 for(int i = 0; i < total_macroblocks; i++)
349 OpticFlowMacroblock *mb = 0;
350 if(macroblocks.size() > i)
352 mb = macroblocks.get(i);
356 mb = new OpticFlowMacroblock;
357 macroblocks.append(mb);
360 mb->x = (i % x_macroblocks) * config.macroblock_size + config.macroblock_size / 2;
361 mb->y = (i / x_macroblocks) * config.macroblock_size + config.macroblock_size / 2;
366 void InterpolateVideo::draw_vectors(int processed)
369 if(config.draw_vectors)
371 create_macroblocks();
373 for(int i = 0; i < total_macroblocks; i++)
375 OpticFlowMacroblock *mb = macroblocks.get(i);
376 // printf("InterpolateVideo::optic_flow %d x=%d y=%d dx=%d dy=%d\n",
380 // mb->dx / OVERSAMPLE,
381 // mb->dy / OVERSAMPLE);
385 draw_arrow(get_output(),
388 mb->x - mb->dx / OVERSAMPLE,
389 mb->y - mb->dy / OVERSAMPLE);
392 // if(mb->is_valid && mb->visible)
394 // draw_arrow(get_output(),
397 // mb->x - mb->dx / OVERSAMPLE + 1,
398 // mb->y - mb->dy / OVERSAMPLE + 1);
403 draw_pixel(get_output(),
409 // Draw center macroblock
410 OpticFlowMacroblock *mb = macroblocks.get(
411 x_macroblocks / 2 + y_macroblocks / 2 * x_macroblocks);
412 draw_rect(get_output(),
413 mb->x - config.macroblock_size / 2,
414 mb->y - config.macroblock_size / 2,
415 mb->x + config.macroblock_size / 2,
416 mb->y + config.macroblock_size / 2);
417 draw_rect(get_output(),
418 mb->x - config.macroblock_size / 2 - config.search_radius,
419 mb->y - config.macroblock_size / 2 - config.search_radius,
420 mb->x + config.macroblock_size / 2 + config.search_radius,
421 mb->y + config.macroblock_size / 2 + config.search_radius);
426 int InterpolateVideo::angles_overlap(float dst2_angle1,
431 if(dst2_angle1 < 0 || dst2_angle2 < 0)
433 dst2_angle1 += 2 * M_PI;
434 dst2_angle2 += 2 * M_PI;
437 if(dst1_angle1 < 0 || dst1_angle2 < 0)
439 dst1_angle1 += 2 * M_PI;
440 dst1_angle2 += 2 * M_PI;
443 if(dst1_angle1 < dst2_angle2 &&
444 dst1_angle2 > dst2_angle1) return 1;
451 void InterpolateVideo::blend_macroblock(int number)
453 OpticFlowMacroblock *src = macroblocks.get(number);
454 struct timeval start_time;
455 gettimeofday(&start_time, 0);
456 // printf("InterpolateVideo::blend_macroblock %d %d\n",
461 // Copy macroblock table to local thread
462 ArrayList<OpticFlowMacroblock*> local_macroblocks;
463 for(int i = 0; i < macroblocks.size(); i++)
465 OpticFlowMacroblock *mb = new OpticFlowMacroblock;
466 mb->copy_from(macroblocks.get(i));
467 local_macroblocks.append(mb);
470 // Get nearest macroblocks
471 for(int i = 0; i < local_macroblocks.size(); i++)
473 OpticFlowMacroblock *dst = local_macroblocks.get(i);
474 if(i != number && dst->is_valid)
477 // rough estimation of angle coverage
478 float angle = atan2(dst->y - src->y, dst->x - src->x);
479 float dist = sqrt(SQR(dst->y - src->y) + SQR(dst->x - src->x));
480 float span = sin((float)config.macroblock_size / dist);
481 dst->angle1 = angle - span / 2;
482 dst->angle2 = angle + span / 2;
484 // All macroblocks start as visible
487 // printf("InterpolateVideo::blend_macroblock %d %d x=%d y=%d span=%f angle1=%f angle2=%f dist=%f\n",
492 // span * 360 / 2 / M_PI,
493 // dst->angle1 * 360 / 2 / M_PI,
494 // dst->angle2 * 360 / 2 / M_PI,
499 for(int i = 0; i < local_macroblocks.size(); i++)
501 // Conceil macroblocks which are hidden
502 OpticFlowMacroblock *dst1 = local_macroblocks.get(i);
503 if(i != number && dst1->is_valid && dst1->visible)
505 // Find macroblock which is obstructing
506 for(int j = 0; j < local_macroblocks.size(); j++)
508 OpticFlowMacroblock *dst2 = local_macroblocks.get(j);
511 dst2->dist < dst1->dist &&
512 angles_overlap(dst2->angle1,
518 j = local_macroblocks.size();
524 // Blend all visible macroblocks
525 // Get distance metrics
530 for(int i = 0; i < local_macroblocks.size(); i++)
532 OpticFlowMacroblock *dst = local_macroblocks.get(i);
533 if(i != number && dst->is_valid && dst->visible)
538 min = max = dst->dist;
543 min = MIN(dst->dist, min);
544 max = MAX(dst->dist, max);
546 // printf("InterpolateVideo::blend_macroblock %d %d x=%d y=%d dist=%f\n",
556 // Invert distances to convert to weights
558 for(int i = 0; i < local_macroblocks.size(); i++)
560 OpticFlowMacroblock *dst = local_macroblocks.get(i);
561 if(i != number && dst->is_valid && dst->visible)
563 dst->dist = max - dst->dist + min;
565 // printf("InterpolateVideo::blend_macroblock %d %d x=%d y=%d dist=%f\n",
570 // max - dst->dist + min);
575 // Add weighted vectors
580 for(int i = 0; i < local_macroblocks.size(); i++)
582 OpticFlowMacroblock *dst = local_macroblocks.get(i);
583 if(i != number && dst->is_valid && dst->visible)
585 dx += dst->dist * dst->dx / total;
586 dy += dst->dist * dst->dy / total;
589 // printf("InterpolateVideo::blend_macroblock %d %d x=%d y=%d dist=%f\n",
594 // max - dst->dist + min);
600 local_macroblocks.remove_all_objects();
602 // printf("InterpolateVideo::blend_macroblock %d total=%f\n",
605 struct timeval end_time;
606 gettimeofday(&end_time, 0);
607 // printf("InterpolateVideo::blend_macroblock %d %d\n",
609 // end_time.tv_sec * 1000 + end_time.tv_usec / 1000 -
610 // start_time.tv_sec * 1000 - start_time.tv_usec / 1000);
615 void InterpolateVideo::optic_flow()
618 create_macroblocks();
622 if(!optic_flow_engine)
624 optic_flow_engine = new OpticFlow(this,
625 PluginClient::get_project_smp() + 1,
626 PluginClient::get_project_smp() + 1);
631 if(motion_number[0] == frame_number[0] &&
632 motion_number[1] == frame_number[1] &&
633 last_macroblock_size == config.macroblock_size &&
634 last_search_radius == config.search_radius)
639 // Calculate new vectors
646 optic_flow_engine->set_package_count(MIN(MAX_PACKAGES, total_macroblocks));
647 optic_flow_engine->process_packages();
649 // Fill in failed macroblocks
650 invalid_blocks.remove_all();
651 for(int i = 0; i < macroblocks.size(); i++)
653 if(!macroblocks.get(i)->is_valid
655 // && i >= 30 * x_macroblocks)
658 invalid_blocks.append(i);
662 if(invalid_blocks.size())
666 blend_engine = new BlendMacroblock(this,
667 PluginClient::get_project_smp() + 1,
668 PluginClient::get_project_smp() + 1);
671 blend_engine->set_package_count(MIN(PluginClient::get_project_smp() + 1,
672 invalid_blocks.size()));
673 blend_engine->process_packages();
679 // for(int i = 0; i < total_macroblocks; i++)
681 // OpticFlowPackage *pkg = (OpticFlowPackage*)optic_flow_engine->get_package(
683 // if((i / x_macroblocks) % 2)
698 warp_engine = new Warp(this,
699 PluginClient::get_project_smp() + 1,
700 PluginClient::get_project_smp() + 1);
703 warp_engine->process_packages();
705 motion_number[0] = frame_number[0];
706 motion_number[1] = frame_number[1];
707 last_macroblock_size = config.macroblock_size;
708 last_search_radius = config.search_radius;
712 // get_output()->copy_from(frames[1]);
719 #define AVERAGE(type, temp_type,components, max) \
721 temp_type fraction0 = (temp_type)(lowest_fraction * max); \
722 temp_type fraction1 = (temp_type)(max - fraction0); \
724 for(int i = 0; i < h; i++) \
726 type *prev_row0 = (type*)frames[0]->get_rows()[i]; \
727 type *next_row0 = (type*)frames[1]->get_rows()[i]; \
728 type *out_row = (type*)frame->get_rows()[i]; \
729 for(int j = 0; j < w * components; j++) \
731 *out_row++ = (*prev_row0++ * fraction0 + *next_row0++ * fraction1) / max; \
737 void InterpolateVideo::average()
739 VFrame *frame = get_output();
740 int w = frame->get_w();
741 int h = frame->get_h();
743 switch(frame->get_color_model())
746 AVERAGE(float, float, 3, 1);
750 AVERAGE(unsigned char, int, 3, 0xff);
753 AVERAGE(float, float, 4, 1);
757 AVERAGE(unsigned char, int, 4, 0xff);
763 int InterpolateVideo::process_buffer(VFrame *frame,
764 int64_t start_position,
767 if(get_direction() == PLAY_REVERSE) start_position--;
768 load_configuration();
772 for(int i = 0; i < 2; i++)
774 frames[i] = new VFrame(0,
778 frame->get_color_model(),
782 //printf("InterpolateVideo::process_buffer 1 %lld %lld\n", range_start, range_end);
784 // Fraction of lowest frame in output
785 int64_t requested_range_start = (int64_t)((double)range_start *
786 frame_rate / active_input_rate);
787 int64_t requested_range_end = (int64_t)((double)range_end *
788 frame_rate / active_input_rate);
789 if(requested_range_start == requested_range_end)
791 read_frame(frame, 0, range_start, active_input_rate, 0);
796 // Fill border frames
797 fill_border(frame_rate, start_position);
799 float highest_fraction = (float)(start_position - requested_range_start) /
800 (requested_range_end - requested_range_start);
802 // Fraction of highest frame in output
803 lowest_fraction = 1.0 - highest_fraction;
805 CLAMP(highest_fraction, 0, 1);
806 CLAMP(lowest_fraction, 0, 1);
808 // printf("InterpolateVideo::process_buffer %lld %lld %lld %f %f %lld %lld %f %f\n",
811 // requested_range_start,
812 // requested_range_end,
814 // config.input_rate,
817 // highest_fraction);
819 if(start_position == (int64_t)(range_start * frame_rate / active_input_rate))
821 //printf("InterpolateVideo::process_buffer %d\n", __LINE__);
822 frame->copy_from(frames[0]);
824 if(config.optic_flow)
830 if(config.optic_flow)
832 //printf("InterpolateVideo::process_buffer %d\n", __LINE__);
845 int InterpolateVideo::is_realtime()
850 NEW_WINDOW_MACRO(InterpolateVideo, InterpolateVideoWindow)
851 const char* InterpolateVideo::plugin_title() { return _("Interpolate Video"); }
853 int InterpolateVideo::load_configuration()
855 KeyFrame *prev_keyframe, *next_keyframe;
856 InterpolateVideoConfig old_config;
857 old_config.copy_from(&config);
859 next_keyframe = get_next_keyframe(get_source_position());
860 prev_keyframe = get_prev_keyframe(get_source_position());
861 // Previous keyframe stays in config object.
862 read_data(prev_keyframe);
865 int64_t prev_position = edl_to_local(prev_keyframe->position);
866 int64_t next_position = edl_to_local(next_keyframe->position);
867 if(prev_position == 0 && next_position == 0)
869 next_position = prev_position = get_source_start();
871 // printf("InterpolateVideo::load_configuration 1 %lld %lld %lld %lld\n",
872 // prev_keyframe->position,
873 // next_keyframe->position,
877 // Get range to average in requested rate
878 range_start = prev_position;
879 range_end = next_position;
882 // Use keyframes to determine range
883 if(config.use_keyframes)
885 active_input_rate = get_framerate();
886 // Between keyframe and edge of range or no keyframes
887 if(range_start == range_end)
889 // Between first keyframe and start of effect
890 if(get_source_position() >= get_source_start() &&
891 get_source_position() < range_start)
893 range_start = get_source_start();
896 // Between last keyframe and end of effect
897 if(get_source_position() >= range_start &&
898 get_source_position() < get_source_start() + get_total_len())
900 // Last frame should be inclusive of current effect
901 range_end = get_source_start() + get_total_len() - 1;
905 // Should never get here
911 // Make requested rate equal to input rate for this mode.
913 // Convert requested rate to input rate
914 // printf("InterpolateVideo::load_configuration 2 %lld %lld %f %f\n",
918 // config.input_rate);
919 // range_start = (int64_t)((double)range_start / get_framerate() * active_input_rate + 0.5);
920 // range_end = (int64_t)((double)range_end / get_framerate() * active_input_rate + 0.5);
925 active_input_rate = config.input_rate;
926 // Convert to input frame rate
927 range_start = (int64_t)(get_source_position() /
930 range_end = (int64_t)(get_source_position() /
932 active_input_rate) + 1;
935 // printf("InterpolateVideo::load_configuration 1 %lld %lld %lld %lld %lld %lld\n",
936 // prev_keyframe->position,
937 // next_keyframe->position,
944 return !config.equivalent(&old_config);
948 void InterpolateVideo::save_data(KeyFrame *keyframe)
952 // cause data to be stored directly in text
953 output.set_shared_output(keyframe->get_data(), MESSAGESIZE);
954 output.tag.set_title("INTERPOLATEVIDEO");
955 output.tag.set_property("INPUT_RATE", config.input_rate);
956 output.tag.set_property("USE_KEYFRAMES", config.use_keyframes);
957 output.tag.set_property("OPTIC_FLOW", config.optic_flow);
958 output.tag.set_property("DRAW_VECTORS", config.draw_vectors);
959 output.tag.set_property("SEARCH_RADIUS", config.search_radius);
960 output.tag.set_property("MACROBLOCK_SIZE", config.macroblock_size);
962 output.terminate_string();
965 void InterpolateVideo::read_data(KeyFrame *keyframe)
969 input.set_shared_input(keyframe->get_data(), strlen(keyframe->get_data()));
971 while(!input.read_tag())
973 if(input.tag.title_is("INTERPOLATEVIDEO"))
975 config.input_rate = input.tag.get_property("INPUT_RATE", config.input_rate);
976 config.input_rate = Units::fix_framerate(config.input_rate);
977 config.use_keyframes = input.tag.get_property("USE_KEYFRAMES", config.use_keyframes);
978 config.optic_flow = input.tag.get_property("OPTIC_FLOW", config.optic_flow);
979 config.draw_vectors = input.tag.get_property("DRAW_VECTORS", config.draw_vectors);
980 config.search_radius = input.tag.get_property("SEARCH_RADIUS", config.search_radius);
981 config.macroblock_size = input.tag.get_property("MACROBLOCK_SIZE", config.macroblock_size);
986 void InterpolateVideo::update_gui()
990 if(load_configuration())
992 thread->window->lock_window("InterpolateVideo::update_gui");
993 ((InterpolateVideoWindow*)thread->window)->rate->update((float)config.input_rate);
994 ((InterpolateVideoWindow*)thread->window)->keyframes->update(config.use_keyframes);
995 ((InterpolateVideoWindow*)thread->window)->flow->update(config.optic_flow);
996 ((InterpolateVideoWindow*)thread->window)->vectors->update(config.draw_vectors);
997 ((InterpolateVideoWindow*)thread->window)->radius->update(config.search_radius);
998 ((InterpolateVideoWindow*)thread->window)->size->update(config.macroblock_size);
999 ((InterpolateVideoWindow*)thread->window)->update_enabled();
1000 thread->window->unlock_window();