upgrade libx265 to 3.1.2, tweak menupopup placement
[goodguy/cinelerra.git] / cinelerra-5.1 / plugins / flowobj / flowobj.C
1 /*
2  * CINELERRA
3  * Copyright (C) 1997-2014 Adam Williams <broadcast at earthling dot net>
4  * 
5  * This program is free software; you can redistribute it and/or modify
6  * it under the terms of the GNU General Public License as published by
7  * the Free Software Foundation; either version 2 of the License, or
8  * (at your option) any later version.
9  * 
10  * This program is distributed in the hope that it will be useful,
11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13  * GNU General Public License for more details.
14  * 
15  * You should have received a copy of the GNU General Public License
16  * along with this program; if not, write to the Free Software
17  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
18  * 
19  */
20
21 #include "clip.h"
22 #include "filexml.h"
23 #include "flowobj.h"
24 #include "flowobjwindow.h"
25 #include "language.h"
26 #include "transportque.inc"
27
28 REGISTER_PLUGIN(FlowObj)
29
30 #define MAX_COUNT 250
31 #define WIN_SIZE 20
32
33 FlowObjConfig::FlowObjConfig()
34 {
35         draw_vectors = 0;
36         do_stabilization = 1;
37         block_size = 20;
38         search_radius = 10;
39         settling_speed = 5;
40 }
41
42 int FlowObjConfig::equivalent(FlowObjConfig &that)
43 {
44         return draw_vectors == that.draw_vectors &&
45                 do_stabilization == that.do_stabilization &&
46                 block_size == that.block_size &&
47                 search_radius == that.search_radius &&
48                 settling_speed == that.settling_speed;
49 }
50
51 void FlowObjConfig::copy_from(FlowObjConfig &that)
52 {
53         draw_vectors = that.draw_vectors;
54         do_stabilization = that.do_stabilization;
55         block_size = that.block_size;
56         search_radius = that.search_radius;
57         settling_speed = that.settling_speed;
58 }
59
60 void FlowObjConfig::interpolate( FlowObjConfig &prev, FlowObjConfig &next, 
61         long prev_frame, long next_frame, long current_frame)
62 {
63         copy_from(next);
64 }
65
66 void FlowObjConfig::limits()
67 {
68         bclamp(block_size, 5, 100);
69         bclamp(search_radius, 1, 100);
70         bclamp(settling_speed, 0, 100);
71 }
72
73
74 FlowObj::FlowObj(PluginServer *server)
75  : PluginVClient(server)
76 {
77         flow_engine = 0;
78         overlay = 0;
79         accum = 0;
80         prev_position = next_position = -1;
81         x_accum = y_accum = 0;
82         angle_accum = 0;
83         prev_corners = 0;
84         next_corners = 0;
85 }
86
87 FlowObj::~FlowObj()
88 {
89         delete flow_engine;
90         delete overlay;
91         delete prev_corners;
92         delete next_corners;
93 }
94
95 const char* FlowObj::plugin_title() { return N_("FlowObj"); }
96 int FlowObj::is_realtime() { return 1; }
97
98 NEW_WINDOW_MACRO(FlowObj, FlowObjWindow);
99 LOAD_CONFIGURATION_MACRO(FlowObj, FlowObjConfig)
100
101 void FlowObj::save_data(KeyFrame *keyframe)
102 {
103         FileXML output;
104
105 // cause data to be stored directly in text
106         output.set_shared_output(keyframe->xbuf);
107         output.tag.set_title("FLOWOBJ");
108         output.tag.set_property("DRAW_VECTORS", config.draw_vectors);
109         output.tag.set_property("DO_STABILIZATION", config.do_stabilization);
110         output.tag.set_property("BLOCK_SIZE", config.block_size);
111         output.tag.set_property("SEARCH_RADIUS", config.search_radius);
112         output.tag.set_property("SETTLING_SPEED", config.settling_speed);
113         output.append_tag();
114         output.append_newline();
115         output.tag.set_title("/FLOWOBJ");
116         output.append_tag();
117         output.append_newline();
118         output.terminate_string();
119 }
120
121 void FlowObj::read_data(KeyFrame *keyframe)
122 {
123         FileXML input;
124         input.set_shared_input(keyframe->xbuf);
125
126         int result = 0;
127         while( !(result = input.read_tag()) ) {
128                 if( input.tag.title_is("FLOWOBJ") ) {
129                         config.draw_vectors = input.tag.get_property("DRAW_VECTORS", config.draw_vectors);
130                         config.do_stabilization = input.tag.get_property("DO_STABILIZATION", config.do_stabilization);
131                         config.block_size = input.tag.get_property("BLOCK_SIZE", config.block_size);
132                         config.search_radius = input.tag.get_property("SEARCH_RADIUS", config.search_radius);
133                         config.settling_speed = input.tag.get_property("SETTLING_SPEED", config.settling_speed);
134                         config.limits();
135                 }
136                 else if( input.tag.title_is("/FLOWOBJ") )
137                         result = 1;
138         }
139 }
140
141 void FlowObj::update_gui()
142 {
143         if( !thread ) return;
144         if( !load_configuration() ) return;
145         thread->window->lock_window("FlowObj::update_gui");
146         FlowObjWindow *window = (FlowObjWindow*)thread->window;
147
148         window->vectors->update(config.draw_vectors);
149         window->do_stabilization->update(config.do_stabilization);
150         window->block_size->update(config.block_size);
151         window->search_radius->update(config.search_radius);
152         window->settling_speed->update(config.settling_speed);
153
154         thread->window->unlock_window();
155 }
156
157 void FlowObj::to_mat(Mat &mat, int mcols, int mrows,
158         VFrame *inp, int ix,int iy, int mcolor_model)
159 {
160         int mcomp = BC_CModels::components(mcolor_model);
161         int mbpp = BC_CModels::calculate_pixelsize(mcolor_model);
162         int psz = mbpp / mcomp;
163         int mdepth = psz < 2 ? CV_8U : psz < 4 ? CV_16U : CV_32F;
164         if( mat.dims != 2 || mat.depth() != mdepth || mat.channels() != mcomp ||
165             mat.cols != mcols || mat.rows != mrows ) {
166                 mat.release();
167         }
168         if( mat.empty() ) {
169                 int type = CV_MAKETYPE(mdepth, mcomp);
170                 mat.create(mrows, mcols, type);
171         }
172         uint8_t *mat_rows[mrows];
173         for( int y=0; y<mrows; ++y ) mat_rows[y] = mat.ptr(y);
174         uint8_t **inp_rows = inp->get_rows();
175         int ibpl = inp->get_bytes_per_line(), mbpl = mcols * mbpp;
176         int icolor_model = inp->get_color_model();
177         BC_CModels::transfer(mat_rows, mcolor_model, 0,0, mcols,mrows, mbpl,
178                 inp_rows, icolor_model, ix,iy, mcols,mrows, ibpl, 0);
179 //      VFrame vfrm(mat_rows[0], -1, mcols,mrows, mcolor_model, mat_rows[1]-mat_rows[0]);
180 //      static int vfrm_no = 0; char vfn[64]; sprintf(vfn,"/tmp/idat/%06d.png", vfrm_no++);
181 //      vfrm.write_png(vfn);
182 }
183
184 void FlowObj::from_mat(VFrame *out, int ox, int oy, int ow, int oh, Mat &mat, int mcolor_model)
185 {
186         int mbpp = BC_CModels::calculate_pixelsize(mcolor_model);
187         int mrows = mat.rows, mcols = mat.cols;
188         uint8_t *mat_rows[mrows];
189         for( int y=0; y<mrows; ++y ) mat_rows[y] = mat.ptr(y);
190         uint8_t **out_rows = out->get_rows();
191         int obpl = out->get_bytes_per_line(), mbpl = mcols * mbpp;
192         int ocolor_model = out->get_color_model();
193         BC_CModels::transfer(out_rows, ocolor_model, ox,oy, ow,oh, obpl,
194                 mat_rows, mcolor_model, 0,0, mcols,mrows, mbpl,  0);
195 //      static int vfrm_no = 0; char vfn[64]; sprintf(vfn,"/tmp/odat/%06d.png", vfrm_no++);
196 //      out->write_png(vfn);
197 }
198
199
200 int FlowObj::process_buffer(VFrame *frame, int64_t start_position, double frame_rate)
201 {
202
203         //int need_reconfigure =
204         load_configuration();
205         input = get_input(0);
206         output = get_output(0);
207         width = input->get_w();
208         height = input->get_h();
209         color_model = input->get_color_model();
210
211         if( accum_matrix.empty() ) {
212                 accum_matrix = Mat::eye(3,3, CV_64F);
213         }
214         if( !prev_corners ) prev_corners = new ptV();
215         if( !next_corners ) next_corners = new ptV();
216
217 // Get the position of previous reference frame.
218         int64_t actual_previous_number = start_position;
219         int skip_current = 0;
220         if( get_direction() == PLAY_REVERSE ) {
221                 if( ++actual_previous_number < get_source_start() + get_total_len() ) {
222                         KeyFrame *keyframe = get_next_keyframe(start_position, 1);
223                         if( keyframe->position > 0 &&
224                             actual_previous_number >= keyframe->position )
225                                 skip_current = 1;
226                 }
227                 else
228                         skip_current = 1;
229         }
230         else {
231                 if( --actual_previous_number >= get_source_start() ) {
232                         KeyFrame *keyframe = get_prev_keyframe(start_position, 1);
233                         if( keyframe->position > 0 &&
234                             actual_previous_number < keyframe->position)
235                                 skip_current = 1;
236                 }
237                 else
238                         skip_current = 1;
239         }
240         
241
242 // move currrent image to previous position
243         if( next_position >= 0 && next_position == actual_previous_number ) {
244                 Mat mat = prev_mat;  prev_mat = next_mat;  next_mat = mat;
245                 ptV *pts = prev_corners;  prev_corners = next_corners;  next_corners = pts;
246                 prev_position = next_position;
247         }
248         else
249 // load previous image
250         if( actual_previous_number >= 0 ) {
251                 read_frame(input, 0, actual_previous_number, frame_rate, 0);
252                 to_mat(prev_mat, width,height, input, 0,0, BC_GREY8);
253         }
254
255         if( skip_current || prev_position != actual_previous_number ) {
256                 skip_current = 1;
257                 accum_matrix = Mat::eye(3,3, CV_64F);
258         }
259
260
261 // load next image
262         next_position = start_position;
263         VFrame *iframe = new_temp(width,height, color_model);
264         read_frame(iframe, 0, start_position, frame_rate, 0);
265         input = iframe;
266         to_mat(next_mat, width,height, iframe, 0,0, BC_GREY8);
267
268         int corner_count = MAX_COUNT;
269         int block_size = config.block_size;
270         int min_distance = config.search_radius;
271
272         goodFeaturesToTrack(next_mat,
273                 *next_corners, corner_count, 0.01,        // quality_level
274                 min_distance, noArray(), block_size,
275                 false,       // use_harris
276                 0.04);       // k
277
278         if( !next_mat.empty() && next_corners->size() > 3 ) {
279                 cornerSubPix(next_mat, *next_corners, Size(WIN_SIZE, WIN_SIZE), Size(-1,-1),
280                         cvTermCriteria(CV_TERMCRIT_ITER | CV_TERMCRIT_EPS, 20, 0.03));
281         }
282
283         Mat flow;
284         if( !prev_mat.empty() && prev_corners->size() > 3 ) {
285 // optical flow
286                 calcOpticalFlowFarneback(prev_mat, next_mat, flow, 0.5, 3, 15, 3, 5, 1.2, 0);
287         }
288
289         if( config.do_stabilization && !flow.empty() ) {
290                 if( !flow_engine )
291                         flow_engine = new FlowObjRemapEngine(this, PluginClient::smp);
292                 flow_mat = &flow;
293                 flow_engine->process_packages();
294         }
295         else
296                 output->copy_from(input);
297
298         if( config.settling_speed > 0 ) {
299                 if( accum && (accum->get_color_model() != color_model ||
300                     accum->get_w() != width || accum->get_h() != height) ) {
301                         delete accum;  accum = 0;
302                 }
303                 if( !accum ) {
304                         accum = new VFrame(width, height, color_model, 0);
305                         accum->clear_frame();
306                 }
307                 if( !overlay )
308                         overlay = new OverlayFrame(PluginClient::smp + 1);
309                 float alpha = 1 - config.settling_speed/100.;
310                 overlay->overlay(accum, output,
311                         0,0, width,height, 0,0, width, height,
312                         alpha, TRANSFER_NORMAL, NEAREST_NEIGHBOR);
313                 output->copy_from(accum);
314         }
315
316         if( !flow.empty() && config.draw_vectors ) {
317                 output->set_pixel_color(GREEN);
318                 int nv = 24; float s = 2;
319                 int gw = width/nv + 1, gh = height/nv + 1;
320                 int sz = bmin(width,height) / 200 + 4;
321                 for( int y=gh/2; y<height; y+=gh ) {
322                         Point2f *row = (Point2f *)flow.ptr(y);
323                         for( int x=gw/2; x<width; x+=gw ) {
324                                 Point2f *ds = row + x;
325                                 float x0 = x+.5, x1 = x+s*ds->x, y0 = y+.5, y1 = y+s*ds->y;
326                                 output->draw_arrow(x0,y0, x1,y1, sz);
327                         }
328                 }
329         }
330
331         return 0;
332 }
333
334
335 FlowObjRemapPackage::FlowObjRemapPackage()
336  : LoadPackage()
337 {
338 }
339
340 FlowObjRemapUnit::FlowObjRemapUnit(FlowObjRemapEngine *engine, FlowObj *plugin)
341  : LoadClient(engine)
342 {
343         this->plugin = plugin;
344 }
345
346 FlowObjRemapUnit::~FlowObjRemapUnit()
347 {
348 }
349
350 void FlowObjRemapUnit::process_package(LoadPackage *package)
351 {
352         FlowObjRemapPackage *pkg = (FlowObjRemapPackage*)package;
353         process_remap(pkg);
354 }
355
356 void FlowObjRemapUnit::process_remap(FlowObjRemapPackage *pkg)
357 {
358         int row1 = pkg->row1;
359         int row2 = pkg->row2;
360         int w = plugin->width, h = plugin->height;
361         int w1 = w-1, h1 = h-1;
362         int color_model = plugin->color_model;
363         int bpp = BC_CModels::calculate_pixelsize(color_model);
364
365 #define FLOW_OBJ_REMAP(type, components) { \
366         unsigned char **in_rows = plugin->input->get_rows(); \
367         type **out_rows = (type**)plugin->output->get_rows(); \
368  \
369         for( int y=row1; y<row2; ++y ) { \
370                 Point2f *ipt = (Point2f *)plugin->flow_mat->ptr(y); \
371                 type *out_row = out_rows[y]; \
372                 for( int x=0; x<w; ++x,++ipt ) { \
373                         int ix = x - ipt->x, iy = y - ipt->y; \
374                         bclamp(ix, 0, w1);  bclamp(iy, 0, h1); \
375                         type *inp_row = (type *)(in_rows[iy] + ix*bpp); \
376                         for( int i=0; i<components; ++i ) \
377                                 *out_row++ = *inp_row++; \
378                 } \
379         } \
380 }
381         switch( color_model ) {
382         case BC_RGB888:     FLOW_OBJ_REMAP(unsigned char, 3);  break;
383         case BC_RGBA8888:   FLOW_OBJ_REMAP(unsigned char, 4);  break;
384         case BC_RGB_FLOAT:  FLOW_OBJ_REMAP(float, 3);          break;
385         case BC_RGBA_FLOAT: FLOW_OBJ_REMAP(float, 4);          break;
386         case BC_YUV888:     FLOW_OBJ_REMAP(unsigned char, 3);  break;
387         case BC_YUVA8888:   FLOW_OBJ_REMAP(unsigned char, 4);  break;
388         }
389 }
390
391
392 FlowObjRemapEngine::FlowObjRemapEngine(FlowObj *plugin, int cpus)
393  : LoadServer(cpus+1, cpus+1)
394 {
395         this->plugin = plugin;
396 }
397
398 FlowObjRemapEngine::~FlowObjRemapEngine()
399 {
400 }
401
402 void FlowObjRemapEngine::init_packages()
403 {
404         int row1 = 0, row2 = 0, n = LoadServer::get_total_packages();
405         for( int i=0; i<n; row1=row2 ) {
406                 FlowObjRemapPackage *package = (FlowObjRemapPackage*)LoadServer::get_package(i);
407                 row2 = plugin->get_input()->get_h() * ++i / n;
408                 package->row1 = row1;  package->row2 = row2;
409         }
410 }
411
412 LoadClient* FlowObjRemapEngine::new_client()
413 {
414         return new FlowObjRemapUnit(this, plugin);
415 }
416
417 LoadPackage* FlowObjRemapEngine::new_package()
418 {
419         return new FlowObjRemapPackage();
420 }
421