dvb chan tuner api upgrade, slip/ripple handle drag keyfrm fix, load menu tweaks
[goodguy/cinelerra.git] / cinelerra-5.1 / plugins / moveobj / moveobj.C
1 /*
2  * CINELERRA
3  * Copyright (C) 1997-2014 Adam Williams <broadcast at earthling dot net>
4  * 
5  * This program is free software; you can redistribute it and/or modify
6  * it under the terms of the GNU General Public License as published by
7  * the Free Software Foundation; either version 2 of the License, or
8  * (at your option) any later version.
9  * 
10  * This program is distributed in the hope that it will be useful,
11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13  * GNU General Public License for more details.
14  * 
15  * You should have received a copy of the GNU General Public License
16  * along with this program; if not, write to the Free Software
17  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
18  * 
19  */
20
21 #include "affine.h"
22 #include "clip.h"
23 #include "filexml.h"
24 #include "moveobj.h"
25 #include "moveobjwindow.h"
26 #include "language.h"
27 #include "transportque.inc"
28
29 REGISTER_PLUGIN(MoveObj)
30
31 #define MAX_COUNT 250
32 #define WIN_SIZE 20
33
34 MoveObjConfig::MoveObjConfig()
35 {
36         draw_vectors = 0;
37         do_stabilization = 1;
38         block_size = 20;
39         search_radius = 10;
40         settling_speed = 5;
41 }
42
43 int MoveObjConfig::equivalent(MoveObjConfig &that)
44 {
45         return draw_vectors == that.draw_vectors &&
46                 do_stabilization == that.do_stabilization &&
47                 block_size == that.block_size &&
48                 search_radius == that.search_radius &&
49                 settling_speed == that.settling_speed;
50 }
51
52 void MoveObjConfig::copy_from(MoveObjConfig &that)
53 {
54         draw_vectors = that.draw_vectors;
55         do_stabilization = that.do_stabilization;
56         block_size = that.block_size;
57         search_radius = that.search_radius;
58         settling_speed = that.settling_speed;
59 }
60
61 void MoveObjConfig::interpolate( MoveObjConfig &prev, MoveObjConfig &next, 
62         long prev_frame, long next_frame, long current_frame)
63 {
64         copy_from(next);
65 }
66
67 void MoveObjConfig::limits()
68 {
69         bclamp(block_size, 5, 100);
70         bclamp(search_radius, 1, 100);
71         bclamp(settling_speed, 0, 100);
72 }
73
74
75 MoveObj::MoveObj(PluginServer *server)
76  : PluginVClient(server)
77 {
78         affine = 0;
79         prev_position = next_position = -1;
80         x_accum = y_accum = 0;
81         angle_accum = 0;
82         prev_corners = 0;
83         next_corners = 0;
84 }
85
86 MoveObj::~MoveObj()
87 {
88         delete affine;
89         delete prev_corners;
90         delete next_corners;
91 }
92
93 const char* MoveObj::plugin_title() { return N_("MoveObj"); }
94 int MoveObj::is_realtime() { return 1; }
95
96 NEW_WINDOW_MACRO(MoveObj, MoveObjWindow);
97 LOAD_CONFIGURATION_MACRO(MoveObj, MoveObjConfig)
98
99 void MoveObj::save_data(KeyFrame *keyframe)
100 {
101         FileXML output;
102
103 // cause data to be stored directly in text
104         output.set_shared_output(keyframe->xbuf);
105         output.tag.set_title("MOVEOBJ");
106         output.tag.set_property("DRAW_VECTORS", config.draw_vectors);
107         output.tag.set_property("DO_STABILIZATION", config.do_stabilization);
108         output.tag.set_property("BLOCK_SIZE", config.block_size);
109         output.tag.set_property("SEARCH_RADIUS", config.search_radius);
110         output.tag.set_property("SETTLING_SPEED", config.settling_speed);
111         output.append_tag();
112         output.append_newline();
113         output.tag.set_title("/MOVEOBJ");
114         output.append_tag();
115         output.append_newline();
116         output.terminate_string();
117 }
118
119 void MoveObj::read_data(KeyFrame *keyframe)
120 {
121         FileXML input;
122         input.set_shared_input(keyframe->xbuf);
123
124         int result = 0;
125         while( !(result = input.read_tag()) ) {
126                 if( input.tag.title_is("MOVEOBJ") ) {
127                         config.draw_vectors = input.tag.get_property("DRAW_VECTORS", config.draw_vectors);
128                         config.do_stabilization = input.tag.get_property("DO_STABILIZATION", config.do_stabilization);
129                         config.block_size = input.tag.get_property("BLOCK_SIZE", config.block_size);
130                         config.search_radius = input.tag.get_property("SEARCH_RADIUS", config.search_radius);
131                         config.settling_speed = input.tag.get_property("SETTLING_SPEED", config.settling_speed);
132                         config.limits();
133                 }
134                 else if( input.tag.title_is("/MOVEOBJ") )
135                         result = 1;
136         }
137 }
138
139 void MoveObj::update_gui()
140 {
141         if( !thread ) return;
142         if( !load_configuration() ) return;
143         thread->window->lock_window("MoveObj::update_gui");
144         MoveObjWindow *window = (MoveObjWindow*)thread->window;
145
146         window->vectors->update(config.draw_vectors);
147         window->do_stabilization->update(config.do_stabilization);
148         window->block_size->update(config.block_size);
149         window->search_radius->update(config.search_radius);
150         window->settling_speed->update(config.settling_speed);
151
152         thread->window->unlock_window();
153 }
154
155 void MoveObj::to_mat(Mat &mat, int mcols, int mrows,
156         VFrame *inp, int ix,int iy, int mcolor_model)
157 {
158         int mcomp = BC_CModels::components(mcolor_model);
159         int mbpp = BC_CModels::calculate_pixelsize(mcolor_model);
160         int psz = mbpp / mcomp;
161         int mdepth = psz < 2 ? CV_8U : psz < 4 ? CV_16U : CV_32F;
162         if( mat.dims != 2 || mat.depth() != mdepth || mat.channels() != mcomp ||
163             mat.cols != mcols || mat.rows != mrows ) {
164                 mat.release();
165         }
166         if( mat.empty() ) {
167                 int type = CV_MAKETYPE(mdepth, mcomp);
168                 mat.create(mrows, mcols, type);
169         }
170         uint8_t *mat_rows[mrows];
171         for( int y=0; y<mrows; ++y ) mat_rows[y] = mat.ptr(y);
172         uint8_t **inp_rows = inp->get_rows();
173         int ibpl = inp->get_bytes_per_line(), obpl = mcols * mbpp;
174         int icolor_model = inp->get_color_model();
175         BC_CModels::transfer(mat_rows, mcolor_model, 0,0, mcols,mrows, obpl,
176                 inp_rows, icolor_model, ix,iy, mcols,mrows, ibpl, 0);
177 //      VFrame vfrm(mat_rows[0], -1, mcols,mrows, mcolor_model, mat_rows[1]-mat_rows[0]);
178 //      static int vfrm_no = 0; char vfn[64]; sprintf(vfn,"/tmp/dat/%06d.png", vfrm_no++);
179 //      vfrm.write_png(vfn);
180 }
181
182 int MoveObj::process_buffer(VFrame *frame, int64_t start_position, double frame_rate)
183 {
184
185         //int need_reconfigure =
186         load_configuration();
187         VFrame *input = get_input(0), *output = get_output(0);
188         int w = input->get_w(), h = input->get_h();
189         int color_model = input->get_color_model();
190
191         if( accum_matrix.empty() ) {
192                 accum_matrix = Mat::eye(3,3, CV_64F);
193         }
194         if( !affine ) {
195                 int cpus1 = PluginClient::get_project_smp() + 1;
196                 affine = new AffineEngine(cpus1, cpus1);
197         }
198         if( !prev_corners ) prev_corners = new ptV();
199         if( !next_corners ) next_corners = new ptV();
200
201 // Get the position of previous reference frame.
202         int64_t actual_previous_number = start_position;
203         int skip_current = 0;
204         if( get_direction() == PLAY_REVERSE ) {
205                 if( ++actual_previous_number < get_source_start() + get_total_len() ) {
206                         KeyFrame *keyframe = get_next_keyframe(start_position, 1);
207                         if( keyframe->position > 0 &&
208                             actual_previous_number >= keyframe->position )
209                                 skip_current = 1;
210                 }
211                 else
212                         skip_current = 1;
213         }
214         else {
215                 if( --actual_previous_number >= get_source_start() ) {
216                         KeyFrame *keyframe = get_prev_keyframe(start_position, 1);
217                         if( keyframe->position > 0 &&
218                             actual_previous_number < keyframe->position)
219                                 skip_current = 1;
220                 }
221                 else
222                         skip_current = 1;
223         }
224         
225
226 // move currrent image to previous position
227         if( next_position >= 0 && next_position == actual_previous_number ) {
228                 Mat mat = prev_mat;  prev_mat = next_mat;  next_mat = mat;
229                 ptV *pts = prev_corners;  prev_corners = next_corners;  next_corners = pts;
230                 prev_position = next_position;
231         }
232         else
233 // load previous image
234         if( actual_previous_number >= 0 ) {
235                 read_frame(input, 0, actual_previous_number, frame_rate, 0);
236                 to_mat(prev_mat, w,h, input, 0,0, BC_GREY8);
237         }
238
239         if( skip_current || prev_position != actual_previous_number ) {
240                 skip_current = 1;
241                 accum_matrix = Mat::eye(3,3, CV_64F);
242         }
243
244
245 // load next image
246         next_position = start_position;
247         VFrame *iframe = !config.do_stabilization ? input : new_temp(w,h, color_model);
248         read_frame(iframe, 0, start_position, frame_rate, 0);
249         to_mat(next_mat, w,h, iframe, 0,0, BC_GREY8);
250
251         int corner_count = MAX_COUNT;
252         int block_size = config.block_size;
253         int min_distance = config.search_radius;
254
255         goodFeaturesToTrack(next_mat,
256                 *next_corners, corner_count, 0.01,        // quality_level
257                 min_distance, noArray(), block_size,
258                 false,       // use_harris
259                 0.04);       // k
260
261         ptV pt1, pt2;
262         if( !next_mat.empty() && next_corners->size() > 3 ) {
263                 cornerSubPix(next_mat, *next_corners, Size(WIN_SIZE, WIN_SIZE), Size(-1,-1),
264                         cvTermCriteria(CV_TERMCRIT_ITER | CV_TERMCRIT_EPS, 20, 0.03));
265         }
266         if( !prev_mat.empty() && prev_corners->size() > 3 ) {
267 // optical flow
268                 Mat st, err;
269                 ptV &prev = *prev_corners, &next = *next_corners;
270                 calcOpticalFlowPyrLK(prev_mat, next_mat, prev, next,
271                         st, err, Size(WIN_SIZE, WIN_SIZE), 5, 
272                         cvTermCriteria(CV_TERMCRIT_ITER | CV_TERMCRIT_EPS, 20, 0.3), 0);
273                 float fails = 0.5 * w + 1;
274                 uint8_t *stp = st.ptr<uint8_t>();
275                 float *errp = err.ptr<float>();
276                 for( int i=0,n=next_corners->size(); i<n; ++i ) {
277                         if( stp[i] == 0 || errp[i] > fails ) continue;
278                         pt1.push_back(next[i]);  
279                         pt2.push_back(prev[i]);
280                 }
281         }
282
283         int points = pt1.size();
284         if( points > 0 && !skip_current ) {
285                 if( config.draw_vectors ) {
286                         int sz = bmin(w,h) / 222 + 2;
287                         for( int i = 0; i < points; ++i )
288                                 iframe->draw_arrow(pt1[i].x,pt1[i].y, pt2[i].x,pt2[i].y, sz);
289                 }
290 #ifdef _RANSAC
291 // ransac
292                 int ninliers = 0;
293                 Mat_<float> translationM =
294                         estimateGlobalMotionRansac(pt1, pt2, MM_TRANSLATION, 
295                                 RansacParams::default2dMotion(MM_TRANSLATION),
296                                 0, &ninliers);
297                 Mat_<float> rotationM =
298                         estimateGlobalMotionRansac(pt1, pt2, MM_ROTATION, 
299                                 RansacParams::default2dMotion(MM_ROTATION),
300                                 0, &ninliers);
301
302                 double temp[9];
303                 Mat temp_matrix = Mat(3, 3, CV_64F, temp);
304                 for( int i=0; i<9; ++i )
305                         temp[i] = i == 2 || i == 5 ?
306                                 translationM(i / 3, i % 3) :
307                                 rotationM(i / 3, i % 3);
308                 accum_matrix = temp_matrix * accum_matrix;
309 #else
310 // homography
311
312                 Mat M1(1, points, CV_32FC2, &pt1[0].x);  
313                 Mat M2(1, points, CV_32FC2, &pt2[0].x);  
314
315 //M2 = H*M1 , old = H*current  
316                 Mat H = findHomography(M1, M2, CV_RANSAC, 2);
317                 if( !H.dims || !H.rows || !H.cols )
318                         printf("MoveObj::process_buffer %d: Find Homography Fail!\n", __LINE__);  
319                 else
320                         accum_matrix = H * accum_matrix;
321 #endif
322         }
323
324         double *amat = accum_matrix.ptr<double>();
325 // deglitch
326 //      if( EQUIV(amat[0], 0) ) {
327 //printf("MoveObj::process_buffer %d\n", __LINE__);
328 //              accum_matrix = Mat::eye(3,3, CV_64F);
329 //      }
330
331         if( config.do_stabilization ) {
332                 Mat identity = Mat::eye(3,3, CV_64F);
333                 double w0 = config.settling_speed/100., w1 = 1.-w0;
334 // interpolate with identity matrix
335                 accum_matrix = w0*identity + w1*accum_matrix;
336
337                 AffineMatrix &matrix = affine->matrix;
338                 for( int i=0,k=0; i<3; ++i )
339                         for( int j=0; j<3; ++j )
340                                 matrix.values[i][j] = amat[k++];
341
342 //printf("MoveObj::process_buffer %d %jd matrix=\n", __LINE__, start_position);
343 //matrix.dump();
344
345 // iframe is always temp, if we get here
346                 output->clear_frame();
347                 affine->process(output, iframe, 0, AffineEngine::TRANSFORM,
348                         0,0, w,0, w,h, 0,h, 1);
349         }
350
351         return 0;
352 }