3 * Copyright (C) 1997-2014 Adam Williams <broadcast at earthling dot net>
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software
17 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
25 #include "moveobjwindow.h"
27 #include "transportque.inc"
29 REGISTER_PLUGIN(MoveObj)
34 MoveObjConfig::MoveObjConfig()
43 int MoveObjConfig::equivalent(MoveObjConfig &that)
45 return draw_vectors == that.draw_vectors &&
46 do_stabilization == that.do_stabilization &&
47 block_size == that.block_size &&
48 search_radius == that.search_radius &&
49 settling_speed == that.settling_speed;
52 void MoveObjConfig::copy_from(MoveObjConfig &that)
54 draw_vectors = that.draw_vectors;
55 do_stabilization = that.do_stabilization;
56 block_size = that.block_size;
57 search_radius = that.search_radius;
58 settling_speed = that.settling_speed;
61 void MoveObjConfig::interpolate( MoveObjConfig &prev, MoveObjConfig &next,
62 long prev_frame, long next_frame, long current_frame)
67 void MoveObjConfig::limits()
69 bclamp(block_size, 5, 100);
70 bclamp(search_radius, 1, 100);
71 bclamp(settling_speed, 0, 100);
75 MoveObj::MoveObj(PluginServer *server)
76 : PluginVClient(server)
79 prev_position = next_position = -1;
80 x_accum = y_accum = 0;
93 const char* MoveObj::plugin_title() { return N_("MoveObj"); }
94 int MoveObj::is_realtime() { return 1; }
96 NEW_WINDOW_MACRO(MoveObj, MoveObjWindow);
97 LOAD_CONFIGURATION_MACRO(MoveObj, MoveObjConfig)
99 void MoveObj::save_data(KeyFrame *keyframe)
103 // cause data to be stored directly in text
104 output.set_shared_output(keyframe->xbuf);
105 output.tag.set_title("MOVEOBJ");
106 output.tag.set_property("DRAW_VECTORS", config.draw_vectors);
107 output.tag.set_property("DO_STABILIZATION", config.do_stabilization);
108 output.tag.set_property("BLOCK_SIZE", config.block_size);
109 output.tag.set_property("SEARCH_RADIUS", config.search_radius);
110 output.tag.set_property("SETTLING_SPEED", config.settling_speed);
112 output.append_newline();
113 output.tag.set_title("/MOVEOBJ");
115 output.append_newline();
116 output.terminate_string();
119 void MoveObj::read_data(KeyFrame *keyframe)
122 input.set_shared_input(keyframe->xbuf);
125 while( !(result = input.read_tag()) ) {
126 if( input.tag.title_is("MOVEOBJ") ) {
127 config.draw_vectors = input.tag.get_property("DRAW_VECTORS", config.draw_vectors);
128 config.do_stabilization = input.tag.get_property("DO_STABILIZATION", config.do_stabilization);
129 config.block_size = input.tag.get_property("BLOCK_SIZE", config.block_size);
130 config.search_radius = input.tag.get_property("SEARCH_RADIUS", config.search_radius);
131 config.settling_speed = input.tag.get_property("SETTLING_SPEED", config.settling_speed);
134 else if( input.tag.title_is("/MOVEOBJ") )
139 void MoveObj::update_gui()
141 if( !thread ) return;
142 if( !load_configuration() ) return;
143 thread->window->lock_window("MoveObj::update_gui");
144 MoveObjWindow *window = (MoveObjWindow*)thread->window;
146 window->vectors->update(config.draw_vectors);
147 window->do_stabilization->update(config.do_stabilization);
148 window->block_size->update(config.block_size);
149 window->search_radius->update(config.search_radius);
150 window->settling_speed->update(config.settling_speed);
152 thread->window->unlock_window();
155 void MoveObj::to_mat(Mat &mat, int mcols, int mrows,
156 VFrame *inp, int ix,int iy, int mcolor_model)
158 int mcomp = BC_CModels::components(mcolor_model);
159 int mbpp = BC_CModels::calculate_pixelsize(mcolor_model);
160 int psz = mbpp / mcomp;
161 int mdepth = psz < 2 ? CV_8U : psz < 4 ? CV_16U : CV_32F;
162 if( mat.dims != 2 || mat.depth() != mdepth || mat.channels() != mcomp ||
163 mat.cols != mcols || mat.rows != mrows ) {
167 int type = CV_MAKETYPE(mdepth, mcomp);
168 mat.create(mrows, mcols, type);
170 uint8_t *mat_rows[mrows];
171 for( int y=0; y<mrows; ++y ) mat_rows[y] = mat.ptr(y);
172 uint8_t **inp_rows = inp->get_rows();
173 int ibpl = inp->get_bytes_per_line(), obpl = mcols * mbpp;
174 int icolor_model = inp->get_color_model();
175 BC_CModels::transfer(mat_rows, mcolor_model, 0,0, mcols,mrows, obpl,
176 inp_rows, icolor_model, ix,iy, mcols,mrows, ibpl, 0);
177 // VFrame vfrm(mat_rows[0], -1, mcols,mrows, mcolor_model, mat_rows[1]-mat_rows[0]);
178 // static int vfrm_no = 0; char vfn[64]; sprintf(vfn,"/tmp/dat/%06d.png", vfrm_no++);
179 // vfrm.write_png(vfn);
182 int MoveObj::process_buffer(VFrame *frame, int64_t start_position, double frame_rate)
185 //int need_reconfigure =
186 load_configuration();
187 VFrame *input = get_input(0), *output = get_output(0);
188 int w = input->get_w(), h = input->get_h();
189 int color_model = input->get_color_model();
191 if( accum_matrix.empty() ) {
192 accum_matrix = Mat::eye(3,3, CV_64F);
195 int cpus1 = PluginClient::get_project_smp() + 1;
196 affine = new AffineEngine(cpus1, cpus1);
198 if( !prev_corners ) prev_corners = new ptV();
199 if( !next_corners ) next_corners = new ptV();
201 // Get the position of previous reference frame.
202 int64_t actual_previous_number = start_position;
203 int skip_current = 0;
204 if( get_direction() == PLAY_REVERSE ) {
205 if( ++actual_previous_number < get_source_start() + get_total_len() ) {
206 KeyFrame *keyframe = get_next_keyframe(start_position, 1);
207 if( keyframe->position > 0 &&
208 actual_previous_number >= keyframe->position )
215 if( --actual_previous_number >= get_source_start() ) {
216 KeyFrame *keyframe = get_prev_keyframe(start_position, 1);
217 if( keyframe->position > 0 &&
218 actual_previous_number < keyframe->position)
226 // move currrent image to previous position
227 if( next_position >= 0 && next_position == actual_previous_number ) {
228 Mat mat = prev_mat; prev_mat = next_mat; next_mat = mat;
229 ptV *pts = prev_corners; prev_corners = next_corners; next_corners = pts;
230 prev_position = next_position;
233 // load previous image
234 if( actual_previous_number >= 0 ) {
235 read_frame(input, 0, actual_previous_number, frame_rate, 0);
236 to_mat(prev_mat, w,h, input, 0,0, BC_GREY8);
239 if( skip_current || prev_position != actual_previous_number ) {
241 accum_matrix = Mat::eye(3,3, CV_64F);
246 next_position = start_position;
247 VFrame *iframe = !config.do_stabilization ? input : new_temp(w,h, color_model);
248 read_frame(iframe, 0, start_position, frame_rate, 0);
249 to_mat(next_mat, w,h, iframe, 0,0, BC_GREY8);
251 int corner_count = MAX_COUNT;
252 int block_size = config.block_size;
253 int min_distance = config.search_radius;
255 goodFeaturesToTrack(next_mat,
256 *next_corners, corner_count, 0.01, // quality_level
257 min_distance, noArray(), block_size,
262 if( !next_mat.empty() && next_corners->size() > 3 ) {
263 cornerSubPix(next_mat, *next_corners, Size(WIN_SIZE, WIN_SIZE), Size(-1,-1),
264 cvTermCriteria(CV_TERMCRIT_ITER | CV_TERMCRIT_EPS, 20, 0.03));
266 if( !prev_mat.empty() && prev_corners->size() > 3 ) {
269 ptV &prev = *prev_corners, &next = *next_corners;
270 calcOpticalFlowPyrLK(prev_mat, next_mat, prev, next,
271 st, err, Size(WIN_SIZE, WIN_SIZE), 5,
272 cvTermCriteria(CV_TERMCRIT_ITER | CV_TERMCRIT_EPS, 20, 0.3), 0);
273 float fails = 0.5 * w + 1;
274 uint8_t *stp = st.ptr<uint8_t>();
275 float *errp = err.ptr<float>();
276 for( int i=0,n=next_corners->size(); i<n; ++i ) {
277 if( stp[i] == 0 || errp[i] > fails ) continue;
278 pt1.push_back(next[i]);
279 pt2.push_back(prev[i]);
283 int points = pt1.size();
284 if( points > 0 && !skip_current ) {
285 if( config.draw_vectors ) {
286 int sz = bmin(w,h) / 222 + 2;
287 for( int i = 0; i < points; ++i )
288 iframe->draw_arrow(pt1[i].x,pt1[i].y, pt2[i].x,pt2[i].y, sz);
293 Mat_<float> translationM =
294 estimateGlobalMotionRansac(pt1, pt2, MM_TRANSLATION,
295 RansacParams::default2dMotion(MM_TRANSLATION),
297 Mat_<float> rotationM =
298 estimateGlobalMotionRansac(pt1, pt2, MM_ROTATION,
299 RansacParams::default2dMotion(MM_ROTATION),
303 Mat temp_matrix = Mat(3, 3, CV_64F, temp);
304 for( int i=0; i<9; ++i )
305 temp[i] = i == 2 || i == 5 ?
306 translationM(i / 3, i % 3) :
307 rotationM(i / 3, i % 3);
308 accum_matrix = temp_matrix * accum_matrix;
312 Mat M1(1, points, CV_32FC2, &pt1[0].x);
313 Mat M2(1, points, CV_32FC2, &pt2[0].x);
315 //M2 = H*M1 , old = H*current
316 Mat H = findHomography(M1, M2, CV_RANSAC, 2);
317 if( !H.dims || !H.rows || !H.cols )
318 printf("MoveObj::process_buffer %d: Find Homography Fail!\n", __LINE__);
320 accum_matrix = H * accum_matrix;
324 double *amat = accum_matrix.ptr<double>();
326 // if( EQUIV(amat[0], 0) ) {
327 //printf("MoveObj::process_buffer %d\n", __LINE__);
328 // accum_matrix = Mat::eye(3,3, CV_64F);
331 if( config.do_stabilization ) {
332 Mat identity = Mat::eye(3,3, CV_64F);
333 double w0 = config.settling_speed/100., w1 = 1.-w0;
334 // interpolate with identity matrix
335 accum_matrix = w0*identity + w1*accum_matrix;
337 AffineMatrix &matrix = affine->matrix;
338 for( int i=0,k=0; i<3; ++i )
339 for( int j=0; j<3; ++j )
340 matrix.values[i][j] = amat[k++];
342 //printf("MoveObj::process_buffer %d %jd matrix=\n", __LINE__, start_position);
345 // iframe is always temp, if we get here
346 output->clear_frame();
347 affine->process(output, iframe, 0, AffineEngine::TRANSFORM,
348 0,0, w,0, w,h, 0,h, 1);