3 * Copyright (C) 1997-2014 Adam Williams <broadcast at earthling dot net>
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software
17 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
24 #include "flowobjwindow.h"
26 #include "transportque.inc"
28 REGISTER_PLUGIN(FlowObj)
33 FlowObjConfig::FlowObjConfig()
42 int FlowObjConfig::equivalent(FlowObjConfig &that)
44 return draw_vectors == that.draw_vectors &&
45 do_stabilization == that.do_stabilization &&
46 block_size == that.block_size &&
47 search_radius == that.search_radius &&
48 settling_speed == that.settling_speed;
51 void FlowObjConfig::copy_from(FlowObjConfig &that)
53 draw_vectors = that.draw_vectors;
54 do_stabilization = that.do_stabilization;
55 block_size = that.block_size;
56 search_radius = that.search_radius;
57 settling_speed = that.settling_speed;
60 void FlowObjConfig::interpolate( FlowObjConfig &prev, FlowObjConfig &next,
61 long prev_frame, long next_frame, long current_frame)
66 void FlowObjConfig::limits()
68 bclamp(block_size, 5, 100);
69 bclamp(search_radius, 1, 100);
70 bclamp(settling_speed, 0, 100);
74 FlowObj::FlowObj(PluginServer *server)
75 : PluginVClient(server)
80 prev_position = next_position = -1;
81 x_accum = y_accum = 0;
95 const char* FlowObj::plugin_title() { return N_("FlowObj"); }
96 int FlowObj::is_realtime() { return 1; }
98 NEW_WINDOW_MACRO(FlowObj, FlowObjWindow);
99 LOAD_CONFIGURATION_MACRO(FlowObj, FlowObjConfig)
101 void FlowObj::save_data(KeyFrame *keyframe)
105 // cause data to be stored directly in text
106 output.set_shared_output(keyframe->xbuf);
107 output.tag.set_title("FLOWOBJ");
108 output.tag.set_property("DRAW_VECTORS", config.draw_vectors);
109 output.tag.set_property("DO_STABILIZATION", config.do_stabilization);
110 output.tag.set_property("BLOCK_SIZE", config.block_size);
111 output.tag.set_property("SEARCH_RADIUS", config.search_radius);
112 output.tag.set_property("SETTLING_SPEED", config.settling_speed);
114 output.append_newline();
115 output.tag.set_title("/FLOWOBJ");
117 output.append_newline();
118 output.terminate_string();
121 void FlowObj::read_data(KeyFrame *keyframe)
124 input.set_shared_input(keyframe->xbuf);
127 while( !(result = input.read_tag()) ) {
128 if( input.tag.title_is("FLOWOBJ") ) {
129 config.draw_vectors = input.tag.get_property("DRAW_VECTORS", config.draw_vectors);
130 config.do_stabilization = input.tag.get_property("DO_STABILIZATION", config.do_stabilization);
131 config.block_size = input.tag.get_property("BLOCK_SIZE", config.block_size);
132 config.search_radius = input.tag.get_property("SEARCH_RADIUS", config.search_radius);
133 config.settling_speed = input.tag.get_property("SETTLING_SPEED", config.settling_speed);
136 else if( input.tag.title_is("/FLOWOBJ") )
141 void FlowObj::update_gui()
143 if( !thread ) return;
144 if( !load_configuration() ) return;
145 thread->window->lock_window("FlowObj::update_gui");
146 FlowObjWindow *window = (FlowObjWindow*)thread->window;
148 window->vectors->update(config.draw_vectors);
149 window->do_stabilization->update(config.do_stabilization);
150 window->block_size->update(config.block_size);
151 window->search_radius->update(config.search_radius);
152 window->settling_speed->update(config.settling_speed);
154 thread->window->unlock_window();
157 void FlowObj::to_mat(Mat &mat, int mcols, int mrows,
158 VFrame *inp, int ix,int iy, int mcolor_model)
160 int mcomp = BC_CModels::components(mcolor_model);
161 int mbpp = BC_CModels::calculate_pixelsize(mcolor_model);
162 int psz = mbpp / mcomp;
163 int mdepth = psz < 2 ? CV_8U : psz < 4 ? CV_16U : CV_32F;
164 if( mat.dims != 2 || mat.depth() != mdepth || mat.channels() != mcomp ||
165 mat.cols != mcols || mat.rows != mrows ) {
169 int type = CV_MAKETYPE(mdepth, mcomp);
170 mat.create(mrows, mcols, type);
172 uint8_t *mat_rows[mrows];
173 for( int y=0; y<mrows; ++y ) mat_rows[y] = mat.ptr(y);
174 uint8_t **inp_rows = inp->get_rows();
175 int ibpl = inp->get_bytes_per_line(), mbpl = mcols * mbpp;
176 int icolor_model = inp->get_color_model();
177 BC_CModels::transfer(mat_rows, mcolor_model, 0,0, mcols,mrows, mbpl,
178 inp_rows, icolor_model, ix,iy, mcols,mrows, ibpl, 0);
179 // VFrame vfrm(mat_rows[0], -1, mcols,mrows, mcolor_model, mat_rows[1]-mat_rows[0]);
180 // static int vfrm_no = 0; char vfn[64]; sprintf(vfn,"/tmp/idat/%06d.png", vfrm_no++);
181 // vfrm.write_png(vfn);
184 void FlowObj::from_mat(VFrame *out, int ox, int oy, int ow, int oh, Mat &mat, int mcolor_model)
186 int mbpp = BC_CModels::calculate_pixelsize(mcolor_model);
187 int mrows = mat.rows, mcols = mat.cols;
188 uint8_t *mat_rows[mrows];
189 for( int y=0; y<mrows; ++y ) mat_rows[y] = mat.ptr(y);
190 uint8_t **out_rows = out->get_rows();
191 int obpl = out->get_bytes_per_line(), mbpl = mcols * mbpp;
192 int ocolor_model = out->get_color_model();
193 BC_CModels::transfer(out_rows, ocolor_model, ox,oy, ow,oh, obpl,
194 mat_rows, mcolor_model, 0,0, mcols,mrows, mbpl, 0);
195 // static int vfrm_no = 0; char vfn[64]; sprintf(vfn,"/tmp/odat/%06d.png", vfrm_no++);
196 // out->write_png(vfn);
200 int FlowObj::process_buffer(VFrame *frame, int64_t start_position, double frame_rate)
203 //int need_reconfigure =
204 load_configuration();
205 input = get_input(0);
206 output = get_output(0);
207 width = input->get_w();
208 height = input->get_h();
209 color_model = input->get_color_model();
211 if( accum_matrix.empty() ) {
212 accum_matrix = Mat::eye(3,3, CV_64F);
214 if( !prev_corners ) prev_corners = new ptV();
215 if( !next_corners ) next_corners = new ptV();
217 // Get the position of previous reference frame.
218 int64_t actual_previous_number = start_position;
219 int skip_current = 0;
220 if( get_direction() == PLAY_REVERSE ) {
221 if( ++actual_previous_number < get_source_start() + get_total_len() ) {
222 KeyFrame *keyframe = get_next_keyframe(start_position, 1);
223 if( keyframe->position > 0 &&
224 actual_previous_number >= keyframe->position )
231 if( --actual_previous_number >= get_source_start() ) {
232 KeyFrame *keyframe = get_prev_keyframe(start_position, 1);
233 if( keyframe->position > 0 &&
234 actual_previous_number < keyframe->position)
242 // move currrent image to previous position
243 if( next_position >= 0 && next_position == actual_previous_number ) {
244 Mat mat = prev_mat; prev_mat = next_mat; next_mat = mat;
245 ptV *pts = prev_corners; prev_corners = next_corners; next_corners = pts;
246 prev_position = next_position;
249 // load previous image
250 if( actual_previous_number >= 0 ) {
251 read_frame(input, 0, actual_previous_number, frame_rate, 0);
252 to_mat(prev_mat, width,height, input, 0,0, BC_GREY8);
255 if( skip_current || prev_position != actual_previous_number ) {
257 accum_matrix = Mat::eye(3,3, CV_64F);
262 next_position = start_position;
263 VFrame *iframe = new_temp(width,height, color_model);
264 read_frame(iframe, 0, start_position, frame_rate, 0);
266 to_mat(next_mat, width,height, iframe, 0,0, BC_GREY8);
268 int corner_count = MAX_COUNT;
269 int block_size = config.block_size;
270 int min_distance = config.search_radius;
272 goodFeaturesToTrack(next_mat,
273 *next_corners, corner_count, 0.01, // quality_level
274 min_distance, noArray(), block_size,
278 if( !next_mat.empty() && next_corners->size() > 3 ) {
279 cornerSubPix(next_mat, *next_corners, Size(WIN_SIZE, WIN_SIZE), Size(-1,-1),
280 cvTermCriteria(CV_TERMCRIT_ITER | CV_TERMCRIT_EPS, 20, 0.03));
284 if( !prev_mat.empty() && prev_corners->size() > 3 ) {
286 calcOpticalFlowFarneback(prev_mat, next_mat, flow, 0.5, 3, 15, 3, 5, 1.2, 0);
289 if( config.do_stabilization && !flow.empty() ) {
291 flow_engine = new FlowObjRemapEngine(this, PluginClient::smp);
293 flow_engine->process_packages();
296 output->copy_from(input);
298 if( config.settling_speed > 0 ) {
299 if( accum && (accum->get_color_model() != color_model ||
300 accum->get_w() != width || accum->get_h() != height) ) {
301 delete accum; accum = 0;
304 accum = new VFrame(width, height, color_model, 0);
305 accum->clear_frame();
308 overlay = new OverlayFrame(PluginClient::smp + 1);
309 float alpha = 1 - config.settling_speed/100.;
310 overlay->overlay(accum, output,
311 0,0, width,height, 0,0, width, height,
312 alpha, TRANSFER_NORMAL, NEAREST_NEIGHBOR);
313 output->copy_from(accum);
316 if( !flow.empty() && config.draw_vectors ) {
317 output->set_pixel_color(GREEN);
318 int nv = 24; float s = 2;
319 int gw = width/nv + 1, gh = height/nv + 1;
320 int sz = bmin(width,height) / 200 + 4;
321 for( int y=gh/2; y<height; y+=gh ) {
322 Point2f *row = (Point2f *)flow.ptr(y);
323 for( int x=gw/2; x<width; x+=gw ) {
324 Point2f *ds = row + x;
325 float x0 = x+.5, x1 = x+s*ds->x, y0 = y+.5, y1 = y+s*ds->y;
326 output->draw_arrow(x0,y0, x1,y1, sz);
335 FlowObjRemapPackage::FlowObjRemapPackage()
340 FlowObjRemapUnit::FlowObjRemapUnit(FlowObjRemapEngine *engine, FlowObj *plugin)
343 this->plugin = plugin;
346 FlowObjRemapUnit::~FlowObjRemapUnit()
350 void FlowObjRemapUnit::process_package(LoadPackage *package)
352 FlowObjRemapPackage *pkg = (FlowObjRemapPackage*)package;
356 void FlowObjRemapUnit::process_remap(FlowObjRemapPackage *pkg)
358 int row1 = pkg->row1;
359 int row2 = pkg->row2;
360 int w = plugin->width, h = plugin->height;
361 int w1 = w-1, h1 = h-1;
362 int color_model = plugin->color_model;
363 int bpp = BC_CModels::calculate_pixelsize(color_model);
365 #define FLOW_OBJ_REMAP(type, components) { \
366 unsigned char **in_rows = plugin->input->get_rows(); \
367 type **out_rows = (type**)plugin->output->get_rows(); \
369 for( int y=row1; y<row2; ++y ) { \
370 Point2f *ipt = (Point2f *)plugin->flow_mat->ptr(y); \
371 type *out_row = out_rows[y]; \
372 for( int x=0; x<w; ++x,++ipt ) { \
373 int ix = x - ipt->x, iy = y - ipt->y; \
374 bclamp(ix, 0, w1); bclamp(iy, 0, h1); \
375 type *inp_row = (type *)(in_rows[iy] + ix*bpp); \
376 for( int i=0; i<components; ++i ) \
377 *out_row++ = *inp_row++; \
381 switch( color_model ) {
382 case BC_RGB888: FLOW_OBJ_REMAP(unsigned char, 3); break;
383 case BC_RGBA8888: FLOW_OBJ_REMAP(unsigned char, 4); break;
384 case BC_RGB_FLOAT: FLOW_OBJ_REMAP(float, 3); break;
385 case BC_RGBA_FLOAT: FLOW_OBJ_REMAP(float, 4); break;
386 case BC_YUV888: FLOW_OBJ_REMAP(unsigned char, 3); break;
387 case BC_YUVA8888: FLOW_OBJ_REMAP(unsigned char, 4); break;
392 FlowObjRemapEngine::FlowObjRemapEngine(FlowObj *plugin, int cpus)
393 : LoadServer(cpus+1, cpus+1)
395 this->plugin = plugin;
398 FlowObjRemapEngine::~FlowObjRemapEngine()
402 void FlowObjRemapEngine::init_packages()
404 int row1 = 0, row2 = 0, n = LoadServer::get_total_packages();
405 for( int i=0; i<n; row1=row2 ) {
406 FlowObjRemapPackage *package = (FlowObjRemapPackage*)LoadServer::get_package(i);
407 row2 = plugin->get_input()->get_h() * ++i / n;
408 package->row1 = row1; package->row2 = row2;
412 LoadClient* FlowObjRemapEngine::new_client()
414 return new FlowObjRemapUnit(this, plugin);
417 LoadPackage* FlowObjRemapEngine::new_package()
419 return new FlowObjRemapPackage();