cinelerra-5.1/plugins/flowobj/flowobj.C

   1 /*
   2  * CINELERRA
   3  * Copyright (C) 1997-2014 Adam Williams <broadcast at earthling dot net>
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation; either version 2 of the License, or
   8  * (at your option) any later version.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13  * GNU General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License
  16  * along with this program; if not, write to the Free Software
  17  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  18  *
  19  */
  20
  21 #include "clip.h"
  22 #include "filexml.h"
  23 #include "flowobj.h"
  24 #include "flowobjwindow.h"
  25 #include "language.h"
  26 #include "transportque.inc"
  27
  28 REGISTER_PLUGIN(FlowObj)
  29
  30 #define MAX_COUNT 250
  31 #define WIN_SIZE 20
  32
  33 FlowObjConfig::FlowObjConfig()
  34 {
  35         draw_vectors = 0;
  36         do_stabilization = 1;
  37         block_size = 20;
  38         search_radius = 10;
  39         settling_speed = 5;
  40 }
  41
  42 int FlowObjConfig::equivalent(FlowObjConfig &that)
  43 {
  44         return draw_vectors == that.draw_vectors &&
  45                 do_stabilization == that.do_stabilization &&
  46                 block_size == that.block_size &&
  47                 search_radius == that.search_radius &&
  48                 settling_speed == that.settling_speed;
  49 }
  50
  51 void FlowObjConfig::copy_from(FlowObjConfig &that)
  52 {
  53         draw_vectors = that.draw_vectors;
  54         do_stabilization = that.do_stabilization;
  55         block_size = that.block_size;
  56         search_radius = that.search_radius;
  57         settling_speed = that.settling_speed;
  58 }
  59
  60 void FlowObjConfig::interpolate( FlowObjConfig &prev, FlowObjConfig &next,
  61         long prev_frame, long next_frame, long current_frame)
  62 {
  63         copy_from(next);
  64 }
  65
  66 void FlowObjConfig::limits()
  67 {
  68         bclamp(block_size, 5, 100);
  69         bclamp(search_radius, 1, 100);
  70         bclamp(settling_speed, 0, 100);
  71 }
  72
  73
  74 FlowObj::FlowObj(PluginServer *server)
  75  : PluginVClient(server)
  76 {
  77         flow_engine = 0;
  78         overlay = 0;
  79         accum = 0;
  80         prev_position = next_position = -1;
  81         x_accum = y_accum = 0;
  82         angle_accum = 0;
  83         prev_corners = 0;
  84         next_corners = 0;
  85 }
  86
  87 FlowObj::~FlowObj()
  88 {
  89         delete flow_engine;
  90         delete overlay;
  91         delete prev_corners;
  92         delete next_corners;
  93 }
  94
  95 const char* FlowObj::plugin_title() { return N_("FlowObj"); }
  96 int FlowObj::is_realtime() { return 1; }
  97
  98 NEW_WINDOW_MACRO(FlowObj, FlowObjWindow);
  99 LOAD_CONFIGURATION_MACRO(FlowObj, FlowObjConfig)
 100
 101 void FlowObj::save_data(KeyFrame *keyframe)
 102 {
 103         FileXML output;
 104
 105 // cause data to be stored directly in text
 106         output.set_shared_output(keyframe->xbuf);
 107         output.tag.set_title("FLOWOBJ");
 108         output.tag.set_property("DRAW_VECTORS", config.draw_vectors);
 109         output.tag.set_property("DO_STABILIZATION", config.do_stabilization);
 110         output.tag.set_property("BLOCK_SIZE", config.block_size);
 111         output.tag.set_property("SEARCH_RADIUS", config.search_radius);
 112         output.tag.set_property("SETTLING_SPEED", config.settling_speed);
 113         output.append_tag();
 114         output.append_newline();
 115         output.tag.set_title("/FLOWOBJ");
 116         output.append_tag();
 117         output.append_newline();
 118         output.terminate_string();
 119 }
 120
 121 void FlowObj::read_data(KeyFrame *keyframe)
 122 {
 123         FileXML input;
 124         input.set_shared_input(keyframe->xbuf);
 125
 126         int result = 0;
 127         while( !(result = input.read_tag()) ) {
 128                 if( input.tag.title_is("FLOWOBJ") ) {
 129                         config.draw_vectors = input.tag.get_property("DRAW_VECTORS", config.draw_vectors);
 130                         config.do_stabilization = input.tag.get_property("DO_STABILIZATION", config.do_stabilization);
 131                         config.block_size = input.tag.get_property("BLOCK_SIZE", config.block_size);
 132                         config.search_radius = input.tag.get_property("SEARCH_RADIUS", config.search_radius);
 133                         config.settling_speed = input.tag.get_property("SETTLING_SPEED", config.settling_speed);
 134                         config.limits();
 135                 }
 136                 else if( input.tag.title_is("/FLOWOBJ") )
 137                         result = 1;
 138         }
 139 }
 140
 141 void FlowObj::update_gui()
 142 {
 143         if( !thread ) return;
 144         if( !load_configuration() ) return;
 145         thread->window->lock_window("FlowObj::update_gui");
 146         FlowObjWindow *window = (FlowObjWindow*)thread->window;
 147
 148         window->vectors->update(config.draw_vectors);
 149         window->do_stabilization->update(config.do_stabilization);
 150         window->block_size->update(config.block_size);
 151         window->search_radius->update(config.search_radius);
 152         window->settling_speed->update(config.settling_speed);
 153
 154         thread->window->unlock_window();
 155 }
 156
 157 void FlowObj::to_mat(Mat &mat, int mcols, int mrows,
 158         VFrame *inp, int ix,int iy, int mcolor_model)
 159 {
 160         int mcomp = BC_CModels::components(mcolor_model);
 161         int mbpp = BC_CModels::calculate_pixelsize(mcolor_model);
 162         int psz = mbpp / mcomp;
 163         int mdepth = psz < 2 ? CV_8U : psz < 4 ? CV_16U : CV_32F;
 164         if( mat.dims != 2 || mat.depth() != mdepth || mat.channels() != mcomp ||
 165             mat.cols != mcols || mat.rows != mrows ) {
 166                 mat.release();
 167         }
 168         if( mat.empty() ) {
 169                 int type = CV_MAKETYPE(mdepth, mcomp);
 170                 mat.create(mrows, mcols, type);
 171         }
 172         uint8_t *mat_rows[mrows];
 173         for( int y=0; y<mrows; ++y ) mat_rows[y] = mat.ptr(y);
 174         uint8_t **inp_rows = inp->get_rows();
 175         int ibpl = inp->get_bytes_per_line(), mbpl = mcols * mbpp;
 176         int icolor_model = inp->get_color_model();
 177         BC_CModels::transfer(mat_rows, mcolor_model, 0,0, mcols,mrows, mbpl,
 178                 inp_rows, icolor_model, ix,iy, mcols,mrows, ibpl, 0);
 179 //      VFrame vfrm(mat_rows[0], -1, mcols,mrows, mcolor_model, mat_rows[1]-mat_rows[0]);
 180 //      static int vfrm_no = 0; char vfn[64]; sprintf(vfn,"/tmp/idat/%06d.png", vfrm_no++);
 181 //      vfrm.write_png(vfn);
 182 }
 183
 184 void FlowObj::from_mat(VFrame *out, int ox, int oy, int ow, int oh, Mat &mat, int mcolor_model)
 185 {
 186         int mbpp = BC_CModels::calculate_pixelsize(mcolor_model);
 187         int mrows = mat.rows, mcols = mat.cols;
 188         uint8_t *mat_rows[mrows];
 189         for( int y=0; y<mrows; ++y ) mat_rows[y] = mat.ptr(y);
 190         uint8_t **out_rows = out->get_rows();
 191         int obpl = out->get_bytes_per_line(), mbpl = mcols * mbpp;
 192         int ocolor_model = out->get_color_model();
 193         BC_CModels::transfer(out_rows, ocolor_model, ox,oy, ow,oh, obpl,
 194                 mat_rows, mcolor_model, 0,0, mcols,mrows, mbpl,  0);
 195 //      static int vfrm_no = 0; char vfn[64]; sprintf(vfn,"/tmp/odat/%06d.png", vfrm_no++);
 196 //      out->write_png(vfn);
 197 }
 198
 199
 200 int FlowObj::process_buffer(VFrame *frame, int64_t start_position, double frame_rate)
 201 {
 202
 203         //int need_reconfigure =
 204         load_configuration();
 205         input = get_input(0);
 206         output = get_output(0);
 207         width = input->get_w();
 208         height = input->get_h();
 209         color_model = input->get_color_model();
 210
 211         if( accum_matrix.empty() ) {
 212                 accum_matrix = Mat::eye(3,3, CV_64F);
 213         }
 214         if( !prev_corners ) prev_corners = new ptV();
 215         if( !next_corners ) next_corners = new ptV();
 216
 217 // Get the position of previous reference frame.
 218         int64_t actual_previous_number = start_position;
 219         int skip_current = 0;
 220         if( get_direction() == PLAY_REVERSE ) {
 221                 if( ++actual_previous_number < get_source_start() + get_total_len() ) {
 222                         KeyFrame *keyframe = get_next_keyframe(start_position, 1);
 223                         if( keyframe->position > 0 &&
 224                             actual_previous_number >= keyframe->position )
 225                                 skip_current = 1;
 226                 }
 227                 else
 228                         skip_current = 1;
 229         }
 230         else {
 231                 if( --actual_previous_number >= get_source_start() ) {
 232                         KeyFrame *keyframe = get_prev_keyframe(start_position, 1);
 233                         if( keyframe->position > 0 &&
 234                             actual_previous_number < keyframe->position)
 235                                 skip_current = 1;
 236                 }
 237                 else
 238                         skip_current = 1;
 239         }
 240
 241
 242 // move currrent image to previous position
 243         if( next_position >= 0 && next_position == actual_previous_number ) {
 244                 Mat mat = prev_mat;  prev_mat = next_mat;  next_mat = mat;
 245                 ptV *pts = prev_corners;  prev_corners = next_corners;  next_corners = pts;
 246                 prev_position = next_position;
 247         }
 248         else
 249 // load previous image
 250         if( actual_previous_number >= 0 ) {
 251                 read_frame(input, 0, actual_previous_number, frame_rate, 0);
 252                 to_mat(prev_mat, width,height, input, 0,0, BC_GREY8);
 253         }
 254
 255         if( skip_current || prev_position != actual_previous_number ) {
 256                 skip_current = 1;
 257                 accum_matrix = Mat::eye(3,3, CV_64F);
 258         }
 259
 260
 261 // load next image
 262         next_position = start_position;
 263         VFrame *iframe = new_temp(width,height, color_model);
 264         read_frame(iframe, 0, start_position, frame_rate, 0);
 265         input = iframe;
 266         to_mat(next_mat, width,height, iframe, 0,0, BC_GREY8);
 267
 268         int corner_count = MAX_COUNT;
 269         int block_size = config.block_size;
 270         int min_distance = config.search_radius;
 271
 272         goodFeaturesToTrack(next_mat,
 273                 *next_corners, corner_count, 0.01,        // quality_level
 274                 min_distance, noArray(), block_size,
 275                 false,       // use_harris
 276                 0.04);       // k
 277
 278         if( !next_mat.empty() && next_corners->size() > 3 ) {
 279                 cornerSubPix(next_mat, *next_corners, Size(WIN_SIZE, WIN_SIZE), Size(-1,-1),
 280                         cvTermCriteria(CV_TERMCRIT_ITER | CV_TERMCRIT_EPS, 20, 0.03));
 281         }
 282
 283         Mat flow;
 284         if( !prev_mat.empty() && prev_corners->size() > 3 ) {
 285 // optical flow
 286                 calcOpticalFlowFarneback(prev_mat, next_mat, flow, 0.5, 3, 15, 3, 5, 1.2, 0);
 287         }
 288
 289         if( config.do_stabilization && !flow.empty() ) {
 290                 if( !flow_engine )
 291                         flow_engine = new FlowObjRemapEngine(this, PluginClient::smp);
 292                 flow_mat = &flow;
 293                 flow_engine->process_packages();
 294         }
 295         else
 296                 output->copy_from(input);
 297
 298         if( config.settling_speed > 0 ) {
 299                 if( accum && (accum->get_color_model() != color_model ||
 300                     accum->get_w() != width || accum->get_h() != height) ) {
 301                         delete accum;  accum = 0;
 302                 }
 303                 if( !accum ) {
 304                         accum = new VFrame(width, height, color_model, 0);
 305                         accum->clear_frame();
 306                 }
 307                 if( !overlay )
 308                         overlay = new OverlayFrame(PluginClient::smp + 1);
 309                 float alpha = 1 - config.settling_speed/100.;
 310                 overlay->overlay(accum, output,
 311                         0,0, width,height, 0,0, width, height,
 312                         alpha, TRANSFER_NORMAL, NEAREST_NEIGHBOR);
 313                 output->copy_from(accum);
 314         }
 315
 316         if( !flow.empty() && config.draw_vectors ) {
 317                 output->set_pixel_color(GREEN);
 318                 int nv = 24; float s = 2;
 319                 int gw = width/nv + 1, gh = height/nv + 1;
 320                 int sz = bmin(width,height) / 200 + 4;
 321                 for( int y=gh/2; y<height; y+=gh ) {
 322                         Point2f *row = (Point2f *)flow.ptr(y);
 323                         for( int x=gw/2; x<width; x+=gw ) {
 324                                 Point2f *ds = row + x;
 325                                 float x0 = x+.5, x1 = x+s*ds->x, y0 = y+.5, y1 = y+s*ds->y;
 326                                 output->draw_arrow(x0,y0, x1,y1, sz);
 327                         }
 328                 }
 329         }
 330
 331         return 0;
 332 }
 333
 334
 335 FlowObjRemapPackage::FlowObjRemapPackage()
 336  : LoadPackage()
 337 {
 338 }
 339
 340 FlowObjRemapUnit::FlowObjRemapUnit(FlowObjRemapEngine *engine, FlowObj *plugin)
 341  : LoadClient(engine)
 342 {
 343         this->plugin = plugin;
 344 }
 345
 346 FlowObjRemapUnit::~FlowObjRemapUnit()
 347 {
 348 }
 349
 350 void FlowObjRemapUnit::process_package(LoadPackage *package)
 351 {
 352         FlowObjRemapPackage *pkg = (FlowObjRemapPackage*)package;
 353         process_remap(pkg);
 354 }
 355
 356 void FlowObjRemapUnit::process_remap(FlowObjRemapPackage *pkg)
 357 {
 358         int row1 = pkg->row1;
 359         int row2 = pkg->row2;
 360         int w = plugin->width, h = plugin->height;
 361         int w1 = w-1, h1 = h-1;
 362         int color_model = plugin->color_model;
 363         int bpp = BC_CModels::calculate_pixelsize(color_model);
 364
 365 #define FLOW_OBJ_REMAP(type, components) { \
 366         unsigned char **in_rows = plugin->input->get_rows(); \
 367         type **out_rows = (type**)plugin->output->get_rows(); \
 368  \
 369         for( int y=row1; y<row2; ++y ) { \
 370                 Point2f *ipt = (Point2f *)plugin->flow_mat->ptr(y); \
 371                 type *out_row = out_rows[y]; \
 372                 for( int x=0; x<w; ++x,++ipt ) { \
 373                         int ix = x - ipt->x, iy = y - ipt->y; \
 374                         bclamp(ix, 0, w1);  bclamp(iy, 0, h1); \
 375                         type *inp_row = (type *)(in_rows[iy] + ix*bpp); \
 376                         for( int i=0; i<components; ++i ) \
 377                                 *out_row++ = *inp_row++; \
 378                 } \
 379         } \
 380 }
 381         switch( color_model ) {
 382         case BC_RGB888:     FLOW_OBJ_REMAP(unsigned char, 3);  break;
 383         case BC_RGBA8888:   FLOW_OBJ_REMAP(unsigned char, 4);  break;
 384         case BC_RGB_FLOAT:  FLOW_OBJ_REMAP(float, 3);          break;
 385         case BC_RGBA_FLOAT: FLOW_OBJ_REMAP(float, 4);          break;
 386         case BC_YUV888:     FLOW_OBJ_REMAP(unsigned char, 3);  break;
 387         case BC_YUVA8888:   FLOW_OBJ_REMAP(unsigned char, 4);  break;
 388         }
 389 }
 390
 391
 392 FlowObjRemapEngine::FlowObjRemapEngine(FlowObj *plugin, int cpus)
 393  : LoadServer(cpus+1, cpus+1)
 394 {
 395         this->plugin = plugin;
 396 }
 397
 398 FlowObjRemapEngine::~FlowObjRemapEngine()
 399 {
 400 }
 401
 402 void FlowObjRemapEngine::init_packages()
 403 {
 404         int row1 = 0, row2 = 0, n = LoadServer::get_total_packages();
 405         for( int i=0; i<n; row1=row2 ) {
 406                 FlowObjRemapPackage *package = (FlowObjRemapPackage*)LoadServer::get_package(i);
 407                 row2 = plugin->get_input()->get_h() * ++i / n;
 408                 package->row1 = row1;  package->row2 = row2;
 409         }
 410 }
 411
 412 LoadClient* FlowObjRemapEngine::new_client()
 413 {
 414         return new FlowObjRemapUnit(this, plugin);
 415 }
 416
 417 LoadPackage* FlowObjRemapEngine::new_package()
 418 {
 419         return new FlowObjRemapPackage();
 420 }
 421