cinelerra-5.1/plugins/moveobj/moveobj.C

   1 /*
   2  * CINELERRA
   3  * Copyright (C) 1997-2014 Adam Williams <broadcast at earthling dot net>
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation; either version 2 of the License, or
   8  * (at your option) any later version.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13  * GNU General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License
  16  * along with this program; if not, write to the Free Software
  17  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  18  *
  19  */
  20
  21 #include "affine.h"
  22 #include "clip.h"
  23 #include "filexml.h"
  24 #include "moveobj.h"
  25 #include "moveobjwindow.h"
  26 #include "language.h"
  27 #include "transportque.inc"
  28
  29 REGISTER_PLUGIN(MoveObj)
  30
  31 #define MAX_COUNT 250
  32 #define WIN_SIZE 20
  33
  34 MoveObjConfig::MoveObjConfig()
  35 {
  36         draw_vectors = 0;
  37         do_stabilization = 1;
  38         block_size = 20;
  39         search_radius = 10;
  40         settling_speed = 5;
  41 }
  42
  43 int MoveObjConfig::equivalent(MoveObjConfig &that)
  44 {
  45         return draw_vectors == that.draw_vectors &&
  46                 do_stabilization == that.do_stabilization &&
  47                 block_size == that.block_size &&
  48                 search_radius == that.search_radius &&
  49                 settling_speed == that.settling_speed;
  50 }
  51
  52 void MoveObjConfig::copy_from(MoveObjConfig &that)
  53 {
  54         draw_vectors = that.draw_vectors;
  55         do_stabilization = that.do_stabilization;
  56         block_size = that.block_size;
  57         search_radius = that.search_radius;
  58         settling_speed = that.settling_speed;
  59 }
  60
  61 void MoveObjConfig::interpolate( MoveObjConfig &prev, MoveObjConfig &next,
  62         long prev_frame, long next_frame, long current_frame)
  63 {
  64         copy_from(next);
  65 }
  66
  67 void MoveObjConfig::limits()
  68 {
  69         bclamp(block_size, 5, 100);
  70         bclamp(search_radius, 1, 100);
  71         bclamp(settling_speed, 0, 100);
  72 }
  73
  74
  75 MoveObj::MoveObj(PluginServer *server)
  76  : PluginVClient(server)
  77 {
  78         affine = 0;
  79         prev_position = next_position = -1;
  80         x_accum = y_accum = 0;
  81         angle_accum = 0;
  82         prev_corners = 0;
  83         next_corners = 0;
  84 }
  85
  86 MoveObj::~MoveObj()
  87 {
  88         delete affine;
  89         delete prev_corners;
  90         delete next_corners;
  91 }
  92
  93 const char* MoveObj::plugin_title() { return N_("MoveObj"); }
  94 int MoveObj::is_realtime() { return 1; }
  95
  96 NEW_WINDOW_MACRO(MoveObj, MoveObjWindow);
  97 LOAD_CONFIGURATION_MACRO(MoveObj, MoveObjConfig)
  98
  99 void MoveObj::save_data(KeyFrame *keyframe)
 100 {
 101         FileXML output;
 102
 103 // cause data to be stored directly in text
 104         output.set_shared_output(keyframe->get_data(), MESSAGESIZE);
 105         output.tag.set_title("MOVEOBJ");
 106         output.tag.set_property("DRAW_VECTORS", config.draw_vectors);
 107         output.tag.set_property("DO_STABILIZATION", config.do_stabilization);
 108         output.tag.set_property("BLOCK_SIZE", config.block_size);
 109         output.tag.set_property("SEARCH_RADIUS", config.search_radius);
 110         output.tag.set_property("SETTLING_SPEED", config.settling_speed);
 111         output.append_tag();
 112         output.append_newline();
 113         output.tag.set_title("/MOVEOBJ");
 114         output.append_tag();
 115         output.append_newline();
 116         output.terminate_string();
 117 }
 118
 119 void MoveObj::read_data(KeyFrame *keyframe)
 120 {
 121         FileXML input;
 122         input.set_shared_input(keyframe->get_data(), strlen(keyframe->get_data()));
 123
 124         int result = 0;
 125         while( !(result = input.read_tag()) ) {
 126                 if( input.tag.title_is("MOVEOBJ") ) {
 127                         config.draw_vectors = input.tag.get_property("DRAW_VECTORS", config.draw_vectors);
 128                         config.do_stabilization = input.tag.get_property("DO_STABILIZATION", config.do_stabilization);
 129                         config.block_size = input.tag.get_property("BLOCK_SIZE", config.block_size);
 130                         config.search_radius = input.tag.get_property("SEARCH_RADIUS", config.search_radius);
 131                         config.settling_speed = input.tag.get_property("SETTLING_SPEED", config.settling_speed);
 132                         config.limits();
 133                 }
 134                 else if( input.tag.title_is("/MOVEOBJ") )
 135                         result = 1;
 136         }
 137 }
 138
 139 void MoveObj::update_gui()
 140 {
 141         if( !thread ) return;
 142         if( !load_configuration() ) return;
 143         thread->window->lock_window("MoveObj::update_gui");
 144         MoveObjWindow *window = (MoveObjWindow*)thread->window;
 145
 146         window->vectors->update(config.draw_vectors);
 147         window->do_stabilization->update(config.do_stabilization);
 148         window->block_size->update(config.block_size);
 149         window->search_radius->update(config.search_radius);
 150         window->settling_speed->update(config.settling_speed);
 151
 152         thread->window->unlock_window();
 153 }
 154
 155 void MoveObj::to_mat(Mat &mat, int mcols, int mrows,
 156         VFrame *inp, int ix,int iy, int mcolor_model)
 157 {
 158         int mcomp = BC_CModels::components(mcolor_model);
 159         int mbpp = BC_CModels::calculate_pixelsize(mcolor_model);
 160         int psz = mbpp / mcomp;
 161         int mdepth = psz < 2 ? CV_8U : psz < 4 ? CV_16U : CV_32F;
 162         if( mat.dims != 2 || mat.depth() != mdepth || mat.channels() != mcomp ||
 163             mat.cols != mcols || mat.rows != mrows ) {
 164                 mat.release();
 165         }
 166         if( mat.empty() ) {
 167                 int type = CV_MAKETYPE(mdepth, mcomp);
 168                 mat.create(mrows, mcols, type);
 169         }
 170         uint8_t *mat_rows[mrows];
 171         for( int y=0; y<mrows; ++y ) mat_rows[y] = mat.ptr(y);
 172         uint8_t **inp_rows = inp->get_rows();
 173         int ibpl = inp->get_bytes_per_line(), obpl = mcols * mbpp;
 174         int icolor_model = inp->get_color_model();
 175         BC_CModels::transfer(mat_rows, mcolor_model, 0,0, mcols,mrows, obpl,
 176                 inp_rows, icolor_model, ix,iy, mcols,mrows, ibpl, 0);
 177 //      VFrame vfrm(mat_rows[0], -1, mcols,mrows, mcolor_model, mat_rows[1]-mat_rows[0]);
 178 //      static int vfrm_no = 0; char vfn[64]; sprintf(vfn,"/tmp/dat/%06d.png", vfrm_no++);
 179 //      vfrm.write_png(vfn);
 180 }
 181
 182 int MoveObj::process_buffer(VFrame *frame, int64_t start_position, double frame_rate)
 183 {
 184
 185         //int need_reconfigure =
 186         load_configuration();
 187         VFrame *input = get_input(0), *output = get_output(0);
 188         int w = input->get_w(), h = input->get_h();
 189         int color_model = input->get_color_model();
 190
 191         if( accum_matrix.empty() ) {
 192                 accum_matrix = Mat::eye(3,3, CV_64F);
 193         }
 194         if( !affine ) {
 195                 int cpus1 = PluginClient::get_project_smp() + 1;
 196                 affine = new AffineEngine(cpus1, cpus1);
 197         }
 198         if( !prev_corners ) prev_corners = new ptV();
 199         if( !next_corners ) next_corners = new ptV();
 200
 201 // Get the position of previous reference frame.
 202         int64_t actual_previous_number = start_position;
 203         int skip_current = 0;
 204         if( get_direction() == PLAY_REVERSE ) {
 205                 if( ++actual_previous_number < get_source_start() + get_total_len() ) {
 206                         KeyFrame *keyframe = get_next_keyframe(start_position, 1);
 207                         if( keyframe->position > 0 &&
 208                             actual_previous_number >= keyframe->position )
 209                                 skip_current = 1;
 210                 }
 211                 else
 212                         skip_current = 1;
 213         }
 214         else {
 215                 if( --actual_previous_number >= get_source_start() ) {
 216                         KeyFrame *keyframe = get_prev_keyframe(start_position, 1);
 217                         if( keyframe->position > 0 &&
 218                             actual_previous_number < keyframe->position)
 219                                 skip_current = 1;
 220                 }
 221                 else
 222                         skip_current = 1;
 223         }
 224
 225
 226 // move currrent image to previous position
 227         if( next_position >= 0 && next_position == actual_previous_number ) {
 228                 Mat mat = prev_mat;  prev_mat = next_mat;  next_mat = mat;
 229                 ptV *pts = prev_corners;  prev_corners = next_corners;  next_corners = pts;
 230                 prev_position = next_position;
 231         }
 232         else
 233 // load previous image
 234         if( actual_previous_number >= 0 ) {
 235                 read_frame(input, 0, actual_previous_number, frame_rate, 0);
 236                 to_mat(prev_mat, w,h, input, 0,0, BC_GREY8);
 237         }
 238
 239         if( skip_current || prev_position != actual_previous_number ) {
 240                 skip_current = 1;
 241                 accum_matrix = Mat::eye(3,3, CV_64F);
 242         }
 243
 244
 245 // load next image
 246         next_position = start_position;
 247         VFrame *iframe = !config.do_stabilization ? input : new_temp(w,h, color_model);
 248         read_frame(iframe, 0, start_position, frame_rate, 0);
 249         to_mat(next_mat, w,h, iframe, 0,0, BC_GREY8);
 250
 251         int corner_count = MAX_COUNT;
 252         int block_size = config.block_size;
 253         int min_distance = config.search_radius;
 254
 255         goodFeaturesToTrack(next_mat,
 256                 *next_corners, corner_count, 0.01,        // quality_level
 257                 min_distance, noArray(), block_size,
 258                 false,       // use_harris
 259                 0.04);       // k
 260
 261         ptV pt1, pt2;
 262         if( !next_mat.empty() && next_corners->size() > 3 ) {
 263                 cornerSubPix(next_mat, *next_corners, Size(WIN_SIZE, WIN_SIZE), Size(-1,-1),
 264                         cvTermCriteria(CV_TERMCRIT_ITER | CV_TERMCRIT_EPS, 20, 0.03));
 265         }
 266         if( !prev_mat.empty() && prev_corners->size() > 3 ) {
 267 // optical flow
 268                 Mat st, err;
 269                 ptV &prev = *prev_corners, &next = *next_corners;
 270                 calcOpticalFlowPyrLK(prev_mat, next_mat, prev, next,
 271                         st, err, Size(WIN_SIZE, WIN_SIZE), 5,
 272                         cvTermCriteria(CV_TERMCRIT_ITER | CV_TERMCRIT_EPS, 20, 0.3), 0);
 273                 float fails = 0.5 * w + 1;
 274                 uint8_t *stp = st.ptr<uint8_t>();
 275                 float *errp = err.ptr<float>();
 276                 for( int i=0,n=next_corners->size(); i<n; ++i ) {
 277                         if( stp[i] == 0 || errp[i] > fails ) continue;
 278                         pt1.push_back(next[i]);
 279                         pt2.push_back(prev[i]);
 280                 }
 281         }
 282
 283         int points = pt1.size();
 284         if( points > 0 && !skip_current ) {
 285                 if( config.draw_vectors ) {
 286                         int sz = bmin(w,h) / 222 + 2;
 287                         for( int i = 0; i < points; ++i )
 288                                 iframe->draw_arrow(pt1[i].x,pt1[i].y, pt2[i].x,pt2[i].y, sz);
 289                 }
 290 #ifdef _RANSAC
 291 // ransac
 292                 int ninliers = 0;
 293                 Mat_<float> translationM =
 294                         estimateGlobalMotionRansac(pt1, pt2, MM_TRANSLATION,
 295                                 RansacParams::default2dMotion(MM_TRANSLATION),
 296                                 0, &ninliers);
 297                 Mat_<float> rotationM =
 298                         estimateGlobalMotionRansac(pt1, pt2, MM_ROTATION,
 299                                 RansacParams::default2dMotion(MM_ROTATION),
 300                                 0, &ninliers);
 301
 302                 double temp[9];
 303                 Mat temp_matrix = Mat(3, 3, CV_64F, temp);
 304                 for( int i=0; i<9; ++i )
 305                         temp[i] = i == 2 || i == 5 ?
 306                                 translationM(i / 3, i % 3) :
 307                                 rotationM(i / 3, i % 3);
 308                 accum_matrix = temp_matrix * accum_matrix;
 309 #else
 310 // homography
 311
 312                 Mat M1(1, points, CV_32FC2, &pt1[0].x);
 313                 Mat M2(1, points, CV_32FC2, &pt2[0].x);
 314
 315 //M2 = H*M1 , old = H*current
 316                 Mat H = findHomography(M1, M2, CV_RANSAC, 2);
 317                 if( !H.dims || !H.rows || !H.cols )
 318                         printf("MoveObj::process_buffer %d: Find Homography Fail!\n", __LINE__);
 319                 else
 320                         accum_matrix = H * accum_matrix;
 321 #endif
 322         }
 323
 324         double *amat = accum_matrix.ptr<double>();
 325 // deglitch
 326 //      if( EQUIV(amat[0], 0) ) {
 327 //printf("MoveObj::process_buffer %d\n", __LINE__);
 328 //              accum_matrix = Mat::eye(3,3, CV_64F);
 329 //      }
 330
 331         if( config.do_stabilization ) {
 332                 Mat identity = Mat::eye(3,3, CV_64F);
 333                 double w0 = config.settling_speed/100., w1 = 1.-w0;
 334 // interpolate with identity matrix
 335                 accum_matrix = w0*identity + w1*accum_matrix;
 336
 337                 AffineMatrix &matrix = affine->matrix;
 338                 for( int i=0,k=0; i<3; ++i )
 339                         for( int j=0; j<3; ++j )
 340                                 matrix.values[i][j] = amat[k++];
 341
 342 //printf("MoveObj::process_buffer %d %jd matrix=\n", __LINE__, start_position);
 343 //matrix.dump();
 344
 345 // iframe is always temp, if we get here
 346                 output->clear_frame();
 347                 affine->process(output, iframe, 0, AffineEngine::TRANSFORM,
 348                         0,0, w,0, w,h, 0,h, 1);
 349         }
 350
 351         return 0;
 352 }