3 * Copyright (C) 2016-2020 William Morrow
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License as published
7 * by the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
10 * This program is distributed in the hope that it will be useful, but
11 * WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 * General Public License for more details.
15 * You should have received a copy of the GNU General Public
16 * License along with this program; if not, write to the Free Software
17 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
29 #include "../db/tdb.h"
38 void write_pgm(uint8_t *tp, int w, int h, const char *fmt, ...)
40 va_list ap; va_start(ap, fmt);
41 char fn[256]; vsnprintf(fn, sizeof(fn), fmt, ap);
43 FILE *fp = !strcmp(fn,"-") ? stdout : fopen(fn,"w");
45 fprintf(fp,"P5\n%d %d\n255\n",w,h);
51 void write_ppm(uint8_t *tp, int w, int h, const char *fmt, ...)
53 va_list ap; va_start(ap, fmt);
54 char fn[256]; vsnprintf(fn, sizeof(fn), fmt, ap);
56 FILE *fp = !strcmp(fn,"-") ? stdout : fopen(fn,"w");
58 fprintf(fp,"P6\n%d %d\n255\n",w,h);
64 static inline int clip(int v, int mn, int mx)
66 return v<mn ? mn : v>mx ? mx : v;
69 void pbm_diff(uint8_t *ap, uint8_t *bp, int w, int h, const char *fmt, ...)
71 va_list va; va_start(va, fmt);
72 char fn[256]; vsnprintf(fn, sizeof(fn), fmt, va);
74 FILE *fp = !strcmp(fn,"-") ? stdout : fopen(fn,"w");
77 uint8_t dat[sz], *dp = dat;
78 for( int i=sz; --i>=0; ++dp,++ap,++bp ) *dp = clip(*ap-*bp+128, 0 ,255);
79 fprintf(fp,"P5\n%d %d\n255\n",w,h);
87 scale(uint8_t *odata, int ow, int oh, int sx, int sy, int sw, int sh)
89 pw = (double)(ow * oh) / (rw * rh);
90 odat = odata + sy*ow + sx;
94 for( int dy=1; dy<=sh; ++dy, odat+=ow ) {
95 iy0 = iy1; yf0 = 1.0 - yf1;
96 double ny = (double)(dy * rh) / oh;
97 iy1 = (int) ny; yf1 = ny-iy1;
101 for( int dx=1; dx<=sw; ++dx ) {
102 ix0 = ix1; xf0 = 1.0 - xf1;
103 double nx = (double)(dx*rw) / ow;
104 ix1 = (int)nx; xf1 = nx-ix1;
105 double px = scalexy();
115 Deletions(int pid, const char *fn)
118 strcpy(this->filepath,fn);
131 if( xml.read_tag() != 0 ) return 1;
132 if( !xml.tag.title_is("DEL") ) break;
133 double time = xml.tag.get_property("TIME", (double)0.0);
134 int action = xml.tag.get_property("ACTION", (int)0);
135 append(new Dele(time, action));
140 Deletions *Deletions::
141 read_dels(const char *filename)
144 if( xml.read_from_file(filename, 1) ) return 0;
146 if( xml.read_tag() ) return 0;
147 } while( !xml.tag.title_is("DELS") );
149 int pid = xml.tag.get_property("PID", (int)-1);
150 const char *file = xml.tag.get_property("FILE");
151 Deletions *dels = new Deletions(pid, file);
152 if( dels->load(xml) || !xml.tag.title_is("/DELS") ) {
153 delete dels; dels = 0;
161 xml.tag.set_title("DELS");
162 xml.tag.set_property("PID", pid);
163 xml.tag.set_property("FILE", filepath);
165 xml.append_newline();
167 for( Dele *del=first; del; del=del->next ) {
168 xml.tag.set_title("DEL");
169 xml.tag.set_property("TIME", del->time);
170 xml.tag.set_property("ACTION", del->action);
172 xml.tag.set_title("/DEL");
174 xml.append_newline();
177 xml.tag.set_title("/DELS");
179 xml.append_newline();
184 write_dels(const char *filename)
186 FILE *fp = fopen(filename, "wb");
187 if( !fp != 0 ) return 1;
190 xml.terminate_string();
191 xml.write_to_file(fp);
198 Clip(int clip_id, double start, double end, int index)
200 this->clip_id = clip_id;
214 Clips *clips = (Clips *)list;
215 if( clips && clips->current == this ) clips->current = 0;
218 Clip *Clips::locate(int id, double start, double end)
220 if( current && current->clip_id == id &&
221 end >= current->start && start < current->end )
223 for( current=first; current!=0; current=current->next ) {
224 if( current->clip_id != id ) continue;
225 if( current->start <= end && current->end > start ) break;
245 new_clip(int clip_id, double start, double end, int index)
247 return append(new Clip(clip_id, start, end, index));
251 get_clip(int idx, int &id, double &start, double &end)
253 int votes = -1, cid = -1;
254 double st = 0, et = 0;
256 for( Clip *clip=first; clip!=0; clip=clip->next ) {
257 if( clip->index != idx ) continue;
258 // must have prefix+suffix frames
259 if( clip->groups != 3 ) continue;
260 if( clip->votes < votes ) continue;
261 if( clip->votes > votes ) {
262 // first/better match
269 // reject if not overlapping
270 if( clip->start > et ) continue;
271 if( clip->end < st ) continue;
272 // expand range on overlap
273 if( clip->start < st ) st = clip->start;
274 if( clip->end > et ) et = clip->end;
278 start = st; end = et; id = cid;
286 check(int clip_id, double start, double end, int group, double err)
288 Clip *clip = locate(clip_id, start, end);
290 if( err < clip->best ) {
292 if( start < clip->start ) clip->start = start;
293 if( end > clip->end ) clip->end = end;
297 clip = new_clip(clip_id, start, end, count++);
298 clip->groups |= group;
299 if( cycle != clip->cycle ) {
302 //printf("clip %d votes %d\n",clip_id,clip->votes);
310 xml.tag.set_title("CLIPS");
311 xml.tag.set_property("PID", pid);
313 xml.append_newline();
315 for( int idx=0; idx<count; ++idx ) {
316 int id; double start, end;
317 if( !get_clip(idx, id, start, end) ) continue;
318 xml.tag.set_title("CLIP");
319 xml.tag.set_property("ID", id);
320 xml.tag.set_property("START", start);
321 xml.tag.set_property("END", end);
323 xml.tag.set_title("/CLIP");
325 xml.append_newline();
328 xml.tag.set_title("/CLIPS");
330 xml.append_newline();
339 if( xml.read_tag() != 0 ) return 1;
340 if( !xml.tag.title_is("CLIP") ) break;
341 int clip_id = xml.tag.get_property("ID", (int)0);
342 double start = xml.tag.get_property("START", (double)0.0);
343 double end = xml.tag.get_property("END", (double)0.0);
344 append(new Clip(clip_id, start, end, i++));
352 Snip(int clip_id, double start, double end, int index)
353 : Clip(clip_id, start, end, index)
373 new_clip(int clip_id, double start, double end, int index)
375 return append(new Snip(clip_id, start, end, index));
389 if( opened ) closeDb();
393 int MediaDb::is_open() { return opened > 0 ? opened : 0; }
394 int MediaDb::transaction() { return db->transaction(); }
396 // clip_set accessors
397 int MediaDb::clip_id(int id) { return db->clip_set.FindId(id); }
398 int MediaDb::clip_id() { return db->clip_set.id(); }
399 int MediaDb::clips_first_id() { return db->clip_set.FirstId(clip_id_loc); }
400 int MediaDb::clips_next_id() { return db->clip_set.NextId(clip_id_loc); }
401 const char *MediaDb::clip_title() { return db->clip_set.Title(); }
402 const char *MediaDb::clip_path() { return db->clip_set.Asset_path(); }
403 double MediaDb::clip_position() { return db->clip_set.Position(); }
404 double MediaDb::clip_framerate() { return db->clip_set.Framerate(); }
405 double MediaDb::clip_average_weight() { return db->clip_set.Average_weight(); }
406 void MediaDb::clip_average_weight(double avg_wt) { return db->clip_set.Average_weight(avg_wt); }
407 int MediaDb::clip_frames() { return db->clip_set.Frames(); }
408 double MediaDb::clip_length() {
409 return clip_framerate()>0 ? clip_frames() / clip_framerate() : 0;
411 int MediaDb::clip_prefix_size() { return db->clip_set.Prefix_size(); }
412 int MediaDb::clip_suffix_size() { return db->clip_set.Suffix_size(); }
413 int MediaDb::clip_size() { return clip_prefix_size()+clip_suffix_size(); }
414 double *MediaDb::clip_weights() { return (double *)db->clip_set._Weights(); }
415 int64_t MediaDb::clip_creation_time() { return db->clip_set.Creation_time(); }
416 int64_t MediaDb::clip_system_time() { return db->clip_set.System_time(); }
418 // MediaDb::timeline accessors
419 int MediaDb::timeline_id(int id) { return db->timeline.FindId(id); }
420 int MediaDb::timeline_id() { return db->timeline.id(); }
421 int MediaDb::timeline_clip_id() { return db->timeline.Clip_id(); }
422 int MediaDb::timeline_sequence_no() { return db->timeline.Sequence_no(); }
423 int MediaDb::timeline_frame_id() { return db->timeline.Frame_id(); }
424 int MediaDb::timeline_group() { return db->timeline.Group(); }
425 double MediaDb::timeline_offset() { return db->timeline.Time_offset(); }
427 // MediaDb::clip_view accessors
428 int MediaDb::views_id(int id) { return db->clip_views.FindId(id); }
429 int MediaDb::views_id() { return db->clip_views.id(); }
430 int MediaDb::views_clip_id(int id) {
431 return Clip_viewsLoc::ikey_Clip_access(db->clip_views,id).Find();
433 int MediaDb::views_clip_id() { return db->clip_views.Access_clip_id(); }
434 int64_t MediaDb::views_access_time() { return db->clip_views.Access_time(); }
435 int MediaDb::views_access_count() { return db->clip_views.Access_count(); }
439 mean(uint8_t *dat, int n, int stride)
442 for( int i=0; i<n; ++i, dat+=stride ) s += *dat;
443 return (double)s / n;
447 variance(uint8_t *dat, double m, int n, int stride)
449 double ss = 0, s = 0;
450 for( int i=0; i<n; ++i, dat+=stride ) {
452 s += dx; ss += dx*dx;
454 return (ss - s*s / n) / (n-1);
458 centroid(uint8_t *dat, int n, int stride)
461 for( int i=0; i<n; dat+=stride ) { s += *dat; ss += ++i * *dat; }
462 return s > 0 ? (double)ss / s : n / 2.;
466 centroid(uint8_t *dat, int w, int h, double &xx, double &yy)
470 for( int i=h; --i>=0; dp+=w ) x += centroid(dp, w);
471 for( int i=w; --i>=0; ) y += centroid(dat+i, h, w);
472 xx = x / h; yy = y / w;
476 deviation(uint8_t *a, uint8_t *b, int sz, int margin,
477 double &dev, int &ofs, int stride)
479 double best = 1e100; int ii = -1;
480 for( int i=-margin; i<=margin; ++i ) {
481 int aofs = i < 0 ? 0 : i*stride;
482 int bofs = i < 0 ? -i*stride : 0;
483 uint8_t *ap = a + aofs, *bp = b + bofs;
484 int ierr = 0; int n = sz - abs(i);
485 for( int j=n; --j>=0; ap+=stride,bp+=stride ) ierr += abs(*ap - *bp);
486 double err = (double)ierr / n;
487 if( err < best ) { best = err; ii = i; }
494 cmpr(uint8_t *ap, uint8_t *bp, int sz)
497 for( int i=sz; --i>=0; ++ap,++bp ) v += abs(*ap-*bp);
502 diff(uint8_t *a, uint8_t *b, int w, int h, int dx, int dy, int bias)
504 int axofs = dx < 0 ? 0 : dx;
505 int ayofs = w * (dy < 0 ? 0 : dy);
506 int aofs = ayofs + axofs;
507 int bxofs = dx < 0 ? -dx : 0;
508 int byofs = w * (dy < 0 ? -dy : 0);
509 int bofs = byofs + bxofs;
510 uint8_t *ap = a + aofs, *bp = b + bofs;
511 int ierr = 0, ww = w-abs(dx), hh = h-abs(dy);
512 for( int y=hh; --y>=0; ap+=w, bp+=w ) {
514 for( int x=ww; --x>=0; ++a, ++b ) { ierr += abs(*a-bias - *b); }
520 fit(int *dat, int n, double &a, double &b)
522 double st2 = 0, sb = 0;
524 for( int i=0; i<n; ++i ) sy += dat[i];
525 double mx = (n-1)/2., my = (double)sy / n;
526 for( int i=0; i<n; ++i ) {
536 eq(uint8_t *sp, uint8_t *dp, int len)
539 int hist[256], wts[256], map[256];
540 for( int i=0; i<256; ++i ) hist[i] = 0;
542 for( int i=len; --i>=0; ++bp ) ++hist[*bp];
544 for( int i=0; i<256; ++i ) { t += hist[i]; wts[i] = t; }
545 int lmn = len/20, lmx = len-lmn;
546 int mn = 0; while( mn < 256 && wts[mn] < lmn ) ++mn;
547 int mx = 255; while( mx > mn && wts[mx] > lmx ) --mx;
549 fit(&wts[mn], mx-mn, a, b);
551 double a1 = (a - b*mn) * r, b1 = b * r;
552 for( int i=0 ; i<256; ++i ) map[i] = clip(a1 + i*b1, 0, 255);
553 for( int i=len; --i>=0; ++sp,++dp ) *dp = map[*sp];
555 if( sp != dp ) memcpy(dp,sp,len);
559 // record constructors
561 new_frame(int clip_id, uint8_t *dat, int no, int group, double offset)
563 //printf("add %d at %f\n",no,tm);
564 int fid, ret = get_media_key(dat, fid, MEDIA_FRAME_DIST, MEDIA_FRAME_ERRLMT);
565 if( ret < 0 ) return -1;
567 db->video_frame.Allocate();
568 uint8_t *fp = db->video_frame._Frame_data(SFRM_SZ);
569 eq(dat, fp, SFRM_SZ);
570 double mn = mean(dat, SFRM_SZ);
571 db->video_frame.Frame_mean(mn);
572 double sd = std_dev(dat, mn, SFRM_SZ);
573 db->video_frame.Frame_std_dev(sd);
574 double cx, cy; centroid(dat, SWIDTH, SHEIGHT, cx, cy);
575 db->video_frame.Frame_cx(cx);
576 db->video_frame.Frame_cy(cy);
577 double moment = cx + cy;
578 db->video_frame.Frame_moment(moment);
579 db->video_frame.Construct();
580 fid = db->video_frame.id();
582 db->timeline.Allocate();
583 db->timeline.Clip_id(clip_id);
584 db->timeline.Sequence_no(no);
585 db->timeline.Frame_id(fid);
586 db->timeline.Group(group);
587 db->timeline.Time_offset(offset);
588 db->timeline.Construct();
593 new_clip_set(const char *title, const char *asset_path, double position,
594 double framerate, int frames, int prefix_size, int suffix_size,
595 int64_t creation_time, int64_t system_time)
597 db->clip_set.Allocate();
598 db->clip_set.Title(title);
599 db->clip_set.Asset_path(asset_path);
600 db->clip_set.Position(position);
601 db->clip_set.Framerate(framerate);
602 db->clip_set.Frames(frames);
603 db->clip_set.Prefix_size(prefix_size);
604 db->clip_set.Suffix_size(suffix_size);
605 db->clip_set._Weights(frames*sizeof(double));
606 db->clip_set.Creation_time(creation_time);
607 db->clip_set.System_time(system_time);
608 db->clip_set.Construct();
610 return new_clip_view(db->clip_set.id(), creation_time);
614 new_clip_view(int clip_id, int64_t access_time)
616 db->clip_views.Allocate();
617 db->clip_views.Access_clip_id(clip_id);
618 db->clip_views.Access_time(access_time);
619 db->clip_views.Access_count(1);
620 db->clip_views.Construct();
625 access_clip(int clip_id, int64_t access_time)
627 if( Clip_viewsLoc::ikey_Clip_access(db->clip_views,clip_id).Find() ) return 1;
628 db->clip_views.Destruct();
629 if( !access_time ) { time_t at; ::time(&at); access_time = (int64_t)at; }
630 db->clip_views.Access_time(access_time);
631 int access_count = db->clip_views.Access_count();
632 db->clip_views.Access_count(access_count+1);
633 db->clip_views.Construct();
638 del_clip_set(int clip_id)
640 if( db->clip_set.FindId(clip_id) ) return 1;
642 db->clip_set.Destruct();
643 db->clip_set.Deallocate();
645 if( Clip_viewsLoc::ikey_Clip_access(db->clip_views,clip_id).Find() ) return 1;
646 db->clip_views.Destruct();
647 db->clip_views.Deallocate();
649 while( !TimelineLoc::ikey_Sequences(db->timeline,clip_id,0).Locate() ) {
650 if( clip_id != (int)db->timeline.Clip_id() ) break;
651 int frame_id = db->timeline.Frame_id();
652 db->timeline.Destruct();
653 db->timeline.Deallocate();
654 // check for more timeline refs
655 if( !TimelineLoc::ikey_Timelines(db->timeline, frame_id).Locate() &&
656 frame_id == (int)db->timeline.Frame_id() ) continue;
657 if( db->video_frame.FindId(frame_id) ) continue;
659 db->video_frame.Destruct();
660 db->video_frame.Deallocate();
666 // db file operations
670 if( opened > 0 || db->create(MEDIA_DB) ) {
671 printf("MediaDb::newDb failed\n");
680 if( opened > 0 ) return 0;
681 if( db->access(MEDIA_DB, MEDIA_SHM_KEY, rw) ) {
682 printf("MediaDb::openDb failed\n");
693 if( access(MEDIA_DB, F_OK) ) result = newDb();
694 if( !result ) result = openDb(rw);
701 if( opened > 0 ) { detachDb(); db->close(); }
708 if( opened > 0 ) db->commit();
714 if( opened > 0 ) db->undo();
720 return opened > 0 ? db->attach(rw) : -1;
726 return opened > 0 ? db->detach() : -1;
731 get_timelines(int frame_id)
733 return TimelineLoc::ikey_Timelines(db->timeline, frame_id).Locate();
739 return TimelineLoc::rkey_Timelines(db->timeline).Next();
743 get_sequences(int clip_id, int seq_no)
745 int ret = TimelineLoc::ikey_Sequences(db->timeline,clip_id,seq_no).Locate();
746 if( !ret && clip_id != (int)db->timeline.Clip_id() ) ret = 1;
753 return TimelineLoc::rkey_Sequences(db->timeline).Next();
758 get_image(int id, uint8_t *dat, int &w, int &h)
761 if( !db->video_frame.FindId(id) ) {
762 memcpy(dat, db->video_frame.Frame_data(), SFRM_SZ);
763 w = SWIDTH; h = SHEIGHT;
770 get_frame_key(uint8_t *dat, int &fid,
771 Clips *clips, double pos, int mask)
773 return get_media_key(dat, fid,
774 MEDIA_SEARCH_DIST, MEDIA_SEARCH_ERRLMT,
779 get_media_key(uint8_t *dat, int &fid, int dist, double errlmt,
780 Clips *clips, double pos, int mask)
782 fmean = mean(dat, SFRM_SZ);
783 if( fmean < 17 ) return -1; // black, forbidden
784 if( fmean > 239 ) return -1; // white, forbidden
786 pixlmt = SFRM_SZ*errlmt;
789 fsd = std_dev(dat, fmean, SFRM_SZ);
790 centroid(dat, SWIDTH, SHEIGHT, fcx, fcy);
792 eq(dat, frm, SFRM_SZ);
793 if( clips ) ++clips->cycle;
798 if( !Video_frameLoc::ikey_Frame_center(db->video_frame, fmoment).Locate() ) {
799 Video_frameLoc prev(db->video_frame), next(db->video_frame);
800 Video_frameLoc::rkey_Frame_center prev_ctr(prev), next_ctr(next);
801 if( !get_key(clips, prev_ctr, next_ctr) ) result = 0;
803 if( !Video_frameLoc::ikey_Frame_weight(db->video_frame, fmean).Locate() ) {
804 Video_frameLoc prev(db->video_frame), next(db->video_frame);
805 Video_frameLoc::rkey_Frame_weight prev_wt(prev), next_wt(next);
806 if( !get_key(clips, prev_wt, next_wt) ) result = 0;
814 get_key(Clips *clips, Db::rKey &prev_rkey, Db::rKey &next_rkey)
816 key_compare(clips, next_rkey.loc);
817 // search outward from search target for frame with least error
818 Db::pgRef next_loc; next_rkey.NextLoc(next_loc);
820 for( int i=1; i<=distance; ++i ) {
821 if( !prev_rkey.Locate(Db::keyLT) )
822 key_compare(clips,prev_rkey.loc);
823 if( !next_rkey.Next(next_loc) )
824 key_compare(clips,next_rkey.loc);
827 return lerr < pixlmt ? 0 : 1;
831 key_compare(Clips *clips, Db::ObjectLoc &loc)
834 if( nid == id ) return 1;
836 Video_frameLoc &frame = *(Video_frameLoc*)&loc;
837 double dm = frame.Frame_mean()-fmean;
838 if( fabs(dm) > MEDIA_MEAN_ERRLMT ) return 1;
839 double ds = frame.Frame_std_dev()-fsd;
840 if( fabs(ds) > MEDIA_STDDEV_ERRLMT ) return 1;
841 double dx = frame.Frame_cx()-fcx;
842 if( fabs(dx) > MEDIA_XCENTER_ERRLMT ) return 1;
843 double dy = frame.Frame_cy()-fcy;
844 if( fabs(dy) > MEDIA_YCENTER_ERRLMT ) return 1;
845 uint8_t *dat = frame._Frame_data();
846 //int64_t err = cmpr(frm, dat, SFRM_SZ);
847 int64_t err = diff(frm, dat, SWIDTH, SHEIGHT, lround(dx), lround(dy), lround(dm));
848 if( lerr > err ) { lerr = err; lid = id; }
849 if( clips && err < pixlmt ) add_timelines(clips, id, err);
854 add_timelines(Clips *clips, int fid, double err)
856 // found frame, find timelines
857 if( get_timelines(fid) ) {
858 printf(_(" find timeline frame_id(%d) failed\n"), fid);
862 // process Timelines which contain Frame_id
864 for( int ret=0; !ret && timeline_frame_id()==fid; ret=next_timelines() ) {
865 int tid = timeline_clip_id();
866 if( tid != cid && clip_id(cid=tid) ) break;
867 double start_time = position - timeline_offset();
868 if( start_time < 0 ) continue;
869 int group = timeline_group();
870 if( (group & group_mask) == 0 ) continue;
871 double length = clip_length();
872 double end_time = start_time + length;
873 //printf("clip=%d, pos=%f, time=%f-%f, %s\n",
874 // cid, position, start_time, end_time, clip_title());
875 clips->check(cid, start_time, end_time, group, err);