cinelerra-5.1/cinelerra/filelist.C

   1
   2 /*
   3  * CINELERRA
   4  * Copyright (C) 1997-2012 Adam Williams <broadcast at earthling dot net>
   5  * Copyright (C) 2003-2016 Cinelerra CV contributors
   6  *
   7  * This program is free software; you can redistribute it and/or modify
   8  * it under the terms of the GNU General Public License as published by
   9  * the Free Software Foundation; either version 2 of the License, or
  10  * (at your option) any later version.
  11  *
  12  * This program is distributed in the hope that it will be useful,
  13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15  * GNU General Public License for more details.
  16  *
  17  * You should have received a copy of the GNU General Public License
  18  * along with this program; if not, write to the Free Software
  19  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  20  *
  21  */
  22
  23 #include "asset.h"
  24 #include "bcsignals.h"
  25 #include "cstrdup.h"
  26 #include "file.h"
  27 #include "filelist.h"
  28 #include "guicast.h"
  29 #include "interlacemodes.h"
  30 #include "mutex.h"
  31 #include "mwindow.inc"
  32 #include "render.h"
  33 #include "renderfarmfsserver.inc"
  34 #include "vframe.h"
  35 #include "mainerror.h"
  36
  37 #include <ctype.h>
  38 #include <errno.h>
  39 #include <fcntl.h>
  40 #include <stdlib.h>
  41 #include <string.h>
  42 #include <sys/stat.h>
  43 #include <unistd.h>
  44
  45
  46 FileList::FileList(Asset *asset,
  47         File *file,
  48         const char *list_prefix,
  49         const char *file_extension,
  50         int frame_type,
  51         int list_type)
  52  : FileBase(asset, file)
  53 {
  54         reset_parameters();
  55         path_list.set_array_delete();
  56         asset->video_data = 1;
  57         this->list_prefix = list_prefix;
  58         this->file_extension = file_extension;
  59         this->frame_type = frame_type;
  60         this->list_type = list_type;
  61         table_lock = new Mutex("FileList::table_lock");
  62 }
  63
  64 FileList::~FileList()
  65 {
  66         close_file();
  67         delete table_lock;
  68 }
  69
  70 int FileList::reset_parameters_derived()
  71 {
  72         data = 0;
  73         writer = 0;
  74         temp = 0;
  75         first_number = 0;
  76         return 0;
  77 }
  78
  79 int FileList::open_file(int rd, int wr)
  80 {
  81         int result = 1;
  82
  83 // skip header for write
  84         if(file->wr)
  85         {
  86                 int fd = open(asset->path, O_CREAT+O_TRUNC+O_RDWR, 0777);
  87                 if( fd >= 0 ) {
  88                         close(fd);
  89                         result = 0;
  90 // Frame files are created in write_frame and list index is created when
  91 // file is closed.
  92 // Look for the starting number in the path but ignore the starting character
  93 // and total digits since these are used by the header.
  94                         Render::get_starting_number(asset->path,
  95                                 first_number, number_start, number_digits);
  96                         path_list.remove_all_objects();
  97                         writer = new FrameWriter(this,
  98                         asset->format == list_type ? file->cpus : 1);
  99                 }
 100                 else
 101                         eprintf(_("Error while opening \"%s\" for writing. \n%m\n"), asset->path);
 102         }
 103         else
 104         if(file->rd)
 105         {
 106 // Determine type of file.
 107 // Header isn't used for background rendering, in which case everything known
 108 // by the file encoder is known by the decoder.
 109 //printf("FileList::open_file %d %d\n", __LINE__, asset->use_header);
 110                 if(asset->use_header)
 111                 {
 112                         FILE *stream = fopen(asset->path, "rb");
 113 //printf("FileList::open_file %d asset->path=%s\n", __LINE__, asset->path);
 114                         if(stream)
 115                         {
 116                                 int width = asset->width, height = asset->height;
 117                                 char string[BCTEXTLEN];
 118                                 int len = strlen(list_prefix);
 119                                 int ret = fread(string, 1, len, stream);
 120                                 fclose(stream);
 121                                 result = len == ret ? 0 : 1;
 122                                 if( !result && !strncasecmp(string, list_prefix, len)) {
 123                                         asset->format = list_type;
 124                                         result = read_list_header();
 125                                         if( !result )
 126                                                 result = read_frame_header(path_list.values[0]);
 127                                 }
 128                                 else if( !result && frame_type != FILE_UNKNOWN ) {
 129                                         asset->format = frame_type;
 130                                         result = read_frame_header(asset->path);
 131                                         asset->video_length = -1;
 132                                 }
 133                                 else
 134                                         result = 1;
 135                                 if( !result ) {
 136                                         asset->actual_width = asset->width;
 137                                         asset->actual_height = asset->height;
 138                                         int scale = asset->proxy_scale;
 139                                         if( scale ) {
 140                                                 asset->width = asset->actual_width * scale;
 141                                                 asset->height = asset->actual_height * scale;
 142                                         }
 143                                         else { // can_scale_input
 144                                                 if( width ) asset->width = width;
 145                                                 if( height ) asset->height = height;
 146                                         }
 147                                         asset->layers = 1;
 148                                         if( !asset->frame_rate )
 149                                                 asset->frame_rate = 10;
 150                                 }
 151                         }
 152                         else
 153                                 eprintf(_("Error while opening \"%s\" for reading. \n%m\n"), asset->path);
 154                 }
 155                 else
 156                 {
 157                         Render::get_starting_number(asset->path,
 158                                 first_number, number_start, number_digits, 6);
 159                         result = 0;
 160                 }
 161         }
 162
 163         file->current_frame = 0;
 164 // Compressed data storage
 165         data = new VFrame;
 166
 167         return result;
 168 }
 169
 170
 171 int FileList::close_file()
 172 {
 173 //      path_list.total, asset->format, list_type, wr);
 174         if(asset->format == list_type && path_list.total)
 175         {
 176                 if(file->wr && asset->use_header) write_list_header();
 177                 path_list.remove_all_objects();
 178         }
 179         if(data) delete data;
 180         if(writer) delete writer;
 181         if(temp) delete temp;
 182         reset_parameters();
 183
 184         FileBase::close_file();
 185         return 0;
 186 }
 187
 188 int FileList::write_list_header()
 189 {
 190         FILE *stream = fopen(asset->path, "w");
 191         if( !stream ) return 1;
 192 // Use sprintf instead of fprintf for VFS.
 193         fprintf(stream, "%s\n", list_prefix);
 194         fprintf(stream, "# First line is always %s\n", list_prefix);
 195         fprintf(stream, "# Frame rate:\n");
 196         fprintf(stream, "%f\n", asset->frame_rate);
 197         fprintf(stream, "# Width:\n");
 198         fprintf(stream, "%d\n", asset->width);
 199         fprintf(stream, "# Height:\n");
 200         fprintf(stream, "%d\n", asset->height);
 201         fprintf(stream, "# List of image files follows\n");
 202
 203         char *cp = strrchr(asset->path, '/');
 204         int dir_len = !cp ? 0 : cp - asset->path;
 205
 206         for(int i = 0; i < path_list.total; i++) {
 207                 const char *path = path_list.values[i];
 208 // Fix path for VFS but leave leading slash
 209                 if( !strncmp(path, RENDERFARM_FS_PREFIX, strlen(RENDERFARM_FS_PREFIX)) )
 210                         path += strlen(RENDERFARM_FS_PREFIX);
 211 // ./path for relative list access
 212                 else if( dir_len > 0 && !strncmp(path, asset->path, dir_len) ) {
 213                         fprintf(stream, ".");  path += dir_len;
 214                 }
 215                 fprintf(stream, "%s\n", path);
 216         }
 217         fclose(stream);
 218         return 0;
 219 }
 220
 221 int FileList::read_list_header()
 222 {
 223         char string[BCTEXTLEN];
 224
 225         FILE *stream = fopen(asset->path, "r");
 226         if( !stream ) return 1;
 227 // Get information about the frames
 228         do {
 229                 if( feof(stream) || !fgets(string, BCTEXTLEN, stream) ) return 1;
 230         } while(string[0] == '#' || string[0] == ' ' || isalpha(string[0]));
 231
 232 // Don't want a user configured frame rate to get destroyed
 233         if(asset->frame_rate == 0)
 234                 asset->frame_rate = atof(string);
 235
 236         do {
 237                 if( feof(stream) || !fgets(string, BCTEXTLEN, stream) ) return 1;
 238         } while(string[0] == '#' || string[0] == ' ');
 239         if( (asset->width = atol(string)) <= 0 ) return 1;
 240
 241         do {
 242                 if( feof(stream) || !fgets(string, BCTEXTLEN, stream) ) return 1;
 243         } while(string[0] == '#' || string[0] == ' ');
 244         if( (asset->height = atol(string)) <= 0 ) return 1;
 245
 246         asset->interlace_mode = ILACE_MODE_UNDETECTED;
 247         asset->layers = 1;
 248         asset->audio_data = 0;
 249         asset->video_data = 1;
 250
 251         char prefix[BCTEXTLEN], *bp = prefix, *cp = strrchr(asset->path, '/');
 252         for( int i=0, n=!cp ? 0 : cp-asset->path; i<n; ++i ) *bp++ = asset->path[i];
 253         *bp = 0;
 254
 255 // Get all the paths, expand relative paths
 256         int missing = 0;
 257         while( !feof(stream) && fgets(string, BCTEXTLEN, stream) ) {
 258                 int len = strlen(string);
 259                 if( !len || string[0] == '#' || string[0] == ' ' ) continue;
 260                 if( string[len-1] == '\n' ) string[len-1] = 0;
 261                 char path[BCTEXTLEN], *pp = path, *ep = pp + sizeof(path)-1;
 262                 if( string[0] == '.' && string[1] == '/' && prefix[0] )
 263                         pp += snprintf(pp, ep-pp, "%s/", prefix);
 264                 snprintf(pp, ep-pp, "%s", string);
 265                 if( access(path, R_OK) && !missing++ )
 266                         eprintf(_("%s:no such file"), path);
 267                 path_list.append(cstrdup(path));
 268         }
 269
 270 //for(int i = 0; i < path_list.total; i++) printf("%s\n", path_list.values[i]);
 271         fclose(stream);
 272         if( !(asset->video_length = path_list.total) )
 273                 eprintf(_("%s:\nlist empty"), asset->path);
 274         if( missing )
 275                 eprintf(_("%s:\n%d files not found"), asset->path, missing);
 276         return 0;
 277 }
 278
 279 int FileList::read_frame(VFrame *frame)
 280 {
 281         int result = 0;
 282
 283 //      PRINT_TRACE
 284 // printf("FileList::read_frame %d %d use_header=%d current_frame=%d total=%d\n",
 285 // __LINE__,
 286 // result,
 287 // asset->use_header,
 288 // file->current_frame,
 289 // path_list.total);
 290
 291         if(file->current_frame < 0 ||
 292                 (asset->use_header && file->current_frame >= path_list.total &&
 293                         asset->format == list_type))
 294                 return 1;
 295
 296         if(asset->format == list_type)
 297         {
 298                 char string[BCTEXTLEN];
 299                 char *path;
 300                 if(asset->use_header)
 301                 {
 302                         path = path_list.values[file->current_frame];
 303                 }
 304                 else
 305                 {
 306                         path = calculate_path(file->current_frame, string);
 307                 }
 308
 309                 FILE *in;
 310
 311 // Fix path for VFS.  Not used anymore.
 312                 if(!strncmp(asset->path, RENDERFARM_FS_PREFIX, strlen(RENDERFARM_FS_PREFIX)))
 313                         sprintf(string, "%s%s", RENDERFARM_FS_PREFIX, path);
 314                 else
 315                         strcpy(string, path);
 316
 317
 318
 319                 if(!use_path() || frame->get_color_model() == BC_COMPRESSED)
 320                 {
 321                         if(!(in = fopen(string, "rb"))) {
 322                                 eprintf(_("Error while opening \"%s\" for reading. \n%m\n"), string);
 323                         }
 324                         else
 325                         {
 326                                 struct stat ostat;
 327                                 stat(string, &ostat);
 328
 329                                 switch(frame->get_color_model())
 330                                 {
 331                                         case BC_COMPRESSED:
 332                                                 frame->allocate_compressed_data(ostat.st_size);
 333                                                 frame->set_compressed_size(ostat.st_size);
 334                                                 (void)fread(frame->get_data(), ostat.st_size, 1, in);
 335                                                 break;
 336                                         default:
 337                                                 data->allocate_compressed_data(ostat.st_size);
 338                                                 data->set_compressed_size(ostat.st_size);
 339                                                 (void)fread(data->get_data(), ostat.st_size, 1, in);
 340                                                 result = read_frame(frame, data);
 341                                                 break;
 342                                 }
 343
 344                                 fclose(in);
 345                         }
 346                 }
 347                 else
 348                 {
 349 //printf("FileList::read_frame %d %s\n", __LINE__, string);
 350                         result = read_frame(frame, string);
 351                 }
 352         }
 353         else
 354         {
 355                 asset->single_frame = 1;
 356 // Allocate and decompress single frame into new temporary
 357 //printf("FileList::read_frame %d\n", frame->get_color_model());
 358                 if( !temp || temp->get_color_model() != frame->get_color_model() ) {
 359                         delete temp;  temp = 0;
 360                         int aw = asset->actual_width, ah = asset->actual_height;
 361                         int color_model = frame->get_color_model();
 362                         switch( color_model ) {
 363                         case BC_YUV420P:
 364                         case BC_YUV420PI:
 365                         case BC_YUV422P:
 366                                 aw = (aw+1) & ~1;  ah = (ah+1) & ~1;
 367                                 break;
 368                         case BC_YUV410P:
 369                         case BC_YUV411P:
 370                                 aw = (aw+3) & ~3;  ah = (ah+3) & ~3;
 371                                 break;
 372                         }
 373                         if( !use_path() || color_model == BC_COMPRESSED ) {
 374                                 FILE *fd = fopen(asset->path, "rb");
 375                                 if( fd ) {
 376                                         struct stat ostat;
 377                                         stat(asset->path, &ostat);
 378
 379                                         switch(frame->get_color_model()) {
 380                                         case BC_COMPRESSED:
 381                                                 frame->allocate_compressed_data(ostat.st_size);
 382                                                 frame->set_compressed_size(ostat.st_size);
 383                                                 (void)fread(frame->get_data(), ostat.st_size, 1, fd);
 384                                                 break;
 385                                         default:
 386                                                 data->allocate_compressed_data(ostat.st_size);
 387                                                 data->set_compressed_size(ostat.st_size);
 388                                                 (void)fread(data->get_data(), ostat.st_size, 1, fd);
 389                                                 temp = new VFrame(aw, ah, color_model);
 390                                                 read_frame(temp, data);
 391                                                 break;
 392                                         }
 393
 394                                         fclose(fd);
 395                                 }
 396                                 else {
 397                                         eprintf(_("Error while opening \"%s\" for reading. \n%m\n"), asset->path);
 398                                         result = 1;
 399                                 }
 400                         }
 401                         else {
 402                                 temp = new VFrame(aw, ah, color_model);
 403                                 read_frame(temp, asset->path);
 404                         }
 405                 }
 406
 407                 if(!temp) return result;
 408
 409 //printf("FileList::read_frame frame=%d temp=%d\n",
 410 // frame->get_color_model(), // temp->get_color_model());
 411                 frame->transfer_from(temp);
 412         }
 413
 414
 415 // printf("FileList::read_frame %d %d\n", __LINE__, result);
 416 //
 417 // if(frame->get_y())
 418 // for(int i = 0; i < 100000; i++)
 419 // {
 420 //      frame->get_y()[i] = 0xff;
 421 // }
 422 // if(frame->get_rows())
 423 // for(int i = 0; i < 100000; i++)
 424 // {
 425 //      frame->get_rows()[0][i] = 0xff;
 426 // }
 427
 428
 429         return result;
 430 }
 431
 432 int FileList::write_frames(VFrame ***frames, int len)
 433 {
 434         return_value = 0;
 435
 436 //printf("FileList::write_frames 1\n");
 437         if(frames[0][0]->get_color_model() == BC_COMPRESSED)
 438         {
 439                 for(int i = 0; i < asset->layers && !return_value; i++)
 440                 {
 441                         for(int j = 0; j < len && !return_value; j++)
 442                         {
 443                                 VFrame *frame = frames[i][j];
 444                                 char *path = create_path(frame->get_number());
 445 //printf("FileList::write_frames %d %jd\n", __LINE__, frame->get_number());
 446
 447
 448                                 FILE *fd = fopen(path, "wb");
 449                                 if(fd)
 450                                 {
 451                                         return_value = !fwrite(frames[i][j]->get_data(),
 452                                                 frames[i][j]->get_compressed_size(),
 453                                                 1,
 454                                                 fd);
 455
 456                                         fclose(fd);
 457                                 }
 458                                 else
 459                                 {
 460                                         eprintf(_("Error while opening \"%s\" for writing. \n%m\n"), asset->path);
 461                                         return_value++;
 462                                 }
 463                         }
 464                 }
 465         }
 466         else
 467         {
 468 //printf("FileList::write_frames 2\n");
 469                 writer->write_frames(frames, len);
 470 //printf("FileList::write_frames 100\n");
 471         }
 472         return return_value;
 473 }
 474
 475
 476
 477
 478
 479
 480
 481
 482
 483 void FileList::add_return_value(int amount)
 484 {
 485         table_lock->lock("FileList::add_return_value");
 486         return_value += amount;
 487         table_lock->unlock();
 488 }
 489
 490 char* FileList::calculate_path(int number, char *string)
 491 {
 492 // Synthesize filename.
 493 // If a header is used, the filename number must be in a different location.
 494         if(asset->use_header)
 495         {
 496                 int k;
 497                 strcpy(string, asset->path);
 498                 for(k = strlen(string) - 1; k > 0 && string[k] != '.'; k--)
 499                         ;
 500                 if(k <= 0) k = strlen(string);
 501
 502                 sprintf(&string[k], "%06d%s",
 503                         number,
 504                         file_extension);
 505         }
 506         else
 507 // Without a header, the original filename can be altered.
 508         {
 509                 Render::create_filename(string,
 510                         asset->path,
 511                         number,
 512                         number_digits,
 513                         number_start);
 514         }
 515
 516         return string;
 517 }
 518
 519 char* FileList::create_path(int number_override)
 520 {
 521         if(asset->format != list_type) return asset->path;
 522
 523         table_lock->lock("FileList::create_path");
 524
 525
 526
 527         char *path = 0;
 528         char output[BCTEXTLEN];
 529         if(file->current_frame >= path_list.total || !asset->use_header)
 530         {
 531                 int number;
 532                 if(number_override < 0)
 533                         number = file->current_frame++;
 534                 else
 535                 {
 536                         number = number_override;
 537                         file->current_frame++;
 538                 }
 539
 540                 if(!asset->use_header)
 541                 {
 542                         number += first_number;
 543                 }
 544
 545                 calculate_path(number, output);
 546
 547                 path = new char[strlen(output) + 1];
 548                 strcpy(path, output);
 549                 path_list.append(path);
 550         }
 551         else
 552         {
 553 // Overwrite an old path
 554                 path = path_list.values[file->current_frame];
 555         }
 556
 557
 558         table_lock->unlock();
 559
 560         return path;
 561 }
 562
 563 FrameWriterUnit* FileList::new_writer_unit(FrameWriter *writer)
 564 {
 565         return new FrameWriterUnit(writer);
 566 }
 567
 568 int64_t FileList::get_memory_usage()
 569 {
 570         int64_t result = 0;
 571         if(data) result += data->get_compressed_allocated();
 572         if(temp) result += temp->get_data_size();
 573 // printf("FileList::get_memory_usage %d %p %s %jd\n",
 574 // __LINE__,
 575 // this,
 576 // file->asset->path,
 577 // result);
 578         return result;
 579 }
 580
 581 int FileList::get_units()
 582 {
 583         return !writer ? 0 : writer->get_total_clients();
 584 }
 585
 586 FrameWriterUnit* FileList::get_unit(int number)
 587 {
 588         return !writer ? 0 : (FrameWriterUnit*)writer->get_client(number);
 589 }
 590
 591 int FileList::use_path()
 592 {
 593         return 0;
 594 }
 595
 596
 597
 598
 599
 600
 601 FrameWriterPackage::FrameWriterPackage()
 602 {
 603 }
 604
 605 FrameWriterPackage::~FrameWriterPackage()
 606 {
 607 }
 608
 609
 610
 611
 612
 613
 614
 615
 616
 617
 618
 619 FrameWriterUnit::FrameWriterUnit(FrameWriter *server)
 620  : LoadClient(server)
 621 {
 622 // Don't use server here since subclasses call this with no server.
 623         this->server = server;
 624         output = new VFrame;
 625 }
 626
 627 FrameWriterUnit::~FrameWriterUnit()
 628 {
 629         delete output;
 630 }
 631
 632 void FrameWriterUnit::process_package(LoadPackage *package)
 633 {
 634 //printf("FrameWriterUnit::process_package 1\n");
 635         FrameWriterPackage *ptr = (FrameWriterPackage*)package;
 636
 637         FILE *file;
 638
 639 //printf("FrameWriterUnit::process_package 2 %s\n", ptr->path);
 640         if(!(file = fopen(ptr->path, "wb")))
 641         {
 642                 eprintf(_("Error while opening \"%s\" for writing. \n%m\n"), ptr->path);
 643                 return;
 644         }
 645 //printf("FrameWriterUnit::process_package 3");
 646
 647
 648         int result = server->file->write_frame(ptr->input, output, this);
 649
 650 //printf("FrameWriterUnit::process_package 4 %s %d\n", ptr->path, output->get_compressed_size());
 651         if(!result) result = !fwrite(output->get_data(), output->get_compressed_size(), 1, file);
 652 //TRACE("FrameWriterUnit::process_package 4");
 653         fclose(file);
 654 //TRACE("FrameWriterUnit::process_package 5");
 655
 656         server->file->add_return_value(result);
 657 //TRACE("FrameWriterUnit::process_package 6");
 658 }
 659
 660
 661
 662
 663
 664
 665
 666
 667
 668
 669
 670 FrameWriter::FrameWriter(FileList *file, int cpus)
 671  : LoadServer(cpus, 0)
 672 {
 673         this->file = file;
 674 }
 675
 676
 677 FrameWriter::~FrameWriter()
 678 {
 679 }
 680
 681 void FrameWriter::init_packages()
 682 {
 683         for(int i = 0, layer = 0, number = 0;
 684                 i < get_total_packages();
 685                 i++)
 686         {
 687                 FrameWriterPackage *package = (FrameWriterPackage*)get_package(i);
 688                 package->input = frames[layer][number];
 689                 package->path = file->create_path(package->input->get_number());
 690 // printf("FrameWriter::init_packages 1 %p %d %s\n",
 691 // package->input,
 692 // package->input->get_number(),
 693 // package->path);
 694                 number++;
 695                 if(number >= len)
 696                 {
 697                         layer++;
 698                         number = 0;
 699                 }
 700         }
 701 }
 702
 703 void FrameWriter::write_frames(VFrame ***frames, int len)
 704 {
 705         this->frames = frames;
 706         this->len = len;
 707         set_package_count(len * file->asset->layers);
 708
 709         process_packages();
 710 }
 711
 712 LoadClient* FrameWriter::new_client()
 713 {
 714         return file->new_writer_unit(this);
 715 }
 716
 717 LoadPackage* FrameWriter::new_package()
 718 {
 719         return new FrameWriterPackage;
 720 }
 721
 722
 723