cinelerra-5.1/cinelerra/filelist.C

   1
   2 /*
   3  * CINELERRA
   4  * Copyright (C) 1997-2012 Adam Williams <broadcast at earthling dot net>
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License as published by
   8  * the Free Software Foundation; either version 2 of the License, or
   9  * (at your option) any later version.
  10  *
  11  * This program is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14  * GNU General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU General Public License
  17  * along with this program; if not, write to the Free Software
  18  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  19  *
  20  */
  21
  22 #include "asset.h"
  23 #include "bcsignals.h"
  24 #include "cstrdup.h"
  25 #include "file.h"
  26 #include "filelist.h"
  27 #include "guicast.h"
  28 #include "interlacemodes.h"
  29 #include "mutex.h"
  30 #include "mwindow.inc"
  31 #include "render.h"
  32 #include "renderfarmfsserver.inc"
  33 #include "vframe.h"
  34 #include "mainerror.h"
  35
  36 #include <ctype.h>
  37 #include <errno.h>
  38 #include <fcntl.h>
  39 #include <stdlib.h>
  40 #include <string.h>
  41 #include <sys/stat.h>
  42 #include <unistd.h>
  43
  44
  45 FileList::FileList(Asset *asset,
  46         File *file,
  47         const char *list_prefix,
  48         const char *file_extension,
  49         int frame_type,
  50         int list_type)
  51  : FileBase(asset, file)
  52 {
  53         reset_parameters();
  54         path_list.set_array_delete();
  55         asset->video_data = 1;
  56         this->list_prefix = list_prefix;
  57         this->file_extension = file_extension;
  58         this->frame_type = frame_type;
  59         this->list_type = list_type;
  60         table_lock = new Mutex("FileList::table_lock");
  61 }
  62
  63 FileList::~FileList()
  64 {
  65         close_file();
  66         delete table_lock;
  67 }
  68
  69 int FileList::reset_parameters_derived()
  70 {
  71         data = 0;
  72         writer = 0;
  73         temp = 0;
  74         first_number = 0;
  75         return 0;
  76 }
  77
  78 int FileList::open_file(int rd, int wr)
  79 {
  80         int result = 1;
  81
  82 // skip header for write
  83         if(file->wr)
  84         {
  85                 int fd = open(asset->path, O_CREAT+O_TRUNC+O_RDWR, 0777);
  86                 if( fd >= 0 ) {
  87                         close(fd);
  88                         result = 0;
  89 // Frame files are created in write_frame and list index is created when
  90 // file is closed.
  91 // Look for the starting number in the path but ignore the starting character
  92 // and total digits since these are used by the header.
  93                         Render::get_starting_number(asset->path,
  94                                 first_number, number_start, number_digits);
  95                         path_list.remove_all_objects();
  96                         writer = new FrameWriter(this,
  97                         asset->format == list_type ? file->cpus : 1);
  98                 }
  99                 else
 100                         eprintf(_("Error while opening \"%s\" for writing. \n%m\n"), asset->path);
 101         }
 102         else
 103         if(file->rd)
 104         {
 105 // Determine type of file.
 106 // Header isn't used for background rendering, in which case everything known
 107 // by the file encoder is known by the decoder.
 108 //printf("FileList::open_file %d %d\n", __LINE__, asset->use_header);
 109                 if(asset->use_header)
 110                 {
 111                         FILE *stream = fopen(asset->path, "rb");
 112 //printf("FileList::open_file %d asset->path=%s\n", __LINE__, asset->path);
 113                         if(stream)
 114                         {
 115                                 char string[BCTEXTLEN];
 116                                 (void)fread(string, strlen(list_prefix), 1, stream);
 117                                 fclose(stream);
 118
 119                                 if(!strncasecmp(string, list_prefix, strlen(list_prefix)))
 120                                 {
 121
 122 //printf("FileList::open_file %d\n", __LINE__);
 123                                         asset->format = list_type;
 124
 125 // Open index here or get frame size from file.
 126                                         result = read_list_header();
 127 //printf("FileList::open_file %d %s\n", __LINE__, path_list.values[0]);
 128                                         if(!result) result = read_frame_header(path_list.values[0]);
 129                                 }
 130                                 else
 131                                 {
 132 //printf("FileList::open_file 2\n", asset->use_header);
 133 //printf("FileList::open_file %d\n", __LINE__);
 134                                         asset->format = frame_type;
 135                                         int width = asset->width;
 136                                         int height = asset->height;
 137                                         result = read_frame_header(asset->path);
 138                                         asset->actual_width = asset->width;
 139                                         if( width ) asset->width = width;
 140                                         asset->actual_height = asset->height;
 141                                         if( height ) asset->height = height;
 142                                         asset->layers = 1;
 143                                         if( !asset->frame_rate )
 144                                                 asset->frame_rate = 1;
 145                                         asset->video_length = -1;
 146                                 }
 147                                 result = 0;
 148                         }
 149                         else
 150                                 eprintf(_("Error while opening \"%s\" for reading. \n%m\n"), asset->path);
 151                 }
 152                 else
 153                 {
 154                         Render::get_starting_number(asset->path,
 155                                 first_number, number_start, number_digits, 6);
 156                         result = 0;
 157                 }
 158         }
 159
 160         file->current_frame = 0;
 161 // Compressed data storage
 162         data = new VFrame;
 163
 164         return result;
 165 }
 166
 167
 168 int FileList::close_file()
 169 {
 170 //      path_list.total, asset->format, list_type, wr);
 171         if(asset->format == list_type && path_list.total)
 172         {
 173                 if(file->wr && asset->use_header) write_list_header();
 174                 path_list.remove_all_objects();
 175         }
 176         if(data) delete data;
 177         if(writer) delete writer;
 178         if(temp) delete temp;
 179         reset_parameters();
 180
 181         FileBase::close_file();
 182         return 0;
 183 }
 184
 185 int FileList::write_list_header()
 186 {
 187         FILE *stream = fopen(asset->path, "w");
 188         if( !stream ) return 1;
 189 // Use sprintf instead of fprintf for VFS.
 190         fprintf(stream, "%s\n", list_prefix);
 191         fprintf(stream, "# First line is always %s\n", list_prefix);
 192         fprintf(stream, "# Frame rate:\n");
 193         fprintf(stream, "%f\n", asset->frame_rate);
 194         fprintf(stream, "# Width:\n");
 195         fprintf(stream, "%d\n", asset->width);
 196         fprintf(stream, "# Height:\n");
 197         fprintf(stream, "%d\n", asset->height);
 198         fprintf(stream, "# List of image files follows\n");
 199
 200         char *cp = strrchr(asset->path, '/');
 201         int dir_len = !cp ? 0 : cp - asset->path;
 202
 203         for(int i = 0; i < path_list.total; i++) {
 204                 const char *path = path_list.values[i];
 205 // Fix path for VFS but leave leading slash
 206                 if( !strncmp(path, RENDERFARM_FS_PREFIX, strlen(RENDERFARM_FS_PREFIX)) )
 207                         path += strlen(RENDERFARM_FS_PREFIX);
 208 // ./path for relative list access
 209                 else if( dir_len > 0 && !strncmp(path, asset->path, dir_len) ) {
 210                         fprintf(stream, ".");  path += dir_len;
 211                 }
 212                 fprintf(stream, "%s\n", path);
 213         }
 214         fclose(stream);
 215         return 0;
 216 }
 217
 218 int FileList::read_list_header()
 219 {
 220         char string[BCTEXTLEN];
 221
 222         FILE *stream = fopen(asset->path, "r");
 223         if( !stream ) return 1;
 224 // Get information about the frames
 225         do {
 226                 if( feof(stream) || !fgets(string, BCTEXTLEN, stream) ) return 1;
 227         } while(string[0] == '#' || string[0] == ' ' || isalpha(string[0]));
 228
 229 // Don't want a user configured frame rate to get destroyed
 230         if(asset->frame_rate == 0)
 231                 asset->frame_rate = atof(string);
 232
 233         do {
 234                 if( feof(stream) || !fgets(string, BCTEXTLEN, stream) ) return 1;
 235         } while(string[0] == '#' || string[0] == ' ');
 236         if( (asset->width = atol(string)) <= 0 ) return 1;
 237
 238         do {
 239                 if( feof(stream) || !fgets(string, BCTEXTLEN, stream) ) return 1;
 240         } while(string[0] == '#' || string[0] == ' ');
 241         if( (asset->height = atol(string)) <= 0 ) return 1;
 242
 243         asset->interlace_mode = ILACE_MODE_UNDETECTED;
 244         asset->layers = 1;
 245         asset->audio_data = 0;
 246         asset->video_data = 1;
 247
 248         char prefix[BCTEXTLEN], *bp = prefix, *cp = strrchr(asset->path, '/');
 249         for( int i=0, n=!cp ? 0 : cp-asset->path; i<n; ++i ) *bp++ = asset->path[i];
 250         *bp = 0;
 251
 252 // Get all the paths, expand relative paths
 253         int missing = 0;
 254         while( !feof(stream) && fgets(string, BCTEXTLEN, stream) ) {
 255                 int len = strlen(string);
 256                 if( !len || string[0] == '#' || string[0] == ' ' ) continue;
 257                 if( string[len-1] == '\n' ) string[len-1] = 0;
 258                 char path[BCTEXTLEN], *pp = path, *ep = pp + sizeof(path)-1;
 259                 if( string[0] == '.' && string[1] == '/' && prefix[0] )
 260                         pp += snprintf(pp, ep-pp, "%s/", prefix);
 261                 snprintf(pp, ep-pp, "%s", string);
 262                 if( access(path, R_OK) && !missing++ )
 263                         eprintf(_("%s:no such file"), path);
 264                 path_list.append(cstrdup(path));
 265         }
 266
 267 //for(int i = 0; i < path_list.total; i++) printf("%s\n", path_list.values[i]);
 268         fclose(stream);
 269         if( !(asset->video_length = path_list.total) )
 270                 eprintf(_("%s:\nlist empty"), asset->path);
 271         if( missing )
 272                 eprintf(_("%s:\n%d files not found"), asset->path, missing);
 273         return 0;
 274 }
 275
 276 int FileList::read_frame(VFrame *frame)
 277 {
 278         int result = 0;
 279
 280 //      PRINT_TRACE
 281 // printf("FileList::read_frame %d %d use_header=%d current_frame=%d total=%d\n",
 282 // __LINE__,
 283 // result,
 284 // asset->use_header,
 285 // file->current_frame,
 286 // path_list.total);
 287
 288         if(file->current_frame < 0 ||
 289                 (asset->use_header && file->current_frame >= path_list.total &&
 290                         asset->format == list_type))
 291                 return 1;
 292
 293         if(asset->format == list_type)
 294         {
 295                 char string[BCTEXTLEN];
 296                 char *path;
 297                 if(asset->use_header)
 298                 {
 299                         path = path_list.values[file->current_frame];
 300                 }
 301                 else
 302                 {
 303                         path = calculate_path(file->current_frame, string);
 304                 }
 305
 306                 FILE *in;
 307
 308 // Fix path for VFS.  Not used anymore.
 309                 if(!strncmp(asset->path, RENDERFARM_FS_PREFIX, strlen(RENDERFARM_FS_PREFIX)))
 310                         sprintf(string, "%s%s", RENDERFARM_FS_PREFIX, path);
 311                 else
 312                         strcpy(string, path);
 313
 314
 315
 316                 if(!use_path() || frame->get_color_model() == BC_COMPRESSED)
 317                 {
 318                         if(!(in = fopen(string, "rb"))) {
 319                                 eprintf(_("Error while opening \"%s\" for reading. \n%m\n"), string);
 320                         }
 321                         else
 322                         {
 323                                 struct stat ostat;
 324                                 stat(string, &ostat);
 325
 326                                 switch(frame->get_color_model())
 327                                 {
 328                                         case BC_COMPRESSED:
 329                                                 frame->allocate_compressed_data(ostat.st_size);
 330                                                 frame->set_compressed_size(ostat.st_size);
 331                                                 (void)fread(frame->get_data(), ostat.st_size, 1, in);
 332                                                 break;
 333                                         default:
 334                                                 data->allocate_compressed_data(ostat.st_size);
 335                                                 data->set_compressed_size(ostat.st_size);
 336                                                 (void)fread(data->get_data(), ostat.st_size, 1, in);
 337                                                 result = read_frame(frame, data);
 338                                                 break;
 339                                 }
 340
 341                                 fclose(in);
 342                         }
 343                 }
 344                 else
 345                 {
 346 //printf("FileList::read_frame %d %s\n", __LINE__, string);
 347                         result = read_frame(frame, string);
 348                 }
 349         }
 350         else
 351         {
 352                 asset->single_frame = 1;
 353 // Allocate and decompress single frame into new temporary
 354 //printf("FileList::read_frame %d\n", frame->get_color_model());
 355                 if( !temp || temp->get_color_model() != frame->get_color_model() ) {
 356                         delete temp;  temp = 0;
 357                         int aw = asset->actual_width, ah = asset->actual_height;
 358                         int color_model = frame->get_color_model();
 359                         switch( color_model ) {
 360                         case BC_YUV420P:
 361                         case BC_YUV420PI:
 362                         case BC_YUV422P:
 363                                 aw = (aw+1) & ~1;  ah = (ah+1) & ~1;
 364                                 break;
 365                         case BC_YUV410P:
 366                         case BC_YUV411P:
 367                                 aw = (aw+3) & ~3;  ah = (ah+3) & ~3;
 368                                 break;
 369                         }
 370                         if( !use_path() || color_model == BC_COMPRESSED ) {
 371                                 FILE *fd = fopen(asset->path, "rb");
 372                                 if( fd ) {
 373                                         struct stat ostat;
 374                                         stat(asset->path, &ostat);
 375
 376                                         switch(frame->get_color_model()) {
 377                                         case BC_COMPRESSED:
 378                                                 frame->allocate_compressed_data(ostat.st_size);
 379                                                 frame->set_compressed_size(ostat.st_size);
 380                                                 (void)fread(frame->get_data(), ostat.st_size, 1, fd);
 381                                                 break;
 382                                         default:
 383                                                 data->allocate_compressed_data(ostat.st_size);
 384                                                 data->set_compressed_size(ostat.st_size);
 385                                                 (void)fread(data->get_data(), ostat.st_size, 1, fd);
 386                                                 temp = new VFrame(aw, ah, color_model);
 387                                                 read_frame(temp, data);
 388                                                 break;
 389                                         }
 390
 391                                         fclose(fd);
 392                                 }
 393                                 else {
 394                                         eprintf(_("Error while opening \"%s\" for reading. \n%m\n"), asset->path);
 395                                         result = 1;
 396                                 }
 397                         }
 398                         else {
 399                                 temp = new VFrame(aw, ah, color_model);
 400                                 read_frame(temp, asset->path);
 401                         }
 402                 }
 403
 404                 if(!temp) return result;
 405
 406 //printf("FileList::read_frame frame=%d temp=%d\n",
 407 // frame->get_color_model(), // temp->get_color_model());
 408                 frame->transfer_from(temp);
 409         }
 410
 411
 412 // printf("FileList::read_frame %d %d\n", __LINE__, result);
 413 //
 414 // if(frame->get_y())
 415 // for(int i = 0; i < 100000; i++)
 416 // {
 417 //      frame->get_y()[i] = 0xff;
 418 // }
 419 // if(frame->get_rows())
 420 // for(int i = 0; i < 100000; i++)
 421 // {
 422 //      frame->get_rows()[0][i] = 0xff;
 423 // }
 424
 425
 426         return result;
 427 }
 428
 429 int FileList::write_frames(VFrame ***frames, int len)
 430 {
 431         return_value = 0;
 432
 433 //printf("FileList::write_frames 1\n");
 434         if(frames[0][0]->get_color_model() == BC_COMPRESSED)
 435         {
 436                 for(int i = 0; i < asset->layers && !return_value; i++)
 437                 {
 438                         for(int j = 0; j < len && !return_value; j++)
 439                         {
 440                                 VFrame *frame = frames[i][j];
 441                                 char *path = create_path(frame->get_number());
 442 //printf("FileList::write_frames %d %jd\n", __LINE__, frame->get_number());
 443
 444
 445                                 FILE *fd = fopen(path, "wb");
 446                                 if(fd)
 447                                 {
 448                                         return_value = !fwrite(frames[i][j]->get_data(),
 449                                                 frames[i][j]->get_compressed_size(),
 450                                                 1,
 451                                                 fd);
 452
 453                                         fclose(fd);
 454                                 }
 455                                 else
 456                                 {
 457                                         eprintf(_("Error while opening \"%s\" for writing. \n%m\n"), asset->path);
 458                                         return_value++;
 459                                 }
 460                         }
 461                 }
 462         }
 463         else
 464         {
 465 //printf("FileList::write_frames 2\n");
 466                 writer->write_frames(frames, len);
 467 //printf("FileList::write_frames 100\n");
 468         }
 469         return return_value;
 470 }
 471
 472
 473
 474
 475
 476
 477
 478
 479
 480 void FileList::add_return_value(int amount)
 481 {
 482         table_lock->lock("FileList::add_return_value");
 483         return_value += amount;
 484         table_lock->unlock();
 485 }
 486
 487 char* FileList::calculate_path(int number, char *string)
 488 {
 489 // Synthesize filename.
 490 // If a header is used, the filename number must be in a different location.
 491         if(asset->use_header)
 492         {
 493                 int k;
 494                 strcpy(string, asset->path);
 495                 for(k = strlen(string) - 1; k > 0 && string[k] != '.'; k--)
 496                         ;
 497                 if(k <= 0) k = strlen(string);
 498
 499                 sprintf(&string[k], "%06d%s",
 500                         number,
 501                         file_extension);
 502         }
 503         else
 504 // Without a header, the original filename can be altered.
 505         {
 506                 Render::create_filename(string,
 507                         asset->path,
 508                         number,
 509                         number_digits,
 510                         number_start);
 511         }
 512
 513         return string;
 514 }
 515
 516 char* FileList::create_path(int number_override)
 517 {
 518         if(asset->format != list_type) return asset->path;
 519
 520         table_lock->lock("FileList::create_path");
 521
 522
 523
 524         char *path = 0;
 525         char output[BCTEXTLEN];
 526         if(file->current_frame >= path_list.total || !asset->use_header)
 527         {
 528                 int number;
 529                 if(number_override < 0)
 530                         number = file->current_frame++;
 531                 else
 532                 {
 533                         number = number_override;
 534                         file->current_frame++;
 535                 }
 536
 537                 if(!asset->use_header)
 538                 {
 539                         number += first_number;
 540                 }
 541
 542                 calculate_path(number, output);
 543
 544                 path = new char[strlen(output) + 1];
 545                 strcpy(path, output);
 546                 path_list.append(path);
 547         }
 548         else
 549         {
 550 // Overwrite an old path
 551                 path = path_list.values[file->current_frame];
 552         }
 553
 554
 555         table_lock->unlock();
 556
 557         return path;
 558 }
 559
 560 FrameWriterUnit* FileList::new_writer_unit(FrameWriter *writer)
 561 {
 562         return new FrameWriterUnit(writer);
 563 }
 564
 565 int64_t FileList::get_memory_usage()
 566 {
 567         int64_t result = 0;
 568         if(data) result += data->get_compressed_allocated();
 569         if(temp) result += temp->get_data_size();
 570 // printf("FileList::get_memory_usage %d %p %s %jd\n",
 571 // __LINE__,
 572 // this,
 573 // file->asset->path,
 574 // result);
 575         return result;
 576 }
 577
 578 int FileList::get_units()
 579 {
 580         return !writer ? 0 : writer->get_total_clients();
 581 }
 582
 583 FrameWriterUnit* FileList::get_unit(int number)
 584 {
 585         return !writer ? 0 : (FrameWriterUnit*)writer->get_client(number);
 586 }
 587
 588 int FileList::use_path()
 589 {
 590         return 0;
 591 }
 592
 593
 594
 595
 596
 597
 598 FrameWriterPackage::FrameWriterPackage()
 599 {
 600 }
 601
 602 FrameWriterPackage::~FrameWriterPackage()
 603 {
 604 }
 605
 606
 607
 608
 609
 610
 611
 612
 613
 614
 615
 616 FrameWriterUnit::FrameWriterUnit(FrameWriter *server)
 617  : LoadClient(server)
 618 {
 619 // Don't use server here since subclasses call this with no server.
 620         this->server = server;
 621         output = new VFrame;
 622 }
 623
 624 FrameWriterUnit::~FrameWriterUnit()
 625 {
 626         delete output;
 627 }
 628
 629 void FrameWriterUnit::process_package(LoadPackage *package)
 630 {
 631 //printf("FrameWriterUnit::process_package 1\n");
 632         FrameWriterPackage *ptr = (FrameWriterPackage*)package;
 633
 634         FILE *file;
 635
 636 //printf("FrameWriterUnit::process_package 2 %s\n", ptr->path);
 637         if(!(file = fopen(ptr->path, "wb")))
 638         {
 639                 eprintf(_("Error while opening \"%s\" for writing. \n%m\n"), ptr->path);
 640                 return;
 641         }
 642 //printf("FrameWriterUnit::process_package 3");
 643
 644
 645         int result = server->file->write_frame(ptr->input, output, this);
 646
 647 //printf("FrameWriterUnit::process_package 4 %s %d\n", ptr->path, output->get_compressed_size());
 648         if(!result) result = !fwrite(output->get_data(), output->get_compressed_size(), 1, file);
 649 //TRACE("FrameWriterUnit::process_package 4");
 650         fclose(file);
 651 //TRACE("FrameWriterUnit::process_package 5");
 652
 653         server->file->add_return_value(result);
 654 //TRACE("FrameWriterUnit::process_package 6");
 655 }
 656
 657
 658
 659
 660
 661
 662
 663
 664
 665
 666
 667 FrameWriter::FrameWriter(FileList *file, int cpus)
 668  : LoadServer(cpus, 0)
 669 {
 670         this->file = file;
 671 }
 672
 673
 674 FrameWriter::~FrameWriter()
 675 {
 676 }
 677
 678 void FrameWriter::init_packages()
 679 {
 680         for(int i = 0, layer = 0, number = 0;
 681                 i < get_total_packages();
 682                 i++)
 683         {
 684                 FrameWriterPackage *package = (FrameWriterPackage*)get_package(i);
 685                 package->input = frames[layer][number];
 686                 package->path = file->create_path(package->input->get_number());
 687 // printf("FrameWriter::init_packages 1 %p %d %s\n",
 688 // package->input,
 689 // package->input->get_number(),
 690 // package->path);
 691                 number++;
 692                 if(number >= len)
 693                 {
 694                         layer++;
 695                         number = 0;
 696                 }
 697         }
 698 }
 699
 700 void FrameWriter::write_frames(VFrame ***frames, int len)
 701 {
 702         this->frames = frames;
 703         this->len = len;
 704         set_package_count(len * file->asset->layers);
 705
 706         process_packages();
 707 }
 708
 709 LoadClient* FrameWriter::new_client()
 710 {
 711         return file->new_writer_unit(this);
 712 }
 713
 714 LoadPackage* FrameWriter::new_package()
 715 {
 716         return new FrameWriterPackage;
 717 }
 718
 719
 720