cinelerra-5.1/cinelerra/filelist.C

   1
   2 /*
   3  * CINELERRA
   4  * Copyright (C) 1997-2012 Adam Williams <broadcast at earthling dot net>
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License as published by
   8  * the Free Software Foundation; either version 2 of the License, or
   9  * (at your option) any later version.
  10  *
  11  * This program is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14  * GNU General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU General Public License
  17  * along with this program; if not, write to the Free Software
  18  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  19  *
  20  */
  21
  22 #include "asset.h"
  23 #include "bcsignals.h"
  24 #include "cstrdup.h"
  25 #include "file.h"
  26 #include "filelist.h"
  27 #include "guicast.h"
  28 #include "interlacemodes.h"
  29 #include "mutex.h"
  30 #include "mwindow.inc"
  31 #include "render.h"
  32 #include "renderfarmfsserver.inc"
  33 #include "vframe.h"
  34 #include "mainerror.h"
  35
  36 #include <ctype.h>
  37 #include <errno.h>
  38 #include <fcntl.h>
  39 #include <stdlib.h>
  40 #include <string.h>
  41 #include <sys/stat.h>
  42 #include <unistd.h>
  43
  44
  45 FileList::FileList(Asset *asset,
  46         File *file,
  47         const char *list_prefix,
  48         const char *file_extension,
  49         int frame_type,
  50         int list_type)
  51  : FileBase(asset, file)
  52 {
  53         reset_parameters();
  54         path_list.set_array_delete();
  55         asset->video_data = 1;
  56         this->list_prefix = list_prefix;
  57         this->file_extension = file_extension;
  58         this->frame_type = frame_type;
  59         this->list_type = list_type;
  60         table_lock = new Mutex("FileList::table_lock");
  61 }
  62
  63 FileList::~FileList()
  64 {
  65         close_file();
  66         delete table_lock;
  67 }
  68
  69 int FileList::reset_parameters_derived()
  70 {
  71         data = 0;
  72         writer = 0;
  73         temp = 0;
  74         first_number = 0;
  75         return 0;
  76 }
  77
  78 int FileList::open_file(int rd, int wr)
  79 {
  80         int result = 1;
  81
  82 // skip header for write
  83         if(file->wr)
  84         {
  85                 int fd = open(asset->path, O_CREAT+O_TRUNC+O_RDWR, 0777);
  86                 if( fd >= 0 ) {
  87                         close(fd);
  88                         result = 0;
  89 // Frame files are created in write_frame and list index is created when
  90 // file is closed.
  91 // Look for the starting number in the path but ignore the starting character
  92 // and total digits since these are used by the header.
  93                         Render::get_starting_number(asset->path,
  94                                 first_number, number_start, number_digits);
  95                         path_list.remove_all_objects();
  96                         writer = new FrameWriter(this,
  97                         asset->format == list_type ? file->cpus : 1);
  98                 }
  99                 else
 100                         eprintf(_("Error while opening \"%s\" for writing. \n%m\n"), asset->path);
 101         }
 102         else
 103         if(file->rd)
 104         {
 105 // Determine type of file.
 106 // Header isn't used for background rendering, in which case everything known
 107 // by the file encoder is known by the decoder.
 108 //printf("FileList::open_file %d %d\n", __LINE__, asset->use_header);
 109                 if(asset->use_header)
 110                 {
 111                         FILE *stream = fopen(asset->path, "rb");
 112 //printf("FileList::open_file %d asset->path=%s\n", __LINE__, asset->path);
 113                         if(stream)
 114                         {
 115                                 char string[BCTEXTLEN];
 116                                 (void)fread(string, strlen(list_prefix), 1, stream);
 117                                 fclose(stream);
 118
 119                                 if(!strncasecmp(string, list_prefix, strlen(list_prefix)))
 120                                 {
 121
 122 //printf("FileList::open_file %d\n", __LINE__);
 123                                         asset->format = list_type;
 124
 125 // Open index here or get frame size from file.
 126                                         result = read_list_header();
 127 //printf("FileList::open_file %d %s\n", __LINE__, path_list.values[0]);
 128                                         if(!result) result = read_frame_header(path_list.values[0]);
 129                                 }
 130                                 else
 131                                 {
 132 //printf("FileList::open_file 2\n", asset->use_header);
 133 //printf("FileList::open_file %d\n", __LINE__);
 134                                         asset->format = frame_type;
 135                                         result = read_frame_header(asset->path);
 136                                         asset->layers = 1;
 137                                         if(!asset->frame_rate)
 138                                                 asset->frame_rate = 1;
 139                                         asset->video_length = -1;
 140                                 }
 141                                 result = 0;
 142                         }
 143                         else
 144                                 eprintf(_("Error while opening \"%s\" for reading. \n%m\n"), asset->path);
 145                 }
 146                 else
 147                 {
 148                         Render::get_starting_number(asset->path,
 149                                 first_number, number_start, number_digits, 6);
 150                         result = 0;
 151                 }
 152         }
 153
 154         file->current_frame = 0;
 155 // Compressed data storage
 156         data = new VFrame;
 157
 158         return result;
 159 }
 160
 161
 162 int FileList::close_file()
 163 {
 164 //      path_list.total, asset->format, list_type, wr);
 165         if(asset->format == list_type && path_list.total)
 166         {
 167                 if(file->wr && asset->use_header) write_list_header();
 168                 path_list.remove_all_objects();
 169         }
 170         if(data) delete data;
 171         if(writer) delete writer;
 172         if(temp) delete temp;
 173         reset_parameters();
 174
 175         FileBase::close_file();
 176         return 0;
 177 }
 178
 179 int FileList::write_list_header()
 180 {
 181         FILE *stream = fopen(asset->path, "w");
 182         if( !stream ) return 1;
 183 // Use sprintf instead of fprintf for VFS.
 184         char string[BCTEXTLEN];
 185         sprintf(string, "%s\n", list_prefix);
 186         fwrite(string, strlen(string), 1, stream);
 187         sprintf(string, "# First line is always %s\n", list_prefix);
 188         fwrite(string, strlen(string), 1, stream);
 189         sprintf(string, "# Frame rate:\n");
 190         fwrite(string, strlen(string), 1, stream);
 191         sprintf(string, "%f\n", asset->frame_rate);
 192         fwrite(string, strlen(string), 1, stream);
 193         sprintf(string, "# Width:\n");
 194         fwrite(string, strlen(string), 1, stream);
 195         sprintf(string, "%d\n", asset->width);
 196         fwrite(string, strlen(string), 1, stream);
 197         sprintf(string, "# Height:\n");
 198         fwrite(string, strlen(string), 1, stream);
 199         sprintf(string, "%d\n", asset->height);
 200         fwrite(string, strlen(string), 1, stream);
 201         sprintf(string, "# List of image files follows\n");
 202         fwrite(string, strlen(string), 1, stream);
 203
 204         for(int i = 0; i < path_list.total; i++)
 205         {
 206 // Fix path for VFS but leave leading slash
 207                 if(!strncmp(path_list.values[i], RENDERFARM_FS_PREFIX, strlen(RENDERFARM_FS_PREFIX)))
 208                         sprintf(string, "%s\n", path_list.values[i] + strlen(RENDERFARM_FS_PREFIX));
 209                 else
 210                         sprintf(string, "%s\n", path_list.values[i]);
 211                 fwrite(string, strlen(string), 1, stream);
 212         }
 213         fclose(stream);
 214         return 0;
 215 }
 216
 217 int FileList::read_list_header()
 218 {
 219         char string[BCTEXTLEN];
 220
 221         FILE *stream = fopen(asset->path, "r");
 222         if( !stream ) return 1;
 223 // Get information about the frames
 224         do {
 225                 if( feof(stream) || !fgets(string, BCTEXTLEN, stream) ) return 1;
 226         } while(string[0] == '#' || string[0] == ' ' || isalpha(string[0]));
 227
 228 // Don't want a user configured frame rate to get destroyed
 229         if(asset->frame_rate == 0)
 230                 asset->frame_rate = atof(string);
 231
 232         do {
 233                 if( feof(stream) || !fgets(string, BCTEXTLEN, stream) ) return 1;
 234         } while(string[0] == '#' || string[0] == ' ');
 235         if( (asset->width = atol(string)) <= 0 ) return 1;
 236
 237         do {
 238                 if( feof(stream) || !fgets(string, BCTEXTLEN, stream) ) return 1;
 239         } while(string[0] == '#' || string[0] == ' ');
 240         if( (asset->height = atol(string)) <= 0 ) return 1;
 241
 242         asset->interlace_mode = ILACE_MODE_UNDETECTED;
 243         asset->layers = 1;
 244         asset->audio_data = 0;
 245         asset->video_data = 1;
 246
 247 // Get all the paths
 248         int missing = 0;
 249         while(!feof(stream) && fgets(string, BCTEXTLEN, stream) ) {
 250                 int len = strlen(string);
 251                 if( !len || string[0] == '#' || string[0] == ' ') continue;
 252                 string[len-1] = 0;
 253                 if( access(string,R_OK) && !missing++ )
 254                         eprintf(_("%s:no such file"), string);
 255                 path_list.append(cstrdup(string));
 256         }
 257
 258 //for(int i = 0; i < path_list.total; i++) printf("%s\n", path_list.values[i]);
 259         fclose(stream);
 260         if( !(asset->video_length = path_list.total) )
 261                 eprintf(_("%s:\nlist empty"), asset->path);
 262         if( missing )
 263                 eprintf(_("%s:\n%d files not found"), asset->path, missing);
 264         return 0;
 265 }
 266
 267 int FileList::read_frame(VFrame *frame)
 268 {
 269         int result = 0;
 270
 271 //      PRINT_TRACE
 272 // printf("FileList::read_frame %d %d use_header=%d current_frame=%d total=%d\n",
 273 // __LINE__,
 274 // result,
 275 // asset->use_header,
 276 // file->current_frame,
 277 // path_list.total);
 278
 279         if(file->current_frame < 0 ||
 280                 (asset->use_header && file->current_frame >= path_list.total &&
 281                         asset->format == list_type))
 282                 return 1;
 283
 284         if(asset->format == list_type)
 285         {
 286                 char string[BCTEXTLEN];
 287                 char *path;
 288                 if(asset->use_header)
 289                 {
 290                         path = path_list.values[file->current_frame];
 291                 }
 292                 else
 293                 {
 294                         path = calculate_path(file->current_frame, string);
 295                 }
 296
 297                 FILE *in;
 298
 299 // Fix path for VFS.  Not used anymore.
 300                 if(!strncmp(asset->path, RENDERFARM_FS_PREFIX, strlen(RENDERFARM_FS_PREFIX)))
 301                         sprintf(string, "%s%s", RENDERFARM_FS_PREFIX, path);
 302                 else
 303                         strcpy(string, path);
 304
 305
 306
 307                 if(!use_path() || frame->get_color_model() == BC_COMPRESSED)
 308                 {
 309                         if(!(in = fopen(string, "rb"))) {
 310                                 eprintf(_("Error while opening \"%s\" for reading. \n%m\n"), string);
 311                         }
 312                         else
 313                         {
 314                                 struct stat ostat;
 315                                 stat(string, &ostat);
 316
 317                                 switch(frame->get_color_model())
 318                                 {
 319                                         case BC_COMPRESSED:
 320                                                 frame->allocate_compressed_data(ostat.st_size);
 321                                                 frame->set_compressed_size(ostat.st_size);
 322                                                 (void)fread(frame->get_data(), ostat.st_size, 1, in);
 323                                                 break;
 324                                         default:
 325                                                 data->allocate_compressed_data(ostat.st_size);
 326                                                 data->set_compressed_size(ostat.st_size);
 327                                                 (void)fread(data->get_data(), ostat.st_size, 1, in);
 328                                                 result = read_frame(frame, data);
 329                                                 break;
 330                                 }
 331
 332                                 fclose(in);
 333                         }
 334                 }
 335                 else
 336                 {
 337 //printf("FileList::read_frame %d %s\n", __LINE__, string);
 338                         result = read_frame(frame, string);
 339                 }
 340         }
 341         else
 342         {
 343                 asset->single_frame = 1;
 344 // Allocate and decompress single frame into new temporary
 345 //printf("FileList::read_frame %d\n", frame->get_color_model());
 346                 if(!temp || temp->get_color_model() != frame->get_color_model())
 347                 {
 348                         if(temp) delete temp;
 349                         temp = 0;
 350
 351
 352                         if(!use_path() || frame->get_color_model() == BC_COMPRESSED)
 353                         {
 354                                 FILE *fd = fopen(asset->path, "rb");
 355                                 if(fd)
 356                                 {
 357                                         struct stat ostat;
 358                                         stat(asset->path, &ostat);
 359
 360                                         switch(frame->get_color_model())
 361                                         {
 362                                                 case BC_COMPRESSED:
 363                                                         frame->allocate_compressed_data(ostat.st_size);
 364                                                         frame->set_compressed_size(ostat.st_size);
 365                                                         (void)fread(frame->get_data(), ostat.st_size, 1, fd);
 366                                                         break;
 367                                                 default:
 368                                                         data->allocate_compressed_data(ostat.st_size);
 369                                                         data->set_compressed_size(ostat.st_size);
 370                                                         (void)fread(data->get_data(), ostat.st_size, 1, fd);
 371                                                         temp = new VFrame(0,
 372                                                                 -1,
 373                                                                 asset->width,
 374                                                                 asset->height,
 375                                                                 frame->get_color_model(),
 376                                                                 -1);
 377                                                         read_frame(temp, data);
 378                                                         break;
 379                                         }
 380
 381                                         fclose(fd);
 382                                 }
 383                                 else
 384                                 {
 385                                         eprintf(_("Error while opening \"%s\" for reading. \n%m\n"), asset->path);
 386                                         result = 1;
 387                                 }
 388                         }
 389                         else
 390                         {
 391                                 temp = new VFrame(0,
 392                                         -1,
 393                                         asset->width,
 394                                         asset->height,
 395                                         frame->get_color_model(),
 396                                         -1);
 397                                 read_frame(temp, asset->path);
 398                         }
 399                 }
 400
 401                 if(!temp) return result;
 402
 403 //printf("FileList::read_frame frame=%d temp=%d\n",
 404 // frame->get_color_model(), // temp->get_color_model());
 405                 frame->transfer_from(temp);
 406         }
 407
 408
 409 // printf("FileList::read_frame %d %d\n", __LINE__, result);
 410 //
 411 // if(frame->get_y())
 412 // for(int i = 0; i < 100000; i++)
 413 // {
 414 //      frame->get_y()[i] = 0xff;
 415 // }
 416 // if(frame->get_rows())
 417 // for(int i = 0; i < 100000; i++)
 418 // {
 419 //      frame->get_rows()[0][i] = 0xff;
 420 // }
 421
 422
 423         return result;
 424 }
 425
 426 int FileList::write_frames(VFrame ***frames, int len)
 427 {
 428         return_value = 0;
 429
 430 //printf("FileList::write_frames 1\n");
 431         if(frames[0][0]->get_color_model() == BC_COMPRESSED)
 432         {
 433                 for(int i = 0; i < asset->layers && !return_value; i++)
 434                 {
 435                         for(int j = 0; j < len && !return_value; j++)
 436                         {
 437                                 VFrame *frame = frames[i][j];
 438                                 char *path = create_path(frame->get_number());
 439 //printf("FileList::write_frames %d %jd\n", __LINE__, frame->get_number());
 440
 441
 442                                 FILE *fd = fopen(path, "wb");
 443                                 if(fd)
 444                                 {
 445                                         return_value = !fwrite(frames[i][j]->get_data(),
 446                                                 frames[i][j]->get_compressed_size(),
 447                                                 1,
 448                                                 fd);
 449
 450                                         fclose(fd);
 451                                 }
 452                                 else
 453                                 {
 454                                         eprintf(_("Error while opening \"%s\" for writing. \n%m\n"), asset->path);
 455                                         return_value++;
 456                                 }
 457                         }
 458                 }
 459         }
 460         else
 461         {
 462 //printf("FileList::write_frames 2\n");
 463                 writer->write_frames(frames, len);
 464 //printf("FileList::write_frames 100\n");
 465         }
 466         return return_value;
 467 }
 468
 469
 470
 471
 472
 473
 474
 475
 476
 477 void FileList::add_return_value(int amount)
 478 {
 479         table_lock->lock("FileList::add_return_value");
 480         return_value += amount;
 481         table_lock->unlock();
 482 }
 483
 484 char* FileList::calculate_path(int number, char *string)
 485 {
 486 // Synthesize filename.
 487 // If a header is used, the filename number must be in a different location.
 488         if(asset->use_header)
 489         {
 490                 int k;
 491                 strcpy(string, asset->path);
 492                 for(k = strlen(string) - 1; k > 0 && string[k] != '.'; k--)
 493                         ;
 494                 if(k <= 0) k = strlen(string);
 495
 496                 sprintf(&string[k], "%06d%s",
 497                         number,
 498                         file_extension);
 499         }
 500         else
 501 // Without a header, the original filename can be altered.
 502         {
 503                 Render::create_filename(string,
 504                         asset->path,
 505                         number,
 506                         number_digits,
 507                         number_start);
 508         }
 509
 510         return string;
 511 }
 512
 513 char* FileList::create_path(int number_override)
 514 {
 515         if(asset->format != list_type) return asset->path;
 516
 517         table_lock->lock("FileList::create_path");
 518
 519
 520
 521         char *path = 0;
 522         char output[BCTEXTLEN];
 523         if(file->current_frame >= path_list.total || !asset->use_header)
 524         {
 525                 int number;
 526                 if(number_override < 0)
 527                         number = file->current_frame++;
 528                 else
 529                 {
 530                         number = number_override;
 531                         file->current_frame++;
 532                 }
 533
 534                 if(!asset->use_header)
 535                 {
 536                         number += first_number;
 537                 }
 538
 539                 calculate_path(number, output);
 540
 541                 path = new char[strlen(output) + 1];
 542                 strcpy(path, output);
 543                 path_list.append(path);
 544         }
 545         else
 546         {
 547 // Overwrite an old path
 548                 path = path_list.values[file->current_frame];
 549         }
 550
 551
 552         table_lock->unlock();
 553
 554         return path;
 555 }
 556
 557 FrameWriterUnit* FileList::new_writer_unit(FrameWriter *writer)
 558 {
 559         return new FrameWriterUnit(writer);
 560 }
 561
 562 int64_t FileList::get_memory_usage()
 563 {
 564         int64_t result = 0;
 565         if(data) result += data->get_compressed_allocated();
 566         if(temp) result += temp->get_data_size();
 567 // printf("FileList::get_memory_usage %d %p %s %jd\n",
 568 // __LINE__,
 569 // this,
 570 // file->asset->path,
 571 // result);
 572         return result;
 573 }
 574
 575 int FileList::get_units()
 576 {
 577         return !writer ? 0 : writer->get_total_clients();
 578 }
 579
 580 FrameWriterUnit* FileList::get_unit(int number)
 581 {
 582         return !writer ? 0 : (FrameWriterUnit*)writer->get_client(number);
 583 }
 584
 585 int FileList::use_path()
 586 {
 587         return 0;
 588 }
 589
 590
 591
 592
 593
 594
 595 FrameWriterPackage::FrameWriterPackage()
 596 {
 597 }
 598
 599 FrameWriterPackage::~FrameWriterPackage()
 600 {
 601 }
 602
 603
 604
 605
 606
 607
 608
 609
 610
 611
 612
 613 FrameWriterUnit::FrameWriterUnit(FrameWriter *server)
 614  : LoadClient(server)
 615 {
 616 // Don't use server here since subclasses call this with no server.
 617         this->server = server;
 618         output = new VFrame;
 619 }
 620
 621 FrameWriterUnit::~FrameWriterUnit()
 622 {
 623         delete output;
 624 }
 625
 626 void FrameWriterUnit::process_package(LoadPackage *package)
 627 {
 628 //printf("FrameWriterUnit::process_package 1\n");
 629         FrameWriterPackage *ptr = (FrameWriterPackage*)package;
 630
 631         FILE *file;
 632
 633 //printf("FrameWriterUnit::process_package 2 %s\n", ptr->path);
 634         if(!(file = fopen(ptr->path, "wb")))
 635         {
 636                 eprintf(_("Error while opening \"%s\" for writing. \n%m\n"), ptr->path);
 637                 return;
 638         }
 639 //printf("FrameWriterUnit::process_package 3");
 640
 641
 642         int result = server->file->write_frame(ptr->input, output, this);
 643
 644 //printf("FrameWriterUnit::process_package 4 %s %d\n", ptr->path, output->get_compressed_size());
 645         if(!result) result = !fwrite(output->get_data(), output->get_compressed_size(), 1, file);
 646 //TRACE("FrameWriterUnit::process_package 4");
 647         fclose(file);
 648 //TRACE("FrameWriterUnit::process_package 5");
 649
 650         server->file->add_return_value(result);
 651 //TRACE("FrameWriterUnit::process_package 6");
 652 }
 653
 654
 655
 656
 657
 658
 659
 660
 661
 662
 663
 664 FrameWriter::FrameWriter(FileList *file, int cpus)
 665  : LoadServer(cpus, 0)
 666 {
 667         this->file = file;
 668 }
 669
 670
 671 FrameWriter::~FrameWriter()
 672 {
 673 }
 674
 675 void FrameWriter::init_packages()
 676 {
 677         for(int i = 0, layer = 0, number = 0;
 678                 i < get_total_packages();
 679                 i++)
 680         {
 681                 FrameWriterPackage *package = (FrameWriterPackage*)get_package(i);
 682                 package->input = frames[layer][number];
 683                 package->path = file->create_path(package->input->get_number());
 684 // printf("FrameWriter::init_packages 1 %p %d %s\n",
 685 // package->input,
 686 // package->input->get_number(),
 687 // package->path);
 688                 number++;
 689                 if(number >= len)
 690                 {
 691                         layer++;
 692                         number = 0;
 693                 }
 694         }
 695 }
 696
 697 void FrameWriter::write_frames(VFrame ***frames, int len)
 698 {
 699         this->frames = frames;
 700         this->len = len;
 701         set_package_count(len * file->asset->layers);
 702
 703         process_packages();
 704 }
 705
 706 LoadClient* FrameWriter::new_client()
 707 {
 708         return file->new_writer_unit(this);
 709 }
 710
 711 LoadPackage* FrameWriter::new_package()
 712 {
 713         return new FrameWriterPackage;
 714 }
 715
 716
 717