minor improvements from Andrew
[goodguy/cinelerra.git] / cinelerra-5.1 / cinelerra / filelist.C
1
2 /*
3  * CINELERRA
4  * Copyright (C) 1997-2012 Adam Williams <broadcast at earthling dot net>
5  *
6  * This program is free software; you can redistribute it and/or modify
7  * it under the terms of the GNU General Public License as published by
8  * the Free Software Foundation; either version 2 of the License, or
9  * (at your option) any later version.
10  *
11  * This program is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
14  * GNU General Public License for more details.
15  *
16  * You should have received a copy of the GNU General Public License
17  * along with this program; if not, write to the Free Software
18  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
19  *
20  */
21
22 #include "asset.h"
23 #include "bcsignals.h"
24 #include "cstrdup.h"
25 #include "file.h"
26 #include "filelist.h"
27 #include "guicast.h"
28 #include "interlacemodes.h"
29 #include "mutex.h"
30 #include "mwindow.inc"
31 #include "render.h"
32 #include "renderfarmfsserver.inc"
33 #include "vframe.h"
34 #include "mainerror.h"
35
36 #include <ctype.h>
37 #include <errno.h>
38 #include <fcntl.h>
39 #include <stdlib.h>
40 #include <string.h>
41 #include <sys/stat.h>
42 #include <unistd.h>
43
44
45 FileList::FileList(Asset *asset,
46         File *file,
47         const char *list_prefix,
48         const char *file_extension,
49         int frame_type,
50         int list_type)
51  : FileBase(asset, file)
52 {
53         reset_parameters();
54         path_list.set_array_delete();
55         asset->video_data = 1;
56         this->list_prefix = list_prefix;
57         this->file_extension = file_extension;
58         this->frame_type = frame_type;
59         this->list_type = list_type;
60         table_lock = new Mutex("FileList::table_lock");
61 }
62
63 FileList::~FileList()
64 {
65         close_file();
66         delete table_lock;
67 }
68
69 int FileList::reset_parameters_derived()
70 {
71         data = 0;
72         writer = 0;
73         temp = 0;
74         first_number = 0;
75         return 0;
76 }
77
78 int FileList::open_file(int rd, int wr)
79 {
80         int result = 1;
81
82 // skip header for write
83         if(file->wr)
84         {
85                 int fd = open(asset->path, O_CREAT+O_TRUNC+O_RDWR, 0777);
86                 if( fd >= 0 ) {
87                         close(fd);
88                         result = 0;
89 // Frame files are created in write_frame and list index is created when
90 // file is closed.
91 // Look for the starting number in the path but ignore the starting character
92 // and total digits since these are used by the header.
93                         Render::get_starting_number(asset->path,
94                                 first_number, number_start, number_digits);
95                         path_list.remove_all_objects();
96                         writer = new FrameWriter(this,
97                         asset->format == list_type ? file->cpus : 1);
98                 }
99                 else
100                         eprintf(_("Error while opening \"%s\" for writing. \n%m\n"), asset->path);
101         }
102         else
103         if(file->rd)
104         {
105 // Determine type of file.
106 // Header isn't used for background rendering, in which case everything known
107 // by the file encoder is known by the decoder.
108 //printf("FileList::open_file %d %d\n", __LINE__, asset->use_header);
109                 if(asset->use_header)
110                 {
111                         FILE *stream = fopen(asset->path, "rb");
112 //printf("FileList::open_file %d asset->path=%s\n", __LINE__, asset->path);
113                         if(stream)
114                         {
115                                 int width = asset->width, height = asset->height;
116                                 char string[BCTEXTLEN];
117                                 int len = strlen(list_prefix);
118                                 int ret = fread(string, 1, strlen(list_prefix), stream);
119                                 fclose(stream);
120                                 result = len == ret ? 0 : 1;
121                                 if( !result && !strncasecmp(string, list_prefix, len)) {
122                                         asset->format = list_type;
123                                         result = read_list_header();
124                                         if( !result )
125                                                 result = read_frame_header(path_list.values[0]);
126                                 }
127                                 else if( !result && frame_type != FILE_UNKNOWN ) {
128                                         asset->format = frame_type;
129                                         result = read_frame_header(asset->path);
130                                         asset->video_length = -1;
131                                 }
132                                 else
133                                         result = 1;
134                                 if( !result ) {
135                                         asset->actual_width = asset->width;
136                                         asset->actual_height = asset->height;
137                                         int scale = asset->proxy_scale;
138                                         if( scale ) {
139                                                 asset->width = asset->actual_width * scale;
140                                                 asset->height = asset->actual_height * scale;
141                                         }
142                                         else { // can_scale_input
143                                                 if( width ) asset->width = width;
144                                                 if( height ) asset->height = height;
145                                         }
146                                         asset->layers = 1;
147                                         if( !asset->frame_rate )
148                                                 asset->frame_rate = 10;
149                                 }
150                         }
151                         else
152                                 eprintf(_("Error while opening \"%s\" for reading. \n%m\n"), asset->path);
153                 }
154                 else
155                 {
156                         Render::get_starting_number(asset->path,
157                                 first_number, number_start, number_digits, 6);
158                         result = 0;
159                 }
160         }
161
162         file->current_frame = 0;
163 // Compressed data storage
164         data = new VFrame;
165
166         return result;
167 }
168
169
170 int FileList::close_file()
171 {
172 //      path_list.total, asset->format, list_type, wr);
173         if(asset->format == list_type && path_list.total)
174         {
175                 if(file->wr && asset->use_header) write_list_header();
176                 path_list.remove_all_objects();
177         }
178         if(data) delete data;
179         if(writer) delete writer;
180         if(temp) delete temp;
181         reset_parameters();
182
183         FileBase::close_file();
184         return 0;
185 }
186
187 int FileList::write_list_header()
188 {
189         FILE *stream = fopen(asset->path, "w");
190         if( !stream ) return 1;
191 // Use sprintf instead of fprintf for VFS.
192         fprintf(stream, "%s\n", list_prefix);
193         fprintf(stream, "# First line is always %s\n", list_prefix);
194         fprintf(stream, "# Frame rate:\n");
195         fprintf(stream, "%f\n", asset->frame_rate);
196         fprintf(stream, "# Width:\n");
197         fprintf(stream, "%d\n", asset->width);
198         fprintf(stream, "# Height:\n");
199         fprintf(stream, "%d\n", asset->height);
200         fprintf(stream, "# List of image files follows\n");
201
202         char *cp = strrchr(asset->path, '/');
203         int dir_len = !cp ? 0 : cp - asset->path;
204
205         for(int i = 0; i < path_list.total; i++) {
206                 const char *path = path_list.values[i];
207 // Fix path for VFS but leave leading slash
208                 if( !strncmp(path, RENDERFARM_FS_PREFIX, strlen(RENDERFARM_FS_PREFIX)) )
209                         path += strlen(RENDERFARM_FS_PREFIX);
210 // ./path for relative list access
211                 else if( dir_len > 0 && !strncmp(path, asset->path, dir_len) ) {
212                         fprintf(stream, ".");  path += dir_len;
213                 }
214                 fprintf(stream, "%s\n", path);
215         }
216         fclose(stream);
217         return 0;
218 }
219
220 int FileList::read_list_header()
221 {
222         char string[BCTEXTLEN];
223
224         FILE *stream = fopen(asset->path, "r");
225         if( !stream ) return 1;
226 // Get information about the frames
227         do {
228                 if( feof(stream) || !fgets(string, BCTEXTLEN, stream) ) return 1;
229         } while(string[0] == '#' || string[0] == ' ' || isalpha(string[0]));
230
231 // Don't want a user configured frame rate to get destroyed
232         if(asset->frame_rate == 0)
233                 asset->frame_rate = atof(string);
234
235         do {
236                 if( feof(stream) || !fgets(string, BCTEXTLEN, stream) ) return 1;
237         } while(string[0] == '#' || string[0] == ' ');
238         if( (asset->width = atol(string)) <= 0 ) return 1;
239
240         do {
241                 if( feof(stream) || !fgets(string, BCTEXTLEN, stream) ) return 1;
242         } while(string[0] == '#' || string[0] == ' ');
243         if( (asset->height = atol(string)) <= 0 ) return 1;
244
245         asset->interlace_mode = ILACE_MODE_UNDETECTED;
246         asset->layers = 1;
247         asset->audio_data = 0;
248         asset->video_data = 1;
249
250         char prefix[BCTEXTLEN], *bp = prefix, *cp = strrchr(asset->path, '/');
251         for( int i=0, n=!cp ? 0 : cp-asset->path; i<n; ++i ) *bp++ = asset->path[i];
252         *bp = 0;
253
254 // Get all the paths, expand relative paths
255         int missing = 0;
256         while( !feof(stream) && fgets(string, BCTEXTLEN, stream) ) {
257                 int len = strlen(string);
258                 if( !len || string[0] == '#' || string[0] == ' ' ) continue;
259                 if( string[len-1] == '\n' ) string[len-1] = 0;
260                 char path[BCTEXTLEN], *pp = path, *ep = pp + sizeof(path)-1;
261                 if( string[0] == '.' && string[1] == '/' && prefix[0] )
262                         pp += snprintf(pp, ep-pp, "%s/", prefix);
263                 snprintf(pp, ep-pp, "%s", string);
264                 if( access(path, R_OK) && !missing++ )
265                         eprintf(_("%s:no such file"), path);
266                 path_list.append(cstrdup(path));
267         }
268
269 //for(int i = 0; i < path_list.total; i++) printf("%s\n", path_list.values[i]);
270         fclose(stream);
271         if( !(asset->video_length = path_list.total) )
272                 eprintf(_("%s:\nlist empty"), asset->path);
273         if( missing )
274                 eprintf(_("%s:\n%d files not found"), asset->path, missing);
275         return 0;
276 }
277
278 int FileList::read_frame(VFrame *frame)
279 {
280         int result = 0;
281
282 //      PRINT_TRACE
283 // printf("FileList::read_frame %d %d use_header=%d current_frame=%d total=%d\n",
284 // __LINE__,
285 // result,
286 // asset->use_header,
287 // file->current_frame,
288 // path_list.total);
289
290         if(file->current_frame < 0 ||
291                 (asset->use_header && file->current_frame >= path_list.total &&
292                         asset->format == list_type))
293                 return 1;
294
295         if(asset->format == list_type)
296         {
297                 char string[BCTEXTLEN];
298                 char *path;
299                 if(asset->use_header)
300                 {
301                         path = path_list.values[file->current_frame];
302                 }
303                 else
304                 {
305                         path = calculate_path(file->current_frame, string);
306                 }
307
308                 FILE *in;
309
310 // Fix path for VFS.  Not used anymore.
311                 if(!strncmp(asset->path, RENDERFARM_FS_PREFIX, strlen(RENDERFARM_FS_PREFIX)))
312                         sprintf(string, "%s%s", RENDERFARM_FS_PREFIX, path);
313                 else
314                         strcpy(string, path);
315
316
317
318                 if(!use_path() || frame->get_color_model() == BC_COMPRESSED)
319                 {
320                         if(!(in = fopen(string, "rb"))) {
321                                 eprintf(_("Error while opening \"%s\" for reading. \n%m\n"), string);
322                         }
323                         else
324                         {
325                                 struct stat ostat;
326                                 stat(string, &ostat);
327
328                                 switch(frame->get_color_model())
329                                 {
330                                         case BC_COMPRESSED:
331                                                 frame->allocate_compressed_data(ostat.st_size);
332                                                 frame->set_compressed_size(ostat.st_size);
333                                                 (void)fread(frame->get_data(), ostat.st_size, 1, in);
334                                                 break;
335                                         default:
336                                                 data->allocate_compressed_data(ostat.st_size);
337                                                 data->set_compressed_size(ostat.st_size);
338                                                 (void)fread(data->get_data(), ostat.st_size, 1, in);
339                                                 result = read_frame(frame, data);
340                                                 break;
341                                 }
342
343                                 fclose(in);
344                         }
345                 }
346                 else
347                 {
348 //printf("FileList::read_frame %d %s\n", __LINE__, string);
349                         result = read_frame(frame, string);
350                 }
351         }
352         else
353         {
354                 asset->single_frame = 1;
355 // Allocate and decompress single frame into new temporary
356 //printf("FileList::read_frame %d\n", frame->get_color_model());
357                 if( !temp || temp->get_color_model() != frame->get_color_model() ) {
358                         delete temp;  temp = 0;
359                         int aw = asset->actual_width, ah = asset->actual_height;
360                         int color_model = frame->get_color_model();
361                         switch( color_model ) {
362                         case BC_YUV420P:
363                         case BC_YUV420PI:
364                         case BC_YUV422P:
365                                 aw = (aw+1) & ~1;  ah = (ah+1) & ~1;
366                                 break;
367                         case BC_YUV410P:
368                         case BC_YUV411P:
369                                 aw = (aw+3) & ~3;  ah = (ah+3) & ~3;
370                                 break;
371                         }
372                         if( !use_path() || color_model == BC_COMPRESSED ) {
373                                 FILE *fd = fopen(asset->path, "rb");
374                                 if( fd ) {
375                                         struct stat ostat;
376                                         stat(asset->path, &ostat);
377
378                                         switch(frame->get_color_model()) {
379                                         case BC_COMPRESSED:
380                                                 frame->allocate_compressed_data(ostat.st_size);
381                                                 frame->set_compressed_size(ostat.st_size);
382                                                 (void)fread(frame->get_data(), ostat.st_size, 1, fd);
383                                                 break;
384                                         default:
385                                                 data->allocate_compressed_data(ostat.st_size);
386                                                 data->set_compressed_size(ostat.st_size);
387                                                 (void)fread(data->get_data(), ostat.st_size, 1, fd);
388                                                 temp = new VFrame(aw, ah, color_model);
389                                                 read_frame(temp, data);
390                                                 break;
391                                         }
392
393                                         fclose(fd);
394                                 }
395                                 else {
396                                         eprintf(_("Error while opening \"%s\" for reading. \n%m\n"), asset->path);
397                                         result = 1;
398                                 }
399                         }
400                         else {
401                                 temp = new VFrame(aw, ah, color_model);
402                                 read_frame(temp, asset->path);
403                         }
404                 }
405
406                 if(!temp) return result;
407
408 //printf("FileList::read_frame frame=%d temp=%d\n",
409 // frame->get_color_model(), // temp->get_color_model());
410                 frame->transfer_from(temp);
411         }
412
413
414 // printf("FileList::read_frame %d %d\n", __LINE__, result);
415 //
416 // if(frame->get_y())
417 // for(int i = 0; i < 100000; i++)
418 // {
419 //      frame->get_y()[i] = 0xff;
420 // }
421 // if(frame->get_rows())
422 // for(int i = 0; i < 100000; i++)
423 // {
424 //      frame->get_rows()[0][i] = 0xff;
425 // }
426
427
428         return result;
429 }
430
431 int FileList::write_frames(VFrame ***frames, int len)
432 {
433         return_value = 0;
434
435 //printf("FileList::write_frames 1\n");
436         if(frames[0][0]->get_color_model() == BC_COMPRESSED)
437         {
438                 for(int i = 0; i < asset->layers && !return_value; i++)
439                 {
440                         for(int j = 0; j < len && !return_value; j++)
441                         {
442                                 VFrame *frame = frames[i][j];
443                                 char *path = create_path(frame->get_number());
444 //printf("FileList::write_frames %d %jd\n", __LINE__, frame->get_number());
445
446
447                                 FILE *fd = fopen(path, "wb");
448                                 if(fd)
449                                 {
450                                         return_value = !fwrite(frames[i][j]->get_data(),
451                                                 frames[i][j]->get_compressed_size(),
452                                                 1,
453                                                 fd);
454
455                                         fclose(fd);
456                                 }
457                                 else
458                                 {
459                                         eprintf(_("Error while opening \"%s\" for writing. \n%m\n"), asset->path);
460                                         return_value++;
461                                 }
462                         }
463                 }
464         }
465         else
466         {
467 //printf("FileList::write_frames 2\n");
468                 writer->write_frames(frames, len);
469 //printf("FileList::write_frames 100\n");
470         }
471         return return_value;
472 }
473
474
475
476
477
478
479
480
481
482 void FileList::add_return_value(int amount)
483 {
484         table_lock->lock("FileList::add_return_value");
485         return_value += amount;
486         table_lock->unlock();
487 }
488
489 char* FileList::calculate_path(int number, char *string)
490 {
491 // Synthesize filename.
492 // If a header is used, the filename number must be in a different location.
493         if(asset->use_header)
494         {
495                 int k;
496                 strcpy(string, asset->path);
497                 for(k = strlen(string) - 1; k > 0 && string[k] != '.'; k--)
498                         ;
499                 if(k <= 0) k = strlen(string);
500
501                 sprintf(&string[k], "%06d%s",
502                         number,
503                         file_extension);
504         }
505         else
506 // Without a header, the original filename can be altered.
507         {
508                 Render::create_filename(string,
509                         asset->path,
510                         number,
511                         number_digits,
512                         number_start);
513         }
514
515         return string;
516 }
517
518 char* FileList::create_path(int number_override)
519 {
520         if(asset->format != list_type) return asset->path;
521
522         table_lock->lock("FileList::create_path");
523
524
525
526         char *path = 0;
527         char output[BCTEXTLEN];
528         if(file->current_frame >= path_list.total || !asset->use_header)
529         {
530                 int number;
531                 if(number_override < 0)
532                         number = file->current_frame++;
533                 else
534                 {
535                         number = number_override;
536                         file->current_frame++;
537                 }
538
539                 if(!asset->use_header)
540                 {
541                         number += first_number;
542                 }
543
544                 calculate_path(number, output);
545
546                 path = new char[strlen(output) + 1];
547                 strcpy(path, output);
548                 path_list.append(path);
549         }
550         else
551         {
552 // Overwrite an old path
553                 path = path_list.values[file->current_frame];
554         }
555
556
557         table_lock->unlock();
558
559         return path;
560 }
561
562 FrameWriterUnit* FileList::new_writer_unit(FrameWriter *writer)
563 {
564         return new FrameWriterUnit(writer);
565 }
566
567 int64_t FileList::get_memory_usage()
568 {
569         int64_t result = 0;
570         if(data) result += data->get_compressed_allocated();
571         if(temp) result += temp->get_data_size();
572 // printf("FileList::get_memory_usage %d %p %s %jd\n",
573 // __LINE__,
574 // this,
575 // file->asset->path,
576 // result);
577         return result;
578 }
579
580 int FileList::get_units()
581 {
582         return !writer ? 0 : writer->get_total_clients();
583 }
584
585 FrameWriterUnit* FileList::get_unit(int number)
586 {
587         return !writer ? 0 : (FrameWriterUnit*)writer->get_client(number);
588 }
589
590 int FileList::use_path()
591 {
592         return 0;
593 }
594
595
596
597
598
599
600 FrameWriterPackage::FrameWriterPackage()
601 {
602 }
603
604 FrameWriterPackage::~FrameWriterPackage()
605 {
606 }
607
608
609
610
611
612
613
614
615
616
617
618 FrameWriterUnit::FrameWriterUnit(FrameWriter *server)
619  : LoadClient(server)
620 {
621 // Don't use server here since subclasses call this with no server.
622         this->server = server;
623         output = new VFrame;
624 }
625
626 FrameWriterUnit::~FrameWriterUnit()
627 {
628         delete output;
629 }
630
631 void FrameWriterUnit::process_package(LoadPackage *package)
632 {
633 //printf("FrameWriterUnit::process_package 1\n");
634         FrameWriterPackage *ptr = (FrameWriterPackage*)package;
635
636         FILE *file;
637
638 //printf("FrameWriterUnit::process_package 2 %s\n", ptr->path);
639         if(!(file = fopen(ptr->path, "wb")))
640         {
641                 eprintf(_("Error while opening \"%s\" for writing. \n%m\n"), ptr->path);
642                 return;
643         }
644 //printf("FrameWriterUnit::process_package 3");
645
646
647         int result = server->file->write_frame(ptr->input, output, this);
648
649 //printf("FrameWriterUnit::process_package 4 %s %d\n", ptr->path, output->get_compressed_size());
650         if(!result) result = !fwrite(output->get_data(), output->get_compressed_size(), 1, file);
651 //TRACE("FrameWriterUnit::process_package 4");
652         fclose(file);
653 //TRACE("FrameWriterUnit::process_package 5");
654
655         server->file->add_return_value(result);
656 //TRACE("FrameWriterUnit::process_package 6");
657 }
658
659
660
661
662
663
664
665
666
667
668
669 FrameWriter::FrameWriter(FileList *file, int cpus)
670  : LoadServer(cpus, 0)
671 {
672         this->file = file;
673 }
674
675
676 FrameWriter::~FrameWriter()
677 {
678 }
679
680 void FrameWriter::init_packages()
681 {
682         for(int i = 0, layer = 0, number = 0;
683                 i < get_total_packages();
684                 i++)
685         {
686                 FrameWriterPackage *package = (FrameWriterPackage*)get_package(i);
687                 package->input = frames[layer][number];
688                 package->path = file->create_path(package->input->get_number());
689 // printf("FrameWriter::init_packages 1 %p %d %s\n",
690 // package->input,
691 // package->input->get_number(),
692 // package->path);
693                 number++;
694                 if(number >= len)
695                 {
696                         layer++;
697                         number = 0;
698                 }
699         }
700 }
701
702 void FrameWriter::write_frames(VFrame ***frames, int len)
703 {
704         this->frames = frames;
705         this->len = len;
706         set_package_count(len * file->asset->layers);
707
708         process_packages();
709 }
710
711 LoadClient* FrameWriter::new_client()
712 {
713         return file->new_writer_unit(this);
714 }
715
716 LoadPackage* FrameWriter::new_package()
717 {
718         return new FrameWriterPackage;
719 }
720
721
722