downgrade dav1d to 5.1 seek fails, bg clr uses alpha=1, renderfarm hang fix, fix...
[goodguy/cinelerra.git] / cinelerra-5.1 / cinelerra / renderfarmclient.C
1
2 /*
3  * CINELERRA
4  * Copyright (C) 2008 Adam Williams <broadcast at earthling dot net>
5  *
6  * This program is free software; you can redistribute it and/or modify
7  * it under the terms of the GNU General Public License as published by
8  * the Free Software Foundation; either version 2 of the License, or
9  * (at your option) any later version.
10  *
11  * This program is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
14  * GNU General Public License for more details.
15  *
16  * You should have received a copy of the GNU General Public License
17  * along with this program; if not, write to the Free Software
18  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
19  *
20  */
21
22 #include "asset.h"
23 #include "assets.h"
24 #include "clip.h"
25 #include "bchash.h"
26 #include "dvbtune.h"
27 #include "edl.h"
28 #include "filesystem.h"
29 #include "filexml.h"
30 #include "language.h"
31 #include "mutex.h"
32 #include "mwindow.h"
33 #include "pluginserver.h"
34 #include "preferences.h"
35 #include "renderfarm.h"
36 #include "renderfarmclient.h"
37 //#include "renderfarmfsclient.h"
38 #include "sighandler.h"
39
40 #include <arpa/inet.h>
41 #include <errno.h>
42 #include <fcntl.h>
43 #include <netdb.h>
44 #include <netinet/in.h>
45 #include <stdio.h>
46 #include <string.h>
47 #include <sys/socket.h>
48 #include <sys/types.h>
49 #include <sys/un.h>
50 #include <sys/wait.h>
51 #include <unistd.h>
52
53
54 #ifndef AF_FILE
55 #define AF_FILE AF_LOCAL
56 #endif
57
58 // The render client waits for connections from the server.
59 // Then it starts a thread for each connection.
60 RenderFarmClient::RenderFarmClient(int port,
61         char *deamon_path,
62         int nice_value,
63         char *config_path)
64 {
65         char string[BCTEXTLEN];
66
67         this->port = port;
68         this->deamon_path = deamon_path;
69         SigHandler *signals = new SigHandler;
70         signals->initialize("/tmp/cinelerra_farm%d.dmp");
71
72         this_pid = getpid();
73         (void)nice(nice_value);
74
75         boot_defaults = 0;
76         MWindow::init_defaults(boot_defaults, config_path);
77         boot_preferences = new Preferences;
78         boot_preferences->load_defaults(boot_defaults);
79         MWindow::init_plugins(0, boot_preferences);
80         BC_Signals::set_catch_segv(boot_preferences->trap_sigsegv);
81         BC_Signals::set_catch_intr(0);
82         if( boot_preferences->trap_sigsegv ) {
83                 BC_Trace::enable_locks();
84         }
85         else {
86                 BC_Trace::disable_locks();
87         }
88
89         strcpy(string, boot_preferences->plugin_dir);
90         strcat(string, "/" FONT_SEARCHPATH);
91         BC_Resources::init_fontconfig(string);
92 }
93
94
95
96
97 RenderFarmClient::~RenderFarmClient()
98 {
99         delete boot_defaults;
100         delete boot_preferences;
101 }
102
103
104 void RenderFarmClient::main_loop()
105 {
106         int socket_fd;
107         BC_WindowBase::get_resources()->vframe_shm = 1;
108
109
110 // Open listening port
111
112         if(!deamon_path)
113         {
114                 struct sockaddr_in addr;
115
116                 addr.sin_family = AF_INET;
117                 addr.sin_port = htons((unsigned short)port);
118                 addr.sin_addr.s_addr = htonl(INADDR_ANY);
119
120                 if((socket_fd = socket(PF_INET, SOCK_STREAM, 0)) < 0)
121                 {
122                         perror(_("RenderFarmClient::main_loop: socket"));
123                         return;
124                 }
125                 struct linger lgr;
126                 lgr.l_onoff = 0;
127                 lgr.l_linger = 0;
128                 if( setsockopt(socket_fd, SOL_SOCKET, SO_LINGER, &lgr, sizeof(lgr)) < 0 )
129                         perror("RenderFarmClient::setsockopt:setlinger 0");
130
131                 if(bind(socket_fd,
132                         (struct sockaddr*)&addr,
133                         sizeof(addr)) < 0)
134                 {
135                         fprintf(stderr,
136                                 _("RenderFarmClient::main_loop: bind port %d: %s"),
137                                 port,
138                                 strerror(errno));
139                         return;
140                 }
141         }
142         else
143         {
144                 struct sockaddr_un addr;
145                 addr.sun_family = AF_FILE;
146                 strcpy(addr.sun_path, deamon_path);
147                 int size = (offsetof(struct sockaddr_un, sun_path) +
148                         strlen(deamon_path) + 1);
149
150                 if((socket_fd = socket(PF_UNIX, SOCK_STREAM, 0)) < 0)
151                 {
152                         perror(_("RenderFarmClient::main_loop: socket"));
153                         return;
154                 }
155                 struct linger lgr;
156                 lgr.l_onoff = 0;
157                 lgr.l_linger = 0;
158                 if( setsockopt(socket_fd, SOL_SOCKET, SO_LINGER, &lgr, sizeof(lgr)) < 0 )
159                         perror("RenderFarmClient::setsockopt:setlinger 1");
160
161                 if(bind(socket_fd,
162                         (struct sockaddr*)&addr,
163                         size) < 0)
164                 {
165                         fprintf(stderr,
166                                 _("RenderFarmClient::main_loop: bind path %s: %s\n"),
167                                 deamon_path,
168                                 strerror(errno));
169                         return;
170                 }
171         }
172
173 // Wait for connections
174         printf("RenderFarmClient::main_loop: client started\n");
175         while(1)
176         {
177                 if(listen(socket_fd, 256) < 0) {
178                         perror(_("RenderFarmClient::main_loop: listen"));
179                         return;
180                 }
181
182                 int new_socket_fd;
183
184                 if(!deamon_path)
185                 {
186                         struct sockaddr_in clientname;
187                         socklen_t size = sizeof(clientname);
188                         if( (new_socket_fd = accept(socket_fd,
189                                 (struct sockaddr*)&clientname, &size) ) < 0 ) {
190                                 perror(_("RenderFarmClient::main_loop: accept"));
191                                 return;
192                         }
193 printf("RenderFarmClient::main_loop: Session started from %s\n", inet_ntoa(clientname.sin_addr));
194                         RenderFarmClientThread *thread =
195                                 new RenderFarmClientThread(this);
196                         thread->main_loop(new_socket_fd);
197                 }
198                 else
199                 {
200                         struct sockaddr_un clientname;
201                         socklen_t size = sizeof(clientname);
202                         if( (new_socket_fd = accept(socket_fd,
203                                         (struct sockaddr*)&clientname, &size)) < 0 ) {
204                                 perror(_("RenderFarmClient::main_loop: accept"));
205                                 return;
206                         }
207 printf("RenderFarmClient::main_loop: Session started from %s\n", clientname.sun_path);
208                         RenderFarmClientThread *thread =
209                                 new RenderFarmClientThread(this);
210                         thread->main_loop(new_socket_fd);
211                 }
212         }
213 }
214
215 void RenderFarmClient::kill_client()
216 {
217 printf("RenderFarmClient::kill_client 1\n");
218         if(deamon_path)
219         {
220 printf("RenderFarmClient::kill_client 2\n");
221                 remove(deamon_path);
222                 kill(this_pid, SIGKILL);
223         }
224 }
225
226
227 // The thread requests jobs from the server until the job table is empty
228 // or the server reports an error.  This thread must poll the server
229 // after every frame for the error status.
230 // Detaches when finished.
231 RenderFarmClientThread::RenderFarmClientThread(RenderFarmClient *client)
232  : Thread(0, 0, 1)
233 {
234         this->client = client;
235         this->edl = 0;
236         frames_per_second = 0;
237         Thread::set_synchronous(0);
238 //      fs_client = 0;
239         mutex_lock = new Mutex("RenderFarmClientThread::mutex_lock");
240         watchdog = 0;
241         keep_alive = 0;
242 }
243
244 RenderFarmClientThread::~RenderFarmClientThread()
245 {
246 //      if(fs_client) delete fs_client;
247         delete mutex_lock;
248         delete watchdog;
249         delete keep_alive;
250 }
251
252
253 int RenderFarmClientThread::send_request_header(int request,
254         int len)
255 {
256         unsigned char datagram[5];
257         datagram[0] = request;
258
259         int i = 1;
260         STORE_INT32(len);
261 // printf("RenderFarmClientThread::send_request_header %d %02x%02x%02x%02x%02x\n",
262 // request, datagram[0], datagram[1], datagram[2], datagram[3], datagram[4]);
263
264         return (write_socket((char*)datagram, 5) != 5);
265 }
266
267 int RenderFarmClientThread::write_socket(char *data, int len)
268 {
269 //printf("RenderFarmClientThread::write_socket 1\n");
270         int result = write(socket_fd, data, len);
271 //printf("RenderFarmClientThread::write_socket 10\n");
272         return result;
273 }
274
275 int RenderFarmClientThread::read_socket(char *data, int len)
276 {
277         int bytes_read = 0;
278         int offset = 0;
279 //printf("RenderFarmClientThread::read_socket 1\n");
280         if( watchdog )
281                 watchdog->begin_request();
282         while(len > 0 && bytes_read >= 0)
283         {
284                 bytes_read = read(socket_fd, data + offset, len);
285                 if(bytes_read > 0)
286                 {
287                         len -= bytes_read;
288                         offset += bytes_read;
289                 }
290                 else
291                 if(bytes_read < 0)
292                 {
293                         break;
294                 }
295         }
296         if( watchdog )
297                 watchdog->end_request();
298 //printf("RenderFarmClientThread::read_socket 10\n");
299
300         return offset;
301 }
302
303 int RenderFarmClientThread::write_int64(int64_t value)
304 {
305         unsigned char data[sizeof(int64_t)];
306         data[0] = (value >> 56) & 0xff;
307         data[1] = (value >> 48) & 0xff;
308         data[2] = (value >> 40) & 0xff;
309         data[3] = (value >> 32) & 0xff;
310         data[4] = (value >> 24) & 0xff;
311         data[5] = (value >> 16) & 0xff;
312         data[6] = (value >> 8) & 0xff;
313         data[7] = value & 0xff;
314         return (write_socket((char*)data, sizeof(int64_t)) != sizeof(int64_t));
315 }
316
317 int64_t RenderFarmClientThread::read_int64(int *error)
318 {
319         int temp = 0;
320         if(!error) error = &temp;
321
322         unsigned char data[sizeof(int64_t)];
323         *error = (read_socket((char*)data, sizeof(int64_t)) != sizeof(int64_t));
324
325 // Make it return 1 if error so it can be used to read a result code from the
326 // server.
327         int64_t result = 1;
328         if(!*error)
329         {
330                 result = (((int64_t)data[0]) << 56) |
331                         (((uint64_t)data[1]) << 48) |
332                         (((uint64_t)data[2]) << 40) |
333                         (((uint64_t)data[3]) << 32) |
334                         (((uint64_t)data[4]) << 24) |
335                         (((uint64_t)data[5]) << 16) |
336                         (((uint64_t)data[6]) << 8)  |
337                         data[7];
338         }
339         return result;
340 }
341
342 void RenderFarmClientThread::read_string(char* &string)
343 {
344         unsigned char header[4];
345         if(read_socket((char*)header, 4) != 4)
346         {
347                 string = 0;
348                 return;
349         }
350
351         int64_t len = (((u_int32_t)header[0]) << 24) |
352                                 (((u_int32_t)header[1]) << 16) |
353                                 (((u_int32_t)header[2]) << 8) |
354                                 ((u_int32_t)header[3]);
355
356         if(len)
357         {
358                 string = new char[len];
359                 if(read_socket(string, len) != len)
360                 {
361                         delete [] string;
362                         string = 0;
363                 }
364         }
365         else
366                 string = 0;
367
368 }
369
370 void RenderFarmClientThread::abort()
371 {
372         send_completion(socket_fd);
373         close(socket_fd);
374         exit(1);
375 }
376
377 void RenderFarmClientThread::lock(const char *location)
378 {
379         mutex_lock->lock(location);
380 }
381
382 void RenderFarmClientThread::unlock()
383 {
384         mutex_lock->unlock();
385 }
386
387 void RenderFarmClientThread::get_command(int socket_fd, int *command)
388 {
389         int error;
390         *command = read_int64(&error);
391         if(error)
392         {
393                 *command = 0;
394                 return;
395         }
396 }
397
398
399 void RenderFarmClientThread::read_preferences(int socket_fd,
400         Preferences *preferences)
401 {
402         lock("RenderFarmClientThread::read_preferences");
403         send_request_header(RENDERFARM_PREFERENCES,
404                 0);
405
406         char *string;
407         read_string(string);
408
409         BC_Hash defaults;
410         defaults.load_string((char*)string);
411         preferences->load_defaults(&defaults);
412
413         delete [] string;
414         unlock();
415 }
416
417
418
419 void RenderFarmClientThread::read_asset(int socket_fd, Asset *asset)
420 {
421         lock("RenderFarmClientThread::read_asset");
422         send_request_header(RENDERFARM_ASSET,
423                 0);
424
425         char *string1;
426         char *string2;
427         read_string(string1);
428         read_string(string2);
429
430
431
432         FileXML file;
433         file.read_from_string((char*)string2);
434         asset->read(&file);
435
436
437
438         BC_Hash defaults;
439         defaults.load_string((char*)string1);
440         asset->load_defaults(&defaults, 0, 1, 1, 1, 1, 1);
441
442 //printf("RenderFarmClientThread::read_asset %d\n", __LINE__);
443 //asset->dump();
444
445         delete [] string1;
446         delete [] string2;
447         unlock();
448 }
449
450 void RenderFarmClientThread::read_edl(int socket_fd,
451                 EDL *edl, Preferences *preferences)
452 {
453         lock("RenderFarmClientThread::read_edl");
454         send_request_header(RENDERFARM_EDL, 0);
455
456         char *string;
457         read_string(string);
458
459         FileXML file;
460         file.read_from_string((char*)string);
461         delete [] string;
462
463         edl->load_xml(&file, LOAD_ALL);
464         unlock();
465 }
466
467 int RenderFarmClientThread::read_package(int socket_fd, RenderPackage *package)
468 {
469         lock("RenderFarmClientThread::read_package");
470         send_request_header(RENDERFARM_PACKAGE,
471                 4);
472
473         unsigned char datagram[5];
474         int i = 0;
475
476
477 // Fails if -ieee isn't set.
478         int64_t fixed = !EQUIV(frames_per_second, 0.0) ?
479                 (int64_t)(frames_per_second * 65536.0) : 0;
480         STORE_INT32(fixed);
481         write_socket((char*)datagram, 4);
482
483
484 //printf("RenderFarmClientThread::read_package 1 %f %ld\n", frames_per_second, fixed);
485         char *data;
486         unsigned char *data_ptr;
487         read_string(data);
488 //printf("RenderFarmClientThread::read_package 2 %p\n", data);
489 // Signifies end of session.
490         if(!data)
491         {
492 //              printf(_("RenderFarmClientThread::read_package no output path received.\n"));
493                 unlock();
494                 return 1;
495         }
496
497 //printf("RenderFarmClientThread::read_package 2\n");
498
499
500         data_ptr = (unsigned char*)data;
501         strcpy(package->path, data);
502         data_ptr += strlen(package->path);
503         data_ptr++;
504         package->audio_start = READ_INT32(data_ptr);
505         data_ptr += 4;
506         package->audio_end = READ_INT32(data_ptr);
507         data_ptr += 4;
508         package->video_start = READ_INT32(data_ptr);
509         data_ptr += 4;
510         package->video_end = READ_INT32(data_ptr);
511         data_ptr += 4;
512         package->use_brender = READ_INT32(data_ptr);
513         data_ptr += 4;
514         package->audio_do = READ_INT32(data_ptr);
515         data_ptr += 4;
516         package->video_do = READ_INT32(data_ptr);
517
518         delete [] data;
519         unlock();
520
521         return 0;
522 }
523
524 int RenderFarmClientThread::send_completion(int socket_fd)
525 {
526         lock("RenderFarmClientThread::send_completion");
527         int result = send_request_header(RENDERFARM_DONE, 0);
528         unlock();
529         return result;
530 }
531
532
533 void RenderFarmClientThread::ping_server()
534 {
535         lock("RenderFarmClientThread::ping_server");
536         send_request_header(RENDERFARM_KEEPALIVE, 0);
537         unlock();
538 }
539
540
541
542 void RenderFarmClientThread::main_loop(int socket_fd)
543 {
544         this->socket_fd = socket_fd;
545
546         Thread::start();
547 }
548
549 void RenderFarmClientThread::run()
550 {
551 // Create new memory space
552         pid = fork();
553         if(pid != 0)
554         {
555                 int return_value;
556                 waitpid(pid, &return_value, 0);
557                 return;
558         }
559
560 // Get the pid of the fork if inside the fork
561         pid = getpid();
562         BC_Signals::set_trap_hook(trap_hook, this);
563         TheList::reset();
564
565         int socket_fd = this->socket_fd;
566         int watchdog_timeout = client->boot_preferences->renderfarm_watchdog_timeout;
567         if( watchdog_timeout > 0 )
568                 init_client_keepalive(watchdog_timeout);
569
570 // Get command to run
571         int command;
572         lock("RenderFarmClientThread::run");
573         get_command(socket_fd, &command);
574         unlock();
575
576 //printf("RenderFarmClientThread::run command=%d\n", command);
577
578         switch(command)
579         {
580                 case RENDERFARM_TUNER:
581                         do_tuner(socket_fd);
582                         break;
583                 case RENDERFARM_PACKAGES:
584                         do_packages(socket_fd);
585                         break;
586         }
587
588         _exit(0);
589 }
590
591
592 void RenderFarmClientThread::init_client_keepalive(int timeout_secs)
593 {
594         keep_alive = new RenderFarmKeepalive(this);
595         keep_alive->start();
596         watchdog = new RenderFarmWatchdog(timeout_secs, 0, this);
597         watchdog->start();
598 }
599
600
601
602 void RenderFarmClientThread::do_tuner(int socket_fd)
603 {
604 // Currently only 1 tuner driver.  Maybe more someday.
605 //      DVBTune server(this);
606 //      server.main_loop();
607 //      ::close(socket_fd);
608 }
609
610
611 void RenderFarmClientThread::do_packages(int socket_fd)
612 {
613         RenderPackage *package;
614         Asset *default_asset;
615         Preferences *preferences;
616
617
618
619         FarmPackageRenderer package_renderer(this, socket_fd);
620         int result = 0;
621
622
623
624 //printf("RenderFarmClientThread::run 2\n");
625 // Read settings
626         preferences = new Preferences;
627         default_asset = new Asset;
628         package = new RenderPackage;
629         edl = new EDL;
630         edl->create_objects();
631
632 //printf("RenderFarmClientThread::run 3\n");
633         read_preferences(socket_fd, preferences);
634 //printf("RenderFarmClientThread::run 4\n");
635         read_asset(socket_fd, default_asset);
636 //printf("RenderFarmClientThread::run 5\n");
637         read_edl(socket_fd, edl, preferences);
638 //edl->dump();
639 //printf("RenderFarmClientThread::run 6\n");
640
641         package_renderer.initialize(0, edl, preferences, default_asset);
642
643 // Read packages
644         while(1)
645         {
646 //printf("RenderFarmClientThread::run 5\n");
647                 result = read_package(socket_fd, package);
648 //printf("RenderFarmClientThread::run 6 %d\n", result);
649
650
651 // Finished list
652                 if(result)
653                 {
654 //printf("RenderFarmClientThread::run 7\n");
655
656                         result = send_completion(socket_fd);
657                         break;
658                 }
659
660 //              Timer timer;
661 //              timer.update();
662
663 // Error
664                 if(package_renderer.render_package(package))
665                 {
666 //printf("RenderFarmClientThread::run 8\n");
667                         result = send_completion(socket_fd);
668                         break;
669                 }
670
671
672                 frames_per_second = package_renderer.frames_per_second;
673 //              frames_per_second = (double)(package->video_end - package->video_start) /
674 //                      ((double)timer.get_difference() / 1000);
675
676 //printf("RenderFarmClientThread::run 9\n");
677
678
679
680         }
681
682
683 //printf("RenderFarmClientThread::run 9\n");
684         default_asset->Garbage::remove_user();
685 //printf("RenderFarmClientThread::run 10\n");
686         edl->Garbage::remove_user();
687         edl = 0;
688 //printf("RenderFarmClientThread::run 11\n");
689         delete preferences;
690 printf(_("RenderFarmClientThread::run: Session finished.\n"));
691 }
692
693 void RenderFarmClientThread::trap_hook(FILE *fp, void *vp)
694 {
695         RenderFarmClientThread *thread = (RenderFarmClientThread*)vp;
696         fprintf(fp, "\nEDL:\n");
697         if( thread->edl ) thread->edl->dump(fp);
698 }
699
700
701
702 RenderFarmKeepalive::RenderFarmKeepalive(
703         RenderFarmClientThread *client_thread)
704  : Thread(1, 0, 0)
705 {
706         this->client_thread = client_thread;
707         done = 0;
708 }
709
710 RenderFarmKeepalive::~RenderFarmKeepalive()
711 {
712         done = 1;
713         cancel();
714         join();
715 }
716
717 void RenderFarmKeepalive::run()
718 {
719         while(!done)
720         {
721                 enable_cancel();
722                 sleep(5);
723                 disable_cancel();
724                 if( !done ) {
725 //printf("RenderFarmKeepalive::run 1\n");
726 // watchdog thread kills this if it gets stuck
727                         client_thread->ping_server();
728 //printf("RenderFarmKeepalive::run 10\n");
729                 }
730         }
731 }
732
733
734 FarmPackageRenderer::FarmPackageRenderer(RenderFarmClientThread *thread,
735                 int socket_fd)
736  : PackageRenderer()
737 {
738         this->thread = thread;
739         this->socket_fd = socket_fd;
740 }
741
742
743 FarmPackageRenderer::~FarmPackageRenderer()
744 {
745 }
746
747
748 int FarmPackageRenderer::get_result()
749 {
750         thread->lock("FarmPackageRenderer::get_result");
751         thread->send_request_header(RENDERFARM_GET_RESULT,
752                 0);
753         unsigned char data[1];
754         data[0] = 1;
755         if(thread->read_socket((char*)data, 1) != 1)
756         {
757                 thread->unlock();
758                 return 1;
759         }
760         thread->unlock();
761         return data[0];
762 }
763
764 void FarmPackageRenderer::set_result(int value)
765 {
766         thread->lock("FarmPackageRenderer::set_result");
767         thread->send_request_header(RENDERFARM_SET_RESULT,
768                 1);
769         unsigned char data[1];
770         data[0] = value;
771         thread->write_socket((char*)data, 1);
772         thread->unlock();
773 }
774
775 void FarmPackageRenderer::set_progress(int64_t total_samples)
776 {
777         thread->lock("FarmPackageRenderer::set_progress");
778         thread->send_request_header(RENDERFARM_PROGRESS,
779                 8);
780         unsigned char datagram[8];
781         int i = 0;
782         STORE_INT32(total_samples);
783
784
785         int64_t fixed = (!EQUIV(frames_per_second, 0.0)) ?
786                 (int64_t)(frames_per_second * 65536.0) : 0;
787         STORE_INT32(fixed);
788
789
790         thread->write_socket((char*)datagram, 8);
791         thread->unlock();
792 }
793
794 int FarmPackageRenderer::set_video_map(int64_t position, int value)
795 {
796         int result = 0;
797         unsigned char datagram[8];
798         char return_value[1];
799         int i = 0;
800
801         thread->lock("FarmPackageRenderer::set_video_map");
802         thread->send_request_header(RENDERFARM_SET_VMAP,
803                 8);
804         STORE_INT32(position);
805         STORE_INT32(value);
806         thread->write_socket((char*)datagram, 8);
807
808 // Get completion since the GUI may be locked for a long time.
809         if(!thread->read_socket(return_value, 1))
810         {
811                 result = 1;
812         }
813
814         thread->unlock();
815         return result;
816 }
817
818