7c50bd5401a177e403ec0728ecb9efc20f01f6a1
[goodguy/cinelerra.git] / cinelerra-5.1 / cinelerra / renderfarmclient.C
1
2 /*
3  * CINELERRA
4  * Copyright (C) 2008 Adam Williams <broadcast at earthling dot net>
5  * Copyright (C) 2003-2016 Cinelerra CV contributors
6  *
7  * This program is free software; you can redistribute it and/or modify
8  * it under the terms of the GNU General Public License as published by
9  * the Free Software Foundation; either version 2 of the License, or
10  * (at your option) any later version.
11  *
12  * This program is distributed in the hope that it will be useful,
13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15  * GNU General Public License for more details.
16  *
17  * You should have received a copy of the GNU General Public License
18  * along with this program; if not, write to the Free Software
19  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
20  *
21  */
22
23 #include "asset.h"
24 #include "assets.h"
25 #include "clip.h"
26 #include "bchash.h"
27 #include "dvbtune.h"
28 #include "edl.h"
29 #include "filesystem.h"
30 #include "filexml.h"
31 #include "language.h"
32 #include "mutex.h"
33 #include "mwindow.h"
34 #include "pluginserver.h"
35 #include "preferences.h"
36 #include "renderfarm.h"
37 #include "renderfarmclient.h"
38 //#include "renderfarmfsclient.h"
39 #include "sighandler.h"
40
41 #include <arpa/inet.h>
42 #include <errno.h>
43 #include <fcntl.h>
44 #include <netdb.h>
45 #include <netinet/in.h>
46 #include <stdio.h>
47 #include <string.h>
48 #include <sys/socket.h>
49 #include <sys/types.h>
50 #include <sys/un.h>
51 #include <sys/wait.h>
52 #include <unistd.h>
53
54
55 #ifndef AF_FILE
56 #define AF_FILE AF_LOCAL
57 #endif
58
59 // The render client waits for connections from the server.
60 // Then it starts a thread for each connection.
61 RenderFarmClient::RenderFarmClient(int port,
62         char *deamon_path,
63         int nice_value,
64         char *config_path)
65 {
66         char string[BCTEXTLEN];
67
68         this->port = port;
69         this->deamon_path = deamon_path;
70         SigHandler *signals = new SigHandler;
71         signals->initialize("/tmp/cinelerra_farm%d.dmp");
72
73         this_pid = getpid();
74         (void)nice(nice_value);
75
76         boot_defaults = 0;
77         MWindow::init_defaults(boot_defaults, config_path);
78         boot_preferences = new Preferences;
79         boot_preferences->load_defaults(boot_defaults);
80         MWindow::init_plugins(0, boot_preferences);
81         MWindow::init_ladspa_plugins(0, boot_preferences);  
82         BC_Signals::set_catch_segv(boot_preferences->trap_sigsegv);
83         BC_Signals::set_catch_intr(0);
84         if( boot_preferences->trap_sigsegv ) {
85                 BC_Trace::enable_locks();
86         }
87         else {
88                 BC_Trace::disable_locks();
89         }
90
91         strcpy(string, boot_preferences->plugin_dir);
92         strcat(string, "/" FONT_SEARCHPATH);
93         BC_Resources::init_fontconfig(string);
94 }
95
96
97
98
99 RenderFarmClient::~RenderFarmClient()
100 {
101         delete boot_defaults;
102         delete boot_preferences;
103 }
104
105
106 void RenderFarmClient::main_loop()
107 {
108         int socket_fd;
109         BC_WindowBase::get_resources()->vframe_shm = 1;
110
111
112 // Open listening port
113
114         if(!deamon_path)
115         {
116                 struct sockaddr_in addr;
117
118                 addr.sin_family = AF_INET;
119                 addr.sin_port = htons((unsigned short)port);
120                 addr.sin_addr.s_addr = htonl(INADDR_ANY);
121
122                 if((socket_fd = socket(PF_INET, SOCK_STREAM, 0)) < 0)
123                 {
124                         perror(_("RenderFarmClient::main_loop: socket"));
125                         return;
126                 }
127                 struct linger lgr;
128                 lgr.l_onoff = 0;
129                 lgr.l_linger = 0;
130                 if( setsockopt(socket_fd, SOL_SOCKET, SO_LINGER, &lgr, sizeof(lgr)) < 0 )
131                         perror("RenderFarmClient::setsockopt:setlinger 0");
132
133                 if(bind(socket_fd,
134                         (struct sockaddr*)&addr,
135                         sizeof(addr)) < 0)
136                 {
137                         fprintf(stderr,
138                                 _("RenderFarmClient::main_loop: bind port %d: %s"),
139                                 port,
140                                 strerror(errno));
141                         return;
142                 }
143         }
144         else
145         {
146                 struct sockaddr_un addr;
147                 addr.sun_family = AF_FILE;
148                 strcpy(addr.sun_path, deamon_path);
149                 int size = (offsetof(struct sockaddr_un, sun_path) +
150                         strlen(deamon_path) + 1);
151
152                 if((socket_fd = socket(PF_UNIX, SOCK_STREAM, 0)) < 0)
153                 {
154                         perror(_("RenderFarmClient::main_loop: socket"));
155                         return;
156                 }
157                 struct linger lgr;
158                 lgr.l_onoff = 0;
159                 lgr.l_linger = 0;
160                 if( setsockopt(socket_fd, SOL_SOCKET, SO_LINGER, &lgr, sizeof(lgr)) < 0 )
161                         perror("RenderFarmClient::setsockopt:setlinger 1");
162
163                 if(bind(socket_fd,
164                         (struct sockaddr*)&addr,
165                         size) < 0)
166                 {
167                         fprintf(stderr,
168                                 _("RenderFarmClient::main_loop: bind path %s: %s\n"),
169                                 deamon_path,
170                                 strerror(errno));
171                         return;
172                 }
173         }
174
175 // Wait for connections
176         printf("RenderFarmClient::main_loop: client started\n");
177         while(1)
178         {
179                 if(listen(socket_fd, 256) < 0) {
180                         perror(_("RenderFarmClient::main_loop: listen"));
181                         return;
182                 }
183
184                 int new_socket_fd;
185
186                 if(!deamon_path)
187                 {
188                         struct sockaddr_in clientname;
189                         socklen_t size = sizeof(clientname);
190                         if( (new_socket_fd = accept(socket_fd,
191                                 (struct sockaddr*)&clientname, &size) ) < 0 ) {
192                                 perror(_("RenderFarmClient::main_loop: accept"));
193                                 return;
194                         }
195 printf("RenderFarmClient::main_loop: Session started from %s\n", inet_ntoa(clientname.sin_addr));
196                         RenderFarmClientThread *thread =
197                                 new RenderFarmClientThread(this);
198                         thread->main_loop(new_socket_fd);
199                 }
200                 else
201                 {
202                         struct sockaddr_un clientname;
203                         socklen_t size = sizeof(clientname);
204                         if( (new_socket_fd = accept(socket_fd,
205                                         (struct sockaddr*)&clientname, &size)) < 0 ) {
206                                 perror(_("RenderFarmClient::main_loop: accept"));
207                                 return;
208                         }
209 printf("RenderFarmClient::main_loop: Session started from %s\n", clientname.sun_path);
210                         RenderFarmClientThread *thread =
211                                 new RenderFarmClientThread(this);
212                         thread->main_loop(new_socket_fd);
213                 }
214         }
215 }
216
217 void RenderFarmClient::kill_client()
218 {
219 printf("RenderFarmClient::kill_client 1\n");
220         if(deamon_path)
221         {
222 printf("RenderFarmClient::kill_client 2\n");
223                 remove(deamon_path);
224                 kill(this_pid, SIGKILL);
225         }
226 }
227
228
229 // The thread requests jobs from the server until the job table is empty
230 // or the server reports an error.  This thread must poll the server
231 // after every frame for the error status.
232 // Detaches when finished.
233 RenderFarmClientThread::RenderFarmClientThread(RenderFarmClient *client)
234  : Thread(0, 0, 1)
235 {
236         this->client = client;
237         this->edl = 0;
238         frames_per_second = 0;
239         Thread::set_synchronous(0);
240 //      fs_client = 0;
241         mutex_lock = new Mutex("RenderFarmClientThread::mutex_lock");
242         watchdog = 0;
243         keep_alive = 0;
244 }
245
246 RenderFarmClientThread::~RenderFarmClientThread()
247 {
248 //      if(fs_client) delete fs_client;
249         delete mutex_lock;
250         delete watchdog;
251         delete keep_alive;
252 }
253
254
255 int RenderFarmClientThread::send_request_header(int request,
256         int len)
257 {
258         unsigned char datagram[5];
259         datagram[0] = request;
260
261         int i = 1;
262         STORE_INT32(len);
263 // printf("RenderFarmClientThread::send_request_header %d %02x%02x%02x%02x%02x\n",
264 // request, datagram[0], datagram[1], datagram[2], datagram[3], datagram[4]);
265
266         return (write_socket((char*)datagram, 5) != 5);
267 }
268
269 int RenderFarmClientThread::write_socket(char *data, int len)
270 {
271 //printf("RenderFarmClientThread::write_socket 1\n");
272         int result = write(socket_fd, data, len);
273 //printf("RenderFarmClientThread::write_socket 10\n");
274         return result;
275 }
276
277 int RenderFarmClientThread::read_socket(char *data, int len)
278 {
279         int bytes_read = 0;
280         int offset = 0;
281 //printf("RenderFarmClientThread::read_socket 1\n");
282         if( watchdog )
283                 watchdog->begin_request();
284         while(len > 0 && bytes_read >= 0)
285         {
286                 bytes_read = read(socket_fd, data + offset, len);
287                 if(bytes_read > 0)
288                 {
289                         len -= bytes_read;
290                         offset += bytes_read;
291                 }
292                 else
293                 if(bytes_read < 0)
294                 {
295                         break;
296                 }
297         }
298         if( watchdog )
299                 watchdog->end_request();
300 //printf("RenderFarmClientThread::read_socket 10\n");
301
302         return offset;
303 }
304
305 int RenderFarmClientThread::write_int64(int64_t value)
306 {
307         unsigned char data[sizeof(int64_t)];
308         data[0] = (value >> 56) & 0xff;
309         data[1] = (value >> 48) & 0xff;
310         data[2] = (value >> 40) & 0xff;
311         data[3] = (value >> 32) & 0xff;
312         data[4] = (value >> 24) & 0xff;
313         data[5] = (value >> 16) & 0xff;
314         data[6] = (value >> 8) & 0xff;
315         data[7] = value & 0xff;
316         return (write_socket((char*)data, sizeof(int64_t)) != sizeof(int64_t));
317 }
318
319 int64_t RenderFarmClientThread::read_int64(int *error)
320 {
321         int temp = 0;
322         if(!error) error = &temp;
323
324         unsigned char data[sizeof(int64_t)];
325         *error = (read_socket((char*)data, sizeof(int64_t)) != sizeof(int64_t));
326
327 // Make it return 1 if error so it can be used to read a result code from the
328 // server.
329         int64_t result = 1;
330         if(!*error)
331         {
332                 result = (((int64_t)data[0]) << 56) |
333                         (((uint64_t)data[1]) << 48) |
334                         (((uint64_t)data[2]) << 40) |
335                         (((uint64_t)data[3]) << 32) |
336                         (((uint64_t)data[4]) << 24) |
337                         (((uint64_t)data[5]) << 16) |
338                         (((uint64_t)data[6]) << 8)  |
339                         data[7];
340         }
341         return result;
342 }
343
344 void RenderFarmClientThread::read_string(char* &string)
345 {
346         unsigned char header[4];
347         if(read_socket((char*)header, 4) != 4)
348         {
349                 string = 0;
350                 return;
351         }
352
353         int64_t len = (((u_int32_t)header[0]) << 24) |
354                                 (((u_int32_t)header[1]) << 16) |
355                                 (((u_int32_t)header[2]) << 8) |
356                                 ((u_int32_t)header[3]);
357
358         if(len)
359         {
360                 string = new char[len];
361                 if(read_socket(string, len) != len)
362                 {
363                         delete [] string;
364                         string = 0;
365                 }
366         }
367         else
368                 string = 0;
369
370 }
371
372 void RenderFarmClientThread::abort()
373 {
374         send_completion(socket_fd);
375         close(socket_fd);
376         exit(1);
377 }
378
379 void RenderFarmClientThread::lock(const char *location)
380 {
381         mutex_lock->lock(location);
382 }
383
384 void RenderFarmClientThread::unlock()
385 {
386         mutex_lock->unlock();
387 }
388
389 void RenderFarmClientThread::get_command(int socket_fd, int *command)
390 {
391         int error;
392         *command = read_int64(&error);
393         if(error)
394         {
395                 *command = 0;
396                 return;
397         }
398 }
399
400
401 void RenderFarmClientThread::read_preferences(int socket_fd,
402         Preferences *preferences)
403 {
404         lock("RenderFarmClientThread::read_preferences");
405         send_request_header(RENDERFARM_PREFERENCES,
406                 0);
407
408         char *string;
409         read_string(string);
410
411         BC_Hash defaults;
412         defaults.load_string((char*)string);
413         preferences->load_defaults(&defaults);
414
415         delete [] string;
416         unlock();
417 }
418
419
420
421 void RenderFarmClientThread::read_asset(int socket_fd, Asset *asset)
422 {
423         lock("RenderFarmClientThread::read_asset");
424         send_request_header(RENDERFARM_ASSET,
425                 0);
426
427         char *string1;
428         char *string2;
429         read_string(string1);
430         read_string(string2);
431
432
433
434         FileXML file;
435         file.read_from_string((char*)string2);
436         asset->read(&file);
437
438
439
440         BC_Hash defaults;
441         defaults.load_string((char*)string1);
442         asset->load_defaults(&defaults, 0, 1, 1, 1, 1, 1);
443
444 //printf("RenderFarmClientThread::read_asset %d\n", __LINE__);
445 //asset->dump();
446
447         delete [] string1;
448         delete [] string2;
449         unlock();
450 }
451
452 void RenderFarmClientThread::read_edl(int socket_fd,
453                 EDL *edl, Preferences *preferences)
454 {
455         lock("RenderFarmClientThread::read_edl");
456         send_request_header(RENDERFARM_EDL, 0);
457
458         char *string;
459         read_string(string);
460
461         FileXML file;
462         file.read_from_string((char*)string);
463         delete [] string;
464
465         edl->load_xml(&file, LOAD_ALL);
466         unlock();
467 }
468
469 int RenderFarmClientThread::read_package(int socket_fd, RenderPackage *package)
470 {
471         lock("RenderFarmClientThread::read_package");
472         send_request_header(RENDERFARM_PACKAGE,
473                 4);
474
475         unsigned char datagram[5];
476         int i = 0;
477
478
479 // Fails if -ieee isn't set.
480         int64_t fixed = !EQUIV(frames_per_second, 0.0) ?
481                 (int64_t)(frames_per_second * 65536.0) : 0;
482         STORE_INT32(fixed);
483         write_socket((char*)datagram, 4);
484
485
486 //printf("RenderFarmClientThread::read_package 1 %f %ld\n", frames_per_second, fixed);
487         char *data;
488         unsigned char *data_ptr;
489         read_string(data);
490 //printf("RenderFarmClientThread::read_package 2 %p\n", data);
491 // Signifies end of session.
492         if(!data)
493         {
494 //              printf(_("RenderFarmClientThread::read_package no output path received.\n"));
495                 unlock();
496                 return 1;
497         }
498
499 //printf("RenderFarmClientThread::read_package 2\n");
500
501
502         data_ptr = (unsigned char*)data;
503         strcpy(package->path, data);
504         data_ptr += strlen(package->path);
505         data_ptr++;
506         package->audio_start = READ_INT32(data_ptr);
507         data_ptr += 4;
508         package->audio_end = READ_INT32(data_ptr);
509         data_ptr += 4;
510         package->video_start = READ_INT32(data_ptr);
511         data_ptr += 4;
512         package->video_end = READ_INT32(data_ptr);
513         data_ptr += 4;
514         package->use_brender = READ_INT32(data_ptr);
515         data_ptr += 4;
516         package->audio_do = READ_INT32(data_ptr);
517         data_ptr += 4;
518         package->video_do = READ_INT32(data_ptr);
519
520         delete [] data;
521         unlock();
522
523         return 0;
524 }
525
526 int RenderFarmClientThread::send_completion(int socket_fd)
527 {
528         lock("RenderFarmClientThread::send_completion");
529         int result = send_request_header(RENDERFARM_DONE, 0);
530         unlock();
531         return result;
532 }
533
534
535 void RenderFarmClientThread::ping_server()
536 {
537         lock("RenderFarmClientThread::ping_server");
538         send_request_header(RENDERFARM_KEEPALIVE, 0);
539         unlock();
540 }
541
542
543
544 void RenderFarmClientThread::main_loop(int socket_fd)
545 {
546         this->socket_fd = socket_fd;
547
548         Thread::start();
549 }
550
551 void RenderFarmClientThread::run()
552 {
553 // Create new memory space
554         pid = fork();
555         if(pid != 0)
556         {
557                 int return_value;
558                 waitpid(pid, &return_value, 0);
559                 return;
560         }
561
562 // Get the pid of the fork if inside the fork
563         pid = getpid();
564         BC_Signals::set_trap_hook(trap_hook, this);
565         TheList::reset();
566
567         int socket_fd = this->socket_fd;
568         int watchdog_timeout = client->boot_preferences->renderfarm_watchdog_timeout;
569         if( watchdog_timeout > 0 )
570                 init_client_keepalive(watchdog_timeout);
571
572 // Get command to run
573         int command;
574         lock("RenderFarmClientThread::run");
575         get_command(socket_fd, &command);
576         unlock();
577
578 //printf("RenderFarmClientThread::run command=%d\n", command);
579
580         switch(command)
581         {
582                 case RENDERFARM_TUNER:
583                         do_tuner(socket_fd);
584                         break;
585                 case RENDERFARM_PACKAGES:
586                         do_packages(socket_fd);
587                         break;
588         }
589
590         _exit(0);
591 }
592
593
594 void RenderFarmClientThread::init_client_keepalive(int timeout_secs)
595 {
596         keep_alive = new RenderFarmKeepalive(this);
597         keep_alive->start();
598         watchdog = new RenderFarmWatchdog(timeout_secs, 0, this);
599         watchdog->start();
600 }
601
602
603
604 void RenderFarmClientThread::do_tuner(int socket_fd)
605 {
606 // Currently only 1 tuner driver.  Maybe more someday.
607 //      DVBTune server(this);
608 //      server.main_loop();
609 //      ::close(socket_fd);
610 }
611
612
613 void RenderFarmClientThread::do_packages(int socket_fd)
614 {
615         RenderPackage *package;
616         Asset *default_asset;
617         Preferences *preferences;
618
619
620
621         FarmPackageRenderer package_renderer(this, socket_fd);
622         int result = 0;
623
624
625
626 //printf("RenderFarmClientThread::run 2\n");
627 // Read settings
628         preferences = new Preferences;
629         default_asset = new Asset;
630         package = new RenderPackage;
631         edl = new EDL;
632         edl->create_objects();
633
634 //printf("RenderFarmClientThread::run 3\n");
635         read_preferences(socket_fd, preferences);
636 //printf("RenderFarmClientThread::run 4\n");
637         read_asset(socket_fd, default_asset);
638 //printf("RenderFarmClientThread::run 5\n");
639         read_edl(socket_fd, edl, preferences);
640 //edl->dump();
641 //printf("RenderFarmClientThread::run 6\n");
642
643         package_renderer.initialize(0, edl, preferences, default_asset);
644
645 // Read packages
646         while(1)
647         {
648 //printf("RenderFarmClientThread::run 5\n");
649                 result = read_package(socket_fd, package);
650 //printf("RenderFarmClientThread::run 6 %d\n", result);
651
652
653 // Finished list
654                 if(result)
655                 {
656 //printf("RenderFarmClientThread::run 7\n");
657
658                         result = send_completion(socket_fd);
659                         break;
660                 }
661
662 //              Timer timer;
663 //              timer.update();
664
665 // Error
666                 if(package_renderer.render_package(package))
667                 {
668 //printf("RenderFarmClientThread::run 8\n");
669                         result = send_completion(socket_fd);
670                         break;
671                 }
672
673
674                 frames_per_second = package_renderer.frames_per_second;
675 //              frames_per_second = (double)(package->video_end - package->video_start) /
676 //                      ((double)timer.get_difference() / 1000);
677
678 //printf("RenderFarmClientThread::run 9\n");
679
680
681
682         }
683
684
685 //printf("RenderFarmClientThread::run 9\n");
686         default_asset->Garbage::remove_user();
687 //printf("RenderFarmClientThread::run 10\n");
688         edl->Garbage::remove_user();
689         edl = 0;
690 //printf("RenderFarmClientThread::run 11\n");
691         delete preferences;
692 printf(_("RenderFarmClientThread::run: Session finished.\n"));
693 }
694
695 void RenderFarmClientThread::trap_hook(FILE *fp, void *vp)
696 {
697         RenderFarmClientThread *thread = (RenderFarmClientThread*)vp;
698         fprintf(fp, "\nEDL:\n");
699         if( thread->edl ) thread->edl->dump(fp);
700 }
701
702
703
704 RenderFarmKeepalive::RenderFarmKeepalive(
705         RenderFarmClientThread *client_thread)
706  : Thread(1, 0, 0)
707 {
708         this->client_thread = client_thread;
709         done = 0;
710 }
711
712 RenderFarmKeepalive::~RenderFarmKeepalive()
713 {
714         done = 1;
715         cancel();
716         join();
717 }
718
719 void RenderFarmKeepalive::run()
720 {
721         while(!done)
722         {
723                 enable_cancel();
724                 sleep(5);
725                 disable_cancel();
726                 if( !done ) {
727 //printf("RenderFarmKeepalive::run 1\n");
728 // watchdog thread kills this if it gets stuck
729                         client_thread->ping_server();
730 //printf("RenderFarmKeepalive::run 10\n");
731                 }
732         }
733 }
734
735
736 FarmPackageRenderer::FarmPackageRenderer(RenderFarmClientThread *thread,
737                 int socket_fd)
738  : PackageRenderer()
739 {
740         this->thread = thread;
741         this->socket_fd = socket_fd;
742 }
743
744
745 FarmPackageRenderer::~FarmPackageRenderer()
746 {
747 }
748
749
750 int FarmPackageRenderer::get_result()
751 {
752         thread->lock("FarmPackageRenderer::get_result");
753         thread->send_request_header(RENDERFARM_GET_RESULT,
754                 0);
755         unsigned char data[1];
756         data[0] = 1;
757         if(thread->read_socket((char*)data, 1) != 1)
758         {
759                 thread->unlock();
760                 return 1;
761         }
762         thread->unlock();
763         return data[0];
764 }
765
766 void FarmPackageRenderer::set_result(int value)
767 {
768         thread->lock("FarmPackageRenderer::set_result");
769         thread->send_request_header(RENDERFARM_SET_RESULT,
770                 1);
771         unsigned char data[1];
772         data[0] = value;
773         thread->write_socket((char*)data, 1);
774         thread->unlock();
775 }
776
777 void FarmPackageRenderer::set_progress(int64_t total_samples)
778 {
779         thread->lock("FarmPackageRenderer::set_progress");
780         thread->send_request_header(RENDERFARM_PROGRESS,
781                 8);
782         unsigned char datagram[8];
783         int i = 0;
784         STORE_INT32(total_samples);
785
786
787         int64_t fixed = (!EQUIV(frames_per_second, 0.0)) ?
788                 (int64_t)(frames_per_second * 65536.0) : 0;
789         STORE_INT32(fixed);
790
791
792         thread->write_socket((char*)datagram, 8);
793         thread->unlock();
794 }
795
796 int FarmPackageRenderer::set_video_map(int64_t position, int value)
797 {
798         int result = 0;
799         unsigned char datagram[8];
800         char return_value[1];
801         int i = 0;
802
803         thread->lock("FarmPackageRenderer::set_video_map");
804         thread->send_request_header(RENDERFARM_SET_VMAP,
805                 8);
806         STORE_INT32(position);
807         STORE_INT32(value);
808         thread->write_socket((char*)datagram, 8);
809
810 // Get completion since the GUI may be locked for a long time.
811         if(!thread->read_socket(return_value, 1))
812         {
813                 result = 1;
814         }
815
816         thread->unlock();
817         return result;
818 }
819
820