initial commit
[goodguy/history.git] / cinelerra-5.0 / cinelerra / renderfarmclient.C
1
2 /*
3  * CINELERRA
4  * Copyright (C) 2008 Adam Williams <broadcast at earthling dot net>
5  * 
6  * This program is free software; you can redistribute it and/or modify
7  * it under the terms of the GNU General Public License as published by
8  * the Free Software Foundation; either version 2 of the License, or
9  * (at your option) any later version.
10  * 
11  * This program is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
14  * GNU General Public License for more details.
15  * 
16  * You should have received a copy of the GNU General Public License
17  * along with this program; if not, write to the Free Software
18  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
19  * 
20  */
21
22 #include "asset.h"
23 #include "assets.h"
24 #include "clip.h"
25 #include "bchash.h"
26 #include "dvbtune.h"
27 #include "edl.h"
28 #include "filesystem.h"
29 #include "filexml.h"
30 #include "language.h"
31 #include "mutex.h"
32 #include "mwindow.h"
33 #include "pluginserver.h"
34 #include "preferences.h"
35 #include "renderfarm.h"
36 #include "renderfarmclient.h"
37 //#include "renderfarmfsclient.h"
38 #include "sighandler.h"
39
40 #include <arpa/inet.h>
41 #include <errno.h>
42 #include <fcntl.h>
43 #include <netdb.h>
44 #include <netinet/in.h>
45 #include <stdio.h>
46 #include <string.h>
47 #include <sys/socket.h>
48 #include <sys/types.h>
49 #include <sys/un.h>
50 #include <sys/wait.h>
51 #include <unistd.h>
52
53
54
55
56 // The render client waits for connections from the server.
57 // Then it starts a thread for each connection.
58 RenderFarmClient::RenderFarmClient(int port, 
59         char *deamon_path, 
60         int nice_value,
61         char *config_path)
62 {
63         this->port = port;
64         this->deamon_path = deamon_path;
65         SigHandler *signals = new SigHandler;
66         signals->initialize();
67
68         this_pid = getpid();
69         (void)nice(nice_value);
70
71
72         MWindow::init_defaults(boot_defaults, config_path);
73         boot_preferences = new Preferences;
74         boot_preferences->load_defaults(boot_defaults);
75         MWindow::init_plugins(0, boot_preferences);
76         MWindow::init_fileserver(boot_preferences);
77 }
78
79
80
81
82 RenderFarmClient::~RenderFarmClient()
83 {
84 //      delete thread;
85         delete boot_defaults;
86         delete boot_preferences;
87         plugindb->remove_all_objects();
88         delete plugindb;
89 }
90
91
92 void RenderFarmClient::main_loop()
93 {
94         int socket_fd;
95         BC_WindowBase::get_resources()->vframe_shm = 1;
96
97
98 // Open listening port
99
100         if(!deamon_path)
101         {
102                 struct sockaddr_in addr;
103
104                 addr.sin_family = AF_INET;
105                 addr.sin_port = htons((unsigned short)port);
106                 addr.sin_addr.s_addr = htonl(INADDR_ANY);
107
108                 if((socket_fd = socket(PF_INET, SOCK_STREAM, 0)) < 0)
109                 {
110                         perror(_("RenderFarmClient::main_loop: socket"));
111                         return;
112                 }
113
114                 if(bind(socket_fd, 
115                         (struct sockaddr*)&addr, 
116                         sizeof(addr)) < 0)
117                 {
118                         fprintf(stderr, 
119                                 _("RenderFarmClient::main_loop: bind port %d: %s"),
120                                 port,
121                                 strerror(errno));
122                         return;
123                 }
124         }
125         else
126         {
127                 struct sockaddr_un addr;
128                 addr.sun_family = AF_FILE;
129                 strcpy(addr.sun_path, deamon_path);
130                 int size = (offsetof(struct sockaddr_un, sun_path) + 
131                         strlen(deamon_path) + 1);
132
133                 if((socket_fd = socket(PF_UNIX, SOCK_STREAM, 0)) < 0)
134                 {
135                         perror(_("RenderFarmClient::main_loop: socket"));
136                         return;
137                 }
138
139                 if(bind(socket_fd, 
140                         (struct sockaddr*)&addr, 
141                         size) < 0)
142                 {
143                         fprintf(stderr, 
144                                 _("RenderFarmClient::main_loop: bind path %s: %s\n"),
145                                 deamon_path,
146                                 strerror(errno));
147                         return;
148                 }
149         }
150
151 // Wait for connections
152         printf("RenderFarmClient::main_loop: client started\n");
153         while(1)
154         {
155                 if(listen(socket_fd, 256) < 0)
156         {
157                 perror(_("RenderFarmClient::main_loop: listen"));
158                 return;
159         }
160
161                 int new_socket_fd;
162
163
164
165                 if(!deamon_path)
166                 {
167                         struct sockaddr_in clientname;
168                         socklen_t size = sizeof(clientname);
169                 if((new_socket_fd = accept(socket_fd,
170                                 (struct sockaddr*)&clientname, 
171                                                 &size)) < 0)
172                 {
173                         perror(_("RenderFarmClient::main_loop: accept"));
174                         return;
175                 }
176                         else
177                         {
178 printf("RenderFarmClient::main_loop: Session started from %s\n", inet_ntoa(clientname.sin_addr));
179                                 RenderFarmClientThread *thread = 
180                                         new RenderFarmClientThread(this);
181                                 thread->main_loop(new_socket_fd);
182                         }
183                 }
184                 else
185                 {
186                         struct sockaddr_un clientname;
187                         socklen_t size = sizeof(clientname);
188                 if((new_socket_fd = accept(socket_fd,
189                                 (struct sockaddr*)&clientname, 
190                                                 &size)) < 0)
191                 {
192                         perror(_("RenderFarmClient::main_loop: accept"));
193                         return;
194                 }
195                         else
196                         {
197 printf("RenderFarmClient::main_loop: Session started from %s\n", clientname.sun_path);
198                                 RenderFarmClientThread *thread = 
199                                         new RenderFarmClientThread(this);
200                                 thread->main_loop(new_socket_fd);
201                         }
202                 }
203         }
204 }
205
206 void RenderFarmClient::kill_client()
207 {
208 printf("RenderFarmClient::kill_client 1\n");
209         if(deamon_path)
210         {
211 printf("RenderFarmClient::kill_client 2\n");
212                 remove(deamon_path);
213                 kill(this_pid, SIGKILL);
214         }
215 }
216
217
218
219
220
221
222
223
224
225
226
227 // The thread requests jobs from the server until the job table is empty
228 // or the server reports an error.  This thread must poll the server
229 // after every frame for the error status.
230 // Detaches when finished.
231 RenderFarmClientThread::RenderFarmClientThread(RenderFarmClient *client)
232  : Thread(0, 0, 1)
233 {
234         this->client = client;
235         frames_per_second = 0;
236         Thread::set_synchronous(0);
237 //      fs_client = 0;
238         mutex_lock = new Mutex("RenderFarmClientThread::mutex_lock");
239         watchdog = 0;
240         keep_alive = 0;
241 }
242
243 RenderFarmClientThread::~RenderFarmClientThread()
244 {
245 //      if(fs_client) delete fs_client;
246         delete mutex_lock;
247         delete watchdog;
248         delete keep_alive;
249 }
250
251
252 int RenderFarmClientThread::send_request_header(int request, 
253         int len)
254 {
255         unsigned char datagram[5];
256         datagram[0] = request;
257
258         int i = 1;
259         STORE_INT32(len);
260 // printf("RenderFarmClientThread::send_request_header %d %02x%02x%02x%02x%02x\n",
261 // request, datagram[0], datagram[1], datagram[2], datagram[3], datagram[4]);
262
263         return (write_socket((char*)datagram, 5) != 5);
264 }
265
266 int RenderFarmClientThread::write_socket(char *data, int len)
267 {
268 //printf("RenderFarmClientThread::write_socket 1\n");
269         int result = write(socket_fd, data, len);
270 //printf("RenderFarmClientThread::write_socket 10\n");
271         return result;
272 }
273
274 int RenderFarmClientThread::read_socket(char *data, int len)
275 {
276         int bytes_read = 0;
277         int offset = 0;
278 //printf("RenderFarmClientThread::read_socket 1\n");
279         watchdog->begin_request();
280         while(len > 0 && bytes_read >= 0)
281         {
282                 bytes_read = read(socket_fd, data + offset, len);
283                 if(bytes_read > 0)
284                 {
285                         len -= bytes_read;
286                         offset += bytes_read;
287                 }
288                 else
289                 if(bytes_read < 0)
290                 {
291                         break;
292                 }
293         }
294         watchdog->end_request();
295 //printf("RenderFarmClientThread::read_socket 10\n");
296
297         return offset;
298 }
299
300 int RenderFarmClientThread::write_int64(int64_t value)
301 {
302         unsigned char data[sizeof(int64_t)];
303         data[0] = (value >> 56) & 0xff;
304         data[1] = (value >> 48) & 0xff;
305         data[2] = (value >> 40) & 0xff;
306         data[3] = (value >> 32) & 0xff;
307         data[4] = (value >> 24) & 0xff;
308         data[5] = (value >> 16) & 0xff;
309         data[6] = (value >> 8) & 0xff;
310         data[7] = value & 0xff;
311         return (write_socket((char*)data, sizeof(int64_t)) != sizeof(int64_t));
312 }
313
314 int64_t RenderFarmClientThread::read_int64(int *error)
315 {
316         int temp = 0;
317         if(!error) error = &temp;
318
319         unsigned char data[sizeof(int64_t)];
320         *error = (read_socket((char*)data, sizeof(int64_t)) != sizeof(int64_t));
321
322 // Make it return 1 if error so it can be used to read a result code from the
323 // server.
324         int64_t result = 1;
325         if(!*error)
326         {
327                 result = (((int64_t)data[0]) << 56) |
328                         (((uint64_t)data[1]) << 48) | 
329                         (((uint64_t)data[2]) << 40) |
330                         (((uint64_t)data[3]) << 32) |
331                         (((uint64_t)data[4]) << 24) |
332                         (((uint64_t)data[5]) << 16) |
333                         (((uint64_t)data[6]) << 8)  |
334                         data[7];
335         }
336         return result;
337 }
338
339 void RenderFarmClientThread::read_string(char* &string)
340 {
341         unsigned char header[4];
342         if(read_socket((char*)header, 4) != 4)
343         {
344                 string = 0;
345                 return;
346         }
347
348         int64_t len = (((u_int32_t)header[0]) << 24) | 
349                                 (((u_int32_t)header[1]) << 16) | 
350                                 (((u_int32_t)header[2]) << 8) | 
351                                 ((u_int32_t)header[3]);
352
353         if(len)
354         {
355                 string = new char[len];
356                 if(read_socket(string, len) != len)
357                 {
358                         delete [] string;
359                         string = 0;
360                 }
361         }
362         else
363                 string = 0;
364
365 }
366
367 void RenderFarmClientThread::abort()
368 {
369         send_completion(socket_fd);
370         close(socket_fd);
371         exit(1);
372 }
373
374 void RenderFarmClientThread::lock(const char *location)
375 {
376         mutex_lock->lock(location);
377 }
378
379 void RenderFarmClientThread::unlock()
380 {
381         mutex_lock->unlock();
382 }
383
384 void RenderFarmClientThread::get_command(int socket_fd, int *command)
385 {
386         int error;
387         *command = read_int64(&error);
388         if(error)
389         {
390                 *command = 0;
391                 return;
392         }
393 }
394
395
396 void RenderFarmClientThread::read_preferences(int socket_fd, 
397         Preferences *preferences)
398 {
399         lock("RenderFarmClientThread::read_preferences");
400         send_request_header(RENDERFARM_PREFERENCES, 
401                 0);
402
403         char *string;
404         read_string(string);
405
406         BC_Hash defaults;
407         defaults.load_string((char*)string);
408         preferences->load_defaults(&defaults);
409
410         delete [] string;
411         unlock();
412 }
413
414
415
416 void RenderFarmClientThread::read_asset(int socket_fd, Asset *asset)
417 {
418         lock("RenderFarmClientThread::read_asset");
419         send_request_header(RENDERFARM_ASSET, 
420                 0);
421
422         char *string1;
423         char *string2;
424         read_string(string1);
425         read_string(string2);
426
427
428
429         FileXML file;
430         file.read_from_string((char*)string2);
431         asset->read(&file);
432         
433
434
435         BC_Hash defaults;
436         defaults.load_string((char*)string1);
437         asset->load_defaults(&defaults,
438                 0,
439                 1,
440                 1,
441                 1,
442                 1,
443                 1);
444
445 //printf("RenderFarmClientThread::read_asset %d\n", __LINE__);
446 //asset->dump();
447
448         delete [] string1;
449         delete [] string2;
450         unlock();
451 }
452
453 void RenderFarmClientThread::read_edl(int socket_fd, 
454         EDL *edl, 
455         Preferences *preferences)
456 {
457         lock("RenderFarmClientThread::read_edl");
458         send_request_header(RENDERFARM_EDL, 
459                 0);
460
461         char *string;
462         read_string(string);
463
464
465         FileXML file;
466         file.read_from_string((char*)string);
467         delete [] string;
468
469
470
471
472
473
474
475
476         edl->load_xml(&file, 
477                 LOAD_ALL);
478
479
480         unlock();
481 }
482
483 int RenderFarmClientThread::read_package(int socket_fd, RenderPackage *package)
484 {
485         lock("RenderFarmClientThread::read_package");
486         send_request_header(RENDERFARM_PACKAGE, 
487                 4);
488
489         unsigned char datagram[5];
490         int i = 0;
491
492
493 // Fails if -ieee isn't set.
494         int64_t fixed = !EQUIV(frames_per_second, 0.0) ? 
495                 (int64_t)(frames_per_second * 65536.0) : 0;
496         STORE_INT32(fixed);
497         write_socket((char*)datagram, 4);
498
499
500 //printf("RenderFarmClientThread::read_package 1 %f %ld\n", frames_per_second, fixed);
501         char *data;
502         unsigned char *data_ptr;
503         read_string(data);
504 //printf("RenderFarmClientThread::read_package 2 %p\n", data);
505 // Signifies end of session.
506         if(!data) 
507         {
508 //              printf(_("RenderFarmClientThread::read_package no output path recieved.\n"));
509                 unlock();
510                 return 1;
511         }
512
513 //printf("RenderFarmClientThread::read_package 2\n");
514
515
516         data_ptr = (unsigned char*)data;
517         strcpy(package->path, data);
518         data_ptr += strlen(package->path);
519         data_ptr++;
520         package->audio_start = READ_INT32(data_ptr);
521         data_ptr += 4;
522         package->audio_end = READ_INT32(data_ptr);
523         data_ptr += 4;
524         package->video_start = READ_INT32(data_ptr);
525         data_ptr += 4;
526         package->video_end = READ_INT32(data_ptr);
527         data_ptr += 4;
528         package->use_brender = READ_INT32(data_ptr);
529
530         delete [] data;
531         unlock();
532
533         return 0;
534 }
535
536 int RenderFarmClientThread::send_completion(int socket_fd)
537 {
538         lock("RenderFarmClientThread::send_completion");
539         int result = send_request_header(RENDERFARM_DONE, 0);
540         unlock();
541         return result;
542 }
543
544
545 void RenderFarmClientThread::ping_server()
546 {
547         lock("RenderFarmClientThread::ping_server");
548         send_request_header(RENDERFARM_KEEPALIVE, 0);
549         unlock();
550 }
551
552
553
554 void RenderFarmClientThread::main_loop(int socket_fd)
555 {
556         this->socket_fd = socket_fd;
557
558         Thread::start();
559 }
560
561 void RenderFarmClientThread::run()
562 {
563 // Create new memory space
564         pid = fork();
565         if(pid != 0)
566         {
567                 int return_value;
568                 waitpid(pid, &return_value, 0);
569                 return;
570         }
571
572 // Get the pid of the fork if inside the fork
573         pid = getpid();
574
575
576
577         int socket_fd = this->socket_fd;
578
579         init_client_keepalive();
580
581 // Get command to run
582         int command;
583         lock("RenderFarmClientThread::run");
584         get_command(socket_fd, &command);
585         unlock();
586
587 //printf("RenderFarmClientThread::run command=%d\n", command);
588
589         switch(command)
590         {
591                 case RENDERFARM_TUNER:
592                         do_tuner(socket_fd);
593                         break;
594                 case RENDERFARM_PACKAGES:
595                         do_packages(socket_fd);
596                         break;
597         }
598
599         _exit(0);
600 }
601
602
603 void RenderFarmClientThread::init_client_keepalive()
604 {
605         keep_alive = new RenderFarmKeepalive(this);
606         keep_alive->start();
607         watchdog = new RenderFarmWatchdog(0, this);
608         watchdog->start();
609 }
610
611
612
613 void RenderFarmClientThread::do_tuner(int socket_fd)
614 {
615 // Currently only 1 tuner driver.  Maybe more someday.
616         DVBTune server(this);
617         server.main_loop();
618         ::close(socket_fd);
619 }
620
621
622 void RenderFarmClientThread::do_packages(int socket_fd)
623 {
624
625         EDL *edl;
626         RenderPackage *package;
627         Asset *default_asset;
628         Preferences *preferences;
629
630
631
632         FarmPackageRenderer package_renderer(this, socket_fd);
633         int result = 0;
634
635
636
637 //printf("RenderFarmClientThread::run 2\n");
638 // Read settings
639         preferences = new Preferences;
640         default_asset = new Asset;
641         package = new RenderPackage;
642         edl = new EDL;
643         edl->create_objects();
644
645
646
647
648
649
650 //printf("RenderFarmClientThread::run 3\n");
651         read_preferences(socket_fd, preferences);
652 //printf("RenderFarmClientThread::run 4\n");
653         read_asset(socket_fd, default_asset);
654 //printf("RenderFarmClientThread::run 5\n");
655         read_edl(socket_fd, edl, preferences);
656 //edl->dump();
657 //printf("RenderFarmClientThread::run 6\n");
658
659
660
661
662
663
664
665
666         package_renderer.initialize(0,
667                         edl, 
668                         preferences, 
669                         default_asset);
670
671 // Read packages
672         while(1)
673         {
674 //printf("RenderFarmClientThread::run 5\n");
675                 result = read_package(socket_fd, package);
676 //printf("RenderFarmClientThread::run 6 %d\n", result);
677
678
679 // Finished list
680                 if(result)
681                 {
682 //printf("RenderFarmClientThread::run 7\n");
683
684                         result = send_completion(socket_fd);
685                         break;
686                 }
687
688 //              Timer timer;
689 //              timer.update();
690
691 // Error
692                 if(package_renderer.render_package(package))
693                 {
694 //printf("RenderFarmClientThread::run 8\n");
695                         result = send_completion(socket_fd);
696                         break;
697                 }
698
699
700                 frames_per_second = package_renderer.frames_per_second;
701 //              frames_per_second = (double)(package->video_end - package->video_start) / 
702 //                      ((double)timer.get_difference() / 1000);
703
704 //printf("RenderFarmClientThread::run 9\n");
705
706
707
708         }
709
710
711 //printf("RenderFarmClientThread::run 9\n");
712         default_asset->Garbage::remove_user();
713 //printf("RenderFarmClientThread::run 10\n");
714         edl->Garbage::remove_user();
715 //printf("RenderFarmClientThread::run 11\n");
716         delete preferences;
717 printf(_("RenderFarmClientThread::run: Session finished.\n"));
718 }
719
720
721
722
723
724
725
726
727
728 RenderFarmKeepalive::RenderFarmKeepalive(
729         RenderFarmClientThread *client_thread)
730  : Thread(1, 0, 0)
731 {
732         this->client_thread = client_thread;
733         done = 0;
734 }
735
736 RenderFarmKeepalive::~RenderFarmKeepalive()
737 {
738         done = 1;
739         cancel();
740         join();
741 }
742
743 void RenderFarmKeepalive::run()
744 {
745         while(!done)
746         {
747                 enable_cancel();
748                 sleep(5);
749                 disable_cancel();
750                 if(!done)
751                 {
752 //printf("RenderFarmKeepalive::run 1\n");
753 // watchdog thread kills this if it gets stuck
754                         client_thread->ping_server();
755 //printf("RenderFarmKeepalive::run 10\n");
756                 }
757         }
758 }
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774 FarmPackageRenderer::FarmPackageRenderer(RenderFarmClientThread *thread,
775                 int socket_fd)
776  : PackageRenderer()
777 {
778         this->thread = thread;
779         this->socket_fd = socket_fd;
780 }
781
782
783
784 FarmPackageRenderer::~FarmPackageRenderer()
785 {
786 }
787
788
789 int FarmPackageRenderer::get_result()
790 {
791         thread->lock("FarmPackageRenderer::get_result");
792         thread->send_request_header(RENDERFARM_GET_RESULT, 
793                 0);
794         unsigned char data[1];
795         data[0] = 1;
796         if(thread->read_socket((char*)data, 1) != 1)
797         {
798                 thread->unlock();
799                 return 1;
800         }
801         thread->unlock();
802         return data[0];
803 }
804
805 void FarmPackageRenderer::set_result(int value)
806 {
807         thread->lock("FarmPackageRenderer::set_result");
808         thread->send_request_header(RENDERFARM_SET_RESULT, 
809                 1);
810         unsigned char data[1];
811         data[0] = value;
812         thread->write_socket((char*)data, 1);
813         thread->unlock();
814 }
815
816 void FarmPackageRenderer::set_progress(int64_t total_samples)
817 {
818         thread->lock("FarmPackageRenderer::set_progress");
819         thread->send_request_header(RENDERFARM_PROGRESS, 
820                 8);
821         unsigned char datagram[8];
822         int i = 0;
823         STORE_INT32(total_samples);
824         
825
826         int64_t fixed = (!EQUIV(frames_per_second, 0.0)) ? 
827                 (int64_t)(frames_per_second * 65536.0) : 0;
828         STORE_INT32(fixed);
829         
830         
831         thread->write_socket((char*)datagram, 8);
832         thread->unlock();
833 }
834
835 int FarmPackageRenderer::set_video_map(int64_t position, int value)
836 {
837         int result = 0;
838         unsigned char datagram[8];
839         char return_value[1];
840         int i = 0;
841
842         thread->lock("FarmPackageRenderer::set_video_map");
843         thread->send_request_header(RENDERFARM_SET_VMAP, 
844                 8);
845         STORE_INT32(position);
846         STORE_INT32(value);
847         thread->write_socket((char*)datagram, 8);
848
849 // Get completion since the GUI may be locked for a long time.
850         if(!thread->read_socket(return_value, 1))
851         {
852                 result = 1;
853         }
854
855         thread->unlock();
856         return result;
857 }
858
859
860
861
862
863
864
865
866
867