3 # debianqueued -- daemon for managing Debian upload queues
5 # Copyright (C) 1997 Roman Hodek <Roman.Hodek@informatik.uni-erlangen.de>
6 # Copyright (C) 2001-2007 Ryan Murray <rmurray@debian.org>
8 # This program is free software. You can redistribute it and/or
9 # modify it under the terms of the GNU General Public License as
10 # published by the Free Software Foundation: either version 2 or
11 # (at your option) any later version.
12 # This program comes with ABSOLUTELY NO WARRANTY!
18 use POSIX qw( strftime sys_stat_h sys_wait_h signal_h );
21 use Socket qw( PF_INET AF_INET SOCK_STREAM );
24 # ---------------------------------------------------------------------------
26 # ---------------------------------------------------------------------------
29 ($conf::queued_dir = (($0 !~ m,^/,) ? POSIX::getcwd()."/" : "") . $0)
31 require "$conf::queued_dir/config";
32 my $junk = $conf::debug; # avoid spurious warnings about unused vars
33 $junk = $conf::ssh_key_file;
34 $junk = $conf::stray_remove_timeout;
35 $junk = $conf::problem_report_timeout;
36 $junk = $conf::queue_delay;
37 $junk = $conf::keep_files;
38 $junk = $conf::valid_files;
39 $junk = $conf::max_upload_retries;
40 $junk = $conf::upload_delay_1;
41 $junk = $conf::upload_delay_2;
47 $junk = $conf::ftpdebug;
48 $junk = $conf::ftptimeout;
49 $junk = $conf::no_changes_timeout;
50 $junk = @conf::nonus_packages;
51 $junk = @conf::test_binaries;
52 $junk = @conf::maintainer_mail;
53 $junk = @conf::targetdir_delayed;
54 $junk = $conf::mail ||= '/usr/sbin/sendmail';
55 $conf::target = "localhost" if $conf::upload_method eq "copy";
58 ($main::progname = $0) =~ s,.*/,,;
62 # extract -r and -k args
64 if (@ARGV == 1 && $ARGV[0] =~ /^-[rk]$/) {
65 $main::arg = ($ARGV[0] eq '-k') ? "kill" : "restart";
69 # test for another instance of the queued already running
70 my ($pid, $delayed_dirs, $adelayedcore);
71 if (open( PIDFILE, "<$conf::pidfile" )) {
72 chomp( $pid = <PIDFILE> );
75 # remove stale pid file
76 unlink( $conf::pidfile );
80 print "Killing running daemon (pid $pid) ...";
83 while( kill( 0, $pid ) && $cnt-- > 0 ) {
87 if (kill( 0, $pid )) {
88 print " failed!\nProcess $pid still running.\n";
92 if (-e "$conf::incoming/core") {
93 unlink( "$conf::incoming/core" );
94 print "(Removed core file)\n";
96 for ($delayed_dirs = 0; $delayed_dirs <= $conf::max_delayed;
98 $adelayedcore = sprintf( "$conf::incoming_delayed/core",
100 if (-e $adelayedcore) {
101 unlink( $adelayedcore );
102 print "(Removed core file)\n";
105 exit 0 if $main::arg eq "kill";
108 die "Another $main::progname is already running (pid $pid)\n"
109 if $pid && kill( 0, $pid );
112 elsif ($main::arg eq "kill") {
113 die "No daemon running\n";
115 elsif ($main::arg eq "restart") {
116 print "(No daemon running; starting anyway)\n";
119 # if started without arguments (initial invocation), then fork
121 # now go to background
122 die "$main::progname: fork failed: $!\n" unless defined( $pid = fork );
124 # parent: wait for signal from child (SIGCHLD or SIGUSR1) and exit
125 my $sigset = POSIX::SigSet->new();
127 $SIG{"CHLD"} = sub { };
128 $SIG{"USR1"} = sub { };
129 POSIX::sigsuspend( $sigset );
130 waitpid( $pid, WNOHANG );
131 if (kill( 0, $pid )) {
132 print "Daemon started in background (pid $pid)\n";
142 if ($conf::upload_method eq "ssh") {
143 # exec an ssh-agent that starts us again
144 # force shell to be /bin/sh, ssh-agent may base its decision
145 # whether to use a fd or a Unix socket on the shell...
146 $ENV{"SHELL"} = "/bin/sh";
147 exec $conf::ssh_agent, $0, "startup", getppid();
148 die "$main::progname: Could not exec $conf::ssh_agent: $!\n";
151 # no need to exec, just set up @ARGV as expected below
152 @ARGV = ("startup", getppid());
156 die "Please start without any arguments.\n"
157 if @ARGV != 2 || $ARGV[0] ne "startup";
158 my $parent_pid = $ARGV[1];
162 ($version = 'Release: 0.9 $Revision: 1.51 $ $Date: 1999/07/08 09:43:21 $ $Author: ftplinux $') =~ s/\$ ?//g;
163 print "debianqueued $version\n";
166 # check if all programs exist
168 foreach $prg ( $conf::gpg, $conf::ssh, $conf::scp, $conf::ssh_agent,
169 $conf::ssh_add, $conf::md5sum, $conf::mail, $conf::mkfifo ) {
170 die "Required program $prg doesn't exist or isn't executable\n"
172 # check for correct upload method
173 die "Bad upload method '$conf::upload_method'.\n"
174 if $conf::upload_method ne "ssh" &&
175 $conf::upload_method ne "ftp" &&
176 $conf::upload_method ne "copy";
177 die "No keyrings\n" if ! @conf::keyrings;
180 die "statusfile path must be absolute."
181 if $conf::statusfile !~ m,^/,;
182 die "upload and target queue paths must be absolute."
183 if $conf::incoming !~ m,^/, ||
184 $conf::incoming_delayed !~ m,^/, ||
185 $conf::targetdir !~ m,^/, ||
186 $conf::targetdir_delayed !~ m,^/,;
189 # ---------------------------------------------------------------------------
191 # ---------------------------------------------------------------------------
196 sub get_filelist_from_known_good_changes($);
197 sub age_delayed_queues();
198 sub process_changes($\@);
199 sub process_commands($);
200 sub age_delayed_queues();
201 sub is_on_target($\@);
202 sub copy_to_target(@);
205 sub check_incoming_writable();
207 sub write_status_file();
208 sub print_status($$$$$$);
209 sub format_status_num(\$$);
210 sub format_status_str(\$$);
222 sub check_incoming_writable();
225 sub is_debian_file($);
226 sub get_maintainer($);
227 sub debian_file_stem($);
233 sub try_to_get_mail_addr($$);
237 sub unblock_signals();
240 sub restart_statusd();
243 $ENV{"PATH"} = "/bin:/usr/bin";
244 $ENV{"IFS"} = "" if defined($ENV{"IFS"} && $ENV{"IFS"} ne "");
257 sub ST_CTIME() { 10 }
258 # fixed lengths of data items passed over status pipe
259 sub STATNUM_LEN() { 30 }
260 sub STATSTR_LEN() { 128 }
262 # init list of signals
263 defined $Config{sig_name} or die "$main::progname: No signal list defined!\n";
266 foreach $name (split( ' ', $Config{sig_name} )) {
267 $main::signo{$name} = $i++;
270 @main::fatal_signals = qw( INT QUIT ILL TRAP ABRT BUS FPE USR2 SEGV PIPE
271 TERM XCPU XFSZ PWR );
273 $main::block_sigset = POSIX::SigSet->new;
274 $main::block_sigset->addset( $main::signo{"INT"} );
275 $main::block_sigset->addset( $main::signo{"TERM"} );
277 # some constant net stuff
278 $main::tcp_proto = (getprotobyname('tcp'))[2]
279 or die "Cannot get protocol number for 'tcp'\n";
280 my $used_service = ($conf::upload_method eq "ssh") ? "ssh" : "ftp";
281 $main::echo_port = (getservbyname($used_service, 'tcp'))[2]
282 or die "Cannot get port number for service '$used_service'\n";
284 # clear queue of stored mails
285 @main::stored_mails = ();
287 # run ssh-add to bring the key into the agent (will use stdin/stdout)
288 if ($conf::upload_method eq "ssh") {
289 system "$conf::ssh_add $conf::ssh_key_file"
290 and die "$main::progname: Running $conf::ssh_add failed ".
291 "(exit status ", $? >> 8, ")\n";
294 # change to queue dir
295 chdir( $conf::incoming )
296 or die "$main::progname: cannot cd to $conf::incoming: $!\n";
298 # needed before /dev/null redirects, some system send a SIGHUP when loosing
299 # the controlling tty
300 $SIG{"HUP"} = "IGNORE";
302 # open logfile, make it unbuffered
303 open( LOG, ">>$conf::logfile" )
304 or die "Cannot open my logfile $conf::logfile: $!\n";
305 chmod( 0644, $conf::logfile )
306 or die "Cannot set modes of $conf::logfile: $!\n";
307 select( (select(LOG), $| = 1)[0] );
310 $SIG{"HUP"} = \&close_log;
312 # redirect stdin, ... to /dev/null
313 open( STDIN, "</dev/null" )
314 or die "$main::progname: Can't redirect stdin to /dev/null: $!\n";
315 open( STDOUT, ">&LOG" )
316 or die "$main::progname: Can't redirect stdout to $conf::logfile: $!\n";
317 open( STDERR, ">&LOG" )
318 or die "$main::progname: Can't redirect stderr to $conf::logfile: $!\n";
319 # ok, from this point usually no "die" anymore, stderr is gone!
320 msg( "log", "daemon (pid $$) started\n" );
322 # initialize variables used by send_status before launching the status daemon
324 format_status_num( $main::next_run, time+10 );
325 format_status_str( $main::current_changes, "" );
327 $main::incoming_writable = 1; # assume this for now
329 # start the daemon watching the 'status' FIFO
330 if ($conf::statusfile && $conf::statusdelay == 0) {
331 $main::statusd_pid = fork_statusd();
332 $SIG{"CHLD"} = \&kid_died; # watch out for dead status daemon
333 # SIGUSR1 triggers status info
334 $SIG{"USR1"} = \&send_status;
336 $main::maind_pid = $$;
338 END { kill( $main::signo{"ABRT"}, $$ ) if defined $main::signo{"ABRT"}; }
341 open( PIDFILE, ">$conf::pidfile" )
342 or msg( "log", "Can't open $conf::pidfile: $!\n" );
343 printf PIDFILE "%5d\n", $$;
345 chmod( 0644, $conf::pidfile )
346 or die "Cannot set modes of $conf::pidfile: $!\n";
348 # other signals will just log an error and exit
349 foreach ( @main::fatal_signals ) {
350 $SIG{$_} = \&fatal_signal;
353 # send signal to user-started process that we're ready and it can exit
354 kill( $main::signo{"USR1"}, $parent_pid );
356 # ---------------------------------------------------------------------------
358 # ---------------------------------------------------------------------------
360 # default to classical incoming/target
361 $main::current_incoming = $conf::incoming;
362 $main::current_targetdir = $conf::targetdir;
365 write_status_file() if $conf::statusdelay;
368 # ping target only if there is the possibility that we'll contact it (but
369 # also don't wait too long).
370 my @have_changes = <*.changes *.commands>;
371 for ( my $delayed_dirs = 0; $delayed_dirs <= $conf::max_delayed;
373 my $adelayeddir = sprintf( "$conf::incoming_delayed",
376 <$adelayeddir/*.changes> );
378 check_alive() if @have_changes || (time - $main::last_ping_time) > 8*60*60;
380 if (@have_changes && $main::target_up) {
381 check_incoming_writable if !$main::incoming_writable;
382 check_dir() if $main::incoming_writable;
385 write_status_file() if $conf::statusdelay;
387 if ($conf::upload_method eq "copy") {
388 age_delayed_queues();
391 # sleep() returns if we received a signal (SIGUSR1 for status FIFO), so
392 # calculate the end time once and wait for it being reached.
393 format_status_num( $main::next_run, time + $conf::queue_delay );
395 while( ($delta = calc_delta()) > 0 ) {
396 debug( "mainloop sleeping $delta secs" );
398 # check if statusd died, if using status FIFO, or update status file
399 if ($conf::statusdelay) {
411 $delta = $main::next_run - time;
412 $delta = $conf::statusdelay
413 if $conf::statusdelay && $conf::statusdelay < $delta;
418 # ---------------------------------------------------------------------------
419 # main working functions
420 # ---------------------------------------------------------------------------
424 # main function for checking the incoming dir
427 my( @files, @changes, @keep_files, @this_keep_files, @stats, $file ,
430 debug( "starting checkdir" );
432 write_status_file() if $conf::statusdelay;
434 # test if needed binaries are available; this is if they're on maybe
435 # slow-mounted NFS filesystems
436 foreach (@conf::test_binaries) {
438 # maybe the mount succeeds now
441 msg( "log", "binary test failed for $_; delaying queue run\n");
445 for ( $adelay=-1; $adelay <= $conf::max_delayed; $adelay++ ) {
446 if ( $adelay == -1 ) {
447 $main::current_incoming = $conf::incoming;
448 $main::current_incoming_short = "";
449 $main::current_targetdir = $conf::targetdir;
452 $main::current_incoming = sprintf( $conf::incoming_delayed,
454 $main::current_incoming_short = sprintf( "DELAYED/%d-day",
456 $main::current_targetdir = sprintf( $conf::targetdir_delayed,
460 # need to clear directory specific variables
461 undef ( @keep_files );
462 undef ( @this_keep_files );
464 chdir ( $main::current_incoming )
466 "Cannot change to dir ".
467 "${main::current_incoming_short}: $!\n" ),
470 # look for *.commands files but not in delayed queues
472 foreach $file ( <*.commands> ) {
475 process_commands( $file );
478 write_status_file() if $conf::statusdelay;
483 or (msg( "log", "Cannot open dir ${main::current_incoming_short}: $!\n" ),
485 @files = readdir( INC );
488 # process all .changes files found
489 @changes = grep /\.changes$/, @files;
490 push( @keep_files, @changes ); # .changes files aren't stray
491 foreach $file ( @changes ) {
493 # wrap in an eval to allow jumpbacks to here with die in case
496 eval { process_changes( $file, @this_keep_files ); };
498 msg( "log,mail", $@ ) if $@;
500 write_status_file() if $conf::statusdelay;
502 # files which are ok in conjunction with this .changes
503 debug( "$file tells to keep @this_keep_files" );
504 push( @keep_files, @this_keep_files );
507 # break out of this loop if the incoming dir has become unwritable
508 goto end_run if !$main::incoming_writable;
510 ftp_close() if $conf::upload_method eq "ftp";
512 # find files which aren't related to any .changes
513 foreach $file ( @files ) {
514 # filter out files we never want to delete
515 next if ! -f $file || # may have disappeared in the meantime
516 $file eq "." || $file eq ".." ||
517 (grep { $_ eq $file } @keep_files) ||
518 $file =~ /$conf::keep_files/;
519 # Delete such files if they're older than
520 # $stray_remove_timeout; they could be part of an
521 # yet-incomplete upload, with the .changes still missing.
522 # Cannot send any notification, since owner unknown.
523 next if !(@stats = stat( $file ));
524 my $age = time - $stats[ST_MTIME];
525 my( $maint, $pattern, @job_files );
526 if ($file =~ /^junk-for-writable-test/ ||
527 $file !~ m,$conf::valid_files, ||
528 $age >= $conf::stray_remove_timeout) {
529 msg( "log", "Deleted stray file ${main::current_incoming_short}/$file\n" ) if rm( $file );
531 elsif ($age > $conf::no_changes_timeout &&
532 is_debian_file( $file ) &&
533 # not already reported
534 !($stats[ST_MODE] & S_ISGID) &&
535 ($pattern = debian_file_stem( $file )) &&
536 (@job_files = glob($pattern)) &&
537 # If a .changes is in the list, it has the same stem as the
538 # found file (probably a .orig.tar.gz). Don't report in this
540 !(grep( /\.changes$/, @job_files ))) {
541 $maint = get_maintainer( $file );
542 # Don't send a mail if this looks like the recompilation of a
543 # package for a non-i386 arch. For those, the maintainer field is
545 if (!grep( /(\.dsc|_(i386|all)\.deb)$/, @job_files )) {
546 msg( "log", "Found an upload without .changes and with no ",
548 msg( "log", "Not sending a report, because probably ",
549 "recompilation job\n" );
553 $main::mail_addr = $maint;
554 $main::mail_addr = $1 if $main::mail_addr =~ /<([^>]*)>/;
555 $main::mail_subject = "Incomplete upload found in ".
556 "Debian upload queue";
557 msg( "mail", "Probably you are the uploader of the following ".
559 msg( "mail", "the Debian upload queue directory:\n " );
560 msg( "mail", join( "\n ", @job_files ), "\n" );
561 msg( "mail", "This looks like an upload, but a .changes file ".
562 "is missing, so the job\n" );
563 msg( "mail", "cannot be processed.\n\n" );
564 msg( "mail", "If no .changes file arrives within ",
565 print_time( $conf::stray_remove_timeout - $age ),
566 ", the files will be deleted.\n\n" );
567 msg( "mail", "If you didn't upload those files, please just ".
568 "ignore this message.\n" );
570 msg( "log", "Sending problem report for an upload without a ".
572 msg( "log", "Maintainer: $maint\n" );
575 msg( "log", "Found an upload without .changes, but can't ".
576 "find a maintainer address\n" );
578 msg( "log", "Files: @job_files\n" );
579 # remember we already have sent a mail regarding this file
580 foreach ( @job_files ) {
582 next if !@st; # file may have disappeared in the meantime
583 chmod +($st[ST_MODE] |= S_ISGID), $_;
587 debug( "found stray file ${main::current_incoming_short}/$file, deleting in ",
588 print_time($conf::stray_remove_timeout - $age) );
592 chdir( $conf::incoming );
596 write_status_file() if $conf::statusdelay;
599 sub get_filelist_from_known_good_changes($) {
605 # parse the .changes file
606 open( CHANGES, "<$changes" )
607 or die "$changes: $!\n";
608 outer_loop: while( <CHANGES> ) {
611 redo outer_loop if !/^\s/;
612 my @field = split( /\s+/ );
614 # forbid shell meta chars in the name, we pass it to a
615 # subshell several times...
616 $field[5] =~ /^([a-zA-Z0-9.+_:@=%-][~a-zA-Z0-9.+_:@=%-]*)/;
617 if ($1 ne $field[5]) {
618 msg( "log", "found suspicious filename $field[5]\n" );
621 push( @filenames, $field[5] );
630 # process one .changes file
632 sub process_changes($\@) {
634 my $keep_list = shift;
635 my( $pgplines, @files, @filenames, @changes_stats, $failure_file,
636 $retries, $last_retry, $upload_time, $file, $do_report, $ls_l,
637 $problems_reported, $errs, $pkgname, $signator );
641 format_status_str( $main::current_changes, "$main::current_incoming_short/$changes" );
643 write_status_file() if $conf::statusdelay;
646 msg( "log", "processing ${main::current_incoming_short}/$changes\n" );
648 # parse the .changes file
649 open( CHANGES, "<$changes" )
650 or die "Cannot open ${main::current_incoming_short}/$changes: $!\n";
652 $main::mail_addr = "";
654 outer_loop: while( <CHANGES> ) {
655 if (/^---+(BEGIN|END) PGP .*---+$/) {
658 elsif (/^Maintainer:\s*/i) {
659 chomp( $main::mail_addr = $' );
660 $main::mail_addr = $1 if $main::mail_addr =~ /<([^>]*)>/;
662 elsif (/^Source:\s*/i) {
663 chomp( $pkgname = $' );
664 $pkgname =~ s/\s+$//;
665 $main::packages{$pkgname}++;
669 redo outer_loop if !/^\s/;
670 my @field = split( /\s+/ );
672 # forbid shell meta chars in the name, we pass it to a
673 # subshell several times...
674 $field[5] =~ /^([a-zA-Z0-9.+_:@=%-][~a-zA-Z0-9.+_:@=%-]*)/;
675 if ($1 ne $field[5]) {
676 msg( "log", "found suspicious filename $field[5]\n" );
677 msg( "mail", "File '$field[5]' mentioned in $main::current_incoming_short/$changes\n",
678 "has bad characters in its name. Removed.\n" );
682 push( @files, { md5 => $field[1],
684 name => $field[5] } );
685 push( @filenames, $field[5] );
686 debug( "includes file $field[5], size $field[2], ",
693 # tell check_dir that the files mentioned in this .changes aren't stray,
694 # we know about them somehow
695 @$keep_list = @filenames;
697 # some consistency checks
698 if (!$main::mail_addr) {
699 msg( "log,mail", "$main::current_incoming_short/$changes doesn't contain a Maintainer: field; ".
700 "cannot process\n" );
701 goto remove_only_changes;
703 if ($main::mail_addr !~ /^(buildd_\S+-\S+|\S+\@\S+\.\S+)/) {
704 # doesn't look like a mail address, maybe only the name
705 my( $new_addr, @addr_list );
706 if ($new_addr = try_to_get_mail_addr( $main::mail_addr, \@addr_list )){
707 # substitute (unique) found addr, but give a warning
708 msg( "mail", "(The Maintainer: field didn't contain a proper ".
710 msg( "mail", "Looking for `$main::mail_addr' in the Debian ".
711 "keyring gave your address\n" );
712 msg( "mail", "as unique result, so I used this.)\n" );
713 msg( "log", "Substituted $new_addr for malformed ".
714 "$main::mail_addr\n" );
715 $main::mail_addr = $new_addr;
718 # not found or not unique: hold the job and inform queue maintainer
719 my $old_addr = $main::mail_addr;
720 $main::mail_addr = $conf::maintainer_mail;
721 msg( "mail", "The job ${main::current_incoming_short}/$changes doesn't have a correct email\n" );
722 msg( "mail", "address in the Maintainer: field:\n" );
723 msg( "mail", " $old_addr\n" );
724 msg( "mail", "A check for this in the Debian keyring gave:\n" );
725 msg( "mail", @addr_list ?
726 " " . join( ", ", @addr_list ) . "\n" :
728 msg( "mail", "Please fix this manually\n" );
729 msg( "log", "Bad Maintainer: field in ${main::current_incoming_short}/$changes: $old_addr\n" );
730 goto remove_only_changes;
734 msg( "log,mail", "$main::current_incoming_short/$changes isn't signed with PGP/GnuPG\n" );
735 msg( "log", "(uploader $main::mail_addr)\n" );
736 goto remove_only_changes;
739 msg( "log,mail", "$main::current_incoming_short/$changes doesn't mention any files\n" );
740 msg( "log", "(uploader $main::mail_addr)\n" );
741 goto remove_only_changes;
744 # check for packages that shouldn't be processed
745 if (grep( $_ eq $pkgname, @conf::nonus_packages )) {
746 msg( "log,mail", "$pkgname is a package that must be uploaded ".
747 "to nonus.debian.org\n" );
748 msg( "log,mail", "instead of target.\n" );
749 msg( "log,mail", "Job rejected and removed all files belonging ".
751 msg( "log,mail", " ", join( ", ", @filenames ), "\n" );
752 rm( $changes, @filenames );
756 $failure_file = $changes . ".failures";
757 $retries = $last_retry = 0;
758 if (-f $failure_file) {
759 open( FAILS, "<$failure_file" )
760 or die "Cannot open $main::current_incoming_short/$failure_file: $!\n";
763 ( $retries, $last_retry ) = ( $1, $2 ) if $line =~ /^(\d+)\s+(\d+)$/;
764 push( @$keep_list, $failure_file );
767 # run PGP on the file to check the signature
768 if (!($signator = pgp_check( $changes ))) {
769 msg( "log,mail", "$main::current_incoming_short/$changes has bad PGP/GnuPG signature!\n" );
770 msg( "log", "(uploader $main::mail_addr)\n" );
772 msg( "log,mail", "Removing $main::current_incoming_short/$changes, but keeping its associated ",
773 "files for now.\n" );
775 # Set SGID bit on associated files, so that the test for Debian files
776 # without a .changes doesn't consider them.
777 foreach ( @filenames ) {
779 next if !@st; # file may have disappeared in the meantime
780 chmod +($st[ST_MODE] |= S_ISGID), $_;
784 elsif ($signator eq "LOCAL ERROR") {
785 # An error has appened when starting pgp... Don't process the file,
786 # but also don't delete it
787 debug( "Can't PGP/GnuPG check $main::current_incoming_short/$changes -- don't process it for now" );
791 die "Cannot stat ${main::current_incoming_short}/$changes (??): $!\n"
792 if !(@changes_stats = stat( $changes ));
793 # Make $upload_time the maximum of all modification times of files
794 # related to this .changes (and the .changes it self). This is the
795 # last time something changes to these files.
796 $upload_time = $changes_stats[ST_MTIME];
797 for $file ( @files ) {
799 next if !(@stats = stat( $file->{"name"} ));
800 $file->{"stats"} = \@stats;
801 $upload_time = $stats[ST_MTIME] if $stats[ST_MTIME] > $upload_time;
804 $do_report = (time - $upload_time) > $conf::problem_report_timeout;
805 $problems_reported = $changes_stats[ST_MODE] & S_ISGID;
806 # if any of the files is newer than the .changes' ctime (the time
807 # we sent a report and set the sticky bit), send new problem reports
808 if ($problems_reported && $changes_stats[ST_CTIME] < $upload_time) {
809 $problems_reported = 0;
810 chmod +($changes_stats[ST_MODE] &= ~S_ISGID), $changes;
811 debug( "upload_time>changes-ctime => resetting problems reported" );
813 debug( "do_report=$do_report problems_reported=$problems_reported" );
815 # now check all files for correct size and md5 sum
816 for $file ( @files ) {
817 my $filename = $file->{"name"};
818 if (!defined( $file->{"stats"} )) {
819 # could be an upload that isn't complete yet, be quiet,
820 # but don't process the file;
821 msg( "log,mail", "$filename doesn't exist\n" )
822 if $do_report && !$problems_reported;
823 msg( "log", "$filename doesn't exist (ignored for now)\n" )
825 msg( "log", "$filename doesn't exist (already reported)\n" )
826 if $problems_reported;
829 elsif ($file->{"stats"}->[ST_SIZE] < $file->{"size"} && !$do_report) {
830 # could be an upload that isn't complete yet, be quiet,
831 # but don't process the file
832 msg( "log", "$filename is too small (ignored for now)\n" );
835 elsif ($file->{"stats"}->[ST_SIZE] != $file->{"size"}) {
836 msg( "log,mail", "$filename has incorrect size; deleting it\n" );
840 elsif (md5sum( $filename ) ne $file->{"md5"}) {
841 msg( "log,mail", "$filename has incorrect md5 checksum; ",
849 if ((time - $upload_time) > $conf::bad_changes_timeout) {
850 # if a .changes fails for a really long time (several days
851 # or so), remove it and all associated files
853 "$main::current_incoming_short/$changes couldn't be processed for ",
854 int($conf::bad_changes_timeout/(60*60)),
855 " hours and is now deleted\n" );
857 "All files it mentions are also removed:\n" );
858 msg( "log,mail", " ", join( ", ", @filenames ), "\n" );
859 rm( $changes, @filenames, $failure_file );
861 elsif ($do_report && !$problems_reported) {
862 # otherwise, send a problem report, if not done already
864 "Due to the errors above, the .changes file couldn't ",
866 "Please fix the problems for the upload to happen.\n" );
867 # remember we already have sent a mail regarding this file
868 debug( "Sending problem report mail and setting SGID bit" );
869 my $mode = $changes_stats[ST_MODE] |= S_ISGID;
870 msg( "log", "chmod failed: $!" ) if (chmod ($mode, $changes) != 1);
877 # if this upload already failed earlier, wait until the delay requirement
879 if ($retries > 0 && (time - $last_retry) <
880 ($retries == 1 ? $conf::upload_delay_1 : $conf::upload_delay_2)) {
881 msg( "log", "delaying retry of upload\n" );
885 if ($conf::upload_method eq "ftp") {
886 return if !ftp_open();
889 # check if the job is already present on target
890 # (moved to here, to avoid bothering target as long as there are errors in
892 if ($ls_l = is_on_target( $changes, @filenames )) {
893 msg( "log,mail", "$main::current_incoming_short/$changes is already present on target host:\n" );
894 msg( "log,mail", "$ls_l\n" );
895 msg( "mail", "Either you already uploaded it, or someone else ",
897 msg( "log,mail", "Job $changes removed.\n" );
898 rm( $changes, @filenames, $failure_file );
902 # clear sgid bit before upload, scp would copy it to target. We don't need
903 # it anymore, we know there are no problems if we come here. Also change
904 # mode of files to 644 if this should be done locally.
905 $changes_stats[ST_MODE] &= ~S_ISGID;
906 if (!$conf::chmod_on_target) {
907 $changes_stats[ST_MODE] &= ~0777;
908 $changes_stats[ST_MODE] |= 0644;
910 chmod +($changes_stats[ST_MODE]), $changes;
912 # try uploading to target
913 if (!copy_to_target( $changes, @filenames )) {
914 # if the upload failed, increment the retry counter and remember the
915 # current time; both things are written to the .failures file. Don't
916 # increment the fail counter if the error was due to incoming
918 return if !$main::incoming_writable;
919 if (++$retries >= $conf::max_upload_retries) {
921 "$changes couldn't be uploaded for $retries times now.\n" );
923 "Giving up and removing it and its associated files:\n" );
924 msg( "log,mail", " ", join( ", ", @filenames ), "\n" );
925 rm( $changes, @filenames, $failure_file );
929 if (open( FAILS, ">$failure_file" )) {
930 print FAILS "$retries $last_retry\n";
932 chmod( 0600, $failure_file )
933 or die "Cannot set modes of $failure_file: $!\n";
935 push( @$keep_list, $failure_file );
936 debug( "now $retries failed uploads" );
938 "The upload will be retried in ",
939 print_time( $retries == 1 ? $conf::upload_delay_1 :
940 $conf::upload_delay_2 ), "\n" );
945 # If the files were uploaded ok, remove them
946 rm( $changes, @filenames, $failure_file );
948 msg( "mail", "$changes uploaded successfully to $conf::target\n" );
949 msg( "mail", "along with the files:\n ",
950 join( "\n ", @filenames ), "\n" );
951 msg( "log", "$changes processed successfully (uploader $main::mail_addr)\n" );
953 # Check for files that have the same stem as the .changes (and weren't
954 # mentioned there) and delete them. It happens often enough that people
955 # upload a .orig.tar.gz where it isn't needed and also not in the
956 # .changes. Explicitly deleting it (and not waiting for the
957 # $stray_remove_timeout) reduces clutter in the queue dir and maybe also
958 # educates uploaders :-)
960 # my $pattern = debian_file_stem( $changes );
961 # my $spattern = substr( $pattern, 0, -1 ); # strip off '*' at end
962 # my @other_files = glob($pattern);
963 # filter out files that have a Debian revision at all and a different
964 # revision. Those belong to a different upload.
965 # if ($changes =~ /^\Q$spattern\E-([\d.+-]+)/) {
967 # @other_files = grep( !/^\Q$spattern\E-([\d.+-]+)/ || $1 eq $this_rev,
970 # Also do not remove those files if a .changes is among them. Then there
971 # is probably a second upload for another version or another architecture.
972 # if (@other_files && !grep( /\.changes$/, @other_files )) {
973 # rm( @other_files );
974 # msg( "mail", "\nThe following file(s) seemed to belong to the same ".
975 # "upload, but weren't listed\n" );
976 # msg( "mail", "in the .changes file:\n " );
977 # msg( "mail", join( "\n ", @other_files ), "\n" );
978 # msg( "mail", "They have been deleted.\n" );
979 # msg( "log", "Deleted files in upload not in $changes: @other_files\n" );
984 # process one .commands file
986 sub process_commands($) {
987 my $commands = shift;
988 my( @cmds, $cmd, $pgplines, $signator );
990 my( @files, $file, @removed, $target_delay );
992 format_status_str( $main::current_changes, $commands );
994 write_status_file() if $conf::statusdelay;
996 msg( "log", "processing $main::current_incoming_short/$commands\n" );
998 # parse the .commands file
999 if (!open( COMMANDS, "<$commands" )) {
1000 msg( "log", "Cannot open $main::current_incoming_short/$commands: $!\n" );
1004 $main::mail_addr = "";
1006 outer_loop: while( <COMMANDS> ) {
1007 if (/^---+(BEGIN|END) PGP .*---+$/) {
1010 elsif (/^Uploader:\s*/i) {
1011 chomp( $main::mail_addr = $' );
1012 $main::mail_addr = $1 if $main::mail_addr =~ /<([^>]*)>/;
1014 elsif (/^Commands:/i) {
1017 s/^\s*(.*)\s*$/$1/; # delete whitespace at both ends
1020 debug( "includes cmd $_" );
1022 last outer_loop if !defined( $_ = scalar(<COMMANDS>) );
1024 redo outer_loop if !/^\s/ || /^$/;
1030 # some consistency checks
1031 if (!$main::mail_addr || $main::mail_addr !~ /^\S+\@\S+\.\S+/) {
1032 msg( "log,mail", "$main::current_incoming_short/$commands contains no or bad Uploader: field: ".
1033 "$main::mail_addr\n" );
1034 msg( "log,mail", "cannot process $main::current_incoming_short/$commands\n" );
1035 $main::mail_addr = "";
1038 msg( "log", "(command uploader $main::mail_addr)\n" );
1040 if ($pgplines < 3) {
1041 msg( "log,mail", "$main::current_incoming_short/$commands isn't signed with PGP/GnuPG\n" );
1042 msg( "mail", "or the uploaded file is broken. Make sure to transfer in binary mode\n" );
1043 msg( "mail", "or better yet - use dcut for commands files\n");
1047 # run PGP on the file to check the signature
1048 if (!($signator = pgp_check( $commands ))) {
1049 msg( "log,mail", "$main::current_incoming_short/$commands has bad PGP/GnuPG signature!\n" );
1051 msg( "log,mail", "Removing $main::current_incoming_short/$commands\n" );
1055 elsif ($signator eq "LOCAL ERROR") {
1056 # An error has appened when starting pgp... Don't process the file,
1057 # but also don't delete it
1058 debug( "Can't PGP/GnuPG check $main::current_incoming_short/$commands -- don't process it for now" );
1061 msg( "log", "(PGP/GnuPG signature by $signator)\n" );
1063 # now process commands
1064 msg( "mail", "Log of processing your commands file $main::current_incoming_short/$commands:\n\n" );
1065 foreach $cmd ( @cmds ) {
1066 my @word = split( /\s+/, $cmd );
1067 msg( "mail,log", "> @word\n" );
1070 if ($word[0] eq "rm") {
1071 foreach ( @word[1..$#word] ) {
1073 msg( "mail,log", "$_: filename may not contain slashes\n" );
1076 # process wildcards but also plain names (for delayed target removal)
1078 my $pat = quotemeta($_);
1079 $pat =~ s/\\\*/.*/g;
1080 $pat =~ s/\\\?/.?/g;
1081 $pat =~ s/\\([][])/$1/g;
1082 opendir( DIR, "." );
1083 push (@thesefiles, grep /^$pat$/, readdir(DIR) );
1085 for ( my($adelay)=0; (! @thesefiles) && $adelay <= $conf::max_delayed; $adelay++ ) {
1086 my($dir) = sprintf( $conf::incoming_delayed,
1088 opendir( DIR, "$dir" );
1089 push( @thesefiles, map ("$dir/$_", grep /^$pat$/, readdir(DIR) ));
1092 push (@files, @thesefiles);
1093 if (! @thesefiles) {
1094 msg( "mail,log", "$_ did not match anything\n" );
1101 push (@thesefiles, $file);
1103 for ( my($adelay)=0; $adelay <= $conf::max_delayed; $adelay++ ) {
1104 my($dir) = sprintf( $conf::incoming_delayed, $adelay );
1105 if (-f "$dir/$file") {
1106 push (@thesefiles, "$dir/$file");
1109 if ($file =~ m/\.changes$/ && $conf::upload_method eq "copy") {
1110 for ( my($adelay)=0; $adelay <= $conf::max_delayed; $adelay++ ) {
1111 my($dir) = sprintf( "$conf::targetdir_delayed",$adelay );
1112 if (-f "$dir/$file") {
1113 push (@thesefiles, "$dir/$file");
1114 push (@thesefiles, map( "$dir/$_",get_filelist_from_known_good_changes("$dir/$file")));
1119 msg( "mail,log", "No file found: $file\n" );
1121 push (@files, @thesefiles);
1125 msg( "mail,log", "No files to delete\n" );
1129 foreach $file ( @files ) {
1131 msg( "mail,log", "$file: no such file\n" );
1133 elsif ($file =~ /$conf::keep_files/) {
1134 msg( "mail,log", "$file is protected, cannot ".
1137 elsif (!unlink( $file )) {
1138 msg( "mail,log", "$file: rm: $!\n" );
1141 push( @removed, $file );
1144 msg( "mail,log", "Files removed: @removed\n" ) if @removed;
1147 elsif ($word[0] eq "mv") {
1149 msg( "mail,log", "Wrong number of arguments\n" );
1151 elsif ($word[1] =~ m,/, || $word[1] !~ m/\.changes/) {
1152 msg( "mail,log", "$word[1]: filename may not contain slashes\n" );
1154 elsif (! (($target_delay) = $word[2] =~ m,^([0-9]+)-day$,) || $target_delay > $conf::max_delayed) {
1155 msg( "mail,log", "$word[2]: target must be #-day with # between 0 and $conf::max_delayed (in particular, no '/' allowed)\n");
1157 elsif ($word[1] =~ /$conf::keep_files/) {
1158 msg( "mail,log", "$word[1] is protected, cannot rename\n" );
1162 for ( $adelay=0; $adelay <= $conf::max_delayed && ! -f (sprintf( "$conf::targetdir_delayed",$adelay )."/$word[1]"); $adelay++ ) {
1164 if ( $adelay > $conf::max_delayed) {
1165 msg( "mail,log", "$word[1] not found\n" );
1167 elsif ($adelay == $target_delay) {
1168 msg( "mail,log", "$word[1] already is in $word[2]\n" );
1172 my($dir) = sprintf( "$conf::targetdir_delayed",$adelay );
1173 my($target_dir) = sprintf( "$conf::targetdir_delayed",$target_delay );
1174 push (@thesefiles, $word[1]);
1175 push (@thesefiles, get_filelist_from_known_good_changes("$dir/$word[1]"));
1176 for my $afile(@thesefiles) {
1177 if (! rename "$dir/$afile","$target_dir/$afile") {
1178 msg( "mail,log", "rename: $!\n" );
1181 msg( "mail,log", "$afile moved to $target_delay-day\n" );
1188 msg( "mail,log", "unknown command $word[0]\n" );
1192 msg( "log", "-- End of $main::current_incoming_short/$commands processing\n" );
1195 sub age_delayed_queues() {
1196 for ( my($adelay)=0 ; $adelay <= $conf::max_delayed ; $adelay++ ) {
1197 my($dir) = sprintf( "$conf::targetdir_delayed",$adelay );
1200 $target_dir = $conf::targetdir;
1203 $target_dir = sprintf( "$conf::targetdir_delayed",$adelay-1 );
1205 for my $achanges (<$dir/*.changes>) {
1206 my $mtime = (stat($achanges))[9];
1207 if ($mtime + 24*60*60 <= time) {
1208 utime undef,undef,($achanges);
1209 my @thesefiles = ($achanges =~ m,.*/([^/]*),);
1210 push (@thesefiles, get_filelist_from_known_good_changes($achanges));
1211 for my $afile(@thesefiles) {
1212 if (! rename "$dir/$afile","$target_dir/$afile") {
1213 msg( "log", "rename: $!\n" );
1216 msg( "log", "$afile moved to $target_dir\n" );
1225 # check if a file is already on target
1227 sub is_on_target($\@) {
1229 my $filelist = shift;
1233 if ($conf::upload_method eq "ssh") {
1234 ($msg, $stat) = ssh_cmd( "ls -l $file" );
1236 elsif ($conf::upload_method eq "ftp") {
1238 ($msg, $err) = ftp_cmd( "dir", $file );
1245 $msg = "ls: no such file\n";
1249 $msg = join( "\n", @$msg );
1253 my @allfiles = ($file);
1254 push ( @allfiles, @$filelist);
1256 $msg = "no such file";
1257 for my $afile(@allfiles) {
1258 if (-f "$conf::targetdir/$afile") {
1263 for ( my($adelay)=0 ; $adelay <= $conf::max_delayed && $stat ; $adelay++ ) {
1264 for my $afile(@allfiles) {
1265 if (-f (sprintf( "$conf::targetdir_delayed",$adelay )."/$afile")) {
1267 $msg = sprintf( "%d-day",$adelay )."/$afile";
1273 debug( "exit status: $stat, output was: $msg" );
1275 return "" if $stat && $msg =~ /no such file/i; # file not present
1276 msg( "log", "strange ls -l output on target:\n", $msg ), return ""
1277 if $stat || $@; # some other error, but still try to upload
1279 # ls -l returned 0 -> file already there
1280 $msg =~ s/\s\s+/ /g; # make multiple spaces into one, to save space
1285 # copy a list of files to target
1287 sub copy_to_target(@) {
1289 my( @md5sum, @expected_files, $sum, $name, $msgs, $stat );
1292 write_status_file() if $conf::statusdelay;
1295 if ($conf::upload_method eq "ssh") {
1296 ($msgs, $stat) = scp_cmd( @files );
1299 elsif ($conf::upload_method eq "ftp") {
1301 if (!$main::FTP_chan->cwd( $main::current_targetdir )) {
1302 msg( "log,mail", "Can't cd to $main::current_targetdir on $conf::target\n" );
1305 foreach $file (@files) {
1306 ($rv, $msgs) = ftp_cmd( "put", $file );
1311 ($msgs, $stat) = local_cmd( "$conf::cp @files $main::current_targetdir", 'NOCD' );
1315 # check md5sums or sizes on target against our own
1316 my $have_md5sums = 1;
1317 if ($conf::upload_method eq "ssh") {
1318 ($msgs, $stat) = ssh_cmd( "md5sum @files" );
1320 @md5sum = split( "\n", $msgs );
1322 elsif ($conf::upload_method eq "ftp") {
1323 my ($rv, $err, $file);
1324 foreach $file (@files) {
1325 ($rv, $err) = ftp_cmd( "quot", "site", "md5sum", $file );
1327 next if ftp_code() == 550; # file not found
1328 if (ftp_code() == 500) { # unimplemented
1330 goto get_sizes_instead;
1335 chomp( my $t = ftp_response() );
1336 push( @md5sum, $t );
1338 if (!$have_md5sums) {
1340 foreach $file (@files) {
1341 ($rv, $err) = ftp_cmd( "size", $file );
1343 next if ftp_code() == 550; # file not found
1347 push( @md5sum, "$rv $file" );
1352 ($msgs, $stat) = local_cmd( "$conf::md5sum @files" );
1354 @md5sum = split( "\n", $msgs );
1357 @expected_files = @files;
1360 ($sum,$name) = split;
1361 next if !grep { $_ eq $name } @files; # a file we didn't upload??
1362 next if $sum eq "md5sum:"; # looks like an error message
1363 if (($have_md5sums && $sum ne md5sum( $name )) ||
1364 (!$have_md5sums && $sum != (-s $name))) {
1365 msg( "log,mail", "Upload of $name to $conf::target failed ",
1366 "(".($have_md5sums ? "md5sum" : "size")." mismatch)\n" );
1369 # seen that file, remove it from expect list
1370 @expected_files = map { $_ eq $name ? () : $_ } @expected_files;
1372 if (@expected_files) {
1373 msg( "log,mail", "Failed to upload the files\n" );
1374 msg( "log,mail", " ", join( ", ", @expected_files ), "\n" );
1375 msg( "log,mail", "(Not present on target after upload)\n" );
1379 if ($conf::chmod_on_target) {
1380 # change file's mode explicitly to 644 on target
1381 if ($conf::upload_method eq "ssh") {
1382 ($msgs, $stat) = ssh_cmd( "chmod 644 @files" );
1385 elsif ($conf::upload_method eq "ftp") {
1387 foreach $file (@files) {
1388 ($rv, $msgs) = ftp_cmd( "quot", "site", "chmod", "644", $file );
1389 msg( "log", "Can't chmod $file on target:\n$msgs" )
1395 ($msgs, $stat) = local_cmd( "$conf::chmod 644 @files" );
1401 write_status_file() if $conf::statusdelay;
1405 msg( "log,mail", "Upload to $conf::target failed",
1406 $? ? ", last exit status ".sprintf( "%s", $?>>8 ) : "", "\n" );
1407 msg( "log,mail", "Error messages:\n", $msgs )
1410 # If "permission denied" was among the errors, test if the incoming is
1412 if ($msgs =~ /(permission denied|read-?only file)/i) {
1413 if (!check_incoming_writable()) {
1414 msg( "log,mail", "(The incoming directory seems to be ",
1419 # remove bad files or an incomplete upload on target
1420 if ($conf::upload_method eq "ssh") {
1421 ssh_cmd( "rm -f @files" );
1423 elsif ($conf::upload_method eq "ftp") {
1425 foreach $file (@files) {
1427 ($rv, $err) = ftp_cmd( "delete", $file );
1428 msg( "log", "Can't delete $file on target:\n$err" )
1433 my @tfiles = map { "$main::current_targetdir/$_" } @files;
1434 debug( "executing unlink(@tfiles)" );
1438 write_status_file() if $conf::statusdelay;
1443 # check if a file is correctly signed with PGP
1454 if (-x $conf::gpg) {
1455 debug( "executing $conf::gpg --no-options --batch ".
1456 "--no-default-keyring --always-trust ".
1457 "--keyring ". join (" --keyring ",@conf::keyrings).
1458 " --verify '$file'" );
1459 if (!open( PIPE, "$conf::gpg --no-options --batch ".
1460 "--no-default-keyring --always-trust ".
1461 "--keyring " . join (" --keyring ",@conf::keyrings).
1462 " --verify '$file'".
1464 msg( "log", "Can't open pipe to $conf::gpg: $!\n" );
1465 return "LOCAL ERROR";
1467 $output .= $_ while( <PIPE> );
1473 msg( "log,mail", "GnuPG signature check failed on $file\n" );
1474 msg( "mail", $output );
1475 msg( "log,mail", "(Exit status ", $stat >> 8, ")\n" );
1479 $output =~ /^(gpg: )?good signature from (user )?"(.*)"\.?$/im;
1480 ($signator = $3) ||= "unknown signator";
1482 debug( "GnuPG signature ok (by $signator)" );
1488 # ---------------------------------------------------------------------------
1490 # ---------------------------------------------------------------------------
1493 # fork a subprocess that watches the 'status' FIFO
1495 # that process blocks until someone opens the FIFO, then sends a
1496 # signal (SIGUSR1) to the main process, expects
1498 sub fork_statusd() {
1504 $statusd_pid = open( STATUSD, "|-" );
1505 die "cannot fork: $!\n" if !defined( $statusd_pid );
1506 # parent just returns
1508 msg( "log", "forked status daemon (pid $statusd_pid)\n" );
1509 return $statusd_pid;
1511 # child: the status FIFO daemon
1513 # ignore SIGPIPE here, in case some closes the FIFO without completely
1515 $SIG{"PIPE"} = "IGNORE";
1516 # also ignore SIGCLD, we don't want to inherit the restart-statusd handler
1518 $SIG{"CHLD"} = "DEFAULT";
1520 rm( $conf::statusfile );
1521 $errs = `$conf::mkfifo $conf::statusfile`;
1522 die "$main::progname: cannot create named pipe $conf::statusfile: $errs"
1524 chmod( 0644, $conf::statusfile )
1525 or die "Cannot set modes of $conf::statusfile: $!\n";
1527 # close log file, so that log rotating works
1533 my( $status, $mup, $incw, $ds, $next_run, $last_ping, $currch, $l );
1535 # open the FIFO for writing; this blocks until someone (probably ftpd)
1536 # opens it for reading
1537 open( STATFIFO, ">$conf::statusfile" )
1538 or die "Cannot open $conf::statusfile\n";
1540 # tell main daemon to send us status infos
1541 kill( $main::signo{"USR1"}, $main_pid );
1543 # get the infos from stdin; must loop until enough bytes received!
1544 my $expect_len = 3 + 2*STATNUM_LEN + STATSTR_LEN;
1545 for( $status = ""; ($l = length($status)) < $expect_len; ) {
1546 sysread( STDIN, $status, $expect_len-$l, $l );
1549 # disassemble the status byte stream
1551 foreach ( [ mup => 1 ], [ incw => 1 ], [ ds => 1 ],
1552 [ next_run => STATNUM_LEN ], [ last_ping => STATNUM_LEN ],
1553 [ currch => STATSTR_LEN ] ) {
1554 eval "\$$_->[0] = substr( \$status, $pos, $_->[1] );";
1557 $currch =~ s/\n+//g;
1559 print_status( $mup, $incw, $ds, $next_run, $last_ping, $currch );
1562 # This sleep is necessary so that we can't reopen the FIFO
1563 # immediately, in case the reader hasn't closed it yet if we get to
1564 # the open again. Is there a better solution for this??
1570 # update the status file, in case we use a plain file and not a FIFO
1572 sub write_status_file() {
1574 return if !$conf::statusfile;
1576 open( STATFILE, ">$conf::statusfile" ) or
1577 (msg( "log", "Could not open $conf::statusfile: $!\n" ), return);
1578 my $oldsel = select( STATFILE );
1580 print_status( $main::target_up, $main::incoming_writable, $main::dstat,
1581 $main::next_run, $main::last_ping_time,
1582 $main::current_changes );
1588 sub print_status($$$$$$) {
1592 my $next_run = shift;
1593 my $last_ping = shift;
1598 ($version = 'Release: 0.9 $Revision: 1.51 $') =~ s/\$ ?//g;
1599 print "debianqueued $version\n";
1601 $approx = $conf::statusdelay ? "approx. " : "";
1604 print "$conf::target is down, queue pausing\n";
1607 elsif ($conf::upload_method ne "copy") {
1608 print "$conf::target seems to be up, last ping $approx",
1609 print_time(time-$last_ping), " ago\n";
1613 print "The incoming directory is not writable, queue pausing\n";
1618 print "Next queue check in $approx",print_time($next_run-time),"\n";
1621 elsif ($ds eq "c") {
1622 print "Checking queue directory\n";
1624 elsif ($ds eq "u") {
1625 print "Uploading to $conf::target\n";
1628 print "Bad status data from daemon: \"$mup$incw$ds\"\n";
1632 print "Current job is $currch\n" if $currch;
1636 # format a number for sending to statusd (fixed length STATNUM_LEN)
1638 sub format_status_num(\$$) {
1642 $$varref = sprintf "%".STATNUM_LEN."d", $num;
1646 # format a string for sending to statusd (fixed length STATSTR_LEN)
1648 sub format_status_str(\$$) {
1652 $$varref = substr( $str, 0, STATSTR_LEN );
1653 $$varref .= "\n" x (STATSTR_LEN - length($$varref));
1657 # send a status string to the status daemon
1659 # Avoid all operations that could call malloc() here! Most libc
1660 # implementations aren't reentrant, so we may not call it from a
1661 # signal handler. So use only already-defined variables.
1664 local $! = 0; # preserve errno
1666 # re-setup handler, in case we have broken SysV signals
1667 $SIG{"USR1"} = \&send_status;
1669 syswrite( STATUSD, $main::target_up, 1 );
1670 syswrite( STATUSD, $main::incoming_writable, 1 );
1671 syswrite( STATUSD, $main::dstat, 1 );
1672 syswrite( STATUSD, $main::next_run, STATNUM_LEN );
1673 syswrite( STATUSD, $main::last_ping_time, STATNUM_LEN );
1674 syswrite( STATUSD, $main::current_changes, STATSTR_LEN );
1678 # ---------------------------------------------------------------------------
1680 # ---------------------------------------------------------------------------
1683 # open FTP connection to target host if not already open
1687 if ($main::FTP_chan) {
1688 # is already open, but might have timed out; test with a cwd
1689 return $main::FTP_chan if $main::FTP_chan->cwd( $main::current_targetdir );
1690 # cwd didn't work, channel is closed, try to reopen it
1691 $main::FTP_chan = undef;
1694 if (!($main::FTP_chan = Net::FTP->new( $conf::target,
1695 Debug => $conf::ftpdebug,
1696 Timeout => $conf::ftptimeout ))) {
1697 msg( "log,mail", "Cannot open FTP server $conf::target\n" );
1700 if (!$main::FTP_chan->login()) {
1701 msg( "log,mail", "Anonymous login on FTP server $conf::target failed\n" );
1704 if (!$main::FTP_chan->binary()) {
1705 msg( "log,mail", "Can't set binary FTP mode on $conf::target\n" );
1708 if (!$main::FTP_chan->cwd( $main::current_targetdir )) {
1709 msg( "log,mail", "Can't cd to $main::current_targetdir on $conf::target\n" );
1712 debug( "opened FTP channel to $conf::target" );
1716 $main::FTP_chan = undef;
1723 my $direct_resp_cmd = ($cmd eq "quot");
1725 debug( "executing FTP::$cmd(".join(", ",@_).")" );
1726 $SIG{"ALRM"} = sub { die "timeout in FTP::$cmd\n" } ;
1727 alarm( $conf::remote_timeout );
1728 eval { $rv = $main::FTP_chan->$cmd( @_ ); };
1731 $rv = (ftp_code() =~ /^2/) ? 1 : 0 if $direct_resp_cmd;
1737 $err = ftp_response();
1743 if ($main::FTP_chan) {
1744 $main::FTP_chan->quit();
1745 $main::FTP_chan = undef;
1750 sub ftp_response() {
1751 return join( '', @{${*$main::FTP_chan}{'net_cmd_resp'}} );
1755 return ${*$main::FTP_chan}{'net_cmd_code'};
1759 my $code = ftp_code();
1760 return ($code =~ /^[45]/) ? 1 : 0;
1763 # ---------------------------------------------------------------------------
1765 # ---------------------------------------------------------------------------
1771 my $ecmd = "$conf::ssh $conf::ssh_options $conf::target ".
1772 "-l $conf::targetlogin \'cd $main::current_targetdir; $cmd\'";
1773 debug( "executing $ecmd" );
1774 $SIG{"ALRM"} = sub { die "timeout in ssh command\n" } ;
1775 alarm( $conf::remote_timeout );
1776 eval { $msg = `$ecmd 2>&1`; };
1785 return ($msg, $stat);
1791 my $ecmd = "$conf::scp $conf::ssh_options @_ ".
1792 "$conf::targetlogin\@$conf::target:$main::current_targetdir";
1793 debug( "executing $ecmd" );
1794 $SIG{"ALRM"} = sub { die "timeout in scp\n" } ;
1795 alarm( $conf::remote_timeout );
1796 eval { $msg = `$ecmd 2>&1`; };
1805 return ($msg, $stat);
1808 sub local_cmd($;$) {
1813 my $ecmd = ($nocd ? "" : "cd $main::current_targetdir; ") . $cmd;
1814 debug( "executing $ecmd" );
1815 $msg = `($ecmd) 2>&1`;
1817 return ($msg, $stat);
1822 # check if target is alive (code stolen from Net::Ping.pm)
1824 sub check_alive(;$) {
1825 my $timeout = shift;
1826 my( $saddr, $ret, $target_ip );
1829 if ($conf::upload_method eq "copy") {
1830 format_status_num( $main::last_ping_time, time );
1831 $main::target_up = 1;
1837 if (!($target_ip = (gethostbyname($conf::target))[4])) {
1838 msg( "log", "Cannot get IP address of $conf::target\n" );
1842 $saddr = pack( 'S n a4 x8', AF_INET, $main::echo_port, $target_ip );
1843 $SIG{'ALRM'} = sub { die } ;
1846 $ret = $main::tcp_proto; # avoid warnings about unused variable
1849 return unless socket( PINGSOCK, PF_INET, SOCK_STREAM, $main::tcp_proto );
1850 return unless connect( PINGSOCK, $saddr );
1855 msg( "log", "pinging $conf::target: " . ($ret ? "ok" : "down") . "\n" );
1857 $main::target_up = $ret ? "1" : "0";
1858 format_status_num( $main::last_ping_time, time );
1859 write_status_file() if $conf::statusdelay;
1863 # check if incoming dir on target is writable
1865 sub check_incoming_writable() {
1866 my $testfile = ".debianqueued-testfile";
1869 if ($conf::upload_method eq "ssh") {
1870 ($msg, $stat) = ssh_cmd( "rm -f $testfile; touch $testfile; ".
1871 "rm -f $testfile" );
1873 elsif ($conf::upload_method eq "ftp") {
1874 my $file = "junk-for-writable-test-".format_time();
1875 $file =~ s/[ :.]/-/g;
1877 open( F, ">$file" ); close( F );
1879 ($rv, $msg) = ftp_cmd( "put", $file );
1881 $msg = "" if !defined $msg;
1883 ftp_cmd( "delete", $file );
1885 elsif ($conf::upload_method eq "copy") {
1886 ($msg, $stat) = local_cmd( "rm -f $testfile; touch $testfile; ".
1887 "rm -f $testfile" );
1890 debug( "exit status: $stat, output was: $msg" );
1893 # change incoming_writable only if ssh didn't return an error
1894 $main::incoming_writable =
1895 ($msg =~ /(permission denied|read-?only file|cannot create)/i) ? "0":"1";
1898 debug( "local error, keeping old status" );
1900 debug( "incoming_writable = $main::incoming_writable" );
1901 write_status_file() if $conf::statusdelay;
1902 return $main::incoming_writable;
1906 # remove a list of files, log failing ones
1912 (unlink $_ and ++$done)
1913 or $! == ENOENT or msg( "log", "Could not delete $_: $!\n" );
1919 # get md5 checksum of a file
1925 chomp( $line = `$conf::md5sum $file` );
1926 debug( "md5sum($file): ", $? ? "exit status $?" :
1927 $line =~ /^(\S+)/ ? $1 : "match failed" );
1928 return $? ? "" : $line =~ /^(\S+)/ ? $1 : "";
1932 # check if a file probably belongs to a Debian upload
1934 sub is_debian_file($) {
1936 return $file =~ /\.(deb|dsc|(diff|tar)\.gz)$/ &&
1937 $file !~ /\.orig\.tar\.gz/;
1941 # try to extract maintainer email address from some a non-.changes file
1942 # return "" if not possible
1944 sub get_maintainer($) {
1946 my $maintainer = "";
1949 if ($file =~ /\.diff\.gz$/) {
1951 open( F, "$conf::gzip -dc '$file' 2>/dev/null |" ) or return "";
1953 # look for header line of a file */debian/control
1954 last if m,^\+\+\+\s+[^/]+/debian/control(\s+|$),;
1957 last if /^---/; # end of control file patch, no Maintainer: found
1958 # inside control file patch look for Maintainer: field
1959 $maintainer = $1, last if /^\+Maintainer:\s*(.*)$/i;
1961 while( <F> ) { } # read to end of file to avoid broken pipe
1962 close( F ) or return "";
1964 elsif ($file =~ /\.(deb|dsc|tar\.gz)$/) {
1965 if ($file =~ /\.deb$/ && $conf::ar) {
1966 # extract control.tar.gz from .deb with ar, then let tar extract
1967 # the control file itself
1968 open( F, "($conf::ar p '$file' control.tar.gz | ".
1969 "$conf::tar -xOf - ".
1970 "--use-compress-program $conf::gzip ".
1971 "control) 2>/dev/null |" )
1974 elsif ($file =~ /\.dsc$/) {
1975 # just do a plain grep
1976 debug( "get_maint: .dsc, no cmd" );
1977 open( F, "<$file" ) or return "";
1979 elsif ($file =~ /\.tar\.gz$/) {
1980 # let tar extract a file */debian/control
1981 open(F, "$conf::tar -xOf '$file' ".
1982 "--use-compress-program $conf::gzip ".
1983 "\\*/debian/control 2>&1 |")
1990 $maintainer = $1, last if /^Maintainer:\s*(.*)$/i;
1992 close( F ) or return "";
1999 # return a pattern that matches all files that probably belong to one job
2001 sub debian_file_stem($) {
2003 my( $pkg, $version );
2006 $file =~ s,\.(deb|dsc|changes|(orig\.)?tar\.gz|diff\.gz)$,,;
2007 # if not is *_* (name_version), can't derive a stem and return just
2009 return $file if !($file =~ /^([^_]+)_([^_]+)/);
2010 ($pkg, $version) = ($1, $2);
2011 # strip Debian revision from version
2012 $version =~ s/^(.*)-[\d.+-]+$/$1/;
2014 return "${pkg}_${version}*";
2018 # output a messages to several destinations
2020 # first arg is a comma-separated list of destinations; valid are "log"
2021 # and "mail"; rest is stuff to be printed, just as with print
2024 my @dest = split( ',', shift );
2026 if (grep /log/, @dest ) {
2027 my $now = format_time();
2028 print LOG "$now ", @_;
2031 if (grep /mail/, @dest ) {
2032 $main::mail_text .= join( '', @_ );
2037 # print a debug messages, if $debug is true
2040 return if !$conf::debug;
2041 my $now = format_time();
2042 print LOG "$now DEBUG ", @_, "\n";
2046 # intialize the "mail" destination of msg() (this clears text,
2047 # address, subject, ...)
2052 $main::mail_addr = "";
2053 $main::mail_text = "";
2054 %main::packages = ();
2055 $main::mail_subject = $file ? "Processing of $file" : "";
2059 # finalize mail to be sent from msg(): check if something present, and
2064 debug( "No mail for $main::mail_addr" )
2065 if $main::mail_addr && !$main::mail_text;
2066 return unless $main::mail_addr && $main::mail_text;
2068 if (!send_mail($main::mail_addr, $main::mail_subject, $main::mail_text)) {
2069 # store this mail in memory so it isn't lost if executing sendmail
2071 push( @main::stored_mails, { addr => $main::mail_addr,
2072 subject => $main::mail_subject,
2073 text => $main::mail_text } );
2077 # try to send out stored mails
2079 while( $mailref = shift(@main::stored_mails) ) {
2080 if (!send_mail( $mailref->{'addr'}, $mailref->{'subject'},
2081 $mailref->{'text'} )) {
2082 unshift( @main::stored_mails, $mailref );
2091 sub send_mail($$$) {
2093 my $subject = shift;
2096 my $package = keys %main::packages ? join(' ', keys %main::packages) : "";
2100 unless (defined($Email::Send::Sendmail::SENDMAIL)) {
2101 $Email::Send::Sendmail::SENDMAIL = $conf::mail;
2104 my $date = sprintf "%s", strftime("%a, %d %b %Y %T %z", (localtime(time)));
2105 my $message = <<__MESSAGE__;
2107 From: Archive Administrator <dak\@ftp-master.debian.org>
2113 if (length $package) {
2114 $message .= "X-Debian-Package: $package\n";
2117 $message .= "\n$text";
2118 $message .= "\nGreetings,\n\n\tYour Debian queue daemon\n";
2120 my $mail = Email::Send->new;
2121 for ( qw[Sendmail SMTP] ) {
2122 $mail->mailer($_) and last if $mail->mailer_available($_);
2125 my $ret = $mail->send($message);
2126 if ($ret && $ret !~ /Message sent|success/) {
2134 # try to find a mail address for a name in the keyrings
2136 sub try_to_get_mail_addr($$) {
2138 my $listref = shift;
2141 open( F, "$conf::gpg --no-options --batch --no-default-keyring ".
2142 "--always-trust --keyring ".
2143 join (" --keyring ",@conf::keyrings).
2147 if (/^pub / && / $name /) {
2149 push( @$listref, $1 );
2154 return (@$listref >= 1) ? $listref->[0] : "";
2158 # return current time as string
2163 # omit weekday and year for brevity
2164 ($t = localtime) =~ /^\w+\s(.*)\s\d+$/;
2170 my $hours = int($secs/(60*60));
2172 $secs -= $hours*60*60;
2173 return sprintf "%d:%02d:%02d", $hours, int($secs/60), $secs % 60;
2177 # block some signals during queue processing
2179 # This is just to avoid data inconsistency or uploads being aborted in the
2180 # middle. Only "soft" signals are blocked, i.e. SIGINT and SIGTERM, try harder
2181 # ones if you really want to kill the daemon at once.
2183 sub block_signals() {
2184 POSIX::sigprocmask( SIG_BLOCK, $main::block_sigset );
2187 sub unblock_signals() {
2188 POSIX::sigprocmask( SIG_UNBLOCK, $main::block_sigset );
2192 # process SIGHUP: close log file and reopen it (for logfile cycling)
2199 open( LOG, ">>$conf::logfile" )
2200 or die "Cannot open my logfile $conf::logfile: $!\n";
2201 chmod( 0644, $conf::logfile )
2202 or msg( "log", "Cannot set modes of $conf::logfile: $!\n" );
2203 select( (select(LOG), $| = 1)[0] );
2205 open( STDOUT, ">&LOG" )
2206 or msg( "log", "$main::progname: Can't redirect stdout to ".
2207 "$conf::logfile: $!\n" );
2208 open( STDERR, ">&LOG" )
2209 or msg( "log", "$main::progname: Can't redirect stderr to ".
2210 "$conf::logfile: $!\n" );
2211 msg( "log", "Restart after SIGHUP\n" );
2215 # process SIGCHLD: check if it was our statusd process
2220 # reap statusd, so that it's no zombie when we try to kill(0) it
2221 waitpid( $main::statusd_pid, WNOHANG );
2223 # Uncomment the following line if your Perl uses unreliable System V signal
2224 # (i.e. if handlers reset to default if the signal is delivered).
2225 # (Unfortunately, the re-setup can't be done in any case, since on some
2226 # systems this will cause the SIGCHLD to be delivered again if there are
2227 # still unreaped children :-(( )
2229 # $SIG{"CHLD"} = \&kid_died; # resetup handler for SysV
2232 sub restart_statusd() {
2233 # restart statusd if it died
2234 if (!kill( 0, $main::statusd_pid)) {
2235 close( STATUSD ); # close out pipe end
2236 $main::statusd_pid = fork_statusd();
2241 # process a fatal signal: cleanup and exit
2243 sub fatal_signal($) {
2244 my $signame = shift;
2247 # avoid recursions of fatal_signal in case of BSD signals
2248 foreach $sig ( qw( ILL ABRT BUS FPE SEGV PIPE ) ) {
2249 $SIG{$sig} = "DEFAULT";
2252 if ($$ == $main::maind_pid) {
2253 # only the main daemon should do this
2254 kill( $main::signo{"TERM"}, $main::statusd_pid )
2255 if defined $main::statusd_pid;
2256 unlink( $conf::statusfile, $conf::pidfile );
2258 msg( "log", "Caught SIG$signame -- exiting (pid $$)\n" );