X-Git-Url: https://git.donarmstrong.com/?a=blobdiff_plain;f=bin%2Flocal-debbugs;h=3e397f38999b8b9126b83e3d6e1e723ccd7cf0ef;hb=6532b246361b5d28b6ce3b44154a71edd3ca9a9e;hp=86f83a97ff9142535fca529548e026f8094b51d2;hpb=20055761efef4215fc55c7708d5240bedeaacc44;p=debbugs.git diff --git a/bin/local-debbugs b/bin/local-debbugs index 86f83a9..3e397f3 100755 --- a/bin/local-debbugs +++ b/bin/local-debbugs @@ -8,7 +8,7 @@ use warnings; use strict; -use Getopt::Long; +use Getopt::Long qw(:config no_ignore_case); use Pod::Usage; =head1 NAME @@ -17,7 +17,7 @@ local-debbugs - use a local mirror of debbugs =head1 SYNOPSIS - [options] + local-debbugs [options] Options: --mirror, -M update local mirror @@ -39,19 +39,19 @@ Update the local mirror of debbugs bugs =item B<--daemon, -D> Start up the daemon on the configured local port to serve bugs which -have been previously retried +have been previously retrieved. =item B<--search, -S> Cause the running daemon to show the pkgreport.cgi page corresponding -to the search by invoking sensible-browser and an appropriate url +to the search by invoking sensible-browser and an appropriate url. =item B<--show, -s> Cause the running daemon to show the bugreport.cgi page corresponding -to the bug by invoking sensible-browser and an appropriate url +to the bug by invoking sensible-browser and an appropriate url. -=item B<--port,-p> +=item B<--port, -p> The port that the daemon is running on (or will be running on.) @@ -63,9 +63,20 @@ the configuration file, or 8080 if nothing is set. File which contains the set of bugs to get. Defaults to ~/.debbugs/bugs_to_get +=item B<--bug-site> + +Hostname for a site which is running a debbugs install. +Defaults to bugs.debian.org + +=item B<--bug-mirror> + +Hostname for a site which is running an rsyncable mirror of the +debbugs install above. +Defaults to bugs-mirror.debian.org + =item B<--debug, -d> -Debug verbosity. (Default 0) +Debug verbosity. =item B<--help, -h> @@ -79,6 +90,21 @@ Display this manual. =head1 EXAMPLES +=over + +=item Update the local mirror + + local-debbugs --mirror + +=item Start up the local-debbugs daemon + + local-debbugs --daemon + +=item Search for bugs with severity serious + + local-debbugs --search severity:serious + +=back =cut @@ -87,10 +113,15 @@ use vars qw($DEBUG); use User; use Config::Simple; +use File::Basename qw(dirname); use File::Temp qw(tempdir); use Params::Validate qw(validate_with :types); use POSIX 'setsid'; -use Debbugs::Common qw(checkpid lockpid); +use SOAP::Lite; +use IPC::Run; +use IO::File; +use File::Path; +use File::Spec; my %options = (debug => 0, help => 0, @@ -98,23 +129,42 @@ my %options = (debug => 0, verbose => 0, quiet => 0, detach => 1, - cgi_bin => '/var/lib/debbugs/www/cgi-bin', - css => '/var/lib/debbugs/www/bugs.css', + git_mode => -d (dirname(__FILE__).'/../.git') ? 1 : 0, + bug_site => 'bugs.debian.org', + bug_mirror => 'bugs-mirror.debian.org', ); my %option_defaults = (port => 8080, debbugs_config => User->Home.'/.debbugs/debbugs_config', - mirror_location => User->Home.'/.debbugs/mirror/', + mirror_location => User->Home.'/.debbugs/mirror', bugs_to_get => User->Home.'/.debbugs/bugs_to_get', ); GetOptions(\%options, - 'daemon|D','show|s','search|select|S','mirror|M', 'stop', + 'daemon|D','show|s','search|select|S','mirror|M', 'stop|exit|quit', 'detach!', 'css=s','cgi_bin|cgi-bin|cgi=s', 'verbose|v+','quiet|q+', + 'bug_site|bug-site=s', + 'bug_mirror|bug-mirror=s', 'debug|d+','help|h|?','man|m'); +if ($options{git_mode}) { + my $base_dir = dirname(File::Spec->rel2abs(dirname(__FILE__))); + $options{cgi_bin} = "$base_dir/cgi" unless defined $options{cgi_bin}; + $options{css} = "$base_dir/html/bugs.css" unless defined $options{css}; + $options{template_dir} = "$base_dir/templates"; + $options{base_dir} = $base_dir; + eval "use lib '$options{base_dir}'"; +} else { + $options{cgi_bin} = '/var/lib/debbugs/www/cgi'; + $options{css} = '/var/lib/debbugs/www/bugs.css'; + $options{template_dir} = "/usr/share/debbugs/templates"; +} + +eval "use Debbugs::Common qw(checkpid lockpid get_hashname)"; +eval "use Debbugs::Mail qw(get_addresses)"; + pod2usage() if $options{help}; pod2usage({verbose=>2}) if $options{man}; @@ -122,7 +172,7 @@ $DEBUG = $options{debug}; my @USAGE_ERRORS; if (1 != grep {exists $options{$_}} qw(daemon show search mirror stop)) { - push @USAGE_ERRORS,"You must pass one (and only one) of --daemon --show --search or --mirror"; + push @USAGE_ERRORS,"You must pass one (and only one) of --daemon --show --search --mirror or --stop"; } $options{verbose} = $options{verbose} - $options{quiet}; @@ -133,6 +183,8 @@ pod2usage(join("\n",@USAGE_ERRORS)) if @USAGE_ERRORS; local_config(\%options); +mkpath($options{mirror_location}); + if ($options{daemon}) { # daemonize, do stuff my $pid = checkpid($options{mirror_location}.'/local-debbugs.pid'); @@ -145,6 +197,25 @@ if ($options{daemon}) { print STDERR "Unable to determine if daemon is running: $!\n"; exit 1; } + my $conf = IO::File->new($options{mirror_location}.'/debbugs_config_local','w') or + die "Unable to open $options{mirror_location}/debbugs_config_local for writing: $!"; + print {$conf} <<"EOF"; +\$gConfigDir = "$options{mirror_location}"; +\$gSpoolDir = "$options{mirror_location}"; +\$gTemplateDir = "$options{template_dir}"; +\$gWebHost = 'localhost:$options{port}'; +\$gPackageSource = '$options{mirror_location}/sources'; +\$gPseudoDescFile = ''; +\$gPseudoMaintFile = ''; +\$gMaintainerFile = '$options{mirror_location}/Maintainers'; +\$gMaintainerFileOverride = ''; +\$config{source_maintainer_file} = '$options{mirror_location}/Source_maintainers'; +\$config{source_maintainer_file_override} = ''; +\$gProject = 'Local Debbugs'; +1; +EOF + close $conf; + $ENV{DEBBUGS_CONFIG_FILE} = $options{mirror_location}.'/debbugs_config_local'; # ok, now lets daemonize # XXX make sure that all paths have been turned into absolute @@ -168,7 +239,8 @@ if ($options{daemon}) { package local_debbugs::server; use IO::File; use HTTP::Server::Simple; - use base qw(HTTP::Server::Simple::CGI); + use File::Basename qw(dirname); + use base qw(HTTP::Server::Simple::CGI HTTP::Server::Simple::CGI::Environment); sub net_server { return 'Net::Server::Fork'; @@ -184,6 +256,7 @@ if ($options{daemon}) { sub handle_request { my ($self,$cgi) = @_; + $ENV{DEBBUGS_CONFIG_FILE} = $options{mirror_location}.'/debbugs_config_local'; my $base_uri = 'http://'.$cgi->virtual_host; if ($cgi->virtual_port ne 80) { $base_uri .= ':'.$cgi->virtual_port; @@ -222,11 +295,32 @@ if ($options{daemon}) { redirect($cgi,$base_uri."/cgi-bin/pkgreport.cgi?pkg=$1"); } elsif ($path =~ m{^/?cgi(?:-bin)?/((?:(?:bug|pkg)report|version)\.cgi)}) { - # dispatch to pkgreport.cgi - print "HTTP/1.1 200 OK\n"; - exec("$options{cgi_bin}/$1") or - die "Unable to execute $options{cgi_bin}/$1"; - } + my @exec_options = "$options{cgi_bin}/$1"; + if ($options{git_mode}) { + unshift @exec_options, + 'perl','-I',$options{base_dir},'-T'; + } + open(my $fh,'-|',@exec_options) or + die "Unable to execute $options{cgi_bin}/$1"; + my $status; + my $cache = ''; + while (<$fh>) { + if (/Status: (\d+\s+.+?)\n?$/) { + $status = $1; + print "HTTP/1.1 $status\n"; + print STDERR "'$status'\n"; + last; + } + $cache .= $_; + if (/^$/) { + print "HTTP/1.1 200 OK\n"; + last; + } + } + print $cache; + print <$fh>; + close($fh) or die "Unable to close"; + } elsif ($path =~ m{^/?css/bugs.css}) { my $fh = IO::File->new($options{css},'r') or die "Unable to open $options{css} for reading: $!"; @@ -264,31 +358,21 @@ elsif ($options{stop}) { elsif ($options{mirror}) { # run the mirror jobies # figure out which bugs we need - my @bugs = select_bugs(\%options); + my $bugs = select_bugs(\%options); # get them - my $tempdir = tempdir(CLEANUP => 1); - my $mirror_log = IO::File->new($options{mirror_location}.'/mirror.log') or + my $tempdir = tempdir();#CLEANUP => 1); + my $mirror_log = IO::File->new($options{mirror_location}.'/mirror.log','>') or die "Unable to open $options{mirror_location}/mirror.log for writing: $!"; - my $inc_fh = IO::File->new("$tempdir/include_list",'w') or - die "Unable to open $tempdir/include_list for writing: $!"; - foreach my $bug (@bugs) { - print {$inc_fh} "*/${bug}.*\n" or - die "Unable to write to $tempdir/include_list: $!"; - } - close $inc_fh or - die "Unable to close $tempdir/include_list: $!"; - my ($wrf,$rfh); + write_bug_list("$tempdir/unarchived_bug_list",$bugs->{unarchived}); + write_bug_list("$tempdir/archived_bug_list",$bugs->{archived}); + my ($wrf,$rfh,$efh); my @common_rsync_options = ('-avz','--partial'); print "Rsyncing bugs\n" if not $options{quiet}; run_rsync(log => $mirror_log, ($options{debug}?(debug => \*STDERR):()), options => [@common_rsync_options, '--delete-after', - '--include-from',"$tempdir/include_list", - # skip things not specifically included - '--exclude','*/*', - # skip the -1,-2,-3.log files - '--exclude','*.log', + '--files-from',"$tempdir/unarchived_bug_list", 'rsync://'.$options{bug_mirror}.'/bts-spool-db/', $options{mirror_location}.'/db-h/'] ); @@ -297,11 +381,7 @@ elsif ($options{mirror}) { ($options{debug}?(debug => \*STDERR):()), options => [@common_rsync_options, '--delete-after', - '--include-from',"$tempdir/include_list", - # skip things not specifically included - '--exclude','*/*', - # skip the -1,-2,-3.log files - '--exclude','*.log', + '--files-from',"$tempdir/archived_bug_list", 'rsync://'.$options{bug_mirror}.'/bts-spool-archive/', $options{mirror_location}.'/archive/', ], @@ -324,7 +404,7 @@ elsif ($options{mirror}) { '--delete-after', '--exclude','*old', '--exclude','*.bak', - 'rsync://'.$options{bug_mirror}.'/bts-spool-versions/', + 'rsync://'.$options{bug_mirror}.'/bts-versions/', $options{mirror_location}.'/versions/', ], ); @@ -350,7 +430,7 @@ elsif ($options{search}) { my $pid = checkpid($options{mirror_location}.'/local-debbugs.pid'); if (not defined $pid or $pid == 0) { print STDERR "Unable to open pidfile or daemon not running: $!\n"; - print STDERR qq(Mr. T: "I pity da fool who tries to show a bug without a running daemon"\n); + print STDERR qq(Mr. T: "I pity da fool who tries to search for bugs without a running daemon"\n); print STDERR "Hint: try the --daemon option first\n"; exit 1; } @@ -387,6 +467,19 @@ sub local_config{ } } +sub write_bug_list { + my ($file,$bug_list) = @_; + my $inc_fh = IO::File->new($file,'w') or + die "Unable to open $file for writing: $!"; + foreach my $bug (keys %{$bug_list}) { + my $file_loc = get_hashname($bug).'/'.$bug; + print {$inc_fh} map {$file_loc.'.'.$_.qq(\n)} qw(log summary report status) or + die "Unable to write to $file: $!"; + } + close $inc_fh or + die "Unable to close $file: $!"; +} + # actually run rsync with the passed options sub run_rsync{ my %param = validate_with(params => \@_, @@ -399,18 +492,18 @@ sub run_rsync{ }, } ); - my ($output_fh,@rsync_options) = @_; - my ($wfh,$rfh); - my $pid = open3($wfh,$rfh, - 'rsync', - @{$param{options}} - ) or die "Unable to start rsync: $!"; - close $wfh or die "Unable to close the writer filehandle $?"; - while (<$rfh>) { - print {$param{log}} $_; - if (exists $param{debug}) { - print {$param{debug}} $_; - } + my ($output,$error) = ('',''); + my $h = IPC::Run::start(['rsync',@{$param{options}}], + \undef,$param{log},$param{log}); + while ($h->pump) { + #print {$param{debug}} $error if defined $param{debug}; + } + $h->finish(); + my $exit = $h->result(0); + # this is suboptimal, but we currently don't know whether we've + # selected an archive or unarchived bug, so.. + if (defined $exit and not ($exit == 0 or $exit == 3 or $exit == 23)) { + print STDERR "Rsync exited with non-zero status: $exit\n"; } } @@ -445,7 +538,7 @@ sub select_bugs{ my $soap = SOAP::Lite -> uri('Debbugs/SOAP/V1') - -> proxy("http://$options{bug_mirror}/cgi-bin/soap.cgi"); + -> proxy("http://$options{bug_site}/cgi-bin/soap.cgi"); my @bugs; my @bug_selections = (); if (not -e $options{bugs_to_get}) { @@ -456,10 +549,10 @@ sub select_bugs{ # maintained by this user, submitted by this user, and rc # bugs push @bug_selections, - ("correspondent:$addr archive:both", - "maint:$addr archive:both", - "submitter:$addr archive:both", - "severity:serious severity:grave severity:critical archive:both", + ("correspondent:$addr archive:0", + "maint:$addr archive:0", + "submitter:$addr archive:0", + "severity:serious severity:grave severity:critical archive:0", ); } else { @@ -474,38 +567,67 @@ sub select_bugs{ elsif (/\s\w+\:/) { push @bug_selections, $_; } - } + } } + # Split archive:both into archive:1 and archive:0 + @bug_selections = + map { + if (m/archive:both/) { + my $y_archive = $_; + my $n_archive = $_; + $y_archive =~ s/archive:both/archive:1/; + $n_archive =~ s/archive:both/archive:0/; + ($y_archive,$n_archive); + } + else { + $_; + } + } @bug_selections; + my %bugs; for my $selection (@bug_selections) { - my @subselects = split /\s+/,$selection; - my %search_parameters; - my %users; - for my $subselect (@subselects) { - my ($key,$value) = split /:/, $subselect, 2; - next unless $key; - if (exists $valid_keys{$key}) { - push @{$search_parameters{$valid_keys{$key}}}, - $value if $value; - } elsif ($key =~/users?$/) { - $users{$value} = 1 if $value; - } - } - my %usertags; - for my $user (keys %users) { - my $ut = $soap->get_usertag($user)->result(); - next unless defined $ut and $ut ne ""; - for my $tag (keys %{$ut}) { - push @{$usertags{$tag}}, - @{$ut->{$tag}}; - } - } - my $bugs = $soap->get_bugs(%search_parameters, - (keys %usertags)?(usertags=>\%usertags):() - )->result(); - push @bugs,@{$bugs} if defined $bugs and @{$bugs}; + my $archived_bugs = "unarchived"; + if ($selection =~ /archive:(\S+)/ and $1) { + $archived_bugs = "archived"; + } + my @subselects = split /\s+/,$selection; + my %search_parameters; + my %users; + for my $subselect (@subselects) { + my ($key,$value) = split /:/, $subselect, 2; + next unless $key; + if (exists $valid_keys{$key}) { + push @{$search_parameters{$valid_keys{$key}}}, + $value if $value; + } elsif ($key =~/users?$/) { + $users{$value} = 1 if $value; + } + } + my %usertags; + for my $user (keys %users) { + my $ut = $soap->get_usertag($user)->result(); + next unless defined $ut and $ut ne ""; + for my $tag (keys %{$ut}) { + push @{$usertags{$tag}}, + @{$ut->{$tag}}; + } + } + my $bugs = $soap->get_bugs(%search_parameters, + (keys %usertags)?(usertags=>\%usertags):() + )->result(); + if (defined $bugs and @{$bugs}) { + $bugs{$archived_bugs}{$_} = 1 for @{$bugs}; + } + } + for my $bug (@bugs) { + $bugs{archived}{$bug} = 1; + $bugs{unarchived}{$bug} = 1; } - return @bugs; + return \%bugs; } __END__ +# Local Variables: +# cperl-indent-level: 4 +# indent-tabs-mode: nil +# End: