2 # local-debbugs is part of debbugs, and is released
3 # under the terms of the GPL version 2, or any later version, at your
4 # option. See the file README and COPYING for more information.
5 # Copyright 2008 by Don Armstrong <don@donarmstrong.com>.
11 use Getopt::Long qw(:config no_ignore_case);
16 local-debbugs - use a local mirror of debbugs
23 --mirror, -M update local mirror
24 --daemon, -D start the daemon
25 --search, -S run a search
27 --debug, -d debugging level (Default 0)
28 --help, -h display this help
29 --man, -m display manual
37 Update the local mirror of debbugs bugs
41 Start up the daemon on the configured local port to serve bugs which
42 have been previously retrieved.
46 Cause the running daemon to show the pkgreport.cgi page corresponding
47 to the search by invoking sensible-browser and an appropriate url.
51 Cause the running daemon to show the bugreport.cgi page corresponding
52 to the bug by invoking sensible-browser and an appropriate url.
56 The port that the daemon is running on (or will be running on.)
58 Defaults to the value of the currently running daemon, the value in
59 the configuration file, or 8080 if nothing is set.
61 =item B<--bugs-to-get>
63 File which contains the set of bugs to get.
64 Defaults to ~/.debbugs/bugs_to_get
68 Hostname for a site which is running a debbugs install.
69 Defaults to bugs.debian.org
73 Hostname for a site which is running an rsyncable mirror of the
74 debbugs install above.
75 Defaults to bugs-mirror.debian.org
83 Display brief useage information.
95 =item Update the local mirror
97 local-debbugs --mirror
99 =item Start up the local-debbugs daemon
101 local-debbugs --daemon
103 =item Search for bugs with severity serious
105 local-debbugs --search severity:serious
116 use File::Basename qw(dirname);
117 use File::Temp qw(tempdir);
118 use Params::Validate qw(validate_with :types);
126 my %options = (debug => 0,
132 git_mode => -d (dirname(__FILE__).'/../.git') ? 1 : 0,
133 bug_site => 'bugs.debian.org',
134 bug_mirror => 'bugs-mirror.debian.org',
137 my %option_defaults = (port => 8080,
138 debbugs_config => User->Home.'/.debbugs/debbugs_config',
139 mirror_location => User->Home.'/.debbugs/mirror',
140 bugs_to_get => User->Home.'/.debbugs/bugs_to_get',
143 GetOptions(\%options,
144 'daemon|D','show|s','search|select|S','mirror|M', 'stop|exit|quit',
146 'css=s','cgi_bin|cgi-bin|cgi=s',
147 'verbose|v+','quiet|q+',
148 'bug_site|bug-site=s',
149 'bug_mirror|bug-mirror=s',
150 'debug|d+','help|h|?','man|m');
152 if ($options{git_mode}) {
153 my $base_dir = File::Spec->rel2abs(dirname(__FILE__).'/..');
154 $options{cgi_bin} = "$base_dir/cgi" unless defined $options{cgi_bin};
155 $options{css} = "$base_dir/html/bugs.css" unless defined $options{css};
156 $options{template_dir} = "$base_dir/templates";
157 eval "use lib '$base_dir'";
159 $options{cgi_bin} = '/var/lib/debbugs/www/cgi';
160 $options{css} = '/var/lib/debbugs/www/bugs.css';
161 $options{template_dir} = "/usr/share/debbugs/templates";
164 eval "use Debbugs::Common qw(checkpid lockpid get_hashname)";
165 eval "use Debbugs::Mail qw(get_addresses)";
167 pod2usage() if $options{help};
168 pod2usage({verbose=>2}) if $options{man};
170 $DEBUG = $options{debug};
173 if (1 != grep {exists $options{$_}} qw(daemon show search mirror stop)) {
174 push @USAGE_ERRORS,"You must pass one (and only one) of --daemon --show --search --mirror or --stop";
176 $options{verbose} = $options{verbose} - $options{quiet};
178 pod2usage(join("\n",@USAGE_ERRORS)) if @USAGE_ERRORS;
181 # munge in local configuration
183 local_config(\%options);
185 mkpath($options{mirror_location});
187 if ($options{daemon}) {
188 # daemonize, do stuff
189 my $pid = checkpid($options{mirror_location}.'/local-debbugs.pid');
190 if (defined $pid and $pid != 0) {
191 print STDERR "Unable to start daemon; it's already running\n";
194 if (-e $options{mirror_location}.'/local-debbugs.pid' and
196 print STDERR "Unable to determine if daemon is running: $!\n";
199 my $conf = IO::File->new($options{mirror_location}.'/debbugs_config_local','w') or
200 die "Unable to open $options{mirror_location}/debbugs_config_local for writing: $!";
201 print {$conf} <<"EOF";
202 \$gConfigDir = "$options{mirror_location}";
203 \$gSpoolDir = "$options{mirror_location}";
204 \$gTemplateDir = "$options{template_dir}";
205 \$gWebHost = 'localhost:$options{port}';
206 \$gPackageSource = '';
207 \$gPseudoDescFile = '';
208 \$gPseudoMaintFile = '';
209 \$gMaintainerFile = '';
210 \$gMaintainerFileOverride = '';
211 \$config{source_maintainer_file} = '';
212 \$config{source_maintainer_file_override} = '';
213 \$gProject = 'Local Debbugs';
217 $ENV{DEBBUGS_CONFIG_FILE} = $options{mirror_location}.'/debbugs_config_local';
218 # ok, now lets daemonize
220 # XXX make sure that all paths have been turned into absolute
222 chdir '/' or die "Can't chdir to /: $!";
223 # allow us not to detach for debugging
224 if ($options{detach}) {
225 open STDIN, '/dev/null' or die "Can't read /dev/null: $!";
226 open STDOUT, '>/dev/null'
227 or die "Can't write to /dev/null: $!";
228 defined(my $pid = fork) or die "Can't fork: $!";
230 setsid or die "Can't start a new session: $!";
231 open STDERR, '>&STDOUT' or die "Can't dup stdout: $!";
233 lockpid($options{mirror_location}.'/local-debbugs.pid') or
234 die "Unable to deal with the pidfile";
235 # this is the subclass of HTTP::Server::Simple::CGI which handles
236 # the "hard" bits of actually running a tiny webserver for us
238 package local_debbugs::server;
240 use HTTP::Server::Simple;
241 use File::Basename qw(dirname);
242 use base qw(HTTP::Server::Simple::CGI HTTP::Server::Simple::CGI::Environment);
245 return 'Net::Server::Fork';
250 print "HTTP/1.1 302 Found\r\n";
251 print "Location: $url\r\n";
254 # here we want to call cgi-bin/pkgreport or cgi-bin/bugreport
256 my ($self,$cgi) = @_;
258 $ENV{DEBBUGS_CONFIG_FILE} = $options{mirror_location}.'/debbugs_config_local';
259 if (-d dirname(__FILE__).'../Debbugs' and
260 -d dirname(__FILE__).'../.git'
262 $ENV{PERL5LIB} = dirname(__FILE__).'/../';
264 my $base_uri = 'http://'.$cgi->virtual_host;
265 if ($cgi->virtual_port ne 80) {
266 $base_uri .= ':'.$cgi->virtual_port;
268 my $path = $cgi->path_info();
269 # RewriteRule ^/[[:space:]]*#?([[:digit:]][[:digit:]][[:digit:]]+)([;&].+)?$ /cgi-bin/bugreport.cgi?bug=$1$2 [L,R,NE]
270 if ($path =~ m{^/?\s*\#?(\d+)((?:[;&].+)?)$}) {
271 redirect($cgi,$base_uri."/cgi-bin/bugreport.cgi?bug=$1$2");
273 # RewriteRule ^/[Ff][Rr][Oo][Mm]:([^/]+\@.+)$ /cgi-bin/pkgreport.cgi?submitter=$1 [L,R,NE]
274 elsif ($path =~ m{^/?\s*from:([^/]+\@.+)$}i) {
275 redirect($cgi,$base_uri."/cgi-bin/pkgreport.cgi?submitter=$1");
277 # RewriteRule ^/([^/]+\@.+)$ /cgi-bin/pkgreport.cgi?maint=$1 [L,R,NE]
278 elsif ($path =~ m{^/?\s*([^/]+\@.+)$}i) {
279 redirect($cgi,$base_uri."/cgi-bin/pkgreport.cgi?maint=$1");
281 # RewriteRule ^/mbox:([[:digit:]][[:digit:]][[:digit:]]+)([;&].+)?$ /cgi-bin/bugreport.cgi?mbox=yes&bug=$1$2 [L,R,NE]
282 elsif ($path =~ m{^/?\s*mbox:\#?(\d+)((?:[;&].+)?)$}i) {
283 redirect($cgi,$base_uri."/cgi-bin/bugreport.cgi?mbox=yes;bug=$1$2");
285 # RewriteRule ^/src:([^/]+)$ /cgi-bin/pkgreport.cgi?src=$1 [L,R,NE]
286 elsif ($path =~ m{^/?src:([^/]+)$}i) {
287 redirect($cgi,$base_uri."/cgi-bin/pkgreport.cgi?src=$1");
289 # RewriteRule ^/severity:([^/]+)$ /cgi-bin/pkgreport.cgi?severity=$1 [L,R,NE]
290 elsif ($path =~ m{^/?severity:([^/]+)$}i) {
291 redirect($cgi,$base_uri."/cgi-bin/pkgreport.cgi?severity=$1");
293 # RewriteRule ^/tag:([^/]+)$ /cgi-bin/pkgreport.cgi?tag=$1 [L,R,NE]
294 elsif ($path =~ m{^/?tag:([^/]+)$}i) {
295 redirect($cgi,$base_uri."/cgi-bin/pkgreport.cgi?tag=$1");
297 # RewriteRule ^/([^/]+)$ /cgi-bin/pkgreport.cgi?pkg=$1 [L,R,NE]
298 elsif ($path =~ m{^/?([^/]+)$}i) {
299 redirect($cgi,$base_uri."/cgi-bin/pkgreport.cgi?pkg=$1");
301 elsif ($path =~ m{^/?cgi(?:-bin)?/((?:(?:bug|pkg)report|version)\.cgi)}) {
302 # dispatch to pkgreport.cgi
303 #print "HTTP/1.1 200 OK\n";
304 open(my $fh,'-|',"$options{cgi_bin}/$1") or
305 die "Unable to execute $options{cgi_bin}/$1";
309 if (/Status: (\d+\s+.+?)\n?$/) {
311 print "HTTP/1.1 $status\n";
312 print STDERR "'$status'\n";
317 print "HTTP/1.1 200 OK\n";
323 close($fh) or die "Unable to close";
325 elsif ($path =~ m{^/?css/bugs.css}) {
326 my $fh = IO::File->new($options{css},'r') or
327 die "Unable to open $options{css} for reading: $!";
328 print "HTTP/1.1 200 OK\n";
329 print "Content-type: text/css\n";
333 elsif ($path =~ m{^/?$}) {
334 redirect($cgi,$base_uri."/cgi-bin/pkgreport.cgi?package=put%20package%20here");
337 print "HTTP/1.1 404 Not Found\n";
338 print "Content-Type: text/html\n";
340 print "<h1>That which you were seeking, found I have not.</h1>\n";
342 # RewriteRule ^/$ /Bugs/ [L,R,NE]
345 my $debbugs_server = local_debbugs::server->new($options{port}) or
346 die "Unable to create debbugs server";
347 $debbugs_server->run() or
348 die 'Unable to run debbugs server';
350 elsif ($options{stop}) {
352 my $pid = checkpid($options{mirror_location}.'/local-debbugs.pid');
353 if (not defined $pid or $pid == 0) {
354 print STDERR "Unable to open pidfile or daemon not running: $!\n";
357 exit !(kill(15,$pid) == 1);
359 elsif ($options{mirror}) {
360 # run the mirror jobies
361 # figure out which bugs we need
362 my $bugs = select_bugs(\%options);
364 my $tempdir = tempdir();#CLEANUP => 1);
365 my $mirror_log = IO::File->new($options{mirror_location}.'/mirror.log','>') or
366 die "Unable to open $options{mirror_location}/mirror.log for writing: $!";
367 write_bug_list("$tempdir/unarchived_bug_list",$bugs->{unarchived});
368 write_bug_list("$tempdir/archived_bug_list",$bugs->{archived});
370 my @common_rsync_options = ('-avz','--partial');
371 print "Rsyncing bugs\n" if not $options{quiet};
372 run_rsync(log => $mirror_log,
373 ($options{debug}?(debug => \*STDERR):()),
374 options => [@common_rsync_options,
376 '--files-from',"$tempdir/unarchived_bug_list",
377 'rsync://'.$options{bug_mirror}.'/bts-spool-db/',
378 $options{mirror_location}.'/db-h/']
380 print "Rsyncing archived bugs\n" if $options{verbose};
381 run_rsync(log => $mirror_log,
382 ($options{debug}?(debug => \*STDERR):()),
383 options => [@common_rsync_options,
385 '--files-from',"$tempdir/archived_bug_list",
386 'rsync://'.$options{bug_mirror}.'/bts-spool-archive/',
387 $options{mirror_location}.'/archive/',
390 print "Rsyncing indexes\n" if $options{verbose};
391 run_rsync(log => $mirror_log,
392 ($options{debug}?(debug => \*STDERR):()),
393 options => [@common_rsync_options,
396 '--exclude','by-reverse*',
397 'rsync://'.$options{bug_mirror}.'/bts-spool-index/',
398 $options{mirror_location}.'/',
401 print "Rsyncing versions\n" if $options{verbose};
402 run_rsync(log => $mirror_log,
403 ($options{debug}?(debug => \*STDERR):()),
404 options => [@common_rsync_options,
408 'rsync://'.$options{bug_mirror}.'/bts-versions/',
409 $options{mirror_location}.'/versions/',
413 elsif ($options{show}) {
415 # see if the daemon is running
416 my $pid = checkpid($options{mirror_location}.'/local-debbugs.pid');
417 if (not defined $pid or $pid == 0) {
418 print STDERR "Unable to open pidfile or daemon not running: $!\n";
419 print STDERR qq(Mr. T: "I pity da fool who tries to show a bug without a running daemon"\n);
420 print STDERR "Hint: try the --daemon option first\n";
424 my $url = qq(http://localhost:$options{port}/$ARGV[0]);
425 exec('/usr/bin/sensible-browser',$url) or
426 die "Unable to run sensible-browser (try feeding me cheetos?)";
428 elsif ($options{search}) {
429 my $url = qq(http://localhost:$options{port}/cgi-bin/pkgreport.cgi?).
430 join(';',map {if (/:/) {s/:/=/; $_;} else {qq(pkg=$_);}} @ARGV);
431 my $pid = checkpid($options{mirror_location}.'/local-debbugs.pid');
432 if (not defined $pid or $pid == 0) {
433 print STDERR "Unable to open pidfile or daemon not running: $!\n";
434 print STDERR qq(Mr. T: "I pity da fool who tries to search for bugs without a running daemon"\n);
435 print STDERR "Hint: try the --daemon option first\n";
439 exec('/usr/bin/sensible-browser',$url) or
440 die "Unable to run sensible-browser (Maybe chorizo is required?)";
443 # you get here, you were an idiot in checking for @USAGE_ERRORS
445 die "No option that we understand was passed (the first check for this is now buggy, so shoot your maintainer)"
449 # determine the local configuration
453 if (-e '/etc/debbugs/local_debbugs.conf') {
454 Config::Simple->import_from('/etc/debbugs/local_debbugs.conf', $config) or
455 die "Unable to read configuration from /etc/debbugs/local_debbugs.conf: $!";
457 if (-e User->Home.'/.debbugs/local_debbugs.conf') {
458 Config::Simple->import_from(User->Home.'/.debbugs/local_debbugs.conf', $config) or
459 die "Unable to read configuration from ".User->Home.'/.debbugs/local_debbugs.conf: '.$!;
461 for (keys %option_defaults) {
462 if (exists $config->{$_} and not defined $options->{$_}) {
463 $options->{$_} = $config->{$_};
465 if (not defined $options->{$_}) {
466 $options->{$_} = $option_defaults{$_};
472 my ($file,$bug_list) = @_;
473 my $inc_fh = IO::File->new($file,'w') or
474 die "Unable to open $file for writing: $!";
475 foreach my $bug (keys %{$bug_list}) {
476 my $file_loc = get_hashname($bug).'/'.$bug;
477 print {$inc_fh} map {$file_loc.'.'.$_.qq(\n)} qw(log summary report status) or
478 die "Unable to write to $file: $!";
481 die "Unable to close $file: $!";
484 # actually run rsync with the passed options
486 my %param = validate_with(params => \@_,
487 spec => {log => {type => HANDLE,
489 debug => {type => HANDLE,
492 options => {type => ARRAYREF,
496 my ($output,$error) = ('','');
497 my $h = IPC::Run::start(['rsync',@{$param{options}}],
498 \undef,$param{log},$param{log});
500 #print {$param{debug}} $error if defined $param{debug};
503 my $exit = $h->result(0);
504 # this is suboptimal, but we currently don't know whether we've
505 # selected an archive or unarchived bug, so..
506 if (defined $exit and not ($exit == 0 or $exit == 3 or $exit == 23)) {
507 print STDERR "Rsync exited with non-zero status: $exit\n";
513 # select a set of bugs
517 my %valid_keys = (package => 'package',
522 maintainer => 'maint',
523 submitter => 'submitter',
532 distribution => 'dist',
534 archive => 'archive',
535 severity => 'severity',
536 correspondent => 'correspondent',
537 affects => 'affects',
540 my $soap = SOAP::Lite
541 -> uri('Debbugs/SOAP/V1')
542 -> proxy("http://$options{bug_site}/cgi-bin/soap.cgi");
544 my @bug_selections = ();
545 if (not -e $options{bugs_to_get}) {
546 my ($addr) = get_addresses(exists $ENV{DEBEMAIL}?
548 (User->Login . '@' . qx(hostname --fqdn)));
549 # by default include bugs talked to by this user packages
550 # maintained by this user, submitted by this user, and rc
552 push @bug_selections,
553 ("correspondent:$addr archive:0",
554 "maint:$addr archive:0",
555 "submitter:$addr archive:0",
556 "severity:serious severity:grave severity:critical archive:0",
560 my $btg_fh = IO::File->new($options{bugs_to_get},'r') or
561 die "unable to open bugs to get file '$options{bugs_to_get}' for reading: $!";
569 push @bug_selections, $_;
573 # Split archive:both into archive:1 and archive:0
576 if (m/archive:both/) {
579 $y_archive =~ s/archive:both/archive:1/;
580 $n_archive =~ s/archive:both/archive:0/;
581 ($y_archive,$n_archive);
588 for my $selection (@bug_selections) {
589 my $archived_bugs = "unarchived";
590 if ($selection =~ /archive:(\S+)/ and $1) {
591 $archived_bugs = "archived";
593 my @subselects = split /\s+/,$selection;
594 my %search_parameters;
596 for my $subselect (@subselects) {
597 my ($key,$value) = split /:/, $subselect, 2;
599 if (exists $valid_keys{$key}) {
600 push @{$search_parameters{$valid_keys{$key}}},
602 } elsif ($key =~/users?$/) {
603 $users{$value} = 1 if $value;
607 for my $user (keys %users) {
608 my $ut = $soap->get_usertag($user)->result();
609 next unless defined $ut and $ut ne "";
610 for my $tag (keys %{$ut}) {
611 push @{$usertags{$tag}},
615 my $bugs = $soap->get_bugs(%search_parameters,
616 (keys %usertags)?(usertags=>\%usertags):()
618 if (defined $bugs and @{$bugs}) {
619 $bugs{$archived_bugs}{$_} = 1 for @{$bugs};
622 for my $bug (@bugs) {
623 $bugs{archived}{$bug} = 1;
624 $bugs{unarchived}{$bug} = 1;