2 # local-debbugs is part of debbugs, and is released
3 # under the terms of the GPL version 2, or any later version, at your
4 # option. See the file README and COPYING for more information.
5 # Copyright 2008 by Don Armstrong <don@donarmstrong.com>.
11 use Getopt::Long qw(:config no_ignore_case);
16 local-debbugs - use a local mirror of debbugs
23 --mirror, -M update local mirror
24 --daemon, -D start the daemon
25 --search, -S run a search
27 --debug, -d debugging level (Default 0)
28 --help, -h display this help
29 --man, -m display manual
37 Update the local mirror of debbugs bugs
41 Start up the daemon on the configured local port to serve bugs which
42 have been previously retrieved.
46 Cause the running daemon to show the pkgreport.cgi page corresponding
47 to the search by invoking sensible-browser and an appropriate url.
51 Cause the running daemon to show the bugreport.cgi page corresponding
52 to the bug by invoking sensible-browser and an appropriate url.
56 The port that the daemon is running on (or will be running on.)
58 Defaults to the value of the currently running daemon, the value in
59 the configuration file, or 8080 if nothing is set.
61 =item B<--bugs-to-get>
63 File which contains the set of bugs to get.
64 Defaults to ~/.debbugs/bugs_to_get
68 Hostname for a site which is running a debbugs install.
69 Defaults to bugs.debian.org
73 Hostname for a site which is running an rsyncable mirror of the
74 debbugs install above.
75 Defaults to bugs-mirror.debian.org
83 Display brief useage information.
95 =item Update the local mirror
97 local-debbugs --mirror
99 =item Start up the local-debbugs daemon
101 local-debbugs --daemon
103 =item Search for bugs with severity serious
105 local-debbugs --search severity:serious
116 use File::Temp qw(tempdir);
117 use Params::Validate qw(validate_with :types);
119 use Debbugs::Common qw(checkpid lockpid get_hashname);
120 use Debbugs::Mail qw(get_addresses);
127 my %options = (debug => 0,
133 cgi_bin => '/var/lib/debbugs/www/cgi',
134 css => '/var/lib/debbugs/www/bugs.css',
135 bug_site => 'bugs.debian.org',
136 bug_mirror => 'bugs-mirror.debian.org',
139 my %option_defaults = (port => 8080,
140 debbugs_config => User->Home.'/.debbugs/debbugs_config',
141 mirror_location => User->Home.'/.debbugs/mirror',
142 bugs_to_get => User->Home.'/.debbugs/bugs_to_get',
145 GetOptions(\%options,
146 'daemon|D','show|s','search|select|S','mirror|M', 'stop|exit|quit',
148 'css=s','cgi_bin|cgi-bin|cgi=s',
149 'verbose|v+','quiet|q+',
150 'bug_site|bug-site=s',
151 'bug_mirror|bug-mirror=s',
152 'debug|d+','help|h|?','man|m');
154 pod2usage() if $options{help};
155 pod2usage({verbose=>2}) if $options{man};
157 $DEBUG = $options{debug};
160 if (1 != grep {exists $options{$_}} qw(daemon show search mirror stop)) {
161 push @USAGE_ERRORS,"You must pass one (and only one) of --daemon --show --search --mirror or --stop";
163 $options{verbose} = $options{verbose} - $options{quiet};
165 pod2usage(join("\n",@USAGE_ERRORS)) if @USAGE_ERRORS;
168 # munge in local configuration
170 local_config(\%options);
172 mkpath($options{mirror_location});
174 if ($options{daemon}) {
175 # daemonize, do stuff
176 my $pid = checkpid($options{mirror_location}.'/local-debbugs.pid');
177 if (defined $pid and $pid != 0) {
178 print STDERR "Unable to start daemon; it's already running\n";
181 if (-e $options{mirror_location}.'/local-debbugs.pid' and
183 print STDERR "Unable to determine if daemon is running: $!\n";
186 my $conf = IO::File->new($options{mirror_location}.'/debbugs_config_local','w') or
187 die "Unable to open $options{mirror_location}/debbugs_config_local for writing: $!";
188 print {$conf} <<"EOF";
189 \$gConfigDir = "$options{mirror_location}";
190 \$gSpoolDir = "$options{mirror_location}";
191 \$gWebHost = 'localhost:$options{port}';
192 \$gPackageSource = '';
193 \$gPseudoDescFile = '';
194 \$gPseudoMaintFile = '';
195 \$gMaintainerFile = '';
196 \$gMaintainerFileOverride = '';
197 \$config{source_maintainer_file} = '';
198 \$config{source_maintainer_file_override} = '';
199 \$gProject = 'Local Debbugs';
203 $ENV{DEBBUGS_CONFIG_FILE} = $options{mirror_location}.'/debbugs_config_local';
204 # ok, now lets daemonize
206 # XXX make sure that all paths have been turned into absolute
208 chdir '/' or die "Can't chdir to /: $!";
209 # allow us not to detach for debugging
210 if ($options{detach}) {
211 open STDIN, '/dev/null' or die "Can't read /dev/null: $!";
212 open STDOUT, '>/dev/null'
213 or die "Can't write to /dev/null: $!";
214 defined(my $pid = fork) or die "Can't fork: $!";
216 setsid or die "Can't start a new session: $!";
217 open STDERR, '>&STDOUT' or die "Can't dup stdout: $!";
219 lockpid($options{mirror_location}.'/local-debbugs.pid') or
220 die "Unable to deal with the pidfile";
221 # this is the subclass of HTTP::Server::Simple::CGI which handles
222 # the "hard" bits of actually running a tiny webserver for us
224 package local_debbugs::server;
226 use HTTP::Server::Simple;
227 use base qw(HTTP::Server::Simple::CGI HTTP::Server::Simple::CGI::Environment);
230 return 'Net::Server::Fork';
235 print "HTTP/1.1 302 Found\r\n";
236 print "Location: $url\r\n";
239 # here we want to call cgi-bin/pkgreport or cgi-bin/bugreport
241 my ($self,$cgi) = @_;
243 $ENV{DEBBUGS_CONFIG_FILE} = $options{mirror_location}.'/debbugs_config_local';
244 my $base_uri = 'http://'.$cgi->virtual_host;
245 if ($cgi->virtual_port ne 80) {
246 $base_uri .= ':'.$cgi->virtual_port;
248 my $path = $cgi->path_info();
249 # RewriteRule ^/[[:space:]]*#?([[:digit:]][[:digit:]][[:digit:]]+)([;&].+)?$ /cgi-bin/bugreport.cgi?bug=$1$2 [L,R,NE]
250 if ($path =~ m{^/?\s*\#?(\d+)((?:[;&].+)?)$}) {
251 redirect($cgi,$base_uri."/cgi-bin/bugreport.cgi?bug=$1$2");
253 # RewriteRule ^/[Ff][Rr][Oo][Mm]:([^/]+\@.+)$ /cgi-bin/pkgreport.cgi?submitter=$1 [L,R,NE]
254 elsif ($path =~ m{^/?\s*from:([^/]+\@.+)$}i) {
255 redirect($cgi,$base_uri."/cgi-bin/pkgreport.cgi?submitter=$1");
257 # RewriteRule ^/([^/]+\@.+)$ /cgi-bin/pkgreport.cgi?maint=$1 [L,R,NE]
258 elsif ($path =~ m{^/?\s*([^/]+\@.+)$}i) {
259 redirect($cgi,$base_uri."/cgi-bin/pkgreport.cgi?maint=$1");
261 # RewriteRule ^/mbox:([[:digit:]][[:digit:]][[:digit:]]+)([;&].+)?$ /cgi-bin/bugreport.cgi?mbox=yes&bug=$1$2 [L,R,NE]
262 elsif ($path =~ m{^/?\s*mbox:\#?(\d+)((?:[;&].+)?)$}i) {
263 redirect($cgi,$base_uri."/cgi-bin/bugreport.cgi?mbox=yes;bug=$1$2");
265 # RewriteRule ^/src:([^/]+)$ /cgi-bin/pkgreport.cgi?src=$1 [L,R,NE]
266 elsif ($path =~ m{^/?src:([^/]+)$}i) {
267 redirect($cgi,$base_uri."/cgi-bin/pkgreport.cgi?src=$1");
269 # RewriteRule ^/severity:([^/]+)$ /cgi-bin/pkgreport.cgi?severity=$1 [L,R,NE]
270 elsif ($path =~ m{^/?severity:([^/]+)$}i) {
271 redirect($cgi,$base_uri."/cgi-bin/pkgreport.cgi?severity=$1");
273 # RewriteRule ^/tag:([^/]+)$ /cgi-bin/pkgreport.cgi?tag=$1 [L,R,NE]
274 elsif ($path =~ m{^/?tag:([^/]+)$}i) {
275 redirect($cgi,$base_uri."/cgi-bin/pkgreport.cgi?tag=$1");
277 # RewriteRule ^/([^/]+)$ /cgi-bin/pkgreport.cgi?pkg=$1 [L,R,NE]
278 elsif ($path =~ m{^/?([^/]+)$}i) {
279 redirect($cgi,$base_uri."/cgi-bin/pkgreport.cgi?pkg=$1");
281 elsif ($path =~ m{^/?cgi(?:-bin)?/((?:(?:bug|pkg)report|version)\.cgi)}) {
282 # dispatch to pkgreport.cgi
283 #print "HTTP/1.1 200 OK\n";
284 open(my $fh,'-|',"$options{cgi_bin}/$1") or
285 die "Unable to execute $options{cgi_bin}/$1";
289 if (/Status: (\d+\s+.+?)\n?$/) {
291 print "HTTP/1.1 $status\n";
292 print STDERR "'$status'\n";
297 print "HTTP/1.1 200 OK\n";
303 close($fh) or die "Unable to close";
305 elsif ($path =~ m{^/?css/bugs.css}) {
306 my $fh = IO::File->new($options{css},'r') or
307 die "Unable to open $options{css} for reading: $!";
308 print "HTTP/1.1 200 OK\n";
309 print "Content-type: text/css\n";
313 elsif ($path =~ m{^/?$}) {
314 redirect($cgi,$base_uri."/cgi-bin/pkgreport.cgi?package=put%20package%20here");
317 print "HTTP/1.1 404 Not Found\n";
318 print "Content-Type: text/html\n";
320 print "<h1>That which you were seeking, found I have not.</h1>\n";
322 # RewriteRule ^/$ /Bugs/ [L,R,NE]
325 my $debbugs_server = local_debbugs::server->new($options{port}) or
326 die "Unable to create debbugs server";
327 $debbugs_server->run() or
328 die 'Unable to run debbugs server';
330 elsif ($options{stop}) {
332 my $pid = checkpid($options{mirror_location}.'/local-debbugs.pid');
333 if (not defined $pid or $pid == 0) {
334 print STDERR "Unable to open pidfile or daemon not running: $!\n";
337 exit !(kill(15,$pid) == 1);
339 elsif ($options{mirror}) {
340 # run the mirror jobies
341 # figure out which bugs we need
342 my $bugs = select_bugs(\%options);
344 my $tempdir = tempdir();#CLEANUP => 1);
345 my $mirror_log = IO::File->new($options{mirror_location}.'/mirror.log','>') or
346 die "Unable to open $options{mirror_location}/mirror.log for writing: $!";
347 write_bug_list("$tempdir/unarchived_bug_list",$bugs->{unarchived});
348 write_bug_list("$tempdir/archived_bug_list",$bugs->{archived});
350 my @common_rsync_options = ('-avz','--partial');
351 print "Rsyncing bugs\n" if not $options{quiet};
352 run_rsync(log => $mirror_log,
353 ($options{debug}?(debug => \*STDERR):()),
354 options => [@common_rsync_options,
356 '--files-from',"$tempdir/unarchived_bug_list",
357 'rsync://'.$options{bug_mirror}.'/bts-spool-db/',
358 $options{mirror_location}.'/db-h/']
360 print "Rsyncing archived bugs\n" if $options{verbose};
361 run_rsync(log => $mirror_log,
362 ($options{debug}?(debug => \*STDERR):()),
363 options => [@common_rsync_options,
365 '--files-from',"$tempdir/archived_bug_list",
366 'rsync://'.$options{bug_mirror}.'/bts-spool-archive/',
367 $options{mirror_location}.'/archive/',
370 print "Rsyncing indexes\n" if $options{verbose};
371 run_rsync(log => $mirror_log,
372 ($options{debug}?(debug => \*STDERR):()),
373 options => [@common_rsync_options,
376 '--exclude','by-reverse*',
377 'rsync://'.$options{bug_mirror}.'/bts-spool-index/',
378 $options{mirror_location}.'/',
381 print "Rsyncing versions\n" if $options{verbose};
382 run_rsync(log => $mirror_log,
383 ($options{debug}?(debug => \*STDERR):()),
384 options => [@common_rsync_options,
388 'rsync://'.$options{bug_mirror}.'/bts-versions/',
389 $options{mirror_location}.'/versions/',
393 elsif ($options{show}) {
395 # see if the daemon is running
396 my $pid = checkpid($options{mirror_location}.'/local-debbugs.pid');
397 if (not defined $pid or $pid == 0) {
398 print STDERR "Unable to open pidfile or daemon not running: $!\n";
399 print STDERR qq(Mr. T: "I pity da fool who tries to show a bug without a running daemon"\n);
400 print STDERR "Hint: try the --daemon option first\n";
404 my $url = qq(http://localhost:$options{port}/$ARGV[0]);
405 exec('/usr/bin/sensible-browser',$url) or
406 die "Unable to run sensible-browser (try feeding me cheetos?)";
408 elsif ($options{search}) {
409 my $url = qq(http://localhost:$options{port}/cgi-bin/pkgreport.cgi?).
410 join(';',map {if (/:/) {s/:/=/; $_;} else {qq(pkg=$_);}} @ARGV);
411 my $pid = checkpid($options{mirror_location}.'/local-debbugs.pid');
412 if (not defined $pid or $pid == 0) {
413 print STDERR "Unable to open pidfile or daemon not running: $!\n";
414 print STDERR qq(Mr. T: "I pity da fool who tries to search for bugs without a running daemon"\n);
415 print STDERR "Hint: try the --daemon option first\n";
419 exec('/usr/bin/sensible-browser',$url) or
420 die "Unable to run sensible-browser (Maybe chorizo is required?)";
423 # you get here, you were an idiot in checking for @USAGE_ERRORS
425 die "No option that we understand was passed (the first check for this is now buggy, so shoot your maintainer)"
429 # determine the local configuration
433 if (-e '/etc/debbugs/local_debbugs.conf') {
434 Config::Simple->import_from('/etc/debbugs/local_debbugs.conf', $config) or
435 die "Unable to read configuration from /etc/debbugs/local_debbugs.conf: $!";
437 if (-e User->Home.'/.debbugs/local_debbugs.conf') {
438 Config::Simple->import_from(User->Home.'/.debbugs/local_debbugs.conf', $config) or
439 die "Unable to read configuration from ".User->Home.'/.debbugs/local_debbugs.conf: '.$!;
441 for (keys %option_defaults) {
442 if (exists $config->{$_} and not defined $options->{$_}) {
443 $options->{$_} = $config->{$_};
445 if (not defined $options->{$_}) {
446 $options->{$_} = $option_defaults{$_};
452 my ($file,$bug_list) = @_;
453 my $inc_fh = IO::File->new($file,'w') or
454 die "Unable to open $file for writing: $!";
455 foreach my $bug (keys %{$bug_list}) {
456 my $file_loc = get_hashname($bug).'/'.$bug;
457 print {$inc_fh} map {$file_loc.'.'.$_.qq(\n)} qw(log summary report status) or
458 die "Unable to write to $file: $!";
461 die "Unable to close $file: $!";
464 # actually run rsync with the passed options
466 my %param = validate_with(params => \@_,
467 spec => {log => {type => HANDLE,
469 debug => {type => HANDLE,
472 options => {type => ARRAYREF,
476 my ($output,$error) = ('','');
477 my $h = IPC::Run::start(['rsync',@{$param{options}}],
478 \undef,$param{log},$param{log});
480 #print {$param{debug}} $error if defined $param{debug};
483 my $exit = $h->result(0);
484 # this is suboptimal, but we currently don't know whether we've
485 # selected an archive or unarchived bug, so..
486 if (defined $exit and not ($exit == 0 or $exit == 3 or $exit == 23)) {
487 print STDERR "Rsync exited with non-zero status: $exit\n";
493 # select a set of bugs
497 my %valid_keys = (package => 'package',
502 maintainer => 'maint',
503 submitter => 'submitter',
512 distribution => 'dist',
514 archive => 'archive',
515 severity => 'severity',
516 correspondent => 'correspondent',
517 affects => 'affects',
520 my $soap = SOAP::Lite
521 -> uri('Debbugs/SOAP/V1')
522 -> proxy("http://$options{bug_site}/cgi-bin/soap.cgi");
524 my @bug_selections = ();
525 if (not -e $options{bugs_to_get}) {
526 my ($addr) = get_addresses(exists $ENV{DEBEMAIL}?
528 (User->Login . '@' . qx(hostname --fqdn)));
529 # by default include bugs talked to by this user packages
530 # maintained by this user, submitted by this user, and rc
532 push @bug_selections,
533 ("correspondent:$addr archive:0",
534 "maint:$addr archive:0",
535 "submitter:$addr archive:0",
536 "severity:serious severity:grave severity:critical archive:0",
540 my $btg_fh = IO::File->new($options{bugs_to_get},'r') or
541 die "unable to open bugs to get file '$options{bugs_to_get}' for reading: $!";
549 push @bug_selections, $_;
553 # Split archive:both into archive:1 and archive:0
556 if (m/archive:both/) {
559 $y_archive =~ s/archive:both/archive:1/;
560 $n_archive =~ s/archive:both/archive:0/;
561 ($y_archive,$n_archive);
568 for my $selection (@bug_selections) {
569 my $archived_bugs = "unarchived";
570 if ($selection =~ /archive:(\S+)/ and $1) {
571 $archived_bugs = "archived";
573 my @subselects = split /\s+/,$selection;
574 my %search_parameters;
576 for my $subselect (@subselects) {
577 my ($key,$value) = split /:/, $subselect, 2;
579 if (exists $valid_keys{$key}) {
580 push @{$search_parameters{$valid_keys{$key}}},
582 } elsif ($key =~/users?$/) {
583 $users{$value} = 1 if $value;
587 for my $user (keys %users) {
588 my $ut = $soap->get_usertag($user)->result();
589 next unless defined $ut and $ut ne "";
590 for my $tag (keys %{$ut}) {
591 push @{$usertags{$tag}},
595 my $bugs = $soap->get_bugs(%search_parameters,
596 (keys %usertags)?(usertags=>\%usertags):()
598 if (defined $bugs and @{$bugs}) {
599 $bugs{$archived_bugs}{$_} = 1 for @{$bugs};