From: Don Armstrong Date: Thu, 21 Mar 2013 17:39:04 +0000 (-0700) Subject: Merge branch 'don/rebuild-index' X-Git-Tag: release/2.6.0~280^2~12 X-Git-Url: https://git.donarmstrong.com/?a=commitdiff_plain;h=cebcfe4a025c588c5f9197b706a1b3c0ca61653e;hp=45c0f862895dfe537b2ed6e55cca35b4565abe63;p=debbugs.git Merge branch 'don/rebuild-index' --- diff --git a/Debbugs/Status.pm b/Debbugs/Status.pm index 6fad22e..f44f9fc 100644 --- a/Debbugs/Status.pm +++ b/Debbugs/Status.pm @@ -70,6 +70,7 @@ BEGIN{ qw(removefoundversions removefixedversions) ], hook => [qw(bughook bughook_archive)], + indexdb => [qw(generate_index_db_line)], fields => [qw(%fields)], ); @EXPORT_OK = (); @@ -207,6 +208,7 @@ sub read_bug{ $log = $status; $log =~ s/\.summary$/.log/; ($location) = $status =~ m/(db-h|db|archive)/; + ($param{bug}) = $status =~ m/(\d+)\.summary$/; } if ($param{lock}) { filelock("$config{spool_dir}/lock/$param{bug}",exists $param{locks}?$param{locks}:()); @@ -1599,6 +1601,39 @@ sub isstrongseverity { return grep { $_ eq $severity } @{$config{strong_severities}}; } +=head1 indexdb + +=head2 generate_index_db_line + + my $data = read_bug(bug => $bug, + location => $initialdir); + # generate_index_db_line hasn't been written yet at all. + my $line = generate_index_db_line($data); + +Returns a line for a bug suitable to be written out to index.db. + +=cut + +sub generate_index_db_line { + my ($data,$bug) = @_; + + # just in case someone has given us a split out data + $data = join_status_fields($data); + + my $whendone = "open"; + my $severity = $config{default_severity}; + (my $pkglist = $data->{package}) =~ s/[,\s]+/,/g; + $pkglist =~ s/^,+//; + $pkglist =~ s/,+$//; + $whendone = "forwarded" if defined $data->{forwarded} and length $data->{forwarded}; + $whendone = "done" if defined $data->{done} and length $data->{done}; + $severity = $data->{severity} if length $data->{severity}; + return sprintf "%s %d %d %s [%s] %s %s\n", + $pkglist, $data->{bug_num}//$bug, $data->{date}, $whendone, + $data->{originator}, $severity, $data->{keywords}; +} + + =head1 PRIVATE FUNCTIONS @@ -1681,19 +1716,7 @@ sub bughook { my $data = $bugs_temp{$bug}; appendfile("$config{spool_dir}/debbugs.trace","$type $bug\n",makestatus($data, 1)); - my $whendone = "open"; - my $severity = $config{default_severity}; - (my $pkglist = $data->{package}) =~ s/[,\s]+/,/g; - $pkglist =~ s/^,+//; - $pkglist =~ s/,+$//; - $whendone = "forwarded" if defined $data->{forwarded} and length $data->{forwarded}; - $whendone = "done" if defined $data->{done} and length $data->{done}; - $severity = $data->{severity} if length $data->{severity}; - - my $k = sprintf "%s %d %d %s [%s] %s %s\n", - $pkglist, $bug, $data->{date}, $whendone, - $data->{originator}, $severity, $data->{keywords}; - $bugs{$bug} = $k; + $bugs{$bug} = generate_index_db_line($data,$bug); } update_realtime("$config{spool_dir}/index.db.realtime", %bugs); diff --git a/bin/debbugs-rebuild-index.db b/bin/debbugs-rebuild-index.db new file mode 100755 index 0000000..49b925b --- /dev/null +++ b/bin/debbugs-rebuild-index.db @@ -0,0 +1,153 @@ +#! /usr/bin/perl +# debbugs-rebuild-index.db is part of debbugs, and is released +# under the terms of the GPL version 2, or any later version, at your +# option. See the file README and COPYING for more information. +# Copyright 2012 by Don Armstrong . + + +use warnings; +use strict; + +use Getopt::Long qw(:config no_ignore_case); +use Pod::Usage; + +=head1 NAME + +debbugs-rebuild-index.db -- rebuild Debbug's index.db + +=head1 SYNOPSIS + +debbugs-rebuild-index.db [options] + + Options: + --spool-dir debbugs spool directory + --debug, -d debugging level (Default 0) + --help, -h display this help + --man, -m display manual + +=head1 OPTIONS + +=over + +=item B<--spool-dir> + +Debbugs spool directory; defaults to the value configured in the +debbugs configuration file. + +=item B<--debug, -d + +Debug verbosity. + +=item B<--help, -h> + +Display brief useage information. + +=item B<--man, -m> + +Display this manual. + +=back + +=head1 EXAMPLES + +Rebuild the index.db for db-h. + + debbugs-rebuild-index.db; + +Rebuild the index.db for archive + + debbugs-rebuild-index.db archive; + + +=cut + + +use vars qw($DEBUG); + +use Debbugs::Common qw(checkpid lockpid get_hashname getparsedaddrs getbugcomponent make_list); +use Debbugs::Config qw(:config); +use Debbugs::Status qw(read_bug split_status_fields generate_index_db_line); + +my %options = (debug => 0, + help => 0, + man => 0, + verbose => 0, + quiet => 0, + quick => 0, + service => 'debbugs', + ); + + +GetOptions(\%options, + 'quick|q', + 'service|s', + 'sysconfdir|c', + 'spool_dir|spool-dir=s', + 'debug|d+','help|h|?','man|m'); + +pod2usage() if $options{help}; +pod2usage({verbose=>2}) if $options{man}; + +$DEBUG = $options{debug}; + +my @USAGE_ERRORS; +$options{verbose} = $options{verbose} - $options{quiet}; + +pod2usage(join("\n",@USAGE_ERRORS)) if @USAGE_ERRORS; + +if (exists $options{spool_dir} and defined $options{spool_dir}) { + $config{spool_dir} = $options{spool_dir}; +} +chdir($config{spool_dir}) or die "chdir $config{spool_dir} failed: $!"; + +my $verbose = $options{debug}; + +my $initialdir = "db-h"; + +if (defined $ARGV[0] and $ARGV[0] eq "archive") { + $initialdir = "archive"; +} + +if (not lockpid($config{spool_dir}.'/lock/debbugs-rebuild-index.db')) { + print STDERR "Another debbugs-rebuild-index.db is running; stopping\n"; + exit 1; +} + +my $file = "index.${initialdir}.realtime"; +my $idx_rebuild = IO::File->new($file.'.rebuild','w') + or die "Couldn't open ${file}.rebuild: $!"; + + +my @dirs = ($initialdir); +my $cnt = 0; +my %bugs; +while (my $dir = shift @dirs) { + printf "Doing dir %s ...\n", $dir if $verbose; + + opendir(DIR, "$dir/.") or die "opendir $dir: $!"; + my @subdirs = readdir(DIR); + closedir(DIR); + + my @list = map { m/^(\d+)\.summary$/?($1):() } @subdirs; + push @dirs, map { m/^(\d+)$/ && -d "$dir/$1"?("$dir/$1"):() } @subdirs; + + for my $bug (@list) { + print "Up to $cnt bugs...\n" if (++$cnt % 100 == 0 && $verbose); + my $stat = stat(getbugcomponent($bug,'summary',$initialdir)); + if (not defined $stat) { + print STDERR "Unable to stat $bug $!\n"; + next; + } + my $data = read_bug(bug => $bug, + location => $initialdir); + my $line = generate_index_db_line($data); + $bugs{$bug} = $line; + } +} +binmode($idx_rebuild,':raw:encoding(UTF-8)'); +print {$idx_rebuild} $bugs{$_} foreach sort keys %bugs; +close($idx_rebuild); +rename("$file.rebuild", $file); + + +__END__ diff --git a/t/15_rebuild_indexdb.t b/t/15_rebuild_indexdb.t new file mode 100644 index 0000000..a3036ec --- /dev/null +++ b/t/15_rebuild_indexdb.t @@ -0,0 +1,128 @@ +# -*- mode: cperl;-*- + +use Test::More tests => 9; + +use warnings; +use strict; + +use utf8; + +# Here, we're going to shoot messages through a set of things that can +# happen. + +# First, we're going to send mesages to receive. +# To do so, we'll first send a message to submit, +# then send messages to the newly created bugnumber. + +use IO::File; +use File::Temp qw(tempdir); +use Cwd qw(getcwd); +use Debbugs::MIME qw(create_mime_message); +use File::Basename qw(dirname basename); +# The test functions are placed here to make things easier +use lib qw(t/lib); +use DebbugsTest qw(:all); +use Data::Dumper; +use Encode qw(decode encode decode_utf8 encode_utf8); + +# HTTP::Server:::Simple defines a SIG{CHLD} handler that breaks system; undef it here. +$SIG{CHLD} = sub {}; +my %config; +eval { + %config = create_debbugs_configuration(debug => exists $ENV{DEBUG}?$ENV{DEBUG}:0); +}; +if ($@) { + BAIL_OUT($@); +} + +my $sendmail_dir = $config{sendmail_dir}; +my $spool_dir = $config{spool_dir}; +my $config_dir = $config{config_dir}; + +END{ + if ($ENV{DEBUG}) { + diag("spool_dir: $spool_dir\n"); + diag("config_dir: $config_dir\n"); + diag("sendmail_dir: $sendmail_dir\n"); + } +} + +# We're going to use create mime message to create these messages, and +# then just send them to receive. + +send_message(to=>'submit@bugs.something', + headers => [To => 'submit@bugs.something', + From => 'föoff@bugs.something', + Subject => 'Submiting a bug', + ], + body => < [{Type=>"text/plain",Charset=>"utf-8",Data=>< '1@bugs.something', + headers => [To => '1@bugs.something', + From => 'föoff@bugs.something', + Subject => 'Sending a message to a bug', + ], + body => < 'control@bugs.something', + headers => [To => 'control@bugs.something', + From => 'föoff@bugs.something', + Subject => 'Munging a bug', + ], + body => <