X-Git-Url: https://git.donarmstrong.com/?a=blobdiff_plain;f=Debbugs%2FBugs.pm;h=5150d11c3c34565c4e7ed59bf0da4e3ce1e78652;hb=0ea01c001c3e07722958665acc783baec2759651;hp=026fce66d3b1e3bc7c74564fcbe6ec4c35ae1267;hpb=a6c428bc8b655a9de967a2794f6fd860e235bc10;p=debbugs.git diff --git a/Debbugs/Bugs.pm b/Debbugs/Bugs.pm index 026fce6..5150d11 100644 --- a/Debbugs/Bugs.pm +++ b/Debbugs/Bugs.pm @@ -37,8 +37,9 @@ incomplete) to slowest (and most complete).] use warnings; use strict; +use feature 'state'; use vars qw($VERSION $DEBUG %EXPORT_TAGS @EXPORT_OK @EXPORT); -use base qw(Exporter); +use Exporter qw(import); BEGIN{ $VERSION = 1.00; @@ -54,11 +55,11 @@ use Debbugs::Config qw(:config); use Params::Validate qw(validate_with :types); use IO::File; use Debbugs::Status qw(splitpackages get_bug_status); -use Debbugs::Packages qw(getsrcpkgs); -use Debbugs::Common qw(getparsedaddrs getmaintainers getmaintainers_reverse make_list); +use Debbugs::Packages qw(getsrcpkgs getpkgsrc); +use Debbugs::Common qw(getparsedaddrs package_maintainer getmaintainers make_list hash_slice); use Fcntl qw(O_RDONLY); use MLDBM qw(DB_File Storable); -use List::Util qw(first); +use List::AllUtils qw(first max); use Carp; =head2 get_bugs @@ -92,6 +93,8 @@ for limited regular expressions, and/or more complex expressions. =item correspondent -- address of someone who sent mail to the log +=item affects -- bugs which affect this package + =item dist -- distribution (I don't know about this one yet) =item bugs -- list of bugs to search within @@ -150,52 +153,68 @@ bug should not. =cut +state $_non_search_key_regex = qr/^(bugs|archive|usertags|schema)$/; + +my %_get_bugs_common_options = + (package => {type => SCALAR|ARRAYREF, + optional => 1, + }, + src => {type => SCALAR|ARRAYREF, + optional => 1, + }, + maint => {type => SCALAR|ARRAYREF, + optional => 1, + }, + submitter => {type => SCALAR|ARRAYREF, + optional => 1, + }, + severity => {type => SCALAR|ARRAYREF, + optional => 1, + }, + status => {type => SCALAR|ARRAYREF, + optional => 1, + }, + tag => {type => SCALAR|ARRAYREF, + optional => 1, + }, + owner => {type => SCALAR|ARRAYREF, + optional => 1, + }, + dist => {type => SCALAR|ARRAYREF, + optional => 1, + }, + correspondent => {type => SCALAR|ARRAYREF, + optional => 1, + }, + affects => {type => SCALAR|ARRAYREF, + optional => 1, + }, + function => {type => CODEREF, + optional => 1, + }, + bugs => {type => SCALAR|ARRAYREF, + optional => 1, + }, + archive => {type => BOOLEAN|SCALAR, + default => 0, + }, + usertags => {type => HASHREF, + optional => 1, + }, + newest => {type => SCALAR|ARRAYREF, + optional => 1, + }, + schema => {type => OBJECT, + optional => 1, + }, + ); + + +state $_get_bugs_options = {%_get_bugs_common_options}; sub get_bugs{ my %param = validate_with(params => \@_, - spec => {package => {type => SCALAR|ARRAYREF, - optional => 1, - }, - src => {type => SCALAR|ARRAYREF, - optional => 1, - }, - maint => {type => SCALAR|ARRAYREF, - optional => 1, - }, - submitter => {type => SCALAR|ARRAYREF, - optional => 1, - }, - severity => {type => SCALAR|ARRAYREF, - optional => 1, - }, - status => {type => SCALAR|ARRAYREF, - optional => 1, - }, - tag => {type => SCALAR|ARRAYREF, - optional => 1, - }, - owner => {type => SCALAR|ARRAYREF, - optional => 1, - }, - dist => {type => SCALAR|ARRAYREF, - optional => 1, - }, - correspondent => {type => SCALAR|ARRAYREF, - optional => 1, - }, - function => {type => CODEREF, - optional => 1, - }, - bugs => {type => SCALAR|ARRAYREF, - optional => 1, - }, - archive => {type => BOOLEAN|SCALAR, - default => 0, - }, - usertags => {type => HASHREF, - optional => 1, - }, - }, - ); + spec => $_get_bugs_options, + ); # Normalize options my %options = %param; @@ -208,7 +227,7 @@ sub get_bugs{ return keys %bugs; } # A configuration option will set an array that we'll use here instead. - for my $routine (qw(Debbugs::Bugs::get_bugs_by_idx Debbugs::Bugs::get_bugs_flatfile)) { + for my $routine (qw(Debbugs::Bugs::get_bugs_by_db Debbugs::Bugs::get_bugs_by_idx Debbugs::Bugs::get_bugs_flatfile)) { my ($package) = $routine =~ m/^(.+)\:\:/; eval "use $package;"; if ($@) { @@ -320,7 +339,7 @@ sub bug_filter { optional => 1, }, repeat_merged => {type => BOOLEAN, - optional => 1, + default => 1, }, include => {type => HASHREF, optional => 1, @@ -383,45 +402,34 @@ searches. =cut + +state $_get_bugs_by_idx_options = + {hash_slice(%_get_bugs_common_options, + (qw(package submitter severity tag archive), + qw(owner src maint bugs correspondent), + qw(affects usertags newest)) + ) + }; sub get_bugs_by_idx{ my %param = validate_with(params => \@_, - spec => {package => {type => SCALAR|ARRAYREF, - optional => 1, - }, - submitter => {type => SCALAR|ARRAYREF, - optional => 1, - }, - severity => {type => SCALAR|ARRAYREF, - optional => 1, - }, - tag => {type => SCALAR|ARRAYREF, - optional => 1, - }, - archive => {type => BOOLEAN, - default => 0, - }, - owner => {type => SCALAR|ARRAYREF, - optional => 1, - }, - src => {type => SCALAR|ARRAYREF, - optional => 1, - }, - maint => {type => SCALAR|ARRAYREF, - optional => 1, - }, - bugs => {type => SCALAR|ARRAYREF, - optional => 1, - }, - correspondent => {type => SCALAR|ARRAYREF, - optional => 1, - }, - usertags => {type => HASHREF, - optional => 1, - }, - }, + spec => $_get_bugs_by_idx_options ); my %bugs = (); + # If we're given an empty maint (unmaintained packages), we can't + # handle it, so bail out here + for my $maint (make_list(exists $param{maint}?$param{maint}:[])) { + if (defined $maint and $maint eq '') { + die "Can't handle empty maint (unmaintained packages) in get_bugs_by_idx"; + } + } + if ($param{newest}) { + my $newest_bug = newest_bug(); + my @bugs = ($newest_bug - max(make_list($param{newest})) + 1) .. $newest_bug; + $param{bugs} = [exists $param{bugs}?make_list($param{bugs}):(), + @bugs, + ]; + } # We handle src packages, maint and maintenc by mapping to the # appropriate binary packages, then removing all packages which # don't match all queries @@ -434,11 +442,11 @@ sub get_bugs_by_idx{ delete @param{qw(maint src)}; $param{package} = [@packages]; } - my $keys = grep {$_ !~ /^(archive|usertags|bugs)$/} keys(%param); + my $keys = grep {$_ !~ $_non_search_key_regex} keys(%param); die "Need at least 1 key to search by" unless $keys; my $arc = $param{archive} ? '-arc':''; my %idx; - for my $key (grep {$_ !~ /^(archive|usertags|bugs)$/} keys %param) { + for my $key (grep {$_ !~ $_non_search_key_regex} keys %param) { my $index = $key; $index = 'submitter-email' if $key eq 'submitter'; $index = "$config{spool_dir}/by-${index}${arc}.idx"; @@ -481,6 +489,199 @@ sub get_bugs_by_idx{ } +=head2 get_bugs_by_db + +This routine uses the database to try to speed up +searches. + + +=cut + +state $_get_bugs_by_db_options = + {hash_slice(%_get_bugs_common_options, + (qw(package submitter severity tag archive), + qw(owner src maint bugs correspondent), + qw(affects usertags newest)) + ), + schema => {type => OBJECT, + }, + }; +sub get_bugs_by_db{ + my %param = validate_with(params => \@_, + spec => $_get_bugs_by_db_options, + ); + my %bugs = (); + + my $s = $param{schema}; + my $keys = grep {$_ !~ $_non_search_key_regex} keys(%param); + die "Need at least 1 key to search by" unless $keys; + my $rs = $s->resultset('Bug'); + if (exists $param{severity}) { + $rs = $rs->search({'severity.severity' => + [make_list($param{severity})], + }, + {join => 'severity'}, + ); + } + for my $key (qw(owner submitter done)) { + if (exists $param{$key}) { + $rs = $rs->search({"${key}.addr" => + [make_list($param{$key})], + }, + {join => $key}, + ); + } + } + if (exists $param{newest}) { + $rs = + $rs->search({}, + {order_by => {-desc => 'me.creation'}, + rows => max(make_list($param{newest})), + }, + ); + } + if (exists $param{correspondent}) { + my $message_rs = + $s->resultset('Message')-> + search({'correspondent.addr' => + [make_list($param{correspondent})], + }, + {join => {message_correspondents => 'correspondent'}, + columns => ['id'], + group_by => ['me.id'], + }, + ); + $rs = $rs->search({'bug_messages.message' => + {-in => $message_rs->get_column('id')->as_query()}, + }, + {join => 'bug_messages', + }, + ); + } + if (exists $param{affects}) { + my @aff_list = make_list($param{affects}); + $rs = $rs->search({-or => {'bin_pkg.pkg' => + [@aff_list], + 'src_pkg.pkg' => + [@aff_list], + 'me.unknown_affects' => + [@aff_list] + }, + }, + {join => [{bug_affects_binpackages => 'bin_pkg'}, + {bug_affects_srcpackages => 'src_pkg'}, + ], + }, + ); + } + if (exists $param{package}) { + $rs = $rs->search({-or => {'bin_pkg.pkg' => + [make_list($param{package})], + 'me.unknown.package' => + [make_list($param{package})]}, + }, + {join => {bug_binpackages => 'bin_pkg'}}); + } + if (exists $param{maint}) { + my @maint_list = + map {$_ eq '' ? undef : $_} + make_list($param{maint}); + my $bin_pkgs_rs = + $s->resultset('BinPkg')-> + search({'correspondent.addr' => [@maint_list]}, + {join => {bin_vers => + {src_ver => + {maintainer => 'correspondent'}}}, + columns => ['id'], + group_by => ['me.id'], + }, + ); + my $src_pkgs_rs = + $s->resultset('SrcPkg')-> + search({'correspondent.addr' => [@maint_list]}, + {join => {src_vers => + {maintainer => 'correspondent'}}, + columns => ['id'], + group_by => ['me.id'], + }, + ); + $rs = $rs->search({-or => {'bug_binpackages.bin_pkg' => + { -in => $bin_pkgs_rs->get_column('id')->as_query}, + 'bug_srcpackages.src_pkg' => + { -in => $bin_pkgs_rs->get_column('id')->as_query}, + }, + }, + {join => ['bug_binpackages', + 'bug_srcpackages', + ]} + ); + } + if (exists $param{src}) { + # identify all of the srcpackages and binpackages that match first + my $src_pkgs_rs = + $s->resultset('SrcPkg')-> + search({-or => [map {('me.pkg' => $_, + )} + make_list($param{src})], + columns => ['id'], + group_by => ['me.id'], + }, + ); + my $bin_pkgs_rs = + $s->resultset('BinPkg')-> + search({-or => [map {('src_pkg.pkg' => $_, + )} + make_list($param{src})], + }, + {join => {bin_vers => {src_ver => 'src_pkg'}}, + columns => ['id'], + group_by => ['me.id'], + }); + $rs = $rs->search({-or => {'bug_binpackages.bin_pkg' => + { -in => $bin_pkgs_rs->get_column('id')->as_query}, + 'bug_srcpackages.src_pkg' => + { -in => $bin_pkgs_rs->get_column('id')->as_query}, + 'me.unknown_package' => + [make_list($param{src})], + }, + }, + {join => ['bug_binpackages', + 'bug_srcpackages', + ]} + ); + } + # tags are very odd, because we must handle usertags. + if (exists $param{tag}) { + # bugs from usertags which matter + my %bugs_matching_usertags; + for my $bug (make_list(grep {defined $_ } + @{$param{usertags}}{make_list($param{tag})})) { + $bugs_matching_usertags{$bug} = 1; + } + # we want all bugs which either match the tag name given in + # param, or have a usertag set which matches one of the tag + # names given in param. + $rs = $rs->search({-or => {map {('tag.tag' => $_)} + make_list($param{tag}), + map {('me.id' => $_)} + keys %bugs_matching_usertags + }, + }, + {join => {bug_tags => 'tag'}}); + } + if (exists $param{bugs}) { + $rs = $rs->search({-or => {map {('me.id' => $_)} + make_list($param{bugs})} + }); + } + # handle archive + if (defined $param{archive} and $param{archive} ne 'both') { + $rs = $rs->search({'me.archived' => $param{archive}}); + } + return $rs->get_column('id')->all(); +} + + =head2 get_bugs_flatfile This is the fallback search routine. It should be able to complete all @@ -488,51 +689,24 @@ searches. [Or at least, that's the idea.] =cut +state $_get_bugs_flatfile_options = + {hash_slice(%_get_bugs_common_options, + map {$_ eq 'dist'?():($_)} keys %_get_bugs_common_options + ) + }; + sub get_bugs_flatfile{ my %param = validate_with(params => \@_, - spec => {package => {type => SCALAR|ARRAYREF, - optional => 1, - }, - src => {type => SCALAR|ARRAYREF, - optional => 1, - }, - maint => {type => SCALAR|ARRAYREF, - optional => 1, - }, - submitter => {type => SCALAR|ARRAYREF, - optional => 1, - }, - severity => {type => SCALAR|ARRAYREF, - optional => 1, - }, - status => {type => SCALAR|ARRAYREF, - optional => 1, - }, - tag => {type => SCALAR|ARRAYREF, - optional => 1, - }, - owner => {type => SCALAR|ARRAYREF, - optional => 1, - }, - correspondent => {type => SCALAR|ARRAYREF, - optional => 1, - }, -# not yet supported -# dist => {type => SCALAR|ARRAYREF, -# optional => 1, -# }, - archive => {type => BOOLEAN, - default => 1, - }, - usertags => {type => HASHREF, - optional => 1, - }, - function => {type => CODEREF, - optional => 1, - }, - }, + spec => $_get_bugs_flatfile_options ); my $flatfile; + if ($param{newest}) { + my $newest_bug = newest_bug(); + my @bugs = ($newest_bug - max(make_list($param{newest})) + 1) .. $newest_bug; + $param{bugs} = [exists $param{bugs}?make_list($param{bugs}):(), + @bugs, + ]; + } if ($param{archive}) { $flatfile = IO::File->new("$config{spool_dir}/index.archive", 'r') or die "Unable to open $config{spool_dir}/index.archive for reading: $!"; @@ -548,6 +722,30 @@ sub get_bugs_flatfile{ @usertag_bugs{make_list(@{$param{usertags}}{make_list($param{tag})}) } = (1) x make_list(@{$param{usertags}}{make_list($param{tag})}); } + my $unmaintained_packages = 0; + # unmaintained packages is a special case + my @maints = make_list(exists $param{maint}?$param{maint}:[]); + $param{maint} = []; + for my $maint (@maints) { + if (defined $maint and $maint eq '' and not $unmaintained_packages) { + $unmaintained_packages = 1; + our %maintainers = %{getmaintainers()}; + $param{function} = [(exists $param{function}? + (ref $param{function}?@{$param{function}}:$param{function}):()), + sub {my %d=@_; + foreach my $try (make_list($d{"pkg"})) { + next unless length $try; + ($try) = $try =~ m/^(?:src:)?(.+)/; + return 1 if not exists $maintainers{$try}; + } + return 0; + } + ]; + } + elsif (defined $maint and $maint ne '') { + push @{$param{maint}},$maint; + } + } # We handle src packages, maint and maintenc by mapping to the # appropriate binary packages, then removing all packages which # don't match all queries @@ -558,7 +756,7 @@ sub get_bugs_flatfile{ exists $param{src} or exists $param{maint}) { delete @param{qw(maint src)}; - $param{package} = [@packages]; + $param{package} = [@packages] if @packages; } my $grep_bugs = 0; my %bugs; @@ -566,15 +764,18 @@ sub get_bugs_flatfile{ $bugs{$_} = 1 for make_list($param{bugs}); $grep_bugs = 1; } - if (exists $param{owner} or exists $param{correspondent}) { - $bugs{$_} = 1 for get_bugs_by_idx(exists $param{correspondent}?(correspondent => $param{correspondent}):(), - exists $param{owner}?(owner => $param{owner}):(), + # These queries have to be handled by get_bugs_by_idx + if (exists $param{owner} + or exists $param{correspondent} + or exists $param{affects}) { + $bugs{$_} = 1 for get_bugs_by_idx(map {exists $param{$_}?($_,$param{$_}):()} + qw(owner correspondent affects), ); $grep_bugs = 1; } my @bugs; - while (<$flatfile>) { - next unless m/^(\S+)\s+(\d+)\s+(\d+)\s+(\S+)\s+\[\s*([^]]*)\s*\]\s+(\w+)\s+(.*)$/; + BUG: while (<$flatfile>) { + next unless m/^(\S+)\s+(\d+)\s+(\d+)\s+(\S+)\s+\[\s*(.*)\s*\]\s+(\w+)\s+(.*)$/; my ($pkg,$bug,$time,$status,$submitter,$severity,$tags) = ($1,$2,$3,$4,$5,$6,$7); next if $grep_bugs and not exists $bugs{$bug}; if (exists $param{package}) { @@ -617,14 +818,16 @@ sub get_bugs_flatfile{ my @bug_tags = split ' ', $tags; my @packages = splitpackages($pkg); my $package = (@packages > 1)?\@packages:$packages[0]; - next unless - $param{function}->(pkg => $package, - bug => $bug, - status => $status, - submitter => $submitter, - severity => $severity, - tags => \@bug_tags, - ); + for my $function (make_list($param{function})) { + next BUG unless + $function->(pkg => $package, + bug => $bug, + status => $status, + submitter => $submitter, + severity => $severity, + tags => \@bug_tags, + ); + } } push @bugs, $bug; } @@ -667,18 +870,34 @@ sub __handle_pkg_src_and_maint{ # We only want to increment the number of keys if there is # something to match my $key_inc = 0; - for my $package ((map { getsrcpkgs($_)} make_list($param{src})),make_list($param{src})) { - $packages{$package}++; + # in case there are binaries with the same name as the + # source + my %_temp_p = (); + for my $package ((map {getsrcpkgs($_)} make_list($param{src}))) { + $packages{$package}++ unless exists $_temp_p{$package}; + $_temp_p{$package} = 1; $key_inc=1; } + for my $package (make_list($param{src})) { + $packages{"src:$package"}++ unless exists $_temp_p{"src:$package"}; + $_temp_p{"src:$package"} = 1; + $key_inc=1; + # As a temporary hack, we will also include $param{src} + # in this list for packages passed which do not have a + # corresponding binary package + if (not exists getpkgsrc()->{$package}) { + $packages{$package}++ unless exists $_temp_p{$package}; + $_temp_p{$package} = 1; + } + } $package_keys += $key_inc; } if (exists $param{maint}) { my $key_inc = 0; - my $maint_rev = getmaintainers_reverse(); - for my $package (map { exists $maint_rev->{$_}?@{$maint_rev->{$_}}:()} - make_list($param{maint})) { - $packages{$package}++; + my %_temp_p = (); + for my $package (package_maintainer(maintainer=>$param{maint})) { + $packages{$package}++ unless exists $_temp_p{$package}; + $_temp_p{$package} = 1; $key_inc = 1; } $package_keys += $key_inc; @@ -686,7 +905,7 @@ sub __handle_pkg_src_and_maint{ return grep {$packages{$_} >= $package_keys} keys %packages; } -my %field_match = ( +state $field_match = { 'subject' => \&__contains_field_match, 'tags' => sub { my ($field, $values, $status) = @_; @@ -698,16 +917,21 @@ my %field_match = ( }, 'severity' => \&__exact_field_match, 'pending' => \&__exact_field_match, + 'package' => \&__exact_field_match, 'originator' => \&__contains_field_match, 'forwarded' => \&__contains_field_match, 'owner' => \&__contains_field_match, -); +}; sub __bug_matches { my ($hash, $status) = @_; foreach my $key( keys( %$hash ) ) { my $value = $hash->{$key}; - my $sub = $field_match{$key}; + next unless exists $field_match->{$key}; + my $sub = $field_match->{$key}; + if (not defined $sub) { + die "No defined subroutine for key: $key"; + } return 1 if ($sub->($key, $value, $status)); } return 0;