From: Don Armstrong Date: Tue, 10 Apr 2018 22:40:26 +0000 (-0700) Subject: move walk_bugs to Debbugs::Common; add bugs option to loadsql X-Git-Url: https://git.donarmstrong.com/?a=commitdiff_plain;h=4c3eca26cf5d3f8173ff55181a532c5b5f5c0bd9;p=debbugs.git move walk_bugs to Debbugs::Common; add bugs option to loadsql - walk_bugs is now in Debbugs::Common with a params_validate interface - bugs now takes a --bugs option in addition to dirs on the command Lines --- diff --git a/Debbugs/Common.pm b/Debbugs/Common.pm index cec2c80..066e965 100644 --- a/Debbugs/Common.pm +++ b/Debbugs/Common.pm @@ -32,6 +32,7 @@ use warnings; use strict; use vars qw($VERSION $DEBUG %EXPORT_TAGS @EXPORT_OK @EXPORT); use Exporter qw(import); +use v5.10; BEGIN{ $VERSION = 1.00; @@ -47,6 +48,7 @@ BEGIN{ qw(package_maintainer), qw(sort_versions), qw(open_compressed_file), + qw(walk_bugs), ], misc => [qw(make_list globify_scalar english_join checkpid), qw(cleanup_eval_fail), @@ -78,6 +80,7 @@ use File::Path qw(mkpath); use File::Basename qw(dirname); use MLDBM qw(DB_File Storable); $MLDBM::DumpMeth='portable'; +use List::AllUtils qw(natatime); use Params::Validate qw(validate_with :types); @@ -280,6 +283,117 @@ sub open_compressed_file { return $fh; } +=head2 walk_bugs + +Walk through directories of bugs, calling a subroutine with a list of bugs +found. + +C sub {print map {qq($_\n)} @_},dirs => [qw(db-h)];> + +=over + +=item callback -- CODEREF of a subroutine to call with a list of bugs + +=item dirs -- ARRAYREF of directories to get bugs from. Like C<[qw(db-h archive)]>. + +=item bugs -- ARRAYREF of bugs to walk through. If both C and C are +provided, both are walked through. + +=item bugs_per_call -- maximum number of bugs to provide to callback + +=item progress_bar -- optional L + +=item bug_file -- bug file to look for (generally C) + +=item logging -- optional filehandle to output logging information + +=back + +=cut + +sub walk_bugs { + state $spec = + {dirs => {type => ARRAYREF, + default => [], + }, + bugs => {type => ARRAYREF, + default => [], + }, + progress_bar => {type => OBJECT|UNDEF, + optional => 1, + }, + bug_file => {type => SCALAR, + default => 'summary', + }, + logging => {type => HANDLE, + optional => 1, + }, + callback => {type => CODEREF, + }, + bugs_per_call => {type => SCALAR, + default => 1, + }, + }; + my %param = validate_with(params => \@_, + spec => $spec + ); + my @dirs = @{$param{dirs}}; + my @initial_bugs = (); + if (@{$param{bugs}}) { + unshift @dirs,''; + @initial_bugs = @{$param{bugs}}; + } + my $tot_dirs = @dirs; + my $done_dirs = 0; + my $avg_subfiles = 0; + my $completed_files = 0; + my $dir; + while ($dir = shift @dirs or defined $dir) { + my @list; + my @subdirs; + if (not length $dir and @initial_bugs) { + push @list,@initial_bugs; + @initial_bugs = (); + } else { + printf {$param{verbose}} "Doing dir %s ...\n", $dir + if defined $param{verbose}; + opendir(my $DIR, "$dir/.") or + die "opendir $dir: $!"; + @subdirs = readdir($DIR) or + die "Unable to readdir $dir: $!"; + closedir($DIR) or + die "Unable to closedir $dir: $!"; + + @list = map { m/^(\d+)\.$param{bug_file}$/?($1):() } @subdirs; + } + $tot_dirs -= @dirs; + push @dirs, map { m/^(\d+)$/ && -d "$dir/$1"?("$dir/$1"):() } @subdirs; + $tot_dirs += @dirs; + if ($param{progress_bar}) { + if ($avg_subfiles == 0) { + $avg_subfiles = @list; + } + $param{progress_bar}-> + target($avg_subfiles*($tot_dirs-$done_dirs)+$completed_files+@list); + $avg_subfiles = ($avg_subfiles * $done_dirs + @list) / ($done_dirs+1); + $done_dirs += 1; + } + + my $it = natatime $param{bugs_per_call},@list; + while (my @bugs = $it->()) { + $param{callback}->(@bugs); + $completed_files += scalar @bugs; + if ($param{progress_bar}) { + $param{progress_bar}->update($completed_files) if $param{progress_bar}; + } + if ($completed_files % 100 == 0 and + defined $param{verbose}) { + print {$param{verbose}} "Up to $completed_files bugs...\n" + } + } + } + $param{progress_bar}->remove() if $param{progress_bar}; +} =head2 getparsedaddrs diff --git a/bin/debbugs-loadsql b/bin/debbugs-loadsql index df17814..56c4f91 100755 --- a/bin/debbugs-loadsql +++ b/bin/debbugs-loadsql @@ -131,7 +131,9 @@ Display this manual. use vars qw($DEBUG); -use Debbugs::Common (qw(checkpid lockpid get_hashname getparsedaddrs getbugcomponent make_list getsourcemaintainers), +use Debbugs::Common (qw(checkpid lockpid get_hashname getparsedaddrs), + qw(getbugcomponent make_list getsourcemaintainers), + qw(walk_bugs), qw(hash_slice open_compressed_file),); use Debbugs::Config qw(:config); use Debbugs::Status qw(read_bug split_status_fields); @@ -180,7 +182,9 @@ $DEBUG = $options{debug}; my %subcommands = ('bugs' => {function => \&add_bugs, - arguments => {'preload' => 0}, + arguments => {'preload' => 0, + 'bugs=s@' => 0, + }, }, 'versions' => {function => \&add_versions, }, @@ -263,11 +267,14 @@ sub add_bugs { my $verbose = $options->{debug}; - my $initialdir = "db-h"; - - if (defined $argv->[0] and $argv->[0] eq "archive") { - $initialdir = "archive"; + my @dirs; + if (@{$argv}) { + @dirs = @{$argv}; + } elsif (not defined $opts->{bugs}) { + @dirs = "db-h" } + $opts->{bugs} //= []; + my $s = db_connect($options); @@ -277,27 +284,26 @@ sub add_bugs { if ($opts->{preload}) { my @bugs; - walk_bugs([(@{$argv}?@{$argv} : $initialdir)], - undef, - 'summary', - undef, - sub { - push @bugs,@_; + walk_bugs(dirs => [@dirs], + bugs => $opts->{bugs}, + callback => sub { + push @bugs,@_; }, - 10000 + bugs_per_call => 10000 ); $s->resultset('Bug')->quick_insert_bugs(@bugs); } - walk_bugs([(@{$argv}?@{$argv} : $initialdir)], - $p, - 'summary', - $verbose, - sub { + walk_bugs(dirs => [@dirs], + bugs => $opts->{bugs}, + progress_bar => $p, + $verbose?(logging=>\*STDERR):(), + callback => + sub { my @bugs = @_; my @bugs_to_update; if ($options{quick}) { @bugs_to_update = - bugs_to_update($s,$initialdir,@bugs); + bugs_to_update($s,@bugs); } else { @bugs_to_update = @bugs; } @@ -316,7 +322,7 @@ sub add_bugs { die "failure while trying to load bug: $@"; } }, - 50 + bugs_per_call => 50 ); handle_load_bug_queue(db => $s, queue => \%queue); @@ -564,11 +570,12 @@ sub add_logs { } my $s = db_connect($options); - walk_bugs([(@{$argv}?@{$argv} : $initialdir)], - $p, - 'log', - $verbose, - sub { + walk_bugs(dirs => [(@{$argv}?@{$argv} : $initialdir)], + progress_bar => $p, + bug_file => 'log', + $verbose?(logging => \*STDERR):(), + callback => + sub { my $bug = shift; my $stat = stat(getbugcomponent($bug,'log',$initialdir)); if (not defined $stat) { @@ -610,16 +617,16 @@ sub add_bugs_and_logs { my %severities; my %queue; - walk_bugs([(@{$argv}?@{$argv} : $initialdir)], - $p, - 'summary', - $verbose, - sub { + walk_bugs(dirs => [(@{$argv}?@{$argv} : $initialdir)], + progress_bar => $p, + $verbose?(logging => \*STDERR):(), + callback => + sub { my @bugs = @_; my @bugs_to_update; if ($options{quick}) { @bugs_to_update = - bugs_to_update($s,$initialdir,@bugs); + bugs_to_update($s,@bugs); } else { @bugs_to_update = @bugs; } @@ -658,7 +665,7 @@ sub add_bugs_and_logs { } } }, - 50 + bugs_per_call => 50, ); handle_load_bug_queue(db=>$s, queue => \%queue, @@ -808,51 +815,12 @@ sub read_release_file { return (\%dist_info,\%p_f); } -sub walk_bugs { - my ($dirs,$p,$what,$verbose,$sub,$n) = @_; - my @dirs = @{$dirs}; - my $tot_dirs = @dirs; - my $done_dirs = 0; - my $avg_subfiles = 0; - my $completed_files = 0; - $n //= 1; - while (my $dir = shift @dirs) { - printf "Doing dir %s ...\n", $dir if $verbose; - - opendir(DIR, "$dir/.") or die "opendir $dir: $!"; - my @subdirs = readdir(DIR); - closedir(DIR); - - my @list = map { m/^(\d+)\.$what$/?($1):() } @subdirs; - $tot_dirs -= @dirs; - push @dirs, map { m/^(\d+)$/ && -d "$dir/$1"?("$dir/$1"):() } @subdirs; - $tot_dirs += @dirs; - if ($avg_subfiles == 0) { - $avg_subfiles = @list; - } - - $p->target($avg_subfiles*($tot_dirs-$done_dirs)+$completed_files+@list) if $p; - $avg_subfiles = ($avg_subfiles * $done_dirs + @list) / ($done_dirs+1); - $done_dirs += 1; - - my $it = natatime $n,@list; - while (my @bugs = $it->()) { - $sub->(@bugs); - $completed_files += scalar @bugs; - $p->update($completed_files) if $p; - print "Up to $completed_files bugs...\n" - if ($completed_files % 100 == 0 && $verbose); - } - } - $p->remove() if $p; -} - sub bugs_to_update { - my ($s,$initialdir,@bugs) = @_; + my ($s,@bugs) = @_; my @bugs_to_update; for my $bug (@bugs) { - my $stat = stat(getbugcomponent($bug,'summary',$initialdir)); + my $stat = stat(getbugcomponent($bug,'summary',getbuglocation($bug,'summary'))); if (not defined $stat) { print STDERR "Unable to stat $bug $!\n"; next;