X-Git-Url: https://git.donarmstrong.com/?a=blobdiff_plain;f=bin%2Fdebbugs-loadsql;h=fe9ce6b0177200f93940143fa34c52eb8f4b4523;hb=a38d37cc8d08e6ea3ff089195f9028f7a5305939;hp=7dd652783b15ef54d28302120f0a88aa2ab6498d;hpb=4a79438a7f45264f948d87d0c7cb819886300439;p=debbugs.git diff --git a/bin/debbugs-loadsql b/bin/debbugs-loadsql index 7dd6527..fe9ce6b 100755 --- a/bin/debbugs-loadsql +++ b/bin/debbugs-loadsql @@ -131,7 +131,17 @@ Display this manual. use vars qw($DEBUG); -use Debbugs::Common (qw(checkpid lockpid get_hashname getparsedaddrs getbugcomponent make_list getsourcemaintainers), +# if we're running out of git, we want to use the git base directory as the +# first INC directory. If you're not running out of git, or someone has given a +# non-absolute INC, don't do that. +use FindBin; +use if (-d $FindBin::Bin.'/../.git/' && $INC[0] =~ m#^/#), + lib => $FindBin::Bin.'/../'; + +use Debbugs::Common (qw(checkpid lockpid get_hashname getparsedaddrs), + qw(getbugcomponent make_list getsourcemaintainers), + qw(getbuglocation), + qw(walk_bugs), qw(hash_slice open_compressed_file),); use Debbugs::Config qw(:config); use Debbugs::Status qw(read_bug split_status_fields); @@ -147,7 +157,7 @@ use IO::Dir; use IO::File; use IO::Uncompress::AnyUncompress; use Encode qw(decode_utf8); -use List::MoreUtils qw(natatime); +use List::AllUtils qw(natatime); my %options = (debug => 0, @@ -180,7 +190,9 @@ $DEBUG = $options{debug}; my %subcommands = ('bugs' => {function => \&add_bugs, - arguments => {'preload' => 0}, + arguments => {'preload' => 0, + 'bugs=s@' => 0, + }, }, 'versions' => {function => \&add_versions, }, @@ -263,11 +275,14 @@ sub add_bugs { my $verbose = $options->{debug}; - my $initialdir = "db-h"; - - if (defined $argv->[0] and $argv->[0] eq "archive") { - $initialdir = "archive"; + my @dirs; + if (@{$argv}) { + @dirs = @{$argv}; + } elsif (not defined $opts->{bugs}) { + @dirs = "db-h" } + $opts->{bugs} //= []; + my $s = db_connect($options); @@ -277,27 +292,26 @@ sub add_bugs { if ($opts->{preload}) { my @bugs; - walk_bugs([(@{$argv}?@{$argv} : $initialdir)], - undef, - 'summary', - undef, - sub { - push @bugs,@_; + walk_bugs(dirs => [@dirs], + bugs => $opts->{bugs}, + callback => sub { + push @bugs,@_; }, - 10000 + bugs_per_call => 10000 ); $s->resultset('Bug')->quick_insert_bugs(@bugs); } - walk_bugs([(@{$argv}?@{$argv} : $initialdir)], - $p, - 'summary', - $verbose, - sub { + walk_bugs(dirs => [@dirs], + bugs => $opts->{bugs}, + progress_bar => $p, + $verbose?(logging=>\*STDERR):(), + callback => + sub { my @bugs = @_; my @bugs_to_update; if ($options{quick}) { @bugs_to_update = - bugs_to_update($s,$initialdir,@bugs); + bugs_to_update($s,@bugs); } else { @bugs_to_update = @bugs; } @@ -316,7 +330,7 @@ sub add_bugs { die "failure while trying to load bug: $@"; } }, - 50 + bugs_per_call => 50 ); handle_load_bug_queue(db => $s, queue => \%queue); @@ -348,7 +362,7 @@ sub add_versions { if (not defined $src_pkgs{$versions[$i][0]}) { $src_pkgs{$versions[$i][0]} = $s->resultset('SrcPkg')-> - get_src_pkg_id($versions[$i][0]); + get_or_create_src_pkg_id($versions[$i][0]); } $sp = $src_pkgs{$versions[$i][0]}; # There's probably something wrong if the source package @@ -399,11 +413,12 @@ sub add_debinfo { while (my @v = $it->()) { my %cache; my @debinfos; - for my $file (@v) { +FILE: for my $file (@v) { my $fh = IO::File->new($file,'r') or die "Unable to open $file for reading: $!"; my $f_stat = stat($file); my $ct_date = DateTime->from_epoch(epoch => $f_stat->ctime); + my @file_debinfos; while (<$fh>) { chomp; next unless length $_; @@ -414,15 +429,29 @@ sub add_debinfo { # the file, so we'll fudge it from the filename. if (not defined $srcver) { ($srcname,$srcver) = ($binarch,$srcname); - ($binarch) = $file =~ /_([^\.]+)\.debinfo/; + ($binarch) = $file =~ /_([a-z0-9-]+)\.debinfo/; } - if (not defined $srcver) { - print STDERR "malformed debinfo (no srcver): $file\n"; - next; + # It turns out that there are debinfo files which are horribly + # screwed up, and have junk in them. We need to discard them + # completely + if (not defined $srcname or + not defined $srcver or + not defined $binname or + not defined $binver or + not defined $binarch or + $srcname !~ /^$config{package_name_re}$/o or + $binname !~ /^$config{package_name_re}$/o or + $srcver !~ /^$config{package_version_re}$/o or + $binver !~ /^$config{package_version_re}$/o + ) { + print STDERR "malformed debinfo: $file\n$_\n"; + next FILE; } - push @debinfos, + push @file_debinfos, [$binname,$binver,$binarch,$srcname,$srcver,$ct_date]; } + push @debinfos, + @file_debinfos; } $s->txn_do( sub { @@ -550,11 +579,12 @@ sub add_logs { } my $s = db_connect($options); - walk_bugs([(@{$argv}?@{$argv} : $initialdir)], - $p, - 'log', - $verbose, - sub { + walk_bugs(dirs => [(@{$argv}?@{$argv} : $initialdir)], + progress_bar => $p, + bug_file => 'log', + $verbose?(logging => \*STDERR):(), + callback => + sub { my $bug = shift; my $stat = stat(getbugcomponent($bug,'log',$initialdir)); if (not defined $stat) { @@ -596,16 +626,16 @@ sub add_bugs_and_logs { my %severities; my %queue; - walk_bugs([(@{$argv}?@{$argv} : $initialdir)], - $p, - 'summary', - $verbose, - sub { + walk_bugs(dirs => [(@{$argv}?@{$argv} : $initialdir)], + progress_bar => $p, + $verbose?(logging => \*STDERR):(), + callback => + sub { my @bugs = @_; my @bugs_to_update; if ($options{quick}) { @bugs_to_update = - bugs_to_update($s,$initialdir,@bugs); + bugs_to_update($s,@bugs); } else { @bugs_to_update = @bugs; } @@ -644,7 +674,7 @@ sub add_bugs_and_logs { } } }, - 50 + bugs_per_call => 50, ); handle_load_bug_queue(db=>$s, queue => \%queue, @@ -794,51 +824,12 @@ sub read_release_file { return (\%dist_info,\%p_f); } -sub walk_bugs { - my ($dirs,$p,$what,$verbose,$sub,$n) = @_; - my @dirs = @{$dirs}; - my $tot_dirs = @dirs; - my $done_dirs = 0; - my $avg_subfiles = 0; - my $completed_files = 0; - $n //= 1; - while (my $dir = shift @dirs) { - printf "Doing dir %s ...\n", $dir if $verbose; - - opendir(DIR, "$dir/.") or die "opendir $dir: $!"; - my @subdirs = readdir(DIR); - closedir(DIR); - - my @list = map { m/^(\d+)\.$what$/?($1):() } @subdirs; - $tot_dirs -= @dirs; - push @dirs, map { m/^(\d+)$/ && -d "$dir/$1"?("$dir/$1"):() } @subdirs; - $tot_dirs += @dirs; - if ($avg_subfiles == 0) { - $avg_subfiles = @list; - } - - $p->target($avg_subfiles*($tot_dirs-$done_dirs)+$completed_files+@list) if $p; - $avg_subfiles = ($avg_subfiles * $done_dirs + @list) / ($done_dirs+1); - $done_dirs += 1; - - my $it = natatime $n,@list; - while (my @bugs = $it->()) { - $sub->(@bugs); - $completed_files += scalar @bugs; - $p->update($completed_files) if $p; - print "Up to $completed_files bugs...\n" - if ($completed_files % 100 == 0 && $verbose); - } - } - $p->remove() if $p; -} - sub bugs_to_update { - my ($s,$initialdir,@bugs) = @_; + my ($s,@bugs) = @_; my @bugs_to_update; for my $bug (@bugs) { - my $stat = stat(getbugcomponent($bug,'summary',$initialdir)); + my $stat = stat(getbugcomponent($bug,'summary',getbuglocation($bug,'summary'))); if (not defined $stat) { print STDERR "Unable to stat $bug $!\n"; next;