X-Git-Url: https://git.donarmstrong.com/?a=blobdiff_plain;f=bin%2Fdebbugs-loadsql;h=e639e0ee27793438f82ffaab54d9671fc847be2d;hb=refs%2Fheads%2Fdatabase;hp=302d3d649f8d0880be1d5af7091d00af14c8b0b7;hpb=44f9d39d67b0bc749038757e9a6291133ba266f6;p=debbugs.git diff --git a/bin/debbugs-loadsql b/bin/debbugs-loadsql index 302d3d6..e639e0e 100755 --- a/bin/debbugs-loadsql +++ b/bin/debbugs-loadsql @@ -142,6 +142,7 @@ use DateTime; use File::stat; use File::Basename; use File::Spec; +use File::Find; use IO::Dir; use IO::File; use IO::Uncompress::AnyUncompress; @@ -163,6 +164,7 @@ Getopt::Long::Configure('pass_through'); GetOptions(\%options, 'quick|q', 'service|s=s', + 'dsn=s', 'sysconfdir|c=s', 'progress!', 'spool_dir|spool-dir=s', @@ -183,7 +185,9 @@ my %subcommands = 'versions' => {function => \&add_versions, }, 'debinfo' => {function => \&add_debinfo, - arguments => {'0|null' => 0}, + arguments => {'0|null' => 0, + 'debinfo_dir|debinfo-dir=s' => 0, + }, }, 'maintainers' => {function => \&add_maintainers, }, @@ -195,6 +199,8 @@ my %subcommands = }, 'logs' => {function => \&add_logs, }, + 'bugs_and_logs' => {function => \&add_bugs_and_logs, + }, 'packages' => {function => \&add_packages, arguments => {'ftpdists=s' => 1, 'suites=s@' => 0, @@ -265,8 +271,6 @@ sub add_bugs { my $s = db_connect($options); - my $time = 0; - my $start_time = time; my %tags; my %severities; my %queue; @@ -292,18 +296,10 @@ sub add_bugs { my @bugs = @_; my @bugs_to_update; if ($options{quick}) { - for my $bug (@bugs) { - my $stat = stat(getbugcomponent($bug,'summary',$initialdir)); - if (not defined $stat) { - print STDERR "Unable to stat $bug $!\n"; - next; - } - my $rs = $s->resultset('Bug')->search({id=>$bug})->single(); - next if defined $rs and $stat->mtime <= $rs->last_modified()->epoch(); - push @bugs_to_update, $bug; - } + @bugs_to_update = + bugs_to_update($s,$initialdir,@bugs); } else { - @bugs_to_update = @bugs; + @bugs_to_update = @bugs; } eval { $s->txn_do(sub { @@ -357,10 +353,11 @@ sub add_versions { $sp = $src_pkgs{$versions[$i][0]}; # There's probably something wrong if the source package # doesn't exist, but we'll skip it for now - next unless defined $sp; + last if not defined $sp; my $sv = $s->resultset('SrcVer')->find({src_pkg=>$sp, ver => $versions[$i][1], }); + last if not defined $sv; if (defined $ancestor_sv and defined $sv and not defined $sv->based_on()) { $sv->update({based_on => $ancestor_sv}) } @@ -375,6 +372,15 @@ sub add_debinfo { my ($options,$opts,$p,$config,$argv) = @_; my @files = @{$argv}; + if (exists $opts->{debinfo_dir} and not @files) { + find(sub { + if (-f $_ and /\.debinfo$/) { + push @files, $File::Find::name; + } + }, + $opts->{debinfo_dir} + ); + } if (not @files) { { local $/ = "\n"; @@ -393,11 +399,12 @@ sub add_debinfo { while (my @v = $it->()) { my %cache; my @debinfos; - for my $file (@v) { +FILE: for my $file (@v) { my $fh = IO::File->new($file,'r') or die "Unable to open $file for reading: $!"; my $f_stat = stat($file); my $ct_date = DateTime->from_epoch(epoch => $f_stat->ctime); + my @file_debinfos; while (<$fh>) { chomp; next unless length $_; @@ -408,15 +415,28 @@ sub add_debinfo { # the file, so we'll fudge it from the filename. if (not defined $srcver) { ($srcname,$srcver) = ($binarch,$srcname); - ($binarch) = $file =~ /_([^\.]+)\.debinfo/; + ($binarch) = $file =~ /_([a-z0-9-]+)\.debinfo/; } - if (not defined $srcver) { - print STDERR "malformed debinfo (no srcver): $file\n"; - next; + # It turns out that there are debinfo files which are horribly + # screwed up, and have junk in them. We need to discard them + # completely + if (not defined $srcname or + not defined $srcver or + not defined $binname or + not defined $binver or + $srcname !~ /^$config{package_name_re}$/o or + $binname !~ /^$config{package_name_re}$/o or + $srcver !~ /^$config{package_version_re}$/o or + $binver !~ /^$config{package_version_re}$/o + ) { + print STDERR "malformed debinfo: $file\n"; + next FILE; } - push @debinfos, + push @file_debinfos, [$binname,$binver,$binarch,$srcname,$srcver,$ct_date]; } + push @debinfos, + @file_debinfos; } $s->txn_do( sub { @@ -433,12 +453,12 @@ sub add_maintainers { my ($options,$opts,$p,$config,$argv) = @_; my $s = db_connect($options); - my $maintainers = getsourcemaintainers(); + my $maintainers = getsourcemaintainers() // {}; $p->target(2) if $p; ## get all of the maintainers, and add the missing ones my $maints = $s->resultset('Maintainer')-> get_maintainers(values %{$maintainers}); - $p->update(); + $p->update() if $p; my @svs = $s->resultset('SrcVer')-> search({maintainer => undef }, @@ -544,10 +564,6 @@ sub add_logs { } my $s = db_connect($options); - - my $time = 0; - my $start_time = time; - walk_bugs([(@{$argv}?@{$argv} : $initialdir)], $p, 'log', @@ -560,8 +576,10 @@ sub add_logs { next; } if ($options{quick}) { - my $rs = $s->resultset('Bug')->search({bug=>$bug})->single(); - next if defined $rs and $stat->mtime <= $rs->last_modified()->epoch(); + my $rs = $s->resultset('Bug')-> + search({id=>$bug})->single(); + return if defined $rs and + $stat->mtime <= $rs->last_modified()->epoch(); } eval { load_bug_log(db => $s, @@ -573,6 +591,81 @@ sub add_logs { }); } +sub add_bugs_and_logs { + my ($options,$opts,$p,$config,$argv) = @_; + + chdir($config->{spool_dir}) or + die "chdir $config->{spool_dir} failed: $!"; + + my $verbose = $options->{debug}; + + my $initialdir = "db-h"; + + if (defined $argv->[0] and $argv->[0] eq "archive") { + $initialdir = "archive"; + } + my $s = db_connect($options); + + my %tags; + my %severities; + my %queue; + + walk_bugs([(@{$argv}?@{$argv} : $initialdir)], + $p, + 'summary', + $verbose, + sub { + my @bugs = @_; + my @bugs_to_update; + if ($options{quick}) { + @bugs_to_update = + bugs_to_update($s,$initialdir,@bugs); + } else { + @bugs_to_update = @bugs; + } + eval { + $s->txn_do(sub { + for my $bug (@bugs_to_update) { + load_bug(db => $s, + bug => $bug, + tags => \%tags, + severities => \%severities, + queue => \%queue); + } + }); + }; + if ($@) { + die "failure while trying to load bug: $@"; + } + for my $bug (@bugs) { + my $stat = stat(getbugcomponent($bug,'log',$initialdir)); + if (not defined $stat) { + print STDERR "Unable to stat $bug $!\n"; + next; + } + if ($options{quick}) { + my $rs = $s->resultset('Bug')-> + search({id=>$bug})->single(); + return if defined $rs and + $stat->mtime <= $rs->last_modified()->epoch(); + } + eval { + load_bug_log(db => $s, + bug => $bug); + }; + if ($@) { + die "failure while trying to load bug log $bug\n$@"; + } + } + }, + 50 + ); + handle_load_bug_queue(db=>$s, + queue => \%queue, + ); + +} + sub add_packages { my ($options,$opts,$p,$config,$argv) = @_; @@ -680,7 +773,8 @@ sub db_connect { my ($options) = @_; # connect to the database; figure out how to handle errors # properly here. - my $s = Debbugs::DB->connect($options->{service}) or + my $s = Debbugs::DB->connect($options->{dsn} // + $options->{service}) or die "Unable to connect to database: "; } @@ -754,5 +848,21 @@ sub walk_bugs { } +sub bugs_to_update { + my ($s,$initialdir,@bugs) = @_; + my @bugs_to_update; + for my $bug (@bugs) { + my $stat = stat(getbugcomponent($bug,'summary',$initialdir)); + if (not defined $stat) { + print STDERR "Unable to stat $bug $!\n"; + next; + } + my $rs = $s->resultset('Bug')->search({id=>$bug})->single(); + next if defined $rs and $stat->mtime <= $rs->last_modified()->epoch(); + push @bugs_to_update, $bug; + } + @bugs_to_update; +} + __END__