allow providing a directory of debinfo files to loadsql

[debbugs.git] / bin / debbugs-loadsql

diff --git a/bin/debbugs-loadsql b/bin/debbugs-loadsql
index f343c97eac9fde3ad8c43a63279967b0780e6e49..b718a05961c862565059a4c7d5c9e7efcd9ac924 100755 (executable)
--- a/bin/debbugs-loadsql
+++ b/bin/debbugs-loadsql
@@ -142,6 +142,7 @@ use DateTime;
 use File::stat;
 use File::Basename;
 use File::Spec;
+use File::Find;
 use IO::Dir;
 use IO::File;
 use IO::Uncompress::AnyUncompress;
@@ -163,6 +164,7 @@ Getopt::Long::Configure('pass_through');
 GetOptions(\%options,
            'quick|q',
            'service|s=s',
+          'dsn=s',
            'sysconfdir|c=s',
            'progress!',
            'spool_dir|spool-dir=s',
@@ -183,7 +185,9 @@ my %subcommands =
      'versions' => {function => \&add_versions,
                    },
      'debinfo' => {function => \&add_debinfo,
-                   arguments => {'0|null' => 0},
+                   arguments => {'0|null' => 0,
+                                'debinfo_dir|debinfo-dir=s' => 0,
+                               },
                   },
      'maintainers' => {function => \&add_maintainers,
                       },
@@ -195,6 +199,8 @@ my %subcommands =
                  },
      'logs' => {function => \&add_logs,
                },
+     'bugs_and_logs' => {function => \&add_bugs_and_logs,
+                       },
      'packages' => {function => \&add_packages,
                    arguments => {'ftpdists=s' => 1,
                                  'suites=s@' => 0,
@@ -265,8 +271,6 @@ sub add_bugs {
     my $s = db_connect($options);
 
 
-    my $time = 0;
-    my $start_time = time;
     my %tags;
     my %severities;
     my %queue;
@@ -292,18 +296,10 @@ sub add_bugs {
                my @bugs = @_;
                my @bugs_to_update;
                if ($options{quick}) {
-                 for my $bug (@bugs) {
-                   my $stat = stat(getbugcomponent($bug,'summary',$initialdir));
-                   if (not defined $stat) {
-                      print STDERR "Unable to stat $bug $!\n";
-                      next;
-                   }
-                   my $rs = $s->resultset('Bug')->search({id=>$bug})->single();
-                   next if defined $rs and $stat->mtime <= $rs->last_modified()->epoch();
-                   push @bugs_to_update, $bug;
-                  }
+                   @bugs_to_update =
+                       bugs_to_update($s,$initialdir,@bugs);
                } else {
-                 @bugs_to_update = @bugs;
+                   @bugs_to_update = @bugs;
                }
                eval {
                  $s->txn_do(sub {
@@ -376,6 +372,15 @@ sub add_debinfo {
     my ($options,$opts,$p,$config,$argv) = @_;
 
     my @files = @{$argv};
+    if (exists $opts->{debinfo_dir} and not @files) {
+       find(sub {
+                if (-f $_ and /\.debinfo$/) {
+                    push @files, $File::Find::name;
+                }
+            },
+            $opts->{debinfo_dir}
+           );
+    }
     if (not @files) {
        {
           local $/ = "\n";
@@ -545,10 +550,6 @@ sub add_logs {
     }
     my $s = db_connect($options);
 
-
-    my $time = 0;
-    my $start_time = time;
-
     walk_bugs([(@{$argv}?@{$argv} : $initialdir)],
               $p,
               'log',
@@ -576,6 +577,81 @@ sub add_logs {
               });
 }
 
+sub add_bugs_and_logs {
+    my ($options,$opts,$p,$config,$argv) = @_;
+
+    chdir($config->{spool_dir}) or
+        die "chdir $config->{spool_dir} failed: $!";
+
+    my $verbose = $options->{debug};
+
+    my $initialdir = "db-h";
+
+    if (defined $argv->[0] and $argv->[0] eq "archive") {
+        $initialdir = "archive";
+    }
+    my $s = db_connect($options);
+
+    my %tags;
+    my %severities;
+    my %queue;
+
+    walk_bugs([(@{$argv}?@{$argv} : $initialdir)],
+              $p,
+              'summary',
+              $verbose,
+              sub {
+                  my @bugs = @_;
+                 my @bugs_to_update;
+                 if ($options{quick}) {
+                     @bugs_to_update =
+                         bugs_to_update($s,$initialdir,@bugs);
+                 } else {
+                     @bugs_to_update = @bugs;
+                 }
+                 eval {
+                     $s->txn_do(sub {
+                                    for my $bug (@bugs_to_update) {
+                                        load_bug(db => $s,
+                                                 bug => $bug,
+                                                 tags => \%tags,
+                                                 severities => \%severities,
+                                                 queue => \%queue);
+                                    }
+                                });
+                 };
+                 if ($@) {
+                     die "failure while trying to load bug: $@";
+                 }
+                 for my $bug (@bugs) {
+                     my $stat = stat(getbugcomponent($bug,'log',$initialdir));
+                     if (not defined $stat) {
+                         print STDERR "Unable to stat $bug $!\n";
+                         next;
+                     }
+                     if ($options{quick}) {
+                         my $rs = $s->resultset('Bug')->
+                             search({id=>$bug})->single();
+                         return if defined $rs and
+                             $stat->mtime <= $rs->last_modified()->epoch();
+                     }
+                     eval {
+                         load_bug_log(db => $s,
+                                      bug => $bug);
+                     };
+                     if ($@) {
+                         die "failure while trying to load bug log $bug\n$@";
+                     }
+                 }
+              },
+             50
+            );
+    handle_load_bug_queue(db=>$s,
+                         queue => \%queue,
+                        );
+
+}
+
 sub add_packages {
     my ($options,$opts,$p,$config,$argv) = @_;
 
@@ -683,7 +759,8 @@ sub db_connect {
     my ($options) = @_;
     # connect to the database; figure out how to handle errors
     # properly here.
-    my $s = Debbugs::DB->connect($options->{service}) or
+    my $s = Debbugs::DB->connect($options->{dsn} //
+                                $options->{service}) or
         die "Unable to connect to database: ";
 }
 
@@ -757,5 +834,21 @@ sub walk_bugs {
 }
 
 
+sub bugs_to_update {
+    my ($s,$initialdir,@bugs) = @_;
+    my @bugs_to_update;
+    for my $bug (@bugs) {
+       my $stat = stat(getbugcomponent($bug,'summary',$initialdir));
+       if (not defined $stat) {
+           print STDERR "Unable to stat $bug $!\n";
+           next;
+       }
+       my $rs = $s->resultset('Bug')->search({id=>$bug})->single();
+       next if defined $rs and $stat->mtime <= $rs->last_modified()->epoch();
+       push @bugs_to_update, $bug;
+    }
+    @bugs_to_update;
+}
+
 
 __END__