support --dsn option in debbugs-loadsql

[debbugs.git] / bin / debbugs-loadsql

diff --git a/bin/debbugs-loadsql b/bin/debbugs-loadsql
index b97432685882e0fc1f9b25c5593d55f19e0fe459..17edf44fa727d67e09afd3b520e4165aa71f1cd8 100755 (executable)
--- a/bin/debbugs-loadsql
+++ b/bin/debbugs-loadsql
@@ -163,6 +163,7 @@ Getopt::Long::Configure('pass_through');
 GetOptions(\%options,
            'quick|q',
            'service|s=s',
+          'dsn=s',
            'sysconfdir|c=s',
            'progress!',
            'spool_dir|spool-dir=s',
@@ -195,6 +196,8 @@ my %subcommands =
                  },
      'logs' => {function => \&add_logs,
                },
+     'bugs_and_logs' => {function => \&add_bugs_and_logs,
+                       },
      'packages' => {function => \&add_packages,
                    arguments => {'ftpdists=s' => 1,
                                  'suites=s@' => 0,
@@ -265,8 +268,6 @@ sub add_bugs {
     my $s = db_connect($options);
 
 
-    my $time = 0;
-    my $start_time = time;
     my %tags;
     my %severities;
     my %queue;
@@ -292,18 +293,10 @@ sub add_bugs {
                my @bugs = @_;
                my @bugs_to_update;
                if ($options{quick}) {
-                 for my $bug (@bugs) {
-                   my $stat = stat(getbugcomponent($bug,'summary',$initialdir));
-                   if (not defined $stat) {
-                      print STDERR "Unable to stat $bug $!\n";
-                      next;
-                   }
-                   my $rs = $s->resultset('Bug')->search({id=>$bug})->single();
-                   next if defined $rs and $stat->mtime <= $rs->last_modified()->epoch();
-                   push @bugs_to_update, $bug;
-                  }
+                   @bugs_to_update =
+                       bugs_to_update($s,$initialdir,@bugs);
                } else {
-                 @bugs_to_update = @bugs;
+                   @bugs_to_update = @bugs;
                }
                eval {
                  $s->txn_do(sub {
@@ -351,19 +344,21 @@ sub add_versions {
             my $sp;
             if (not defined $src_pkgs{$versions[$i][0]}) {
                 $src_pkgs{$versions[$i][0]} =
-                    $s->resultset('SrcPkg')->find_or_create({pkg => $versions[$i][0]});
+                    $s->resultset('SrcPkg')->
+                   get_src_pkg_id($versions[$i][0]);
             }
             $sp = $src_pkgs{$versions[$i][0]};
             # There's probably something wrong if the source package
             # doesn't exist, but we'll skip it for now
-            next unless defined $sp;
-            my $sv = $s->resultset('SrcVer')->find({src_pkg=>$sp->id(),
+            last if not defined $sp;
+            my $sv = $s->resultset('SrcVer')->find({src_pkg=>$sp,
                                                     ver => $versions[$i][1],
                                                    });
+           last if not defined $sv;
             if (defined $ancestor_sv and defined $sv and not defined $sv->based_on()) {
-                $sv->update({based_on => $ancestor_sv->id()})
+                $sv->update({based_on => $ancestor_sv})
             }
-            $ancestor_sv = $sv;
+            $ancestor_sv = $sv->id();
         }
         $p->update() if $p;
     }
@@ -387,10 +382,10 @@ sub add_debinfo {
     }
     return unless @files;
     my $s = db_connect($options);
-    my %cache;
     $p->target(scalar @files) if $p;
     my $it = natatime 100, @files;
     while (my @v = $it->()) {
+       my %cache;
        my @debinfos;
        for my $file (@v) {
            my $fh = IO::File->new($file,'r') or
@@ -409,6 +404,10 @@ sub add_debinfo {
                    ($srcname,$srcver) = ($binarch,$srcname);
                    ($binarch) = $file =~ /_([^\.]+)\.debinfo/;
                }
+               if (not defined $srcver) {
+                   print STDERR "malformed debinfo (no srcver): $file\n";
+                   next;
+               }
                push @debinfos,
                    [$binname,$binver,$binarch,$srcname,$srcver,$ct_date];
            }
@@ -539,10 +538,6 @@ sub add_logs {
     }
     my $s = db_connect($options);
 
-
-    my $time = 0;
-    my $start_time = time;
-
     walk_bugs([(@{$argv}?@{$argv} : $initialdir)],
               $p,
               'log',
@@ -555,8 +550,10 @@ sub add_logs {
                       next;
                   }
                   if ($options{quick}) {
-                      my $rs = $s->resultset('Bug')->search({bug=>$bug})->single();
-                      next if defined $rs and $stat->mtime <= $rs->last_modified()->epoch();
+                      my $rs = $s->resultset('Bug')->
+                         search({id=>$bug})->single();
+                      return if defined $rs and
+                         $stat->mtime <= $rs->last_modified()->epoch();
                   }
                   eval {
                       load_bug_log(db => $s,
@@ -568,6 +565,81 @@ sub add_logs {
               });
 }
 
+sub add_bugs_and_logs {
+    my ($options,$opts,$p,$config,$argv) = @_;
+
+    chdir($config->{spool_dir}) or
+        die "chdir $config->{spool_dir} failed: $!";
+
+    my $verbose = $options->{debug};
+
+    my $initialdir = "db-h";
+
+    if (defined $argv->[0] and $argv->[0] eq "archive") {
+        $initialdir = "archive";
+    }
+    my $s = db_connect($options);
+
+    my %tags;
+    my %severities;
+    my %queue;
+
+    walk_bugs([(@{$argv}?@{$argv} : $initialdir)],
+              $p,
+              'summary',
+              $verbose,
+              sub {
+                  my @bugs = @_;
+                 my @bugs_to_update;
+                 if ($options{quick}) {
+                     @bugs_to_update =
+                         bugs_to_update($s,$initialdir,@bugs);
+                 } else {
+                     @bugs_to_update = @bugs;
+                 }
+                 eval {
+                     $s->txn_do(sub {
+                                    for my $bug (@bugs_to_update) {
+                                        load_bug(db => $s,
+                                                 bug => $bug,
+                                                 tags => \%tags,
+                                                 severities => \%severities,
+                                                 queue => \%queue);
+                                    }
+                                });
+                 };
+                 if ($@) {
+                     die "failure while trying to load bug: $@";
+                 }
+                 for my $bug (@bugs) {
+                     my $stat = stat(getbugcomponent($bug,'log',$initialdir));
+                     if (not defined $stat) {
+                         print STDERR "Unable to stat $bug $!\n";
+                         next;
+                     }
+                     if ($options{quick}) {
+                         my $rs = $s->resultset('Bug')->
+                             search({id=>$bug})->single();
+                         return if defined $rs and
+                             $stat->mtime <= $rs->last_modified()->epoch();
+                     }
+                     eval {
+                         load_bug_log(db => $s,
+                                      bug => $bug);
+                     };
+                     if ($@) {
+                         die "failure while trying to load bug log $bug\n$@";
+                     }
+                 }
+              },
+             50
+            );
+    handle_load_bug_queue(db=>$s,
+                         queue => \%queue,
+                        );
+
+}
+
 sub add_packages {
     my ($options,$opts,$p,$config,$argv) = @_;
 
@@ -675,7 +747,8 @@ sub db_connect {
     my ($options) = @_;
     # connect to the database; figure out how to handle errors
     # properly here.
-    my $s = Debbugs::DB->connect($options->{service}) or
+    my $s = Debbugs::DB->connect($options->{dsn} //
+                                $options->{service}) or
         die "Unable to connect to database: ";
 }
 
@@ -749,5 +822,21 @@ sub walk_bugs {
 }
 
 
+sub bugs_to_update {
+    my ($s,$initialdir,@bugs) = @_;
+    my @bugs_to_update;
+    for my $bug (@bugs) {
+       my $stat = stat(getbugcomponent($bug,'summary',$initialdir));
+       if (not defined $stat) {
+           print STDERR "Unable to stat $bug $!\n";
+           next;
+       }
+       my $rs = $s->resultset('Bug')->search({id=>$bug})->single();
+       next if defined $rs and $stat->mtime <= $rs->last_modified()->epoch();
+       push @bugs_to_update, $bug;
+    }
+    @bugs_to_update;
+}
+
 
 __END__