add initial code for updating bug status cache

[debbugs.git] / bin / debbugs-loadsql

diff --git a/bin/debbugs-loadsql b/bin/debbugs-loadsql
index e9b62ef9016ed7b9d5620d32c58fc00894d7e040..59a8502d4bc14dc6d2cb261b66cbe43b78b24bcc 100755 (executable)
--- a/bin/debbugs-loadsql
+++ b/bin/debbugs-loadsql
@@ -104,7 +104,7 @@ use Debbugs::Config qw(:config);
 use Debbugs::Status qw(read_bug split_status_fields);
 use Debbugs::Log;
 use Debbugs::DB;
-use Debbugs::DB::Load qw(load_bug handle_load_bug_queue :load_package :load_suite);
+use Debbugs::DB::Load qw(:load_bug :load_package :load_suite);
 use DateTime;
 use File::stat;
 use File::Basename;
@@ -144,6 +144,7 @@ $DEBUG = $options{debug};
 
 my %subcommands =
     ('bugs' => {function => \&add_bugs,
+               arguments => {'preload' => 0},
                },
      'versions' => {function => \&add_versions,
                    },
@@ -236,6 +237,17 @@ sub add_bugs {
     my %severities;
     my %queue;
 
+    if ($opts->{preload}) {
+       my @bugs;
+       walk_bugs([(@{$argv}?@{$argv} : $initialdir)],
+                 undef,
+                 'summary',
+                 undef,
+                 sub {
+                     push @bugs,shift;
+                 });
+       $s->resultset('Bug')->quick_insert_bugs(@bugs);
+    }
     walk_bugs([(@{$argv}?@{$argv} : $initialdir)],
               $p,
               'summary',
@@ -387,42 +399,36 @@ sub add_maintainers {
 
     my $s = db_connect($options);
     my $maintainers = getsourcemaintainers();
-    $p->target(scalar keys %{$maintainers}) if $p;
-    for my $pkg (keys %{$maintainers}) {
-        my $maint = $maintainers->{$pkg};
-        # see if a maintainer already exists; if so, we don't do
-        # anything here
-        my $maint_r = $s->resultset('Maintainer')->
-            find({name => $maint});
-        if (not defined $maint_r) {
-            # get e-mail address of maintainer
-            my $addr = getparsedaddrs($maint);
-            my $e_mail = $addr->address();
-            my $full_name = $addr->phrase();
-            $full_name =~ s/^\"|\"$//g;
-            $full_name =~ s/^\s+|\s+$//g;
-            # find correspondent
-            my $correspondent = $s->resultset('Correspondent')->
-                find_or_create({addr => $e_mail});
-            if (length $full_name) {
-                my $c_full_name = $correspondent->find_or_create_related('correspondent_full_names',
-                                                                        {full_name => $full_name}) if length $full_name;
-                $c_full_name->update({last_seen => 'NOW()'});
-            }
-            $maint_r =
-                $s->resultset('Maintainer')->
-                find_or_create({name => $maint,
-                                correspondent => $correspondent,
-                               });
-        }
-        # add the maintainer to the source package for packages with
-        # no maintainer
-        $s->txn_do(sub {
-                      $s->resultset('SrcPkg')->search({pkg => $pkg})->
-                          search_related_rs('src_vers',{ maintainer => undef})->
-                          update_all({maintainer => $maint_r->id()});
-                  });
-        $p->update() if $p;
+    $p->target(2) if $p;
+    ## get all of the maintainers, and add the missing ones
+    my $maints = $s->resultset('Maintainer')->
+       get_maintainers(values %{$maintainers});
+    $p->update();
+    my @svs = $s->resultset('SrcVer')->
+       search({maintainer => undef
+              },
+             {join => 'src_pkg',
+              group_by => 'me.src_pkg, src_pkg.pkg',
+              result_class => 'DBIx::Class::ResultClass::HashRefInflator',
+              columns => [qw(me.src_pkg src_pkg.pkg)],
+             }
+             )->all();
+    $p->target(2+@svs) if $p;
+    $p->update() if $p;
+    for my $sv (@svs) {
+       if (exists $maintainers->{$sv->{src_pkg}{pkg}}) {
+           my $pkg = $sv->{src_pkg}{pkg};
+           my $maint = $maints->
+              {$maintainers->{$pkg}};
+           $s->txn_do(sub {$s->resultset('SrcVer')->
+                               search({maintainer => undef,
+                                       'src_pkg.pkg' => $pkg
+                                      },
+                                     {join => 'src_pkg'}
+                                     )->update({maintainer => $maint})
+                                 });
+       }
+       $p->update() if $p;
     }
     $p->remove() if $p;
 }
@@ -434,25 +440,35 @@ sub add_configuration {
 
     # tags
     # add all tags
+    my %tags;
+    for my $tag (@{$config{tags}}) {
+       $tags{$tag} = 1;
+       $s->resultset('Tag')->find_or_create({tag => $tag});
+    }
     # mark obsolete tags
+    for my $tag ($s->resultset('Tag')->search_rs()->all()) {
+       next if exists $tags{$tag->tag};
+       $tag->obsolete(1);
+       $tag->update;
+    }
 
     # severities
     my %sev_names;
-    my $order = 0;
-    for my $sev_name (@{$config{severities}}) {
+    my $order = -1;
+    for my $sev_name (($config{default_severity},@{$config{severity_list}})) {
         # add all severitites
         my $sev = $s->resultset('Severity')->find_or_create({severity => $sev_name});
         # mark strong severities
         if (grep {$_ eq $sev_name} @{$config{strong_severities}}) {
             $sev->strong(1);
         }
-        $sev->order($order);
+        $sev->ordering($order);
         $sev->update();
         $order++;
         $sev_names{$sev_name} = 1;
     }
     # mark obsolete severities
-    for my $sev ($s->resultset('Severity')->find()) {
+    for my $sev ($s->resultset('Severity')->search_rs()->all()) {
         next if exists $sev_names{$sev->severity()};
         $sev->obsolete(1);
         $sev->update();
@@ -503,7 +519,16 @@ sub add_logs {
               $verbose,
               sub {
                   my $bug = shift;
-                  eval { 
+                 my $stat = stat(getbugcomponent($bug,'log',$initialdir));
+                  if (not defined $stat) {
+                      print STDERR "Unable to stat $bug $!\n";
+                      next;
+                  }
+                  if ($options{quick}) {
+                      my $rs = $s->resultset('Bug')->search({bug=>$bug})->single();
+                      next if defined $rs and $stat->mtime < $rs->last_modified()->epoch();
+                  }
+                  eval {
                       load_bug_log(db => $s,
                                    bug => $bug);
                   };
@@ -543,7 +568,6 @@ sub add_packages {
     my $completed_pkgs=0;
     # parse packages files
     for my $suite (keys %s_p) {
-       print STDERR "working on $suite\n";
        my @pkgs;
        for my $component (keys %{$s_p{$suite}}) {
            my @archs = keys %{$s_p{$suite}{$component}};
@@ -573,12 +597,10 @@ sub add_packages {
        }
         $p->target($avg_pkgs*($tot_suites-$done_suites-1)+
                   $completed_pkgs+@pkgs) if $p;
-       $s->txn_do(sub {
-                      Debbugs::DB::Load::load_packages($s,
-                                                       $suite,
-                                                       \@pkgs,
-                                                       $p)
-                      });
+       load_packages($s,
+                     $suite,
+                     \@pkgs,
+                     $p);
        $avg_pkgs=($avg_pkgs*$done_suites + @pkgs)/($done_suites+1);
        $completed_pkgs += @pkgs;
        $done_suites++;