X-Git-Url: https://git.donarmstrong.com/?a=blobdiff_plain;f=bin%2Fdebbugs-loadsql;h=1dd7a6aaa81e40dd5b9a5aef375228e320ab6e9f;hb=4b191ef548a2c738cbe9d1abc9ea2cab782cf74f;hp=b36a4998747a22513eab8d0b2aeb8d1fb740aad7;hpb=bce30f8c7af943f8072e5674f0bdebd915d4b5ce;p=debbugs.git

diff --git a/bin/debbugs-loadsql b/bin/debbugs-loadsql
index b36a499..1dd7a6a 100755
--- a/bin/debbugs-loadsql
+++ b/bin/debbugs-loadsql
@@ -147,6 +147,8 @@ my %subcommands =
                       },
      'configuration' => {function => \&add_configuration,
                         },
+     'suites' => {function => \&add_suites,
+                 },
      'logs' => {function => \&add_logs,
                },
      'help' => {function => sub {pod2usage({verbose => 2});}}
@@ -213,66 +215,41 @@ sub add_bugs {
 
     my $time = 0;
     my $start_time = time;
-
-
-    my @dirs = (@{$argv}?@{$argv} : $initialdir);
-    my $cnt = 0;
     my %tags;
     my %severities;
     my %queue;
-    my $tot_dirs = @{$argv}? @{$argv} : 0;
-    my $done_dirs = 0;
-    my $avg_subfiles = 0;
-    my $completed_files = 0;
-    while (my $dir = shift @dirs) {
-        printf "Doing dir %s ...\n", $dir if $verbose;
-
-        opendir(DIR, "$dir/.") or die "opendir $dir: $!";
-        my @subdirs = readdir(DIR);
-        closedir(DIR);
 
-        my @list = map { m/^(\d+)\.summary$/?($1):() } @subdirs;
-        $tot_dirs -= @dirs;
-        push @dirs, map { m/^(\d+)$/ && -d "$dir/$1"?("$dir/$1"):() } @subdirs;
-        $tot_dirs += @dirs;
-        if ($avg_subfiles == 0) {
-            $avg_subfiles = @list;
-        }
-
-        $p->target($avg_subfiles*($tot_dirs-$done_dirs)+$completed_files+@list) if $p;
-        $avg_subfiles = ($avg_subfiles * $done_dirs + @list) / ($done_dirs+1);
-        $done_dirs += 1;
-
-        for my $bug (@list) {
-            $completed_files++;
-            $p->update($completed_files) if $p;
-            print "Up to $cnt bugs...\n" if (++$cnt % 100 == 0 && $verbose);
-            my $stat = stat(getbugcomponent($bug,'summary',$initialdir));
-            if (not defined $stat) {
-                print STDERR "Unable to stat $bug $!\n";
-                next;
-            }
-            if ($options{quick}) {
-                my $rs = $s->resultset('Bug')->search({bug=>$bug})->single();
-                next if defined $rs and $stat->mtime < $rs->last_modified()->epoch();
-            }
-            my $data = read_bug(bug => $bug,
-                                location => $initialdir);
-            eval {
-                load_bug(db => $s,
-                         data => split_status_fields($data),
-                         tags => \%tags,
-                         severities => \%severities,
-                         queue => \%queue);
-            };
-            if ($@) {
-                use Data::Dumper;
-                print STDERR Dumper($data) if $DEBUG;
-                die "failure while trying to load bug $bug\n$@";
-            }
-        }
-    }
-    $p->remove() if $p;
+    walk_bugs([(@{$argv}?@{$argv} : $initialdir)],
+              $p,
+              'summary',
+              $verbose,
+              sub {
+                  my $bug = shift;
+                  my $stat = stat(getbugcomponent($bug,'summary',$initialdir));
+                  if (not defined $stat) {
+                      print STDERR "Unable to stat $bug $!\n";
+                      next;
+                  }
+                  if ($options{quick}) {
+                      my $rs = $s->resultset('Bug')->search({bug=>$bug})->single();
+                      next if defined $rs and $stat->mtime < $rs->last_modified()->epoch();
+                  }
+                  my $data = read_bug(bug => $bug,
+                                      location => $initialdir);
+                  eval {
+                      load_bug(db => $s,
+                               data => split_status_fields($data),
+                               tags => \%tags,
+                               severities => \%severities,
+                               queue => \%queue);
+                  };
+                  if ($@) {
+                      use Data::Dumper;
+                      print STDERR Dumper($data) if $DEBUG;
+                      die "failure while trying to load bug $bug\n$@";
+                  }
+              }
+             );
     handle_load_bug_queue(db => $s,
                           queue => \%queue);
 }
@@ -346,8 +323,19 @@ sub add_debinfo {
                 ($binarch) = $file =~ /_([^\.]+)\.debinfo/;
             }
             my $sp = $s->resultset('SrcPkg')->find_or_create({pkg => $srcname});
+            # update the creation date if the data we have is earlier
+            my $ct_date = DateTime->from_epoch(epoch => $f_stat->ctime);
+            if ($ct_date < $sp->creation) {
+                $sp->creation($ct_date);
+                $sp->last_modified(DateTime->now);
+                $sp->update;
+            }
             my $sv = $s->resultset('SrcVer')->find_or_create({src_pkg =>$sp->id(),
                                                               ver => $srcver});
+            if ($ct_date < $sv->upload_date()) {
+                $sv->upload_date($ct_date);
+                $sv->update;
+            }
             my $arch;
             if (defined $arch{$binarch}) {
                 $arch = $arch{$binarch};
@@ -414,10 +402,79 @@ sub add_maintainers {
 
 sub add_configuration {
     my ($options,$opts,$p,$config,$argv) = @_;
+
+    my $s = db_connect($options);
+
+    # tags
+    # add all tags
+    # mark obsolete tags
+
+    # severities
+    my %sev_names;
+    my $order = 0;
+    for my $sev_name (@{$config{severities}}) {
+        # add all severitites
+        my $sev = $s->resultset('Severity')->find_or_create({severity => $sev_name});
+        # mark strong severities
+        if (grep {$_ eq $sev_name} @{$config{strong_severities}}) {
+            $sev->strong(1);
+        }
+        $sev->order($order);
+        $sev->update();
+        $order++;
+        $sev_names{$sev_name} = 1;
+    }
+    # mark obsolete severities
+    for my $sev ($s->resultset('Severity')->find()) {
+        next if exists $sev_names{$sev->severity()};
+        $sev->obsolete(1);
+        $sev->update();
+    }
+}
+
+sub add_suite {
+    my ($options,$opts,$p,$config,$argv) = @_;
+    # suites
+    die "add_suite is currently not implemented; modify suites manually using SQL."
 }
 
 sub add_logs {
     my ($options,$opts,$p,$config,$argv) = @_;
+
+    chdir($config->{spool_dir}) or
+        die "chdir $config->{spool_dir} failed: $!";
+
+    my $verbose = $options->{debug};
+
+    my $initialdir = "db-h";
+
+    if (defined $argv->[0] and $argv->[0] eq "archive") {
+        $initialdir = "archive";
+    }
+    my $s = db_connect($options);
+
+
+    my $time = 0;
+    my $start_time = time;
+
+    walk_bugs([(@{$argv}?@{$argv} : $initialdir)],
+              $p,
+              'log',
+              $verbose,
+              sub {
+                  my $bug = shift;
+                  eval { 
+                      load_bug_log(db => $s,
+                                   bug => $bug);
+                  };
+                  if ($@) {
+                      die "failure while trying to load bug log $bug\n$@";
+                  }
+              });
+}
+
+sub add_packages {
+
 }
 
 sub handle_subcommand_arguments {
@@ -461,6 +518,42 @@ sub db_connect {
         die "Unable to connect to database: ";
 }
 
+sub walk_bugs {
+    my ($dirs,$p,$what,$verbose,$sub) = @_;
+    my @dirs = @{$dirs};
+    my $tot_dirs = @dirs;
+    my $done_dirs = 0;
+    my $avg_subfiles = 0;
+    my $completed_files = 0;
+    while (my $dir = shift @dirs) {
+        printf "Doing dir %s ...\n", $dir if $verbose;
+
+        opendir(DIR, "$dir/.") or die "opendir $dir: $!";
+        my @subdirs = readdir(DIR);
+        closedir(DIR);
+
+        my @list = map { m/^(\d+)\.$what$/?($1):() } @subdirs;
+        $tot_dirs -= @dirs;
+        push @dirs, map { m/^(\d+)$/ && -d "$dir/$1"?("$dir/$1"):() } @subdirs;
+        $tot_dirs += @dirs;
+        if ($avg_subfiles == 0) {
+            $avg_subfiles = @list;
+        }
+
+        $p->target($avg_subfiles*($tot_dirs-$done_dirs)+$completed_files+@list) if $p;
+        $avg_subfiles = ($avg_subfiles * $done_dirs + @list) / ($done_dirs+1);
+        $done_dirs += 1;
+
+        for my $bug (@list) {
+            $completed_files++;
+            $p->update($completed_files) if $p;
+            print "Up to $completed_files bugs...\n" if ($completed_files % 100 == 0 && $verbose);
+            $sub->($bug);
+        }
+    }
+    $p->remove() if $p;
+}
+
 
 
 __END__