merge master

[debbugs.git] / bin / debbugs-loadsql

diff --git a/bin/debbugs-loadsql b/bin/debbugs-loadsql
index a9c3399066e7243db85df5d6f35048dfe7768d79..6b7d68b1da1f5019bd7f0497bd1874cd5d215e32 100755 (executable)
--- a/bin/debbugs-loadsql
+++ b/bin/debbugs-loadsql
@@ -21,6 +21,7 @@ debbugs-loadsql [options]
 
  Options:
   --quick, -q only load changed bugs
+  --progress Show progress bar
   --service, -s service name
   --sysconfdir, -c postgresql service config dir
   --spool-dir debbugs spool directory
@@ -36,6 +37,10 @@ debbugs-loadsql [options]
 
 Only load changed bugs
 
+=item B<--progress>
+
+Show progress bar (requires Term::ProgressBar)
+
 =item B<--service,-s>
 
 Postgreql service to use; defaults to debbugs
@@ -50,6 +55,11 @@ postgresql default. [Operates by setting PGSYSCONFDIR]
 Debbugs spool directory; defaults to the value configured in the
 debbugs configuration file.
 
+=item B<--verbose>
+
+Output more information about what is happening. Probably not useful
+if you also set --progress.
+
 =item B<--debug, -d
 
 Debug verbosity.
@@ -80,22 +90,27 @@ use DateTime;
 use File::stat;
 
 
-my %options = (debug           => 0,
-              help            => 0,
-              man             => 0,
-              verbose         => 0,
-              quiet           => 0,
-              quick           => 0,
-              service         => 'debbugs',
-             );
+my %options =
+    (debug           => 0,
+     help            => 0,
+     man             => 0,
+     verbose         => 0,
+     quiet           => 0,
+     quick           => 0,
+     service         => 'debbugs',
+     progress        => 0,
+    );
 
 
 GetOptions(\%options,
-          'quick|q',
-          'service|s',
-          'sysconfdir|c',
-          'spool_dir|spool-dir=s',
-          'debug|d+','help|h|?','man|m');
+           'quick|q',
+           'service|s',
+           'sysconfdir|c',
+           'progress!',
+           'spool_dir|spool-dir=s',
+           'verbose|v+',
+           'quiet+',
+           'debug|d+','help|h|?','man|m');
 
 pod2usage() if $options{help};
 pod2usage({verbose=>2}) if $options{man};
@@ -105,6 +120,12 @@ $DEBUG = $options{debug};
 my @USAGE_ERRORS;
 $options{verbose} = $options{verbose} - $options{quiet};
 
+if ($options{progress}) {
+    eval "use Term::ProgressBar";
+    push @USAGE_ERRORS, "You asked for a progress bar, but Term::ProgressBar isn't installed" if $@;
+}
+
+
 pod2usage(join("\n",@USAGE_ERRORS)) if @USAGE_ERRORS;
 
 if (exists $options{sysconfdir}) {
@@ -138,6 +159,7 @@ if (not lockpid($config{spool_dir}.'/lock/debbugs-loadsql')) {
      exit 1;
 }
 
+
 # connect to the database; figure out how to handle errors properly
 # here.
 my $schema = Debbugs::DB->connect('dbi:Pg:service='.$options{service}) or
@@ -147,10 +169,20 @@ my $time = 0;
 my $start_time = time;
 
 
-my @dirs = ($initialdir);
+my @dirs = (@ARGV?@ARGV : $initialdir);
 my $cnt = 0;
 my %tags;
+my %severities;
 my %queue;
+my $tot_dirs = @ARGV ? @ARGV : 0;
+my $done_dirs = 0;
+my $avg_subfiles = 0;
+my $completed_files = 0;
+my $p;
+if ($options{progress}) {
+    $p = eval "Term::ProgressBar->new({count => 1,ETA=>q(linear)})";
+    warn "Unable to initialize progress bar: $@" if not $p;
+}
 while (my $dir = shift @dirs) {
     printf "Doing dir %s ...\n", $dir if $verbose;
 
@@ -159,9 +191,20 @@ while (my $dir = shift @dirs) {
     closedir(DIR);
 
     my @list = map { m/^(\d+)\.summary$/?($1):() } @subdirs;
+    $tot_dirs -= @dirs;
     push @dirs, map { m/^(\d+)$/ && -d "$dir/$1"?("$dir/$1"):() } @subdirs;
+    $tot_dirs += @dirs;
+    if ($avg_subfiles == 0) {
+        $avg_subfiles = @list;
+    }
+
+    $p->target($avg_subfiles*($tot_dirs-$done_dirs)+$completed_files+@list) if $p;
+    $avg_subfiles = ($avg_subfiles * $done_dirs + @list) / ($done_dirs+1);
+    $done_dirs += 1;
 
     for my $bug (@list) {
+        $completed_files++;
+        $p->update($completed_files) if $p;
        print "Up to $cnt bugs...\n" if (++$cnt % 100 == 0 && $verbose);
        my $stat = stat(getbugcomponent($bug,'summary',$initialdir));
        if (not defined $stat) {
@@ -171,14 +214,24 @@ while (my $dir = shift @dirs) {
        next if $stat->mtime < $time;
        my $data = read_bug(bug => $bug,
                            location => $initialdir);
-       load_bug(db => $schema,
-                 data => split_status_fields($data),
-                 tags => \%tags,
-                 queue => \%queue);
+        eval {
+            load_bug(db => $schema,
+                     data => split_status_fields($data),
+                     tags => \%tags,
+                     severities => \%severities,
+                     queue => \%queue);
+        };
+        if ($@) {
+            use Data::Dumper;
+            print STDERR Dumper($data) if $DEBUG;
+            die "failure while trying to load bug $bug\n$@";
+        }
     }
 }
+$p->remove() if $p;
 handle_load_bug_queue(db => $schema,
                       queue => \%queue);
 
 
+
 __END__