Options:
--quick, -q only load changed bugs
+ --progress Show progress bar
--service, -s service name
--sysconfdir, -c postgresql service config dir
--spool-dir debbugs spool directory
Only load changed bugs
+=item B<--progress>
+
+Show progress bar (requires Term::ProgressBar)
+
=item B<--service,-s>
Postgreql service to use; defaults to debbugs
Debbugs spool directory; defaults to the value configured in the
debbugs configuration file.
+=item B<--verbose>
+
+Output more information about what is happening. Probably not useful
+if you also set --progress.
+
=item B<--debug, -d
Debug verbosity.
use Debbugs::Status qw(read_bug split_status_fields);
use Debbugs::Log;
use Debbugs::DB;
+use Debbugs::DB::Load qw(load_bug handle_load_bug_queue);
use DateTime;
use File::stat;
-my %options = (debug => 0,
- help => 0,
- man => 0,
- verbose => 0,
- quiet => 0,
- quick => 0,
- service => 'debbugs',
- );
+my %options =
+ (debug => 0,
+ help => 0,
+ man => 0,
+ verbose => 0,
+ quiet => 0,
+ quick => 0,
+ service => 'debbugs',
+ progress => 0,
+ );
GetOptions(\%options,
- 'quick|q',
- 'service|s',
- 'sysconfdir|c',
- 'spool_dir|spool-dir=s',
- 'debug|d+','help|h|?','man|m');
+ 'quick|q',
+ 'service|s',
+ 'sysconfdir|c',
+ 'progress!',
+ 'spool_dir|spool-dir=s',
+ 'verbose|v+',
+ 'quiet+',
+ 'debug|d+','help|h|?','man|m');
pod2usage() if $options{help};
pod2usage({verbose=>2}) if $options{man};
my @USAGE_ERRORS;
$options{verbose} = $options{verbose} - $options{quiet};
+if ($options{progress}) {
+ eval "use Term::ProgressBar";
+ push @USAGE_ERRORS, "You asked for a progress bar, but Term::ProgressBar isn't installed" if $@;
+}
+
+
pod2usage(join("\n",@USAGE_ERRORS)) if @USAGE_ERRORS;
if (exists $options{sysconfdir}) {
exit 1;
}
+
# connect to the database; figure out how to handle errors properly
# here.
my $schema = Debbugs::DB->connect('dbi:Pg:service='.$options{service}) or
my $start_time = time;
-my @dirs = ($initialdir);
+my @dirs = (@ARGV?@ARGV : $initialdir);
my $cnt = 0;
my %tags;
+my %severities;
my %queue;
+my $tot_dirs = @ARGV ? @ARGV : 0;
+my $done_dirs = 0;
+my $avg_subfiles = 0;
+my $completed_files = 0;
+my $p;
+if ($options{progress}) {
+ $p = eval "Term::ProgressBar->new({count => 1,ETA=>q(linear)})";
+ warn "Unable to initialize progress bar: $@" if not $p;
+}
while (my $dir = shift @dirs) {
printf "Doing dir %s ...\n", $dir if $verbose;
closedir(DIR);
my @list = map { m/^(\d+)\.summary$/?($1):() } @subdirs;
+ $tot_dirs -= @dirs;
push @dirs, map { m/^(\d+)$/ && -d "$dir/$1"?("$dir/$1"):() } @subdirs;
+ $tot_dirs += @dirs;
+ if ($avg_subfiles == 0) {
+ $avg_subfiles = @list;
+ }
+
+ $p->target($avg_subfiles*($tot_dirs-$done_dirs)+$completed_files+@list) if $p;
+ $avg_subfiles = ($avg_subfiles * $done_dirs + @list) / ($done_dirs+1);
+ $done_dirs += 1;
for my $bug (@list) {
+ $completed_files++;
+ $p->update($completed_files) if $p;
print "Up to $cnt bugs...\n" if (++$cnt % 100 == 0 && $verbose);
my $stat = stat(getbugcomponent($bug,'summary',$initialdir));
if (not defined $stat) {
next if $stat->mtime < $time;
my $data = read_bug(bug => $bug,
location => $initialdir);
- load_bug($schema,split_status_fields($data),\%tags,\%queue);
- }
-}
-hanlde_queue($schema,\%queue);
-
-sub load_bug {
- my ($s,$data,$tags,$queue) = @_;
- my $s_data = split_status_fields($data);
- my @tags;
- for my $tag (make_list($s_data->{keywords})) {
- next unless defined $tag and length $tag;
- # this allows for invalid tags. But we'll use this to try to
- # find those bugs and clean them up
- if (not exists $tags->{$tag}) {
- $tags->{$tag} = $s->resultset('Tag')->find_or_create({tag => $tag});
- }
- push @tags, $tags->{$tag};
+ eval {
+ load_bug(db => $schema,
+ data => split_status_fields($data),
+ tags => \%tags,
+ severities => \%severities,
+ queue => \%queue);
+ };
+ if ($@) {
+ use Data::Dumper;
+ print STDERR Dumper($data) if $DEBUG;
+ die "failure while trying to load bug $bug\n$@";
+ }
}
- my $bug = {id => $data->{bug_num},
- creation => DateTime->from_epoch(epoch => $data->{date}),
- log_modified => DateTime->from_epoch(epoch => $data->{log_modified}),
- last_modified => DateTime->from_epoch(epoch => $data->{last_modified}),
- archived => $data->{archived},
- (defined $data->{unarchived} and length($data->{unarchived}))?(unarchived => DateTime->from_epoch(epoch => $data->{unarchived})):(),
- forwarded => $data->{forwarded} // '',
- summary => $data->{summary} // '',
- outlook => $data->{outlook} // '',
- subject => $data->{subject} // '',
- done => $data->{done} // '',
- owner => $data->{owner} // '',
- severity => length($data->{severity}) ? $data->{severity} : $config{default_severity},
- };
- $s->resultset('Bug')->update_or_create($bug);
- $s->txn_do(sub {
- for my $ff (qw(found fixed)) {
- my @elements = $s->resultset('BugVer')->search({bug_id => $data->{bug_num},
- found => $ff eq 'found'?1:0,
- });
- my %elements_to_delete = map {($elements[$_]->ver_string(),$_)} 0..$#elements;
- my @elements_to_add;
- for my $version (@{$data->{"${ff}_versions"}}) {
- if (exists $elements_to_delete{$version}) {
- delete $elements_to_delete{$version};
- } else {
- push @elements_to_add,$version;
- }
- }
- for my $element (keys %elements_to_delete) {
- $elements_to_delete{$element}->delete();
- }
- for my $element (@elements_to_add) {
- # find source package and source version id
- my $ne = $s->resultset('BugVer')->new_result({bug_id => $data->{bug_num},
- ver_string => $element,
- found => $ff eq 'found'?1:0,
- }
- );
- if (my ($src_pkg,$src_ver) = $element =~ m{^([^\/]+)/(.+)$}) {
- my $src_pkg_e = $s->resultset('SrcPkg')->single({pkg => $src_pkg});
- if (defined $src_pkg_e) {
- $ne->src_pkg_id($src_pkg_e->id());
- my $src_ver_e = $s->resultset('SrcVer')->single({src_pkg_id => $src_pkg_e->id(),
- ver => $src_ver
- });
- $ne->src_ver_id($src_ver_e->id()) if defined $src_ver_e;
- }
- }
- $ne->insert();
- }
- }
- });
- $s->txn_do(sub {
- $s->resultset('BugTag')->search({bug_id => $data->{bug_num}})->delete();
- $s->populate(BugTag => [[qw(bug_id tag_id)], map {[$data->{bug_num}, $_->id()]} @tags]);
- });
- # because these bugs reference other bugs which might not exist
- # yet, we can't handle them until we've loaded all bugs. queue
- # them up.
- $queue->{merged}{$data->{bug_num}} = [@{$data->{mergedwith}}];
- $queue->{blocks}{$data->{bug_num}} = [@{$data->{blocks}}];
-
- print STDERR "Handled $data->{bug_num}\n";
- # still need to handle merges, versions, etc.
}
+$p->remove() if $p;
+handle_load_bug_queue(db => $schema,
+ queue => \%queue);
-sub handle_queue{
- my ($s,$queue) = @_;
- my %queue_types =
- (merged => {set => 'BugMerged',
- columns => [qw(bug_id merged)],
- bug_id => 'bug_id',
- },
- blocks => {set => 'BugBlock',
- columns => [qw(bug_id blocks)],
- bug_id => 'bug_id',
- },
- );
- for my $queue_type (keys %queue_types) {
- for my $bug (%{$queue->{$queue_type}}) {
- my $qt = $queue_types{$queue_type};
- $s->txn_do(sub {
- $s->resultset($qt->{set})->search({$qt->{bug_id},$bug})->delete();
- $s->populate($qt->{set},[[@{$qt->{columns}}],map {[$bug,$_]} @{$queue->{$queue_type}{$bug}}]) if
- @{$queue->{$queue_type}{$bug}};
- }
- );
- }
- }
-}
__END__
projects / debbugs.git / blobdiff