From 01657bf61b205931f9e263f626f80f5b34682b0c Mon Sep 17 00:00:00 2001 From: Don Armstrong Date: Tue, 8 Aug 2017 07:19:46 -0700 Subject: [PATCH] use get_*_id in load_debinfo; handle -0 correctly there too --- Debbugs/DB/Load.pm | 58 +++++++++++++++++++++++++------ bin/debbugs-loadsql | 85 +++++++++++++++++++-------------------------- 2 files changed, 83 insertions(+), 60 deletions(-) diff --git a/Debbugs/DB/Load.pm b/Debbugs/DB/Load.pm index 040e2a7..86662f3 100644 --- a/Debbugs/DB/Load.pm +++ b/Debbugs/DB/Load.pm @@ -445,17 +445,53 @@ Commands to handle src and package version loading from debinfo files =cut sub load_debinfo { - my ($schema,$binname, $binver, $binarch, $srcname, $srcver) = @_; - my $sp = $schema->resultset('SrcPkg')->find_or_create({pkg => $srcname}); - my $sv = $schema->resultset('SrcVer')->find_or_create({src_pkg=>$sp->id(), - ver => $srcver}); - my $arch = $schema->resultset('Arch')->find_or_create({arch => $binarch}); - my $bp = $schema->resultset('BinPkg')->find_or_create({pkg => $binname}); - $schema->resultset('BinVer')->find_or_create({bin_pkg_id => $bp->id(), - src_ver_id => $sv->id(), - arch_id => $arch->id(), - ver => $binver, - }); + my ($s,$binname, $binver, $binarch, $srcname, $srcver,$ct_date,$cache) = @_; + $cache //= {}; + my $sp; + if (not defined $cache->{sp}{$srcname}) { + $cache->{sp}{$srcname} = + $s->resultset('SrcPkg')->find_or_create({pkg => $srcname}); + } + $sp = $cache->{sp}{$srcname}; + # update the creation date if the data we have is earlier + if (defined $ct_date and + (not defined $sp->creation or + $ct_date < $sp->creation)) { + $sp->creation($ct_date); + $sp->last_modified(DateTime->now); + $sp->update; + } + my $sv; + if (not defined $cache->{sv}{$srcname}{$srcver}) { + $cache->{sv}{$srcname}{$srcver} = + $s->resultset('SrcVer')-> + find_or_create({src_pkg =>$sp->id(), + ver => $srcver}); + } + $sv = $cache->{sv}{$srcname}{$srcver}; + if (defined $ct_date and + (not defined $sv->upload_date() or $ct_date < $sv->upload_date())) { + $sv->upload_date($ct_date); + $sv->update; + } + my $arch; + if (not defined $cache->{arch}{$binarch}) { + $cache->{arch}{$binarch} = + $s->resultset('Arch')-> + find_or_create({arch => $binarch}, + {result_class => 'DBIx::Class::ResultClass::HashRefInflator'}, + )->{id}; + } + $arch = $cache->{arch}{$binarch}; + my $bp; + if (not defined $cache->{bp}{$binname}) { + $cache->{bp}{$binname} = + $s->resultset('BinPkg')-> + get_bin_pkg_id($binname); + } + $bp = $cache->{bp}{$binname}; + $s->resultset('BinVer')-> + get_bin_ver_id($bp,$binver,$arch,$sv->id()); } diff --git a/bin/debbugs-loadsql b/bin/debbugs-loadsql index c583af0..b974326 100755 --- a/bin/debbugs-loadsql +++ b/bin/debbugs-loadsql @@ -376,63 +376,50 @@ sub add_debinfo { my @files = @{$argv}; if (not @files) { { - if ($opts->{0}) { - local $/ = "\0"; - } + local $/ = "\n"; + local $/ = "\0" if $opts->{0}; while () { + s/\n$// unless $opts->{0}; + s/\0$// if $opts->{0}; push @files, $_; } } } return unless @files; my $s = db_connect($options); - my %arch; + my %cache; $p->target(scalar @files) if $p; - for my $file (@files) { - my $fh = IO::File->new($file,'r') or - die "Unable to open $file for reading: $!"; - my $f_stat = stat($file); - while (<$fh>) { - chomp; - next unless length $_; - my ($binname, $binver, $binarch, $srcname, $srcver) = split; - # if $srcver is not defined, this is probably a broken - # .debinfo file [they were causing #686106, see commit - # 49c85ab8 in dak.] Basically, $binarch didn't get put into - # the file, so we'll fudge it from the filename. - if (not defined $srcver) { - ($srcname,$srcver) = ($binarch,$srcname); - ($binarch) = $file =~ /_([^\.]+)\.debinfo/; - } - my $sp = $s->resultset('SrcPkg')->find_or_create({pkg => $srcname}); - # update the creation date if the data we have is earlier - my $ct_date = DateTime->from_epoch(epoch => $f_stat->ctime); - if ($ct_date < $sp->creation) { - $sp->creation($ct_date); - $sp->last_modified(DateTime->now); - $sp->update; - } - my $sv = $s->resultset('SrcVer')->find_or_create({src_pkg =>$sp->id(), - ver => $srcver}); - if (not defined $sv->upload_date() or $ct_date < $sv->upload_date()) { - $sv->upload_date($ct_date); - $sv->update; - } - my $arch; - if (defined $arch{$binarch}) { - $arch = $arch{$binarch}; - } else { - $arch = $s->resultset('Arch')->find_or_create({arch => $binarch}); - $arch{$binarch} = $arch; - } - my $bp = $s->resultset('BinPkg')->find_or_create({pkg => $binname}); - $s->resultset('BinVer')->find_or_create({bin_pkg => $bp->id(), - src_ver => $sv->id(), - arch => $arch->id(), - ver => $binver, - }); - } - $p->update() if $p; + my $it = natatime 100, @files; + while (my @v = $it->()) { + my @debinfos; + for my $file (@v) { + my $fh = IO::File->new($file,'r') or + die "Unable to open $file for reading: $!"; + my $f_stat = stat($file); + my $ct_date = DateTime->from_epoch(epoch => $f_stat->ctime); + while (<$fh>) { + chomp; + next unless length $_; + my ($binname, $binver, $binarch, $srcname, $srcver) = split; + # if $srcver is not defined, this is probably a broken + # .debinfo file [they were causing #686106, see commit + # 49c85ab8 in dak.] Basically, $binarch didn't get put into + # the file, so we'll fudge it from the filename. + if (not defined $srcver) { + ($srcname,$srcver) = ($binarch,$srcname); + ($binarch) = $file =~ /_([^\.]+)\.debinfo/; + } + push @debinfos, + [$binname,$binver,$binarch,$srcname,$srcver,$ct_date]; + } + } + $s->txn_do( + sub { + for my $di (@debinfos) { + Debbugs::DB::Load::load_debinfo($s,@{$di}[0..5],\%cache); + } + }); + $p->update($p->last_update()+@v) if $p; } $p->remove() if $p; } -- 2.39.2