X-Git-Url: https://git.donarmstrong.com/?a=blobdiff_plain;f=Debbugs%2FDB%2FLoad.pm;h=03ab770636eb8ec3a1d554ad78815f1f17794d8f;hb=466f7faff129a5699c7674f59900a92aa256175d;hp=4697e62fef7dd9862cfa4fada809262fed1508e2;hpb=d3e7ad7325629a06ffec12f47482b9da38f63a92;p=debbugs.git diff --git a/Debbugs/DB/Load.pm b/Debbugs/DB/Load.pm index 4697e62..03ab770 100644 --- a/Debbugs/DB/Load.pm +++ b/Debbugs/DB/Load.pm @@ -23,6 +23,7 @@ None known. use warnings; use strict; +use v5.10; use vars qw($VERSION $DEBUG %EXPORT_TAGS @EXPORT_OK @EXPORT); use base qw(Exporter); @@ -33,7 +34,7 @@ BEGIN{ @EXPORT = (); %EXPORT_TAGS = (load_bug => [qw(load_bug handle_load_bug_queue load_bug_log)], load_debinfo => [qw(load_debinfo)], - load_package => [qw(load_package)], + load_package => [qw(load_packages)], load_suite => [qw(load_suite)], ); @EXPORT_OK = (); @@ -42,12 +43,15 @@ BEGIN{ } use Params::Validate qw(validate_with :types); +use List::AllUtils qw(natatime); use Debbugs::Status qw(read_bug split_status_fields); use Debbugs::DB; use DateTime; use Debbugs::Common qw(make_list getparsedaddrs); use Debbugs::Config qw(:config); +use Debbugs::MIME qw(parse_to_mime_entity decode_rfc1522); +use DateTime::Format::Mail; use Carp; =head2 Bug loading @@ -98,6 +102,10 @@ sub load_bug { }, queue => {type => HASHREF, optional => 1}, + packages => {type => HASHREF, + default => sub {return {}}, + optional => 1, + }, }); my $s = $param{db}; if (not exists $param{data} and not exists $param{bug}) { @@ -116,8 +124,8 @@ sub load_bug { $queue = {}; } my %tags; - my $s_data = split_status_fields($data); - for my $tag (make_list($s_data->{keywords})) { + $data = split_status_fields($data); + for my $tag (make_list($data->{keywords})) { next unless defined $tag and length $tag; # this allows for invalid tags. But we'll use this to try to # find those bugs and clean them up @@ -127,20 +135,23 @@ sub load_bug { } $tags{$tag} = $tags->{$tag}; } - my $severity = length($data->{severity}) ? $data->{severity} : $config{default_severity}; - if (exists $severities->{$severity}) { - $severity = $severities->{$severity}; - } else { - $severity = $s->resultset('Severity')-> - find_or_create({severity => $severity}); + my $severity = length($data->{severity}) ? $data->{severity} : + $config{default_severity}; + if (not exists $severities->{$severity}) { + $severities->{$severity} = + $s->resultset('Severity')-> + find_or_create({severity => $severity}, + ); } + $severity = $severities->{$severity}; my $bug = {id => $data->{bug_num}, creation => DateTime->from_epoch(epoch => $data->{date}), log_modified => DateTime->from_epoch(epoch => $data->{log_modified}), last_modified => DateTime->from_epoch(epoch => $data->{last_modified}), archived => $data->{archived}, - (defined $data->{unarchived} and length($data->{unarchived}))?(unarchived => DateTime->from_epoch(epoch => $data->{unarchived})):(), + (defined $data->{unarchived} and length($data->{unarchived}))? + (unarchived => DateTime->from_epoch(epoch => $data->{unarchived})):(), forwarded => $data->{forwarded} // '', summary => $data->{summary} // '', outlook => $data->{outlook} // '', @@ -156,28 +167,60 @@ sub load_bug { submitter => 'originator', ); for my $addr_type (keys %addr_map) { - my @addrs = getparsedaddrs($data->{$addr_map{$addr_type}} // ''); - next unless @addrs; - $bug->{$addr_type} = $s->resultset('Correspondent')->find_or_create({addr => lc($addrs[0]->address())}); - # insert the full name as well - my $full_name = $addrs[0]->phrase(); - $full_name =~ s/^\"|\"$//g; - $full_name =~ s/^\s+|\s+$//g; - if (length $full_name) { - $bug->{$addr_type}-> - update_or_create_related('correspondent_full_names', - {full_name=>$full_name, - last_seen => 'NOW()'}); - } + $bug->{$addr_type} = undef; + next unless defined $data->{$addr_map{$addr_type}} and + length($data->{$addr_map{$addr_type}}); + $bug->{$addr_type} = + $s->resultset('Correspondent')-> + get_correspondent_id($data->{$addr_map{$addr_type}}) } my $b = $s->resultset('Bug')->update_or_create($bug) or die "Unable to update or create bug $bug->{id}"; - $s->txn_do(sub { - for my $ff (qw(found fixed)) { + $s->txn_do(sub { + my @unknown_packages; + my @unknown_affects_packages; + push @unknown_packages, + $b->set_related_packages('binpackages', + [grep {defined $_ and + length $_ and $_ !~ /^src:/} + make_list($data->{package})], + $param{packages}, + ); + push @unknown_packages, + $b->set_related_packages('srcpackages', + [map {s/src://; + $_} + grep {defined $_ and + $_ =~ /^src:/} + make_list($data->{package})], + $param{packages}, + ); + push @unknown_affects_packages, + $b->set_related_packages('affects_binpackages', + [grep {defined $_ and + length $_ and $_ !~ /^src:/} + make_list($data->{affects}) + ], + $param{packages}, + ); + push @unknown_affects_packages, + $b->set_related_packages('affects_srcpackages', + [map {s/src://; + $_} + grep {defined $_ and + $_ =~ /^src:/} + make_list($data->{affects})], + $param{packages}, + ); + $b->unknown_packages(join(', ',@unknown_packages)); + $b->unknown_affects(join(', ',@unknown_affects_packages)); + $b->update(); + for my $ff (qw(found fixed)) { my @elements = $s->resultset('BugVer')->search({bug => $data->{bug_num}, found => $ff eq 'found'?1:0, }); - my %elements_to_delete = map {($elements[$_]->ver_string(),$elements[$_])} 0..$#elements; + my %elements_to_delete = map {($elements[$_]->ver_string(), + $elements[$_])} 0..$#elements; my %elements_to_add; my @elements_to_keep; for my $version (@{$data->{"${ff}_versions"}}) { @@ -214,27 +257,27 @@ sub load_bug { } } }); - $s->txn_do(sub { - my $t = $s->resultset('BugTag')->search({bug => $data->{bug_num}}); - $t->delete() if defined $t; - $s->populate(BugTag => [[qw(bug tag)], map {[$data->{bug_num}, $_->id()]} values %tags]); - }); + ### set bug tags + $s->txn_do(sub {$b->set_tags([values %tags ] )}); # because these bugs reference other bugs which might not exist # yet, we can't handle them until we've loaded all bugs. queue # them up. - for my $merge_block (qw(merged block)) { - my $data_key = $merge_block; - $data_key .= 'with' if $merge_block eq 'merged'; - if (@{$data->{$data_key}||[]}) { - my $count = $s->resultset('Bug')->search({id => [@{$data->{$data_key}}]})->count(); - if ($count == @{$data->{$data_key}}) { - handle_load_bug_queue(db=>$s, - queue => {$merge_block, - {$data->{bug_num},[@{$data->{$data_key}}]} - }); - } else { - $queue->{$merge_block}{$data->{bug_num}} = [@{$data->{$data_key}}]; - } + for my $merge_block (qw(mergedwith blocks)) { + my $count = 0; + if (@{$data->{$merge_block}}) { + $count = + $s->resultset('Bug')-> + search({id => [@{$data->{$merge_block}}]})-> + count(); + } + # if all of the bugs exist, immediately fix the merge/blocks + if ($count == @{$data->{$merge_block}}) { + handle_load_bug_queue(db=>$s, + queue => {$merge_block, + {$data->{bug_num},[@{$data->{$merge_block}}]} + }); + } else { + $queue->{$merge_block}{$data->{bug_num}} = [@{$data->{$merge_block}}]; } } @@ -266,26 +309,33 @@ sub handle_load_bug_queue{ my $s = $param{db}; my $queue = $param{queue}; my %queue_types = - (merged => {set => 'BugMerged', - columns => [qw(bug merged)], - bug => 'bug', - }, + (mergedwith => {set => 'BugMerged', + columns => [qw(bug merged)], + bug => 'bug', + }, blocks => {set => 'BugBlock', - columns => [qw(bug blocks)], - bug => 'bug', - }, + columns => [qw(bug blocks)], + bug => 'bug', + }, ); for my $queue_type (keys %queue_types) { - for my $bug (%{$queue->{$queue_type}}) { - my $qt = $queue_types{$queue_type}; - $s->txn_do(sub { - $s->resultset($qt->{set})->search({$qt->{bug},$bug})->delete(); - $s->populate($qt->{set},[[@{$qt->{columns}}], - map {[$bug,$_]} @{$queue->{$queue_type}{$bug}}]) if - @{$queue->{$queue_type}{$bug}//[]}; + my $qt = $queue_types{$queue_type}; + my @bugs = keys %{$queue->{$queue_type}}; + next unless @bugs; + my @entries; + for my $bug (@bugs) { + push @entries, + map {[$bug,$_]} + @{$queue->{$queue_type}{$bug}}; + } + $s->txn_do(sub { + $s->resultset($qt->{set})-> + search({$qt->{bug}=>\@bugs})->delete(); + $s->resultset($qt->{set})-> + populate([[@{$qt->{columns}}], + @entries]) if @entries; } - ); - } + ); } } @@ -325,18 +375,84 @@ sub load_bug_log { next if defined $msg_id and exists $seen_msg_ids{$msg_id}; $seen_msg_ids{$msg_id} = 1 if defined $msg_id; next if defined $msg_id and $msg_id =~ /handler\..+\.ack(?:info)?\@/; - my $message = parse($record->{text}); + my $entity = parse_to_mime_entity($record); # search for a message with this message id in the database - - # check to see if the subject, to, and from match. if so, it's + $msg_id = $entity->head->get('Message-Id') // + $entity->head->get('Resent-Message-ID') // + ''; + $msg_id =~ s/^\s*\\s*$//; + # check to see if the subject, to, and from match. if so, it's # probably the same message. - - # if not, create a new message - - # add correspondents if necessary - + my $subject = decode_rfc1522($entity->head->get('Subject')//''); + $subject =~ s/\n(?:(\s)\s*|\s*$)//g; + my $to = decode_rfc1522($entity->head->get('To')//''); + $to =~ s/\n(?:(\s)\s*|\s*$)//g; + my $from = decode_rfc1522($entity->head->get('From')//''); + $from =~ s/\n(?:(\s)\s*|\s*$)//g; + my $m = $s->resultset('Message')-> + find({msgid => $msg_id, + from_complete => $from, + to_complete => $to, + subject => $subject + }); + if (not defined $m) { + # if not, create a new message + $m = $s->resultset('Message')-> + find_or_create({msgid => $msg_id, + from_complete => $from, + to_complete => $to, + subject => $subject + }); + eval { + my $date = DateTime::Format::Mail-> + parse_datetime($entity->head->get('Date',0)); + if (abs($date->offset) >= 60 * 60 * 12) { + $date = $date->set_time_zone('UTC'); + } + $m->sent_date($date); + }; + my $spam = $entity->head->get('X-Spam-Status',0)//''; + if ($spam=~ /score=([\d\.]+)/) { + $m->spam_score($1); + } + my %corr; + @{$corr{from}} = getparsedaddrs($from); + @{$corr{to}} = getparsedaddrs($to); + @{$corr{cc}} = getparsedaddrs($entity->head->get('Cc')); + # add correspondents if necessary + my @cors; + for my $type (keys %corr) { + for my $addr (@{$corr{$type}}) { + my $cor = $s->resultset('Correspondent')-> + get_correspondent_id($addr); + next unless defined $cor; + push @cors, + {correspondent => $cor, + correspondent_type => $type, + }; + } + } + $m->update(); + $s->txn_do(sub { + $m->message_correspondents()->delete(); + $m->add_to_message_correspondents(@cors) if + @cors; + } + ); + } + my $recv; + if ($entity->head->get('Received',0) + =~ /via spool by (\S+)/) { + $recv = $s->resultset('Correspondent')-> + get_correspondent_id($1); + $m->add_to_message_correspondents({correspondent=>$recv, + correspondent_type => 'recv'}); + } # link message to bugs if necessary - + $m->find_or_create_related('bug_messages', + {bug=>$param{bug}, + message_number => $msg_num}); } } @@ -358,23 +474,58 @@ Commands to handle src and package version loading from debinfo files =cut sub load_debinfo { - my ($schema,$binname, $binver, $binarch, $srcname, $srcver) = @_; - my $sp = $schema->resultset('SrcPkg')->find_or_create({pkg => $srcname}); - my $sv = $schema->resultset('SrcVer')->find_or_create({src_pkg=>$sp->id(), - ver => $srcver}); - my $arch = $schema->resultset('Arch')->find_or_create({arch => $binarch}); - my $bp = $schema->resultset('BinPkg')->find_or_create({pkg => $binname}); - $schema->resultset('BinVer')->find_or_create({bin_pkg_id => $bp->id(), - src_ver_id => $sv->id(), - arch_id => $arch->id(), - ver => $binver, - }); + my ($s,$binname, $binver, $binarch, $srcname, $srcver,$ct_date,$cache) = @_; + $cache //= {}; + my $sp; + if (not defined $cache->{sp}{$srcname}) { + $cache->{sp}{$srcname} = + $s->resultset('SrcPkg')->find_or_create({pkg => $srcname}); + } + $sp = $cache->{sp}{$srcname}; + # update the creation date if the data we have is earlier + if (defined $ct_date and + (not defined $sp->creation or + $ct_date < $sp->creation)) { + $sp->creation($ct_date); + $sp->last_modified(DateTime->now); + $sp->update; + } + my $sv; + if (not defined $cache->{sv}{$srcname}{$srcver}) { + $cache->{sv}{$srcname}{$srcver} = + $s->resultset('SrcVer')-> + find_or_create({src_pkg =>$sp->id(), + ver => $srcver}); + } + $sv = $cache->{sv}{$srcname}{$srcver}; + if (defined $ct_date and + (not defined $sv->upload_date() or $ct_date < $sv->upload_date())) { + $sv->upload_date($ct_date); + $sv->update; + } + my $arch; + if (not defined $cache->{arch}{$binarch}) { + $cache->{arch}{$binarch} = + $s->resultset('Arch')-> + find_or_create({arch => $binarch}, + )->id(); + } + $arch = $cache->{arch}{$binarch}; + my $bp; + if (not defined $cache->{bp}{$binname}) { + $cache->{bp}{$binname} = + $s->resultset('BinPkg')-> + get_or_create_bin_pkg_id($binname); + } + $bp = $cache->{bp}{$binname}; + $s->resultset('BinVer')-> + get_bin_ver_id($bp,$binver,$arch,$sv->id()); } =back -=head Packages +=head2 Packages =over @@ -384,71 +535,203 @@ sub load_debinfo { =cut -sub load_package { - my ($schema,$suite,$component,$arch,$pkg) = @_; - if ($arch eq 'source') { - my $sp = $schema->resultset('SrcPkg')->find_or_create({pkg => $pkg->{Package}}); - my $suite = $schema->resultset('Suite')->find_or_create({suite_name => $suite}); - my $sv = $schema->resultset('SrcVer')->find_or_create({src_pkg =>$sp->id, - ver => $pkg->{Version}}); - my @addrs = getparsedaddrs($pkg->{Maintainer} // ''); - if (@addrs) { - my $mc = $schema->resultset('Correspondent')-> - find_or_create({addr => lc($addrs[0]->address())}); - my $full_name = $addrs[0]->phrase(); - $full_name =~ s/^\"|\"$//g; - $full_name =~ s/^\s+|\s+$//g; - $sv->discard_changes; - $sv->find_or_create_related('maintainer', - {name => $full_name, - correspondent => $mc->id}, - ); - $mc->update_or_create_related('correspondent_full_names', - {full_name=>$full_name, - last_seen => 'NOW()'}); +sub load_packages { + my ($schema,$suite,$pkgs,$p) = @_; + my $suite_id = $schema->resultset('Suite')-> + find_or_create({codename => $suite})->id; + my %maint_cache; + my %arch_cache; + my %source_cache; + my $src_max_last_modified = $schema->resultset('SrcAssociation')-> + search_rs({suite => $suite_id}, + {order_by => {-desc => ['me.modified']}, + rows => 1, + page => 1 + } + )->single(); + my $bin_max_last_modified = $schema->resultset('BinAssociation')-> + search_rs({suite => $suite_id}, + {order_by => {-desc => ['me.modified']}, + rows => 1, + page => 1 + } + )->single(); + my %maints; + my %sources; + my %bins; + for my $pkg_tuple (@{$pkgs}) { + my ($arch,$component,$pkg) = @{$pkg_tuple}; + $maints{$pkg->{Maintainer}} = $pkg->{Maintainer}; + if ($arch eq 'source') { + my $source = $pkg->{Package}; + my $source_ver = $pkg->{Version}; + $sources{$source}{$source_ver} = $pkg->{Maintainer}; + } else { + my $source = $pkg->{Source} // $pkg->{Package}; + my $source_ver = $pkg->{Version}; + if ($source =~ /^\s*(\S+) \(([^\)]+)\)\s*$/) { + ($source,$source_ver) = ($1,$2); + } + $sources{$source}{$source_ver} = $pkg->{Maintainer}; + $bins{$arch}{$pkg->{Package}} = + {arch => $arch, + bin => $pkg->{Package}, + bin_ver => $pkg->{Version}, + src_ver => $source_ver, + source => $source, + maint => $pkg->{Maintainer}, + }; } - # update the link for this source package - $schema-> - txndo(sub { - # delete associations for this source package in this - # suite - $schema->resultset('SrcAssociations')-> - search_rs({suite => $suite->id,})-> - search_related_rs('src_pkg', - {src_pkg => $sp->id})->delete; - $schema->resultset('SrcAssociations')-> - create({suite => $suite->id, - source => $sv->id, - }); - }); - } else { - my $bp = $schema->resultset('BinPkg')->find_or_create({pkg => $pkg->{Package}}); - my $suite = $schema->resultset('Suite')->find_or_create({suite_name => $suite}); - my ($bv) = $bp->search_related('bin_vers',{ver => $pkg->{Version}}); - # if there isn't already a binary version for this package, we don't - # know what source it belongs to, so we can't associate it with a - # release - return if (not defined $bv); - $schema-> - txndo(sub { - $schema->resultset('BinAssociations')-> - search_rs({suite => $suite->id,})-> - search_related_rs('bin_pkg', - {bin_pkg_id => $bp->id} - )->delete; - $schema->resultset('BinAssociations')-> - create({suite => $suite->id, - bin => $bv->id - }); - }); } + # Retrieve and Insert new maintainers + my $maints = + $schema->resultset('Maintainer')-> + get_maintainers(keys %maints); + my $archs = + $schema->resultset('Arch')-> + get_archs(keys %bins); + # We want all of the source package/versions which are in this suite to + # start with + my @sa_to_add; + my @sa_to_del; + my %included_sa; + # Calculate which source packages are no longer in this suite + for my $s ($schema->resultset('SrcPkg')-> + src_pkg_and_ver_in_suite($suite)) { + if (not exists $sources{$s->{pkg}} or + not exists $sources{$s->{pkg}}{$s->{src_vers}{ver}} + ) { + push @sa_to_del, + $s->{src_associations}{id}; + } + $included_sa{$s->{pkg}}{$s->{src_vers}} = 1; + } + # Calculate which source packages are newly in this suite + for my $s (keys %sources) { + for my $v (keys %{$sources{$s}}) { + if (not exists $included_sa{$s} and + not $included_sa{$s}{$v}) { + push @sa_to_add, + [$s,$v,$sources{$s}{$v}]; + } else { + $p->update() if defined $p; + } + } + } + # add new source packages + my $it = natatime 100, @sa_to_add; + while (my @v = $it->()) { + $schema->txn_do( + sub { + for my $svm (@_) { + my $s_id = $schema->resultset('SrcPkg')-> + get_or_create_src_pkg_id($svm->[0]); + my $sv_id = $schema->resultset('SrcVer')-> + get_src_ver_id($s_id,$svm->[1],$maints->{$svm->[2]}); + $schema->resultset('SrcAssociation')-> + insert_suite_src_ver_association($suite_id,$sv_id); + } + }, + @v + ); + $p->update($p->last_update()+ + scalar @v) if defined $p; + } + # remove associations for packages not in this suite + if (@sa_to_del) { + $it = natatime 1000, @sa_to_del; + while (my @v = $it->()) { + $schema-> + txn_do(sub { + $schema->resultset('SrcAssociation')-> + search_rs({id => \@v})-> + delete(); + }); + } + } + # update packages in this suite to have a modification time of now + $schema->resultset('SrcAssociation')-> + search_rs({suite => $suite_id})-> + update({modified => 'NOW()'}); + ## Handle binary packages + my @bin_to_del; + my @bin_to_add; + my %included_bin; + # calculate which binary packages are no longer in this suite + for my $b ($schema->resultset('BinPkg')-> + bin_pkg_and_ver_in_suite($suite)) { + if (not exists $bins{$b->{arch}{arch}} or + not exists $bins{$b->{arch}{arch}}{$b->{pkg}} or + ($bins{$b->{arch}{arch}}{$b->{pkg}}{bin_ver} ne + $b->{bin_vers}{ver} + ) + ) { + push @bin_to_del, + $b->{bin_associations}{id}; + } + $included_bin{$b->{arch}{arch}}{$b->{pkg}} = + $b->{bin_vers}{ver}; + } + # calculate which binary packages are newly in this suite + for my $a (keys %bins) { + for my $pkg (keys %{$bins{$a}}) { + if (not exists $included_bin{$a} or + not exists $included_bin{$a}{$pkg} or + $bins{$a}{$pkg}{bin_ver} ne + $included_bin{$a}{$pkg}) { + push @bin_to_add, + $bins{$a}{$pkg}; + } else { + $p->update() if defined $p; + } + } + } + $it = natatime 100, @bin_to_add; + while (my @v = $it->()) { + $schema->txn_do( + sub { + for my $bvm (@_) { + my $s_id = $schema->resultset('SrcPkg')-> + get_or_create_src_pkg_id($bvm->{source}); + my $sv_id = $schema->resultset('SrcVer')-> + get_src_ver_id($s_id,$bvm->{src_ver},$maints->{$bvm->{maint}}); + my $b_id = $schema->resultset('BinPkg')-> + get_or_create_bin_pkg_id($bvm->{bin}); + my $bv_id = $schema->resultset('BinVer')-> + get_bin_ver_id($b_id,$bvm->{bin_ver}, + $archs->{$bvm->{arch}},$sv_id); + $schema->resultset('BinAssociation')-> + insert_suite_bin_ver_association($suite_id,$bv_id); + } + }, + @v + ); + $p->update($p->last_update()+ + scalar @v) if defined $p; + } + if (@bin_to_del) { + $it = natatime 1000, @bin_to_del; + while (my @v = $it->()) { + $schema-> + txn_do(sub { + $schema->resultset('BinAssociation')-> + search_rs({id => \@v})-> + delete(); + }); + } + } + $schema->resultset('BinAssociation')-> + search_rs({suite => $suite_id})-> + update({modified => 'NOW()'}); + } + =back =cut -=head Suites +=head2 Suites =over @@ -482,3 +765,7 @@ sub load_suite { __END__ +# Local Variables: +# indent-tabs-mode: nil +# cperl-indent-level: 4 +# End: