1 # This module is part of debbugs, and is released
2 # under the terms of the GPL version 2, or any later version. See the
3 # file README and COPYING for more information.
4 # Copyright 2013 by Don Armstrong <don@donarmstrong.com>.
6 package Debbugs::DB::Load;
10 Debbugs::DB::Load -- Utility routines for loading the database
27 use vars qw($VERSION $DEBUG %EXPORT_TAGS @EXPORT_OK @EXPORT);
28 use base qw(Exporter);
31 ($VERSION) = q$Revision$ =~ /^Revision:\s+([^\s+])/;
32 $DEBUG = 0 unless defined $DEBUG;
35 %EXPORT_TAGS = (load_bug => [qw(load_bug handle_load_bug_queue load_bug_log)],
36 load_debinfo => [qw(load_debinfo)],
37 load_package => [qw(load_packages)],
38 load_suite => [qw(load_suite)],
41 Exporter::export_ok_tags(keys %EXPORT_TAGS);
42 $EXPORT_TAGS{all} = [@EXPORT_OK];
45 use Params::Validate qw(validate_with :types);
46 use List::AllUtils qw(natatime);
48 use Debbugs::Status qw(read_bug split_status_fields);
51 use Debbugs::Common qw(make_list getparsedaddrs);
52 use Debbugs::Config qw(:config);
53 use Debbugs::MIME qw(parse_to_mime_entity decode_rfc1522);
54 use DateTime::Format::Mail;
59 Routines to load bug; exported with :load_bug
65 load_bug(db => $schema,
66 data => split_status_fields($data),
70 Loads a bug's metadata into the database. (Does not load any messages)
74 =item db -- Debbugs::DB object
76 =item data -- Bug data (from read_bug) which has been split with split_status_fields
78 =item tags -- tag cache (hashref); optional
80 =item queue -- queue of operations to perform after bug is loaded; optional.
87 my %param = validate_with(params => \@_,
88 spec => {db => {type => OBJECT,
90 data => {type => HASHREF,
93 bug => {type => SCALAR,
96 tags => {type => HASHREF,
97 default => sub {return {}},
99 severities => {type => HASHREF,
100 default => sub {return {}},
103 queue => {type => HASHREF,
105 packages => {type => HASHREF,
106 default => sub {return {}},
111 if (not exists $param{data} and not exists $param{bug}) {
112 croak "One of data or bug must be provided to load_bug";
114 if (not exists $param{data}) {
115 $param{data} = read_bug(bug => $param{bug});
117 my $data = $param{data};
118 my $tags = $param{tags};
119 my $queue = $param{queue};
120 my $severities = $param{severities};
122 if (not defined $queue) {
127 $data = split_status_fields($data);
128 for my $tag (make_list($data->{keywords})) {
129 next unless defined $tag and length $tag;
130 # this allows for invalid tags. But we'll use this to try to
131 # find those bugs and clean them up
132 if (not exists $tags->{$tag}) {
133 $tags->{$tag} = $s->resultset('Tag')->
134 find_or_create({tag => $tag});
136 $tags{$tag} = $tags->{$tag};
138 my $severity = length($data->{severity}) ? $data->{severity} :
139 $config{default_severity};
140 if (not exists $severities->{$severity}) {
141 $severities->{$severity} =
142 $s->resultset('Severity')->
143 find_or_create({severity => $severity},
146 $severity = $severities->{$severity};
148 {id => $data->{bug_num},
149 creation => DateTime->from_epoch(epoch => $data->{date}),
150 log_modified => DateTime->from_epoch(epoch => $data->{log_modified}),
151 last_modified => DateTime->from_epoch(epoch => $data->{last_modified}),
152 archived => $data->{archived},
153 (defined $data->{unarchived} and length($data->{unarchived}))?
154 (unarchived => DateTime->from_epoch(epoch => $data->{unarchived})):(),
155 forwarded => $data->{forwarded} // '',
156 summary => $data->{summary} // '',
157 outlook => $data->{outlook} // '',
158 subject => $data->{subject} // '',
159 done_full => $data->{done} // '',
160 severity => $severity,
161 owner_full => $data->{owner} // '',
162 submitter_full => $data->{originator} // '',
167 submitter => 'originator',
169 for my $addr_type (keys %addr_map) {
170 $bug->{$addr_type} = undef;
171 next unless defined $data->{$addr_map{$addr_type}} and
172 length($data->{$addr_map{$addr_type}});
174 $s->resultset('Correspondent')->
175 get_correspondent_id($data->{$addr_map{$addr_type}})
177 my $b = $s->resultset('Bug')->update_or_create($bug) or
178 die "Unable to update or create bug $bug->{id}";
180 my @unknown_packages;
181 my @unknown_affects_packages;
182 push @unknown_packages,
183 $b->set_related_packages('binpackages',
184 [grep {defined $_ and
185 length $_ and $_ !~ /^src:/}
186 make_list($data->{package})],
189 push @unknown_packages,
190 $b->set_related_packages('srcpackages',
195 make_list($data->{package})],
198 push @unknown_affects_packages,
199 $b->set_related_packages('affects_binpackages',
200 [grep {defined $_ and
201 length $_ and $_ !~ /^src:/}
202 make_list($data->{affects})
206 push @unknown_affects_packages,
207 $b->set_related_packages('affects_srcpackages',
212 make_list($data->{affects})],
215 $b->unknown_packages(join(', ',@unknown_packages));
216 $b->unknown_affects(join(', ',@unknown_affects_packages));
218 for my $ff (qw(found fixed)) {
219 my @elements = $s->resultset('BugVer')->search({bug => $data->{bug_num},
220 found => $ff eq 'found'?1:0,
222 my %elements_to_delete = map {($elements[$_]->ver_string(),
223 $elements[$_])} 0..$#elements;
225 my @elements_to_keep;
226 for my $version (@{$data->{"${ff}_versions"}}) {
227 if (exists $elements_to_delete{$version}) {
228 push @elements_to_keep,$version;
230 $elements_to_add{$version} = 1;
233 for my $version (@elements_to_keep) {
234 delete $elements_to_delete{$version};
236 for my $element (keys %elements_to_delete) {
237 $elements_to_delete{$element}->delete();
239 for my $element (keys %elements_to_add) {
240 # find source package and source version id
241 my $ne = $s->resultset('BugVer')->new_result({bug => $data->{bug_num},
242 ver_string => $element,
243 found => $ff eq 'found'?1:0,
246 if (my ($src_pkg,$src_ver) = $element =~ m{^([^\/]+)/(.+)$}) {
247 my $src_pkg_e = $s->resultset('SrcPkg')->single({pkg => $src_pkg});
248 if (defined $src_pkg_e) {
249 $ne->src_pkg($src_pkg_e->id());
250 my $src_ver_e = $s->resultset('SrcVer')->single({src_pkg => $src_pkg_e->id(),
253 $ne->src_ver($src_ver_e->id()) if defined $src_ver_e;
261 $s->txn_do(sub {$b->set_tags([values %tags ] )});
262 # because these bugs reference other bugs which might not exist
263 # yet, we can't handle them until we've loaded all bugs. queue
265 for my $merge_block (qw(mergedwith blocks)) {
267 if (@{$data->{$merge_block}}) {
269 $s->resultset('Bug')->
270 search({id => [@{$data->{$merge_block}}]})->
273 # if all of the bugs exist, immediately fix the merge/blocks
274 if ($count == @{$data->{$merge_block}}) {
275 handle_load_bug_queue(db=>$s,
276 queue => {$merge_block,
277 {$data->{bug_num},[@{$data->{$merge_block}}]}
280 $queue->{$merge_block}{$data->{bug_num}} = [@{$data->{$merge_block}}];
284 if (not $can_queue and keys %{$queue}) {
285 handle_load_bug_queue(db => $s,queue => $queue);
288 # still need to handle merges, versions, etc.
291 =item handle_load_bug_queue
293 handle_load_bug_queue(db => $schema,queue => $queue);
295 Handles a queue of operations created by load bug. [These operations
296 are used to handle cases where a bug referenced by a loaded bug may
297 not exist yet. In cases where the bugs should exist, the queue is
298 cleared automatically by load_bug if queue is undefined.
302 sub handle_load_bug_queue{
303 my %param = validate_with(params => \@_,
304 spec => {db => {type => OBJECT,
306 queue => {type => HASHREF,
310 my $queue = $param{queue};
312 (mergedwith => {set => 'BugMerged',
313 columns => [qw(bug merged)],
316 blocks => {set => 'BugBlock',
317 columns => [qw(bug blocks)],
321 for my $queue_type (keys %queue_types) {
322 my $qt = $queue_types{$queue_type};
323 my @bugs = keys %{$queue->{$queue_type}};
326 for my $bug (@bugs) {
329 @{$queue->{$queue_type}{$bug}};
332 $s->resultset($qt->{set})->
333 search({$qt->{bug}=>\@bugs})->delete();
334 $s->resultset($qt->{set})->
335 populate([[@{$qt->{columns}}],
336 @entries]) if @entries;
342 =item load_bug_log -- load bug logs
344 load_bug_log(db => $s,
352 =item bug -- bug whose log should be loaded
359 my %param = validate_with(params => \@_,
360 spec => {db => {type => OBJECT,
362 bug => {type => SCALAR,
364 queue => {type => HASHREF,
370 my $log = Debbugs::Log->new(bug_num => $param{bug}) or
371 die "Unable to open log for $param{bug} for reading: $!";
372 while (my $record = $log->read_record()) {
373 next unless $record->{type} eq 'incoming-recv';
374 my ($msg_id) = $record->{text} =~ /^Message-Id:\s+<(.+)>/im;
375 next if defined $msg_id and exists $seen_msg_ids{$msg_id};
376 $seen_msg_ids{$msg_id} = 1 if defined $msg_id;
377 next if defined $msg_id and $msg_id =~ /handler\..+\.ack(?:info)?\@/;
378 my $entity = parse_to_mime_entity($record);
379 # search for a message with this message id in the database
380 $msg_id = $entity->head->get('Message-Id') //
381 $entity->head->get('Resent-Message-ID') //
383 $msg_id =~ s/^\s*\<//;
384 $msg_id =~ s/>\s*$//;
385 # check to see if the subject, to, and from match. if so, it's
386 # probably the same message.
387 my $subject = decode_rfc1522($entity->head->get('Subject')//'');
388 $subject =~ s/\n(?:(\s)\s*|\s*$)//g;
389 my $to = decode_rfc1522($entity->head->get('To')//'');
390 $to =~ s/\n(?:(\s)\s*|\s*$)//g;
391 my $from = decode_rfc1522($entity->head->get('From')//'');
392 $from =~ s/\n(?:(\s)\s*|\s*$)//g;
393 my $m = $s->resultset('Message')->
394 find({msgid => $msg_id,
395 from_complete => $from,
399 if (not defined $m) {
400 # if not, create a new message
401 $m = $s->resultset('Message')->
402 find_or_create({msgid => $msg_id,
403 from_complete => $from,
408 my $date = DateTime::Format::Mail->
409 parse_datetime($entity->head->get('Date',0));
410 if (abs($date->offset) >= 60 * 60 * 12) {
411 $date = $date->set_time_zone('UTC');
413 $m->sent_date($date);
415 my $spam = $entity->head->get('X-Spam-Status',0)//'';
416 if ($spam=~ /score=([\d\.]+)/) {
420 @{$corr{from}} = getparsedaddrs($from);
421 @{$corr{to}} = getparsedaddrs($to);
422 @{$corr{cc}} = getparsedaddrs($entity->head->get('Cc'));
423 # add correspondents if necessary
425 for my $type (keys %corr) {
426 for my $addr (@{$corr{$type}}) {
427 my $cor = $s->resultset('Correspondent')->
428 get_correspondent_id($addr);
429 next unless defined $cor;
431 {correspondent => $cor,
432 correspondent_type => $type,
438 $m->message_correspondents()->delete();
439 $m->add_to_message_correspondents(@cors) if
445 if ($entity->head->get('Received',0)
446 =~ /via spool by (\S+)/) {
447 $recv = $s->resultset('Correspondent')->
448 get_correspondent_id($1);
449 $m->add_to_message_correspondents({correspondent=>$recv,
450 correspondent_type => 'recv'});
452 # link message to bugs if necessary
453 $m->find_or_create_related('bug_messages',
455 message_number => $msg_num});
464 Commands to handle src and package version loading from debinfo files
470 load_debinfo($schema,$binname, $binver, $binarch, $srcname, $srcver);
477 my ($s,$binname, $binver, $binarch, $srcname, $srcver,$ct_date,$cache) = @_;
480 if (not defined $cache->{sp}{$srcname}) {
481 $cache->{sp}{$srcname} =
482 $s->resultset('SrcPkg')->find_or_create({pkg => $srcname});
484 $sp = $cache->{sp}{$srcname};
485 # update the creation date if the data we have is earlier
486 if (defined $ct_date and
487 (not defined $sp->creation or
488 $ct_date < $sp->creation)) {
489 $sp->creation($ct_date);
490 $sp->last_modified(DateTime->now);
494 if (not defined $cache->{sv}{$srcname}{$srcver}) {
495 $cache->{sv}{$srcname}{$srcver} =
496 $s->resultset('SrcVer')->
497 find_or_create({src_pkg =>$sp->id(),
500 $sv = $cache->{sv}{$srcname}{$srcver};
501 if (defined $ct_date and
502 (not defined $sv->upload_date() or $ct_date < $sv->upload_date())) {
503 $sv->upload_date($ct_date);
507 if (not defined $cache->{arch}{$binarch}) {
508 $cache->{arch}{$binarch} =
509 $s->resultset('Arch')->
510 find_or_create({arch => $binarch},
513 $arch = $cache->{arch}{$binarch};
515 if (not defined $cache->{bp}{$binname}) {
516 $cache->{bp}{$binname} =
517 $s->resultset('BinPkg')->
518 get_or_create_bin_pkg_id($binname);
520 $bp = $cache->{bp}{$binname};
521 $s->resultset('BinVer')->
522 get_bin_ver_id($bp,$binver,$arch,$sv->id());
534 load_package($schema,$suite,$component,$arch,$pkg)
539 my ($schema,$suite,$pkgs,$p) = @_;
540 my $suite_id = $schema->resultset('Suite')->
541 find_or_create({codename => $suite})->id;
545 my $src_max_last_modified = $schema->resultset('SrcAssociation')->
546 search_rs({suite => $suite_id},
547 {order_by => {-desc => ['me.modified']},
552 my $bin_max_last_modified = $schema->resultset('BinAssociation')->
553 search_rs({suite => $suite_id},
554 {order_by => {-desc => ['me.modified']},
562 for my $pkg_tuple (@{$pkgs}) {
563 my ($arch,$component,$pkg) = @{$pkg_tuple};
564 $maints{$pkg->{Maintainer}} = $pkg->{Maintainer};
565 if ($arch eq 'source') {
566 my $source = $pkg->{Package};
567 my $source_ver = $pkg->{Version};
568 $sources{$source}{$source_ver} = $pkg->{Maintainer};
570 my $source = $pkg->{Source} // $pkg->{Package};
571 my $source_ver = $pkg->{Version};
572 if ($source =~ /^\s*(\S+) \(([^\)]+)\)\s*$/) {
573 ($source,$source_ver) = ($1,$2);
575 $sources{$source}{$source_ver} = $pkg->{Maintainer};
576 $bins{$arch}{$pkg->{Package}} =
578 bin => $pkg->{Package},
579 bin_ver => $pkg->{Version},
580 src_ver => $source_ver,
582 maint => $pkg->{Maintainer},
586 # Retrieve and Insert new maintainers
588 $schema->resultset('Maintainer')->
589 get_maintainers(keys %maints);
591 $schema->resultset('Arch')->
592 get_archs(keys %bins);
593 # We want all of the source package/versions which are in this suite to
598 # Calculate which source packages are no longer in this suite
599 for my $s ($schema->resultset('SrcPkg')->
600 src_pkg_and_ver_in_suite($suite)) {
601 if (not exists $sources{$s->{pkg}} or
602 not exists $sources{$s->{pkg}}{$s->{src_vers}{ver}}
605 $s->{src_associations}{id};
607 $included_sa{$s->{pkg}}{$s->{src_vers}} = 1;
609 # Calculate which source packages are newly in this suite
610 for my $s (keys %sources) {
611 for my $v (keys %{$sources{$s}}) {
612 if (not exists $included_sa{$s} and
613 not $included_sa{$s}{$v}) {
615 [$s,$v,$sources{$s}{$v}];
617 $p->update() if defined $p;
621 # add new source packages
622 my $it = natatime 100, @sa_to_add;
623 while (my @v = $it->()) {
627 my $s_id = $schema->resultset('SrcPkg')->
628 get_or_create_src_pkg_id($svm->[0]);
629 my $sv_id = $schema->resultset('SrcVer')->
630 get_src_ver_id($s_id,$svm->[1],$maints->{$svm->[2]});
631 $schema->resultset('SrcAssociation')->
632 insert_suite_src_ver_association($suite_id,$sv_id);
637 $p->update($p->last_update()+
638 scalar @v) if defined $p;
640 # remove associations for packages not in this suite
642 $it = natatime 1000, @sa_to_del;
643 while (my @v = $it->()) {
646 $schema->resultset('SrcAssociation')->
647 search_rs({id => \@v})->
652 # update packages in this suite to have a modification time of now
653 $schema->resultset('SrcAssociation')->
654 search_rs({suite => $suite_id})->
655 update({modified => 'NOW()'});
656 ## Handle binary packages
660 # calculate which binary packages are no longer in this suite
661 for my $b ($schema->resultset('BinPkg')->
662 bin_pkg_and_ver_in_suite($suite)) {
663 if (not exists $bins{$b->{arch}{arch}} or
664 not exists $bins{$b->{arch}{arch}}{$b->{pkg}} or
665 ($bins{$b->{arch}{arch}}{$b->{pkg}}{bin_ver} ne
670 $b->{bin_associations}{id};
672 $included_bin{$b->{arch}{arch}}{$b->{pkg}} =
675 # calculate which binary packages are newly in this suite
676 for my $a (keys %bins) {
677 for my $pkg (keys %{$bins{$a}}) {
678 if (not exists $included_bin{$a} or
679 not exists $included_bin{$a}{$pkg} or
680 $bins{$a}{$pkg}{bin_ver} ne
681 $included_bin{$a}{$pkg}) {
685 $p->update() if defined $p;
689 $it = natatime 100, @bin_to_add;
690 while (my @v = $it->()) {
694 my $s_id = $schema->resultset('SrcPkg')->
695 get_or_create_src_pkg_id($bvm->{source});
696 my $sv_id = $schema->resultset('SrcVer')->
697 get_src_ver_id($s_id,$bvm->{src_ver},$maints->{$bvm->{maint}});
698 my $b_id = $schema->resultset('BinPkg')->
699 get_or_create_bin_pkg_id($bvm->{bin});
700 my $bv_id = $schema->resultset('BinVer')->
701 get_bin_ver_id($b_id,$bvm->{bin_ver},
702 $archs->{$bvm->{arch}},$sv_id);
703 $schema->resultset('BinAssociation')->
704 insert_suite_bin_ver_association($suite_id,$bv_id);
709 $p->update($p->last_update()+
710 scalar @v) if defined $p;
713 $it = natatime 1000, @bin_to_del;
714 while (my @v = $it->()) {
717 $schema->resultset('BinAssociation')->
718 search_rs({id => \@v})->
723 $schema->resultset('BinAssociation')->
724 search_rs({suite => $suite_id})->
725 update({modified => 'NOW()'});
740 load_suite($schema,$codename,$suite,$version,$active);
745 my ($schema,$codename,$suite,$version,$active) = @_;
746 if (ref($codename)) {
747 ($codename,$suite,$version) =
748 @{$codename}{qw(Codename Suite Version)};
751 my $s = $schema->resultset('Suite')->find_or_create({codename => $codename});
752 $s->suite_name($suite);
753 $s->version($version);
769 # indent-tabs-mode: nil
770 # cperl-indent-level: 4