1 # This module is part of debbugs, and is released
2 # under the terms of the GPL version 2, or any later version. See the
3 # file README and COPYING for more information.
4 # Copyright 2013 by Don Armstrong <don@donarmstrong.com>.
6 package Debbugs::DB::Load;
10 Debbugs::DB::Load -- Utility routines for loading the database
26 use vars qw($VERSION $DEBUG %EXPORT_TAGS @EXPORT_OK @EXPORT);
27 use base qw(Exporter);
30 ($VERSION) = q$Revision$ =~ /^Revision:\s+([^\s+])/;
31 $DEBUG = 0 unless defined $DEBUG;
34 %EXPORT_TAGS = (load_bug => [qw(load_bug handle_load_bug_queue load_bug_log)],
35 load_debinfo => [qw(load_debinfo)],
36 load_package => [qw(load_packages)],
37 load_suite => [qw(load_suite)],
40 Exporter::export_ok_tags(keys %EXPORT_TAGS);
41 $EXPORT_TAGS{all} = [@EXPORT_OK];
44 use Params::Validate qw(validate_with :types);
45 use List::MoreUtils qw(natatime);
47 use Debbugs::Status qw(read_bug split_status_fields);
50 use Debbugs::Common qw(make_list getparsedaddrs);
51 use Debbugs::Config qw(:config);
56 Routines to load bug; exported with :load_bug
62 load_bug(db => $schema,
63 data => split_status_fields($data),
67 Loads a bug's metadata into the database. (Does not load any messages)
71 =item db -- Debbugs::DB object
73 =item data -- Bug data (from read_bug) which has been split with split_status_fields
75 =item tags -- tag cache (hashref); optional
77 =item queue -- queue of operations to perform after bug is loaded; optional.
84 my %param = validate_with(params => \@_,
85 spec => {db => {type => OBJECT,
87 data => {type => HASHREF,
90 bug => {type => SCALAR,
93 tags => {type => HASHREF,
94 default => sub {return {}},
96 severities => {type => HASHREF,
97 default => sub {return {}},
100 queue => {type => HASHREF,
104 if (not exists $param{data} and not exists $param{bug}) {
105 croak "One of data or bug must be provided to load_bug";
107 if (not exists $param{data}) {
108 $param{data} = read_bug(bug => $param{bug});
110 my $data = $param{data};
111 my $tags = $param{tags};
112 my $queue = $param{queue};
113 my $severities = $param{severities};
115 if (not defined $queue) {
120 my $s_data = split_status_fields($data);
121 for my $tag (make_list($s_data->{keywords})) {
122 next unless defined $tag and length $tag;
123 # this allows for invalid tags. But we'll use this to try to
124 # find those bugs and clean them up
125 if (not exists $tags->{$tag}) {
126 $tags->{$tag} = $s->resultset('Tag')->
127 find_or_create({tag => $tag});
129 $tags{$tag} = $tags->{$tag};
131 my $severity = length($data->{severity}) ? $data->{severity} : $config{default_severity};
132 if (exists $severities->{$severity}) {
133 $severity = $severities->{$severity};
135 $severity = $s->resultset('Severity')->
136 find_or_create({severity => $severity});
139 {id => $data->{bug_num},
140 creation => DateTime->from_epoch(epoch => $data->{date}),
141 log_modified => DateTime->from_epoch(epoch => $data->{log_modified}),
142 last_modified => DateTime->from_epoch(epoch => $data->{last_modified}),
143 archived => $data->{archived},
144 (defined $data->{unarchived} and length($data->{unarchived}))?(unarchived => DateTime->from_epoch(epoch => $data->{unarchived})):(),
145 forwarded => $data->{forwarded} // '',
146 summary => $data->{summary} // '',
147 outlook => $data->{outlook} // '',
148 subject => $data->{subject} // '',
149 done_full => $data->{done} // '',
150 severity => $severity,
151 owner_full => $data->{owner} // '',
152 submitter_full => $data->{originator} // '',
157 submitter => 'originator',
159 for my $addr_type (keys %addr_map) {
160 my @addrs = getparsedaddrs($data->{$addr_map{$addr_type}} // '');
162 $bug->{$addr_type} = $s->resultset('Correspondent')->find_or_create({addr => lc($addrs[0]->address())});
163 # insert the full name as well
164 my $full_name = $addrs[0]->phrase();
165 $full_name =~ s/^\"|\"$//g;
166 $full_name =~ s/^\s+|\s+$//g;
167 if (length $full_name) {
169 update_or_create_related('correspondent_full_names',
170 {full_name=>$full_name,
171 last_seen => 'NOW()'});
174 my $b = $s->resultset('Bug')->update_or_create($bug) or
175 die "Unable to update or create bug $bug->{id}";
177 for my $ff (qw(found fixed)) {
178 my @elements = $s->resultset('BugVer')->search({bug => $data->{bug_num},
179 found => $ff eq 'found'?1:0,
181 my %elements_to_delete = map {($elements[$_]->ver_string(),$elements[$_])} 0..$#elements;
183 my @elements_to_keep;
184 for my $version (@{$data->{"${ff}_versions"}}) {
185 if (exists $elements_to_delete{$version}) {
186 push @elements_to_keep,$version;
188 $elements_to_add{$version} = 1;
191 for my $version (@elements_to_keep) {
192 delete $elements_to_delete{$version};
194 for my $element (keys %elements_to_delete) {
195 $elements_to_delete{$element}->delete();
197 for my $element (keys %elements_to_add) {
198 # find source package and source version id
199 my $ne = $s->resultset('BugVer')->new_result({bug => $data->{bug_num},
200 ver_string => $element,
201 found => $ff eq 'found'?1:0,
204 if (my ($src_pkg,$src_ver) = $element =~ m{^([^\/]+)/(.+)$}) {
205 my $src_pkg_e = $s->resultset('SrcPkg')->single({pkg => $src_pkg});
206 if (defined $src_pkg_e) {
207 $ne->src_pkg($src_pkg_e->id());
208 my $src_ver_e = $s->resultset('SrcVer')->single({src_pkg => $src_pkg_e->id(),
211 $ne->src_ver($src_ver_e->id()) if defined $src_ver_e;
219 my $t = $s->resultset('BugTag')->search({bug => $data->{bug_num}});
220 $t->delete() if defined $t;
221 $s->populate(BugTag => [[qw(bug tag)], map {[$data->{bug_num}, $_->id()]} values %tags]);
223 # because these bugs reference other bugs which might not exist
224 # yet, we can't handle them until we've loaded all bugs. queue
226 for my $merge_block (qw(merged block)) {
227 my $data_key = $merge_block;
228 $data_key .= 'with' if $merge_block eq 'merged';
229 if (@{$data->{$data_key}||[]}) {
230 my $count = $s->resultset('Bug')->search({id => [@{$data->{$data_key}}]})->count();
231 if ($count == @{$data->{$data_key}}) {
232 handle_load_bug_queue(db=>$s,
233 queue => {$merge_block,
234 {$data->{bug_num},[@{$data->{$data_key}}]}
237 $queue->{$merge_block}{$data->{bug_num}} = [@{$data->{$data_key}}];
242 if (not $can_queue and keys %{$queue}) {
243 handle_load_bug_queue(db => $s,queue => $queue);
246 # still need to handle merges, versions, etc.
249 =item handle_load_bug_queue
251 handle_load_bug_queue(db => $schema,queue => $queue);
253 Handles a queue of operations created by load bug. [These operations
254 are used to handle cases where a bug referenced by a loaded bug may
255 not exist yet. In cases where the bugs should exist, the queue is
256 cleared automatically by load_bug if queue is undefined.
260 sub handle_load_bug_queue{
261 my %param = validate_with(params => \@_,
262 spec => {db => {type => OBJECT,
264 queue => {type => HASHREF,
268 my $queue = $param{queue};
270 (merged => {set => 'BugMerged',
271 columns => [qw(bug merged)],
274 blocks => {set => 'BugBlock',
275 columns => [qw(bug blocks)],
279 for my $queue_type (keys %queue_types) {
280 for my $bug (%{$queue->{$queue_type}}) {
281 my $qt = $queue_types{$queue_type};
283 $s->resultset($qt->{set})->search({$qt->{bug},$bug})->delete();
284 $s->populate($qt->{set},[[@{$qt->{columns}}],
285 map {[$bug,$_]} @{$queue->{$queue_type}{$bug}}]) if
286 @{$queue->{$queue_type}{$bug}//[]};
293 =item load_bug_log -- load bug logs
295 load_bug_log(db => $s,
303 =item bug -- bug whose log should be loaded
310 my %param = validate_with(params => \@_,
311 spec => {db => {type => OBJECT,
313 bug => {type => SCALAR,
315 queue => {type => HASHREF,
321 my $log = Debbugs::Log->new(bug_num => $param{bug}) or
322 die "Unable to open log for $param{bug} for reading: $!";
323 while (my $record = $log->read_record()) {
324 next unless $record->{type} eq 'incoming-recv';
325 my ($msg_id) = $record->{text} =~ /^Message-Id:\s+<(.+)>/im;
326 next if defined $msg_id and exists $seen_msg_ids{$msg_id};
327 $seen_msg_ids{$msg_id} = 1 if defined $msg_id;
328 next if defined $msg_id and $msg_id =~ /handler\..+\.ack(?:info)?\@/;
329 my $message = parse($record->{text});
330 # search for a message with this message id in the database
332 # check to see if the subject, to, and from match. if so, it's
333 # probably the same message.
335 # if not, create a new message
337 # add correspondents if necessary
339 # link message to bugs if necessary
349 Commands to handle src and package version loading from debinfo files
355 load_debinfo($schema,$binname, $binver, $binarch, $srcname, $srcver);
362 my ($schema,$binname, $binver, $binarch, $srcname, $srcver) = @_;
363 my $sp = $schema->resultset('SrcPkg')->find_or_create({pkg => $srcname});
364 my $sv = $schema->resultset('SrcVer')->find_or_create({src_pkg=>$sp->id(),
366 my $arch = $schema->resultset('Arch')->find_or_create({arch => $binarch});
367 my $bp = $schema->resultset('BinPkg')->find_or_create({pkg => $binname});
368 $schema->resultset('BinVer')->find_or_create({bin_pkg_id => $bp->id(),
369 src_ver_id => $sv->id(),
370 arch_id => $arch->id(),
384 load_package($schema,$suite,$component,$arch,$pkg)
389 my ($schema,$suite,$pkgs,$p) = @_;
390 my $suite_id = $schema->resultset('Suite')->
391 find_or_create({codename => $suite})->id;
395 my $src_max_last_modified = $schema->resultset('SrcAssociation')->
396 search_rs({suite => $suite_id},
397 {order_by => {-desc => ['me.modified']},
402 my $bin_max_last_modified = $schema->resultset('BinAssociation')->
403 search_rs({suite => $suite_id},
404 {order_by => {-desc => ['me.modified']},
412 for my $pkg_tuple (@{$pkgs}) {
413 my ($arch,$component,$pkg) = @{$pkg_tuple};
414 $maints{$pkg->{Maintainer}} = $pkg->{Maintainer};
415 if ($arch eq 'source') {
416 my $source = $pkg->{Package};
417 my $source_ver = $pkg->{Version};
418 $sources{$source}{$source_ver} = $pkg->{Maintainer};
420 my $source = $pkg->{Source} // $pkg->{Package};
421 my $source_ver = $pkg->{Version};
422 if ($source =~ /^\s*(\S+) \(([^\)]+)\)\s*$/) {
423 ($source,$source_ver) = ($1,$2);
425 $sources{$source}{$source_ver} = $pkg->{Maintainer};
426 $bins{$arch}{$pkg->{Package}} =
428 bin => $pkg->{Package},
429 bin_ver => $pkg->{Version},
430 src_ver => $source_ver,
432 maint => $pkg->{Maintainer},
436 # Retrieve and Insert new maintainers
438 $schema->resultset('Maintainer')->
439 get_maintainers(keys %maints);
441 $schema->resultset('Arch')->
442 get_archs(keys %bins);
443 # We want all of the source package/versions which are in this suite to
448 # Calculate which source packages are no longer in this suite
449 for my $s ($schema->resultset('SrcPkg')->
450 src_pkg_and_ver_in_suite($suite)) {
451 if (not exists $sources{$s->{pkg}} or
452 not exists $sources{$s->{pkg}}{$s->{src_vers}{ver}}
455 $s->{src_associations}{id};
457 $included_sa{$s->{pkg}}{$s->{src_vers}} = 1;
459 # Calculate which source packages are newly in this suite
460 for my $s (keys %sources) {
461 for my $v (keys %{$sources{$s}}) {
462 if (not exists $included_sa{$s} and
463 not $included_sa{$s}{$v}) {
465 [$s,$v,$sources{$s}{$v}];
467 $p->update() if defined $p;
471 # add new source packages
472 my $it = natatime 100, @sa_to_add;
473 while (my @v = $it->()) {
477 my $s_id = $schema->resultset('SrcPkg')->
478 get_src_pkg_id($svm->[0]);
479 my $sv_id = $schema->resultset('SrcVer')->
480 get_src_ver_id($s_id,$svm->[1],$maints->{$svm->[2]});
481 $schema->resultset('SrcAssociation')->
482 insert_suite_src_ver_association($suite_id,$sv_id);
487 $p->update($p->last_update()+
488 scalar @v) if defined $p;
490 # remove associations for packages not in this suite
492 $schema->resultset('SrcAssociation')->
493 search_rs({id => \@sa_to_del})->delete();
495 # update packages in this suite to have a modification time of now
496 $schema->resultset('SrcAssociation')->
497 search_rs({suite => $suite_id})->
498 update({modified => 'NOW()'});
499 ## Handle binary packages
503 # calculate which binary packages are no longer in this suite
504 for my $b ($schema->resultset('BinPkg')->
505 bin_pkg_and_ver_in_suite($suite)) {
506 if (not exists $bins{$b->{arch}{arch}} or
507 not exists $bins{$b->{arch}{arch}}{$b->{pkg}} or
508 ($bins{$b->{arch}{arch}}{$b->{pkg}}{bin_ver} ne
513 $b->{bin_associations}{id};
515 $included_bin{$b->{arch}{arch}}{$b->{pkg}} =
518 # calculate which binary packages are newly in this suite
519 for my $a (keys %bins) {
520 for my $pkg (keys %{$bins{$a}}) {
521 if (not exists $included_bin{$a} or
522 not exists $included_bin{$a}{$pkg} or
523 $bins{$a}{$pkg}{bin_ver} ne
524 $included_bin{$a}{$pkg}) {
528 $p->update() if defined $p;
532 $it = natatime 100, @bin_to_add;
533 while (my @v = $it->()) {
537 my $s_id = $schema->resultset('SrcPkg')->
538 get_src_pkg_id($bvm->{source});
539 my $sv_id = $schema->resultset('SrcVer')->
540 get_src_ver_id($s_id,$bvm->{src_ver},$maints->{$bvm->{maint}});
541 my $b_id = $schema->resultset('BinPkg')->
542 get_bin_pkg_id($bvm->{bin});
543 my $bv_id = $schema->resultset('BinVer')->
544 get_bin_ver_id($b_id,$bvm->{bin_ver},
545 $archs->{$bvm->{arch}},$sv_id);
546 $schema->resultset('BinAssociation')->
547 insert_suite_bin_ver_association($suite_id,$bv_id);
552 $p->update($p->last_update()+
553 scalar @v) if defined $p;
556 $schema->resultset('BinAssociation')->
557 search_rs({id => \@bin_to_del})->delete();
559 $schema->resultset('BinAssociation')->
560 search_rs({suite => $suite_id})->
561 update({modified => 'NOW()'});
576 load_suite($schema,$codename,$suite,$version,$active);
581 my ($schema,$codename,$suite,$version,$active) = @_;
582 if (ref($codename)) {
583 ($codename,$suite,$version) =
584 @{$codename}{qw(Codename Suite Version)};
587 my $s = $schema->resultset('Suite')->find_or_create({codename => $codename});
588 $s->suite_name($suite);
589 $s->version($version);