Debbugs/DB/Load.pm

   1 # This module is part of debbugs, and is released
   2 # under the terms of the GPL version 2, or any later version. See the
   3 # file README and COPYING for more information.
   4 # Copyright 2013 by Don Armstrong <don@donarmstrong.com>.
   5
   6 package Debbugs::DB::Load;
   7
   8 =head1 NAME
   9
  10 Debbugs::DB::Load -- Utility routines for loading the database
  11
  12 =head1 SYNOPSIS
  13
  14
  15 =head1 DESCRIPTION
  16
  17
  18 =head1 BUGS
  19
  20 None known.
  21
  22 =cut
  23
  24 use warnings;
  25 use strict;
  26 use vars qw($VERSION $DEBUG %EXPORT_TAGS @EXPORT_OK @EXPORT);
  27 use base qw(Exporter);
  28
  29 BEGIN{
  30      ($VERSION) = q$Revision$ =~ /^Revision:\s+([^\s+])/;
  31      $DEBUG = 0 unless defined $DEBUG;
  32
  33      @EXPORT = ();
  34      %EXPORT_TAGS = (load_bug    => [qw(load_bug handle_load_bug_queue load_bug_log)],
  35                      load_debinfo => [qw(load_debinfo)],
  36                      load_package => [qw(load_packages)],
  37                      load_suite => [qw(load_suite)],
  38                     );
  39      @EXPORT_OK = ();
  40      Exporter::export_ok_tags(keys %EXPORT_TAGS);
  41      $EXPORT_TAGS{all} = [@EXPORT_OK];
  42 }
  43
  44 use Params::Validate qw(validate_with :types);
  45 use List::MoreUtils qw(natatime);
  46
  47 use Debbugs::Status qw(read_bug split_status_fields);
  48 use Debbugs::DB;
  49 use DateTime;
  50 use Debbugs::Common qw(make_list getparsedaddrs);
  51 use Debbugs::Config qw(:config);
  52 use Debbugs::MIME qw(parse_to_mime_entity decode_rfc1522);
  53 use DateTime::Format::Mail;
  54 use Carp;
  55
  56 =head2 Bug loading
  57
  58 Routines to load bug; exported with :load_bug
  59
  60 =over
  61
  62 =item load_bug
  63
  64      load_bug(db => $schema,
  65               data => split_status_fields($data),
  66               tags => \%tags,
  67               queue => \%queue);
  68
  69 Loads a bug's metadata into the database. (Does not load any messages)
  70
  71 =over
  72
  73 =item db -- Debbugs::DB object
  74
  75 =item data -- Bug data (from read_bug) which has been split with split_status_fields
  76
  77 =item tags -- tag cache (hashref); optional
  78
  79 =item queue -- queue of operations to perform after bug is loaded; optional.
  80
  81 =back
  82
  83 =cut
  84
  85 sub load_bug {
  86     my %param = validate_with(params => \@_,
  87                               spec => {db => {type => OBJECT,
  88                                              },
  89                                        data => {type => HASHREF,
  90                                                 optional => 1,
  91                                                },
  92                                        bug => {type => SCALAR,
  93                                                optional => 1,
  94                                               },
  95                                        tags => {type => HASHREF,
  96                                                 default => sub {return {}},
  97                                                 optional => 1},
  98                                        severities => {type => HASHREF,
  99                                                       default => sub {return {}},
 100                                                       optional => 1,
 101                                                      },
 102                                        queue => {type => HASHREF,
 103                                                  optional => 1},
 104                                       });
 105     my $s = $param{db};
 106     if (not exists $param{data} and not exists $param{bug}) {
 107         croak "One of data or bug must be provided to load_bug";
 108     }
 109     if (not exists $param{data}) {
 110         $param{data} = read_bug(bug => $param{bug});
 111     }
 112     my $data = $param{data};
 113     my $tags = $param{tags};
 114     my $queue = $param{queue};
 115     my $severities = $param{severities};
 116     my $can_queue = 1;
 117     if (not defined $queue) {
 118         $can_queue = 0;
 119         $queue = {};
 120     }
 121     my %tags;
 122     my $s_data = split_status_fields($data);
 123     for my $tag (make_list($s_data->{keywords})) {
 124         next unless defined $tag and length $tag;
 125         # this allows for invalid tags. But we'll use this to try to
 126         # find those bugs and clean them up
 127         if (not exists $tags->{$tag}) {
 128             $tags->{$tag} = $s->resultset('Tag')->
 129             find_or_create({tag => $tag});
 130         }
 131         $tags{$tag} = $tags->{$tag};
 132     }
 133     my $severity = length($data->{severity}) ? $data->{severity} :
 134         $config{default_severity};
 135     if (not exists $severities->{$severity}) {
 136         $severities->{$severity} =
 137             $s->resultset('Severity')->
 138             find_or_create({severity => $severity},
 139                           );
 140     }
 141     $severity = $severities->{$severity};
 142     my $bug =
 143         {id => $data->{bug_num},
 144          creation => DateTime->from_epoch(epoch => $data->{date}),
 145          log_modified => DateTime->from_epoch(epoch => $data->{log_modified}),
 146          last_modified => DateTime->from_epoch(epoch => $data->{last_modified}),
 147          archived => $data->{archived},
 148          (defined $data->{unarchived} and length($data->{unarchived}))?
 149          (unarchived => DateTime->from_epoch(epoch => $data->{unarchived})):(),
 150          forwarded => $data->{forwarded} // '',
 151          summary => $data->{summary} // '',
 152          outlook => $data->{outlook} // '',
 153          subject => $data->{subject} // '',
 154          done_full => $data->{done} // '',
 155          severity => $severity,
 156          owner_full => $data->{owner} // '',
 157          submitter_full => $data->{originator} // '',
 158         };
 159     my %addr_map =
 160         (done => 'done',
 161          owner => 'owner',
 162          submitter => 'originator',
 163         );
 164     for my $addr_type (keys %addr_map) {
 165         $bug->{$addr_type} = undef;
 166         next unless defined $data->{$addr_map{$addr_type}} and
 167             length($data->{$addr_map{$addr_type}});
 168         $bug->{$addr_type} =
 169             $s->resultset('Correspondent')->
 170             get_correspondent_id($data->{$addr_map{$addr_type}})
 171     }
 172     my $b = $s->resultset('Bug')->update_or_create($bug) or
 173         die "Unable to update or create bug $bug->{id}";
 174      $s->txn_do(sub {
 175                    for my $ff (qw(found fixed)) {
 176                        my @elements = $s->resultset('BugVer')->search({bug => $data->{bug_num},
 177                                                                        found  => $ff eq 'found'?1:0,
 178                                                                       });
 179                        my %elements_to_delete = map {($elements[$_]->ver_string(),
 180                                                       $elements[$_])} 0..$#elements;
 181                        my %elements_to_add;
 182                        my @elements_to_keep;
 183                        for my $version (@{$data->{"${ff}_versions"}}) {
 184                            if (exists $elements_to_delete{$version}) {
 185                                push @elements_to_keep,$version;
 186                            } else {
 187                                $elements_to_add{$version} = 1;
 188                            }
 189                        }
 190                        for my $version (@elements_to_keep) {
 191                            delete $elements_to_delete{$version};
 192                        }
 193                        for my $element (keys %elements_to_delete) {
 194                            $elements_to_delete{$element}->delete();
 195                        }
 196                        for my $element (keys %elements_to_add) {
 197                            # find source package and source version id
 198                            my $ne = $s->resultset('BugVer')->new_result({bug => $data->{bug_num},
 199                                                                          ver_string => $element,
 200                                                                          found => $ff eq 'found'?1:0,
 201                                                                         }
 202                                                                        );
 203                            if (my ($src_pkg,$src_ver) = $element =~ m{^([^\/]+)/(.+)$}) {
 204                                my $src_pkg_e = $s->resultset('SrcPkg')->single({pkg => $src_pkg});
 205                                if (defined $src_pkg_e) {
 206                                    $ne->src_pkg($src_pkg_e->id());
 207                                    my $src_ver_e = $s->resultset('SrcVer')->single({src_pkg => $src_pkg_e->id(),
 208                                                                                     ver => $src_ver
 209                                                                                    });
 210                                    $ne->src_ver($src_ver_e->id()) if defined $src_ver_e;
 211                                }
 212                            }
 213                            $ne->insert();
 214                        }
 215                    }
 216                });
 217     ### set bug tags
 218     $s->txn_do(sub {$b->set_tags([values %tags ] )});
 219     # because these bugs reference other bugs which might not exist
 220     # yet, we can't handle them until we've loaded all bugs. queue
 221     # them up.
 222     for my $merge_block (qw(merged block)) {
 223         my $data_key = $merge_block;
 224         $data_key .= 'with' if $merge_block eq 'merged';
 225         if (@{$data->{$data_key}||[]}) {
 226             my $count = $s->resultset('Bug')->search({id => [@{$data->{$data_key}}]})->count();
 227             if ($count == @{$data->{$data_key}}) {
 228                 handle_load_bug_queue(db=>$s,
 229                                       queue => {$merge_block,
 230                                                {$data->{bug_num},[@{$data->{$data_key}}]}
 231                                                });
 232             } else {
 233                 $queue->{$merge_block}{$data->{bug_num}} = [@{$data->{$data_key}}];
 234             }
 235         }
 236     }
 237
 238     if (not $can_queue and keys %{$queue}) {
 239         handle_load_bug_queue(db => $s,queue => $queue);
 240     }
 241
 242     # still need to handle merges, versions, etc.
 243 }
 244
 245 =item handle_load_bug_queue
 246
 247      handle_load_bug_queue(db => $schema,queue => $queue);
 248
 249 Handles a queue of operations created by load bug. [These operations
 250 are used to handle cases where a bug referenced by a loaded bug may
 251 not exist yet. In cases where the bugs should exist, the queue is
 252 cleared automatically by load_bug if queue is undefined.
 253
 254 =cut
 255
 256 sub handle_load_bug_queue{
 257     my %param = validate_with(params => \@_,
 258                               spec => {db => {type => OBJECT,
 259                                              },
 260                                        queue => {type => HASHREF,
 261                                                 },
 262                                       });
 263     my $s = $param{db};
 264     my $queue = $param{queue};
 265     my %queue_types =
 266         (merged => {set => 'BugMerged',
 267                     columns => [qw(bug merged)],
 268                     bug => 'bug',
 269                    },
 270          blocks => {set => 'BugBlock',
 271                     columns => [qw(bug blocks)],
 272                     bug => 'bug',
 273                    },
 274         );
 275     for my $queue_type (keys %queue_types) {
 276         for my $bug (%{$queue->{$queue_type}}) {
 277             my $qt = $queue_types{$queue_type};
 278             $s->txn_do(sub {
 279                            $s->resultset($qt->{set})->search({$qt->{bug},$bug})->delete();
 280                            $s->populate($qt->{set},[[@{$qt->{columns}}],
 281                                                     map {[$bug,$_]} @{$queue->{$queue_type}{$bug}}]) if
 282                                @{$queue->{$queue_type}{$bug}//[]};
 283                        }
 284                       );
 285         }
 286     }
 287 }
 288
 289 =item load_bug_log -- load bug logs
 290
 291        load_bug_log(db  => $s,
 292                     bug => $bug);
 293
 294
 295 =over
 296
 297 =item db -- database
 298
 299 =item bug -- bug whose log should be loaded
 300
 301 =back
 302
 303 =cut
 304
 305 sub load_bug_log {
 306     my %param = validate_with(params => \@_,
 307                               spec => {db => {type => OBJECT,
 308                                              },
 309                                        bug => {type => SCALAR,
 310                                               },
 311                                        queue => {type => HASHREF,
 312                                                  optional => 1},
 313                                       });
 314     my $s = $param{db};
 315     my $msg_num=0;
 316     my %seen_msg_ids;
 317     my $log = Debbugs::Log->new(bug_num => $param{bug}) or
 318         die "Unable to open log for $param{bug} for reading: $!";
 319     while (my $record = $log->read_record()) {
 320         next unless $record->{type} eq 'incoming-recv';
 321         my ($msg_id) = $record->{text} =~ /^Message-Id:\s+<(.+)>/im;
 322         next if defined $msg_id and exists $seen_msg_ids{$msg_id};
 323         $seen_msg_ids{$msg_id} = 1 if defined $msg_id;
 324         next if defined $msg_id and $msg_id =~ /handler\..+\.ack(?:info)?\@/;
 325         my $entity = parse_to_mime_entity($record);
 326         # search for a message with this message id in the database
 327         $msg_id = $entity->head->get('Message-Id:');
 328         $msg_id =~ s/^\s*\<//;
 329         $msg_id =~ s/>\s*$//;
 330         # check to see if the subject, to, and from match. if so, it's
 331         # probably the same message.
 332         my $subject = decode_rfc1522($entity->head->get('Subject:'));
 333         $subject =~ s/\n(?:(\s)\s*|\s*$)/$1/g;
 334         my $to = decode_rfc1522($entity->head->get('To:'));
 335         $to =~ s/\n(?:(\s)\s*|\s*$)/$1/g;
 336         my $from = decode_rfc1522($entity->head->get('From:'));
 337         $from =~ s/\n(?:(\s)\s*|\s*$)/$1/g;
 338         my $m = $s->resultset('Message')->
 339             find({msgid => $msg_id,
 340                   from_complete => $from,
 341                   to_complete => $to,
 342                   subject => $subject
 343                  });
 344         if (not defined $m) {
 345             # if not, create a new message
 346             $m = $s->resultset('Message')->
 347                 find_or_create({msgid => $msg_id,
 348                                 from_complete => $from,
 349                                 to_complete => $to,
 350                                 subject => $subject
 351                                });
 352             eval {
 353                 $m->sent_date(DateTime::Format::Mail->
 354                               parse_datetime($entity->head->get('Date:',0)));
 355             };
 356             my $spam = $entity->head->get('X-Spam-Status:',0);
 357             if ($spam=~ /score=([\d\.]+)/) {
 358                 $m->spam_score($1);
 359             }
 360             my %corr;
 361             @{$corr{from}} = getparsedaddrs($from);
 362             @{$corr{to}} = getparsedaddrs($to);
 363             @{$corr{cc}} = getparsedaddrs($entity->head->get('Cc:'));
 364             # add correspondents if necessary
 365             my @cors;
 366             for my $type (keys %corr) {
 367                 for my $addr (@{$corr{$type}}) {
 368                     push @cors,
 369                         {correspondent => $s->resultset('Correspondent')->
 370                          get_correspondent_id($addr),
 371                          correspondent_type => $type,
 372                         };
 373                 }
 374             }
 375             $m->update();
 376             $s->txn_do(sub {
 377                            $m->message_correspondents()->delete();
 378                            $m->add_to_message_correspondents(@cors);
 379                        }
 380                       );
 381         }
 382         my $recv;
 383         if ($entity->head->get('Received:',0)
 384             =~ /via spool by (\S+)/) {
 385             $recv = $s->resultset('Correspondent')->
 386                 get_correspondent_id($1);
 387             $m->add_to_message_correspondents({correspondent=>$recv,
 388                                                correspondent_type => 'recv'});
 389         }
 390         # link message to bugs if necessary
 391         $m->find_or_create_related('bug_messages',
 392                                   {bug=>$param{bug},
 393                                    message_number => $msg_num});
 394     }
 395
 396 }
 397
 398 =back
 399
 400 =head2 Debinfo
 401
 402 Commands to handle src and package version loading from debinfo files
 403
 404 =over
 405
 406 =item load_debinfo
 407
 408      load_debinfo($schema,$binname, $binver, $binarch, $srcname, $srcver);
 409
 410
 411
 412 =cut
 413
 414 sub load_debinfo {
 415     my ($schema,$binname, $binver, $binarch, $srcname, $srcver) = @_;
 416     my $sp = $schema->resultset('SrcPkg')->find_or_create({pkg => $srcname});
 417     my $sv = $schema->resultset('SrcVer')->find_or_create({src_pkg=>$sp->id(),
 418                                                            ver => $srcver});
 419     my $arch = $schema->resultset('Arch')->find_or_create({arch => $binarch});
 420     my $bp = $schema->resultset('BinPkg')->find_or_create({pkg => $binname});
 421     $schema->resultset('BinVer')->find_or_create({bin_pkg_id => $bp->id(),
 422                                                   src_ver_id => $sv->id(),
 423                                                   arch_id    => $arch->id(),
 424                                                   ver        => $binver,
 425                                                  });
 426 }
 427
 428
 429 =back
 430
 431 =head Packages
 432
 433 =over
 434
 435 =item load_package
 436
 437      load_package($schema,$suite,$component,$arch,$pkg)
 438
 439 =cut
 440
 441 sub load_packages {
 442     my ($schema,$suite,$pkgs,$p) = @_;
 443     my $suite_id = $schema->resultset('Suite')->
 444         find_or_create({codename => $suite})->id;
 445     my %maint_cache;
 446     my %arch_cache;
 447     my %source_cache;
 448     my $src_max_last_modified = $schema->resultset('SrcAssociation')->
 449         search_rs({suite => $suite_id},
 450                  {order_by => {-desc => ['me.modified']},
 451                   rows => 1,
 452                   page => 1
 453                  }
 454                  )->single();
 455     my $bin_max_last_modified = $schema->resultset('BinAssociation')->
 456         search_rs({suite => $suite_id},
 457                  {order_by => {-desc => ['me.modified']},
 458                   rows => 1,
 459                   page => 1
 460                  }
 461                  )->single();
 462     my %maints;
 463     my %sources;
 464     my %bins;
 465     for my $pkg_tuple (@{$pkgs}) {
 466         my ($arch,$component,$pkg) = @{$pkg_tuple};
 467         $maints{$pkg->{Maintainer}} = $pkg->{Maintainer};
 468         if ($arch eq 'source') {
 469             my $source = $pkg->{Package};
 470             my $source_ver = $pkg->{Version};
 471             $sources{$source}{$source_ver} = $pkg->{Maintainer};
 472         } else {
 473             my $source = $pkg->{Source} // $pkg->{Package};
 474             my $source_ver = $pkg->{Version};
 475             if ($source =~ /^\s*(\S+) \(([^\)]+)\)\s*$/) {
 476                 ($source,$source_ver) = ($1,$2);
 477             }
 478             $sources{$source}{$source_ver} = $pkg->{Maintainer};
 479             $bins{$arch}{$pkg->{Package}} =
 480                {arch => $arch,
 481                 bin => $pkg->{Package},
 482                 bin_ver => $pkg->{Version},
 483                 src_ver => $source_ver,
 484                 source  => $source,
 485                 maint   => $pkg->{Maintainer},
 486                };
 487         }
 488     }
 489     # Retrieve and Insert new maintainers
 490     my $maints =
 491         $schema->resultset('Maintainer')->
 492         get_maintainers(keys %maints);
 493     my $archs =
 494         $schema->resultset('Arch')->
 495         get_archs(keys %bins);
 496     # We want all of the source package/versions which are in this suite to
 497     # start with
 498     my @sa_to_add;
 499     my @sa_to_del;
 500     my %included_sa;
 501     # Calculate which source packages are no longer in this suite
 502     for my $s ($schema->resultset('SrcPkg')->
 503                src_pkg_and_ver_in_suite($suite)) {
 504         if (not exists $sources{$s->{pkg}} or
 505             not exists $sources{$s->{pkg}}{$s->{src_vers}{ver}}
 506            ) {
 507             push @sa_to_del,
 508                 $s->{src_associations}{id};
 509         }
 510         $included_sa{$s->{pkg}}{$s->{src_vers}} = 1;
 511     }
 512     # Calculate which source packages are newly in this suite
 513     for my $s (keys %sources) {
 514         for my $v (keys %{$sources{$s}}) {
 515             if (not exists $included_sa{$s} and
 516                 not $included_sa{$s}{$v}) {
 517                 push @sa_to_add,
 518                     [$s,$v,$sources{$s}{$v}];
 519             } else {
 520                 $p->update() if defined $p;
 521             }
 522         }
 523     }
 524     # add new source packages
 525     my $it = natatime 100, @sa_to_add;
 526     while (my @v = $it->()) {
 527         $schema->txn_do(
 528             sub {
 529                 for my $svm (@_) {
 530                     my $s_id = $schema->resultset('SrcPkg')->
 531                         get_src_pkg_id($svm->[0]);
 532                     my $sv_id = $schema->resultset('SrcVer')->
 533                         get_src_ver_id($s_id,$svm->[1],$maints->{$svm->[2]});
 534                     $schema->resultset('SrcAssociation')->
 535                         insert_suite_src_ver_association($suite_id,$sv_id);
 536                 }
 537             },
 538                         @v
 539                        );
 540         $p->update($p->last_update()+
 541                    scalar @v) if defined $p;
 542     }
 543     # remove associations for packages not in this suite
 544     if (@sa_to_del) {
 545         $schema->resultset('SrcAssociation')->
 546             search_rs({id => \@sa_to_del})->delete();
 547     }
 548     # update packages in this suite to have a modification time of now
 549     $schema->resultset('SrcAssociation')->
 550         search_rs({suite => $suite_id})->
 551         update({modified => 'NOW()'});
 552     ## Handle binary packages
 553     my @bin_to_del;
 554     my @bin_to_add;
 555     my %included_bin;
 556     # calculate which binary packages are no longer in this suite
 557     for my $b ($schema->resultset('BinPkg')->
 558                bin_pkg_and_ver_in_suite($suite)) {
 559         if (not exists $bins{$b->{arch}{arch}} or
 560             not exists $bins{$b->{arch}{arch}}{$b->{pkg}} or
 561             ($bins{$b->{arch}{arch}}{$b->{pkg}}{bin_ver} ne
 562              $b->{bin_vers}{ver}
 563             )
 564            ) {
 565             push @bin_to_del,
 566                 $b->{bin_associations}{id};
 567         }
 568         $included_bin{$b->{arch}{arch}}{$b->{pkg}} =
 569             $b->{bin_vers}{ver};
 570     }
 571     # calculate which binary packages are newly in this suite
 572     for my $a (keys %bins) {
 573         for my $pkg (keys %{$bins{$a}}) {
 574             if (not exists $included_bin{$a} or
 575                 not exists $included_bin{$a}{$pkg} or
 576                 $bins{$a}{$pkg}{bin_ver} ne
 577                 $included_bin{$a}{$pkg}) {
 578                 push @bin_to_add,
 579                     $bins{$a}{$pkg};
 580             } else {
 581                 $p->update() if defined $p;
 582             }
 583         }
 584     }
 585     $it = natatime 100, @bin_to_add;
 586     while (my @v = $it->()) {
 587         $schema->txn_do(
 588         sub {
 589             for my $bvm (@_) {
 590                 my $s_id = $schema->resultset('SrcPkg')->
 591                     get_src_pkg_id($bvm->{source});
 592                 my $sv_id = $schema->resultset('SrcVer')->
 593                     get_src_ver_id($s_id,$bvm->{src_ver},$maints->{$bvm->{maint}});
 594                 my $b_id = $schema->resultset('BinPkg')->
 595                     get_bin_pkg_id($bvm->{bin});
 596                 my $bv_id = $schema->resultset('BinVer')->
 597                     get_bin_ver_id($b_id,$bvm->{bin_ver},
 598                                    $archs->{$bvm->{arch}},$sv_id);
 599                 $schema->resultset('BinAssociation')->
 600                     insert_suite_bin_ver_association($suite_id,$bv_id);
 601             }
 602         },
 603                         @v
 604                        );
 605         $p->update($p->last_update()+
 606                    scalar @v) if defined $p;
 607     }
 608     if (@bin_to_del) {
 609         $schema->resultset('BinAssociation')->
 610             search_rs({id => \@bin_to_del})->delete();
 611     }
 612     $schema->resultset('BinAssociation')->
 613         search_rs({suite => $suite_id})->
 614         update({modified => 'NOW()'});
 615
 616 }
 617
 618
 619 =back
 620
 621 =cut
 622
 623 =head Suites
 624
 625 =over
 626
 627 =item load_suite
 628
 629      load_suite($schema,$codename,$suite,$version,$active);
 630
 631 =cut
 632
 633 sub load_suite {
 634     my ($schema,$codename,$suite,$version,$active) = @_;
 635     if (ref($codename)) {
 636         ($codename,$suite,$version) =
 637             @{$codename}{qw(Codename Suite Version)};
 638         $active = 1;
 639     }
 640     my $s = $schema->resultset('Suite')->find_or_create({codename => $codename});
 641     $s->suite_name($suite);
 642     $s->version($version);
 643     $s->active($active);
 644     $s->update();
 645     return $s;
 646
 647 }
 648
 649 =back
 650
 651 =cut
 652
 653 1;
 654
 655
 656 __END__