Debbugs/DB/Load.pm

   1 # This module is part of debbugs, and is released
   2 # under the terms of the GPL version 2, or any later version. See the
   3 # file README and COPYING for more information.
   4 # Copyright 2013 by Don Armstrong <don@donarmstrong.com>.
   5
   6 package Debbugs::DB::Load;
   7
   8 =head1 NAME
   9
  10 Debbugs::DB::Load -- Utility routines for loading the database
  11
  12 =head1 SYNOPSIS
  13
  14
  15 =head1 DESCRIPTION
  16
  17
  18 =head1 BUGS
  19
  20 None known.
  21
  22 =cut
  23
  24 use warnings;
  25 use strict;
  26 use vars qw($VERSION $DEBUG %EXPORT_TAGS @EXPORT_OK @EXPORT);
  27 use base qw(Exporter);
  28
  29 BEGIN{
  30      ($VERSION) = q$Revision$ =~ /^Revision:\s+([^\s+])/;
  31      $DEBUG = 0 unless defined $DEBUG;
  32
  33      @EXPORT = ();
  34      %EXPORT_TAGS = (load_bug    => [qw(load_bug handle_load_bug_queue load_bug_log)],
  35                      load_debinfo => [qw(load_debinfo)],
  36                      load_package => [qw(load_packages)],
  37                      load_suite => [qw(load_suite)],
  38                     );
  39      @EXPORT_OK = ();
  40      Exporter::export_ok_tags(keys %EXPORT_TAGS);
  41      $EXPORT_TAGS{all} = [@EXPORT_OK];
  42 }
  43
  44 use Params::Validate qw(validate_with :types);
  45 use List::MoreUtils qw(natatime);
  46
  47 use Debbugs::Status qw(read_bug split_status_fields);
  48 use Debbugs::DB;
  49 use DateTime;
  50 use Debbugs::Common qw(make_list getparsedaddrs);
  51 use Debbugs::Config qw(:config);
  52 use Carp;
  53
  54 =head2 Bug loading
  55
  56 Routines to load bug; exported with :load_bug
  57
  58 =over
  59
  60 =item load_bug
  61
  62      load_bug(db => $schema,
  63               data => split_status_fields($data),
  64               tags => \%tags,
  65               queue => \%queue);
  66
  67 Loads a bug's metadata into the database. (Does not load any messages)
  68
  69 =over
  70
  71 =item db -- Debbugs::DB object
  72
  73 =item data -- Bug data (from read_bug) which has been split with split_status_fields
  74
  75 =item tags -- tag cache (hashref); optional
  76
  77 =item queue -- queue of operations to perform after bug is loaded; optional.
  78
  79 =back
  80
  81 =cut
  82
  83 sub load_bug {
  84     my %param = validate_with(params => \@_,
  85                               spec => {db => {type => OBJECT,
  86                                              },
  87                                        data => {type => HASHREF,
  88                                                 optional => 1,
  89                                                },
  90                                        bug => {type => SCALAR,
  91                                                optional => 1,
  92                                               },
  93                                        tags => {type => HASHREF,
  94                                                 default => sub {return {}},
  95                                                 optional => 1},
  96                                        severities => {type => HASHREF,
  97                                                       default => sub {return {}},
  98                                                       optional => 1,
  99                                                      },
 100                                        queue => {type => HASHREF,
 101                                                  optional => 1},
 102                                       });
 103     my $s = $param{db};
 104     if (not exists $param{data} and not exists $param{bug}) {
 105         croak "One of data or bug must be provided to load_bug";
 106     }
 107     if (not exists $param{data}) {
 108         $param{data} = read_bug(bug => $param{bug});
 109     }
 110     my $data = $param{data};
 111     my $tags = $param{tags};
 112     my $queue = $param{queue};
 113     my $severities = $param{severities};
 114     my $can_queue = 1;
 115     if (not defined $queue) {
 116         $can_queue = 0;
 117         $queue = {};
 118     }
 119     my %tags;
 120     my $s_data = split_status_fields($data);
 121     for my $tag (make_list($s_data->{keywords})) {
 122         next unless defined $tag and length $tag;
 123         # this allows for invalid tags. But we'll use this to try to
 124         # find those bugs and clean them up
 125         if (not exists $tags->{$tag}) {
 126             $tags->{$tag} = $s->resultset('Tag')->
 127             find_or_create({tag => $tag});
 128         }
 129         $tags{$tag} = $tags->{$tag};
 130     }
 131     my $severity = length($data->{severity}) ? $data->{severity} : $config{default_severity};
 132     if (exists $severities->{$severity}) {
 133         $severity = $severities->{$severity};
 134     } else {
 135         $severity = $s->resultset('Severity')->
 136             find_or_create({severity => $severity});
 137     }
 138     my $bug =
 139         {id => $data->{bug_num},
 140          creation => DateTime->from_epoch(epoch => $data->{date}),
 141          log_modified => DateTime->from_epoch(epoch => $data->{log_modified}),
 142          last_modified => DateTime->from_epoch(epoch => $data->{last_modified}),
 143          archived => $data->{archived},
 144          (defined $data->{unarchived} and length($data->{unarchived}))?(unarchived => DateTime->from_epoch(epoch => $data->{unarchived})):(),
 145          forwarded => $data->{forwarded} // '',
 146          summary => $data->{summary} // '',
 147          outlook => $data->{outlook} // '',
 148          subject => $data->{subject} // '',
 149          done_full => $data->{done} // '',
 150          severity => $severity,
 151          owner_full => $data->{owner} // '',
 152          submitter_full => $data->{originator} // '',
 153         };
 154     my %addr_map =
 155         (done => 'done',
 156          owner => 'owner',
 157          submitter => 'originator',
 158         );
 159     for my $addr_type (keys %addr_map) {
 160         my @addrs = getparsedaddrs($data->{$addr_map{$addr_type}} // '');
 161         next unless @addrs;
 162         $bug->{$addr_type} = $s->resultset('Correspondent')->find_or_create({addr => lc($addrs[0]->address())});
 163         # insert the full name as well
 164         my $full_name = $addrs[0]->phrase();
 165         $full_name =~ s/^\"|\"$//g;
 166         $full_name =~ s/^\s+|\s+$//g;
 167         if (length $full_name) {
 168             $bug->{$addr_type}->
 169                 update_or_create_related('correspondent_full_names',
 170                                         {full_name=>$full_name,
 171                                          last_seen => 'NOW()'});
 172         }
 173     }
 174     my $b = $s->resultset('Bug')->update_or_create($bug) or
 175         die "Unable to update or create bug $bug->{id}";
 176      $s->txn_do(sub {
 177                    for my $ff (qw(found fixed)) {
 178                        my @elements = $s->resultset('BugVer')->search({bug => $data->{bug_num},
 179                                                                        found  => $ff eq 'found'?1:0,
 180                                                                       });
 181                        my %elements_to_delete = map {($elements[$_]->ver_string(),$elements[$_])} 0..$#elements;
 182                        my %elements_to_add;
 183                        my @elements_to_keep;
 184                        for my $version (@{$data->{"${ff}_versions"}}) {
 185                            if (exists $elements_to_delete{$version}) {
 186                                push @elements_to_keep,$version;
 187                            } else {
 188                                $elements_to_add{$version} = 1;
 189                            }
 190                        }
 191                        for my $version (@elements_to_keep) {
 192                            delete $elements_to_delete{$version};
 193                        }
 194                        for my $element (keys %elements_to_delete) {
 195                            $elements_to_delete{$element}->delete();
 196                        }
 197                        for my $element (keys %elements_to_add) {
 198                            # find source package and source version id
 199                            my $ne = $s->resultset('BugVer')->new_result({bug => $data->{bug_num},
 200                                                                          ver_string => $element,
 201                                                                          found => $ff eq 'found'?1:0,
 202                                                                         }
 203                                                                        );
 204                            if (my ($src_pkg,$src_ver) = $element =~ m{^([^\/]+)/(.+)$}) {
 205                                my $src_pkg_e = $s->resultset('SrcPkg')->single({pkg => $src_pkg});
 206                                if (defined $src_pkg_e) {
 207                                    $ne->src_pkg($src_pkg_e->id());
 208                                    my $src_ver_e = $s->resultset('SrcVer')->single({src_pkg => $src_pkg_e->id(),
 209                                                                                     ver => $src_ver
 210                                                                                    });
 211                                    $ne->src_ver($src_ver_e->id()) if defined $src_ver_e;
 212                                }
 213                            }
 214                            $ne->insert();
 215                        }
 216                    }
 217                });
 218     $s->txn_do(sub {
 219                    my $t = $s->resultset('BugTag')->search({bug => $data->{bug_num}});
 220                    $t->delete() if defined $t;
 221                    $s->populate(BugTag => [[qw(bug tag)], map {[$data->{bug_num}, $_->id()]} values %tags]);
 222                });
 223     # because these bugs reference other bugs which might not exist
 224     # yet, we can't handle them until we've loaded all bugs. queue
 225     # them up.
 226     for my $merge_block (qw(merged block)) {
 227         my $data_key = $merge_block;
 228         $data_key .= 'with' if $merge_block eq 'merged';
 229         if (@{$data->{$data_key}||[]}) {
 230             my $count = $s->resultset('Bug')->search({id => [@{$data->{$data_key}}]})->count();
 231             if ($count == @{$data->{$data_key}}) {
 232                 handle_load_bug_queue(db=>$s,
 233                                       queue => {$merge_block,
 234                                                {$data->{bug_num},[@{$data->{$data_key}}]}
 235                                                });
 236             } else {
 237                 $queue->{$merge_block}{$data->{bug_num}} = [@{$data->{$data_key}}];
 238             }
 239         }
 240     }
 241
 242     if (not $can_queue and keys %{$queue}) {
 243         handle_load_bug_queue(db => $s,queue => $queue);
 244     }
 245
 246     # still need to handle merges, versions, etc.
 247 }
 248
 249 =item handle_load_bug_queue
 250
 251      handle_load_bug_queue(db => $schema,queue => $queue);
 252
 253 Handles a queue of operations created by load bug. [These operations
 254 are used to handle cases where a bug referenced by a loaded bug may
 255 not exist yet. In cases where the bugs should exist, the queue is
 256 cleared automatically by load_bug if queue is undefined.
 257
 258 =cut
 259
 260 sub handle_load_bug_queue{
 261     my %param = validate_with(params => \@_,
 262                               spec => {db => {type => OBJECT,
 263                                              },
 264                                        queue => {type => HASHREF,
 265                                                 },
 266                                       });
 267     my $s = $param{db};
 268     my $queue = $param{queue};
 269     my %queue_types =
 270         (merged => {set => 'BugMerged',
 271                     columns => [qw(bug merged)],
 272                     bug => 'bug',
 273                    },
 274          blocks => {set => 'BugBlock',
 275                     columns => [qw(bug blocks)],
 276                     bug => 'bug',
 277                    },
 278         );
 279     for my $queue_type (keys %queue_types) {
 280         for my $bug (%{$queue->{$queue_type}}) {
 281             my $qt = $queue_types{$queue_type};
 282             $s->txn_do(sub {
 283                            $s->resultset($qt->{set})->search({$qt->{bug},$bug})->delete();
 284                            $s->populate($qt->{set},[[@{$qt->{columns}}],
 285                                                     map {[$bug,$_]} @{$queue->{$queue_type}{$bug}}]) if
 286                                @{$queue->{$queue_type}{$bug}//[]};
 287                        }
 288                       );
 289         }
 290     }
 291 }
 292
 293 =item load_bug_log -- load bug logs
 294
 295        load_bug_log(db  => $s,
 296                     bug => $bug);
 297
 298
 299 =over
 300
 301 =item db -- database
 302
 303 =item bug -- bug whose log should be loaded
 304
 305 =back
 306
 307 =cut
 308
 309 sub load_bug_log {
 310     my %param = validate_with(params => \@_,
 311                               spec => {db => {type => OBJECT,
 312                                              },
 313                                        bug => {type => SCALAR,
 314                                               },
 315                                        queue => {type => HASHREF,
 316                                                  optional => 1},
 317                                       });
 318     my $s = $param{db};
 319     my $msg_num=0;
 320     my %seen_msg_ids;
 321     my $log = Debbugs::Log->new(bug_num => $param{bug}) or
 322         die "Unable to open log for $param{bug} for reading: $!";
 323     while (my $record = $log->read_record()) {
 324         next unless $record->{type} eq 'incoming-recv';
 325         my ($msg_id) = $record->{text} =~ /^Message-Id:\s+<(.+)>/im;
 326         next if defined $msg_id and exists $seen_msg_ids{$msg_id};
 327         $seen_msg_ids{$msg_id} = 1 if defined $msg_id;
 328         next if defined $msg_id and $msg_id =~ /handler\..+\.ack(?:info)?\@/;
 329         my $message = parse($record->{text});
 330         # search for a message with this message id in the database
 331
 332         # check to see if the subject, to, and from match. if so, it's
 333         # probably the same message.
 334
 335         # if not, create a new message
 336
 337         # add correspondents if necessary
 338
 339         # link message to bugs if necessary
 340
 341     }
 342
 343 }
 344
 345 =back
 346
 347 =head2 Debinfo
 348
 349 Commands to handle src and package version loading from debinfo files
 350
 351 =over
 352
 353 =item load_debinfo
 354
 355      load_debinfo($schema,$binname, $binver, $binarch, $srcname, $srcver);
 356
 357
 358
 359 =cut
 360
 361 sub load_debinfo {
 362     my ($schema,$binname, $binver, $binarch, $srcname, $srcver) = @_;
 363     my $sp = $schema->resultset('SrcPkg')->find_or_create({pkg => $srcname});
 364     my $sv = $schema->resultset('SrcVer')->find_or_create({src_pkg=>$sp->id(),
 365                                                            ver => $srcver});
 366     my $arch = $schema->resultset('Arch')->find_or_create({arch => $binarch});
 367     my $bp = $schema->resultset('BinPkg')->find_or_create({pkg => $binname});
 368     $schema->resultset('BinVer')->find_or_create({bin_pkg_id => $bp->id(),
 369                                                   src_ver_id => $sv->id(),
 370                                                   arch_id    => $arch->id(),
 371                                                   ver        => $binver,
 372                                                  });
 373 }
 374
 375
 376 =back
 377
 378 =head Packages
 379
 380 =over
 381
 382 =item load_package
 383
 384      load_package($schema,$suite,$component,$arch,$pkg)
 385
 386 =cut
 387
 388 sub load_packages {
 389     my ($schema,$suite,$pkgs,$p) = @_;
 390     my $suite_id = $schema->resultset('Suite')->
 391         find_or_create({codename => $suite})->id;
 392     my %maint_cache;
 393     my %arch_cache;
 394     my %source_cache;
 395     my $src_max_last_modified = $schema->resultset('SrcAssociation')->
 396         search_rs({suite => $suite_id},
 397                  {order_by => {-desc => ['me.modified']},
 398                   rows => 1,
 399                   page => 1
 400                  }
 401                  )->single();
 402     my $bin_max_last_modified = $schema->resultset('BinAssociation')->
 403         search_rs({suite => $suite_id},
 404                  {order_by => {-desc => ['me.modified']},
 405                   rows => 1,
 406                   page => 1
 407                  }
 408                  )->single();
 409     my %maints;
 410     my %sources;
 411     my %bins;
 412     for my $pkg_tuple (@{$pkgs}) {
 413         my ($arch,$component,$pkg) = @{$pkg_tuple};
 414         $maints{$pkg->{Maintainer}} = $pkg->{Maintainer};
 415         if ($arch eq 'source') {
 416             my $source = $pkg->{Package};
 417             my $source_ver = $pkg->{Version};
 418             $sources{$source}{$source_ver} = $pkg->{Maintainer};
 419         } else {
 420             my $source = $pkg->{Source} // $pkg->{Package};
 421             my $source_ver = $pkg->{Version};
 422             if ($source =~ /^\s*(\S+) \(([^\)]+)\)\s*$/) {
 423                 ($source,$source_ver) = ($1,$2);
 424             }
 425             $sources{$source}{$source_ver} = $pkg->{Maintainer};
 426             $bins{$arch}{$pkg->{Package}} =
 427                {arch => $arch,
 428                 bin => $pkg->{Package},
 429                 bin_ver => $pkg->{Version},
 430                 src_ver => $source_ver,
 431                 source  => $source,
 432                 maint   => $pkg->{Maintainer},
 433                };
 434         }
 435     }
 436     # Retrieve and Insert new maintainers
 437     my $maints =
 438         $schema->resultset('Maintainer')->
 439         get_maintainers(keys %maints);
 440     my $archs =
 441         $schema->resultset('Arch')->
 442         get_archs(keys %bins);
 443     # We want all of the source package/versions which are in this suite to
 444     # start with
 445     my @sa_to_add;
 446     my @sa_to_del;
 447     my %included_sa;
 448     # Calculate which source packages are no longer in this suite
 449     for my $s ($schema->resultset('SrcPkg')->
 450                src_pkg_and_ver_in_suite($suite)) {
 451         if (not exists $sources{$s->{pkg}} or
 452             not exists $sources{$s->{pkg}}{$s->{src_vers}{ver}}
 453            ) {
 454             push @sa_to_del,
 455                 $s->{src_associations}{id};
 456         }
 457         $included_sa{$s->{pkg}}{$s->{src_vers}} = 1;
 458     }
 459     # Calculate which source packages are newly in this suite
 460     for my $s (keys %sources) {
 461         for my $v (keys %{$sources{$s}}) {
 462             if (not exists $included_sa{$s} and
 463                 not $included_sa{$s}{$v}) {
 464                 push @sa_to_add,
 465                     [$s,$v,$sources{$s}{$v}];
 466             } else {
 467                 $p->update() if defined $p;
 468             }
 469         }
 470     }
 471     # add new source packages
 472     my $it = natatime 100, @sa_to_add;
 473     while (my @v = $it->()) {
 474         $schema->txn_do(
 475             sub {
 476                 for my $svm (@_) {
 477                     my $s_id = $schema->resultset('SrcPkg')->
 478                         get_src_pkg_id($svm->[0]);
 479                     my $sv_id = $schema->resultset('SrcVer')->
 480                         get_src_ver_id($s_id,$svm->[1],$maints->{$svm->[2]});
 481                     $schema->resultset('SrcAssociation')->
 482                         insert_suite_src_ver_association($suite_id,$sv_id);
 483                 }
 484             },
 485                         @v
 486                        );
 487         $p->update($p->last_update()+
 488                    scalar @v) if defined $p;
 489     }
 490     # remove associations for packages not in this suite
 491     if (@sa_to_del) {
 492         $schema->resultset('SrcAssociation')->
 493             search_rs({id => \@sa_to_del})->delete();
 494     }
 495     # update packages in this suite to have a modification time of now
 496     $schema->resultset('SrcAssociation')->
 497         search_rs({suite => $suite_id})->
 498         update({modified => 'NOW()'});
 499     ## Handle binary packages
 500     my @bin_to_del;
 501     my @bin_to_add;
 502     my %included_bin;
 503     # calculate which binary packages are no longer in this suite
 504     for my $b ($schema->resultset('BinPkg')->
 505                bin_pkg_and_ver_in_suite($suite)) {
 506         if (not exists $bins{$b->{arch}{arch}} or
 507             not exists $bins{$b->{arch}{arch}}{$b->{pkg}} or
 508             ($bins{$b->{arch}{arch}}{$b->{pkg}}{bin_ver} ne
 509              $b->{bin_vers}{ver}
 510             )
 511            ) {
 512             push @bin_to_del,
 513                 $b->{bin_associations}{id};
 514         }
 515         $included_bin{$b->{arch}{arch}}{$b->{pkg}} =
 516             $b->{bin_vers}{ver};
 517     }
 518     # calculate which binary packages are newly in this suite
 519     for my $a (keys %bins) {
 520         for my $pkg (keys %{$bins{$a}}) {
 521             if (not exists $included_bin{$a} or
 522                 not exists $included_bin{$a}{$pkg} or
 523                 $bins{$a}{$pkg}{bin_ver} ne
 524                 $included_bin{$a}{$pkg}) {
 525                 push @bin_to_add,
 526                     $bins{$a}{$pkg};
 527             } else {
 528                 $p->update() if defined $p;
 529             }
 530         }
 531     }
 532     $it = natatime 100, @bin_to_add;
 533     while (my @v = $it->()) {
 534         $schema->txn_do(
 535         sub {
 536             for my $bvm (@_) {
 537                 my $s_id = $schema->resultset('SrcPkg')->
 538                     get_src_pkg_id($bvm->{source});
 539                 my $sv_id = $schema->resultset('SrcVer')->
 540                     get_src_ver_id($s_id,$bvm->{src_ver},$maints->{$bvm->{maint}});
 541                 my $b_id = $schema->resultset('BinPkg')->
 542                     get_bin_pkg_id($bvm->{bin});
 543                 my $bv_id = $schema->resultset('BinVer')->
 544                     get_bin_ver_id($b_id,$bvm->{bin_ver},
 545                                    $archs->{$bvm->{arch}},$sv_id);
 546                 $schema->resultset('BinAssociation')->
 547                     insert_suite_bin_ver_association($suite_id,$bv_id);
 548             }
 549         },
 550                         @v
 551                        );
 552         $p->update($p->last_update()+
 553                    scalar @v) if defined $p;
 554     }
 555     if (@bin_to_del) {
 556         $schema->resultset('BinAssociation')->
 557             search_rs({id => \@bin_to_del})->delete();
 558     }
 559     $schema->resultset('BinAssociation')->
 560         search_rs({suite => $suite_id})->
 561         update({modified => 'NOW()'});
 562
 563 }
 564
 565
 566 =back
 567
 568 =cut
 569
 570 =head Suites
 571
 572 =over
 573
 574 =item load_suite
 575
 576      load_suite($schema,$codename,$suite,$version,$active);
 577
 578 =cut
 579
 580 sub load_suite {
 581     my ($schema,$codename,$suite,$version,$active) = @_;
 582     if (ref($codename)) {
 583         ($codename,$suite,$version) =
 584             @{$codename}{qw(Codename Suite Version)};
 585         $active = 1;
 586     }
 587     my $s = $schema->resultset('Suite')->find_or_create({codename => $codename});
 588     $s->suite_name($suite);
 589     $s->version($version);
 590     $s->active($active);
 591     $s->update();
 592     return $s;
 593
 594 }
 595
 596 =back
 597
 598 =cut
 599
 600 1;
 601
 602
 603 __END__