Debbugs/DB/Load.pm

   1 # This module is part of debbugs, and is released
   2 # under the terms of the GPL version 2, or any later version. See the
   3 # file README and COPYING for more information.
   4 # Copyright 2013 by Don Armstrong <don@donarmstrong.com>.
   5
   6 package Debbugs::DB::Load;
   7
   8 =head1 NAME
   9
  10 Debbugs::DB::Load -- Utility routines for loading the database
  11
  12 =head1 SYNOPSIS
  13
  14
  15 =head1 DESCRIPTION
  16
  17
  18 =head1 BUGS
  19
  20 None known.
  21
  22 =cut
  23
  24 use warnings;
  25 use strict;
  26 use vars qw($VERSION $DEBUG %EXPORT_TAGS @EXPORT_OK @EXPORT);
  27 use base qw(Exporter);
  28
  29 BEGIN{
  30      ($VERSION) = q$Revision$ =~ /^Revision:\s+([^\s+])/;
  31      $DEBUG = 0 unless defined $DEBUG;
  32
  33      @EXPORT = ();
  34      %EXPORT_TAGS = (load_bug    => [qw(load_bug handle_load_bug_queue load_bug_log)],
  35                      load_debinfo => [qw(load_debinfo)],
  36                      load_package => [qw(load_package)],
  37                      load_suite => [qw(load_suite)],
  38                     );
  39      @EXPORT_OK = ();
  40      Exporter::export_ok_tags(keys %EXPORT_TAGS);
  41      $EXPORT_TAGS{all} = [@EXPORT_OK];
  42 }
  43
  44 use Params::Validate qw(validate_with :types);
  45
  46 use Debbugs::Status qw(read_bug split_status_fields);
  47 use Debbugs::DB;
  48 use DateTime;
  49 use Debbugs::Common qw(make_list getparsedaddrs);
  50 use Debbugs::Config qw(:config);
  51 use Carp;
  52
  53 =head2 Bug loading
  54
  55 Routines to load bug; exported with :load_bug
  56
  57 =over
  58
  59 =item load_bug
  60
  61      load_bug(db => $schema,
  62               data => split_status_fields($data),
  63               tags => \%tags,
  64               queue => \%queue);
  65
  66 Loads a bug's metadata into the database. (Does not load any messages)
  67
  68 =over
  69
  70 =item db -- Debbugs::DB object
  71
  72 =item data -- Bug data (from read_bug) which has been split with split_status_fields
  73
  74 =item tags -- tag cache (hashref); optional
  75
  76 =item queue -- queue of operations to perform after bug is loaded; optional.
  77
  78 =back
  79
  80 =cut
  81
  82 sub load_bug {
  83     my %param = validate_with(params => \@_,
  84                               spec => {db => {type => OBJECT,
  85                                              },
  86                                        data => {type => HASHREF,
  87                                                 optional => 1,
  88                                                },
  89                                        bug => {type => SCALAR,
  90                                                optional => 1,
  91                                               },
  92                                        tags => {type => HASHREF,
  93                                                 default => sub {return {}},
  94                                                 optional => 1},
  95                                        severities => {type => HASHREF,
  96                                                       default => sub {return {}},
  97                                                       optional => 1,
  98                                                      },
  99                                        queue => {type => HASHREF,
 100                                                  optional => 1},
 101                                       });
 102     my $s = $param{db};
 103     if (not exists $param{data} and not exists $param{bug}) {
 104         croak "One of data or bug must be provided to load_bug";
 105     }
 106     if (not exists $param{data}) {
 107         $param{data} = read_bug(bug => $param{bug});
 108     }
 109     my $data = $param{data};
 110     my $tags = $param{tags};
 111     my $queue = $param{queue};
 112     my $severities = $param{severities};
 113     my $can_queue = 1;
 114     if (not defined $queue) {
 115         $can_queue = 0;
 116         $queue = {};
 117     }
 118     my %tags;
 119     my $s_data = split_status_fields($data);
 120     for my $tag (make_list($s_data->{keywords})) {
 121         next unless defined $tag and length $tag;
 122         # this allows for invalid tags. But we'll use this to try to
 123         # find those bugs and clean them up
 124         if (not exists $tags->{$tag}) {
 125             $tags->{$tag} = $s->resultset('Tag')->
 126             find_or_create({tag => $tag});
 127         }
 128         $tags{$tag} = $tags->{$tag};
 129     }
 130     my $severity = length($data->{severity}) ? $data->{severity} : $config{default_severity};
 131     if (exists $severities->{$severity}) {
 132         $severity = $severities->{$severity};
 133     } else {
 134         $severity = $s->resultset('Severity')->
 135             find_or_create({severity => $severity});
 136     }
 137     my $bug =
 138         {id => $data->{bug_num},
 139          creation => DateTime->from_epoch(epoch => $data->{date}),
 140          log_modified => DateTime->from_epoch(epoch => $data->{log_modified}),
 141          last_modified => DateTime->from_epoch(epoch => $data->{last_modified}),
 142          archived => $data->{archived},
 143          (defined $data->{unarchived} and length($data->{unarchived}))?(unarchived => DateTime->from_epoch(epoch => $data->{unarchived})):(),
 144          forwarded => $data->{forwarded} // '',
 145          summary => $data->{summary} // '',
 146          outlook => $data->{outlook} // '',
 147          subject => $data->{subject} // '',
 148          done_full => $data->{done} // '',
 149          severity => $severity,
 150          owner_full => $data->{owner} // '',
 151          submitter_full => $data->{originator} // '',
 152         };
 153     my %addr_map =
 154         (done => 'done',
 155          owner => 'owner',
 156          submitter => 'originator',
 157         );
 158     for my $addr_type (keys %addr_map) {
 159         my @addrs = getparsedaddrs($data->{$addr_map{$addr_type}} // '');
 160         next unless @addrs;
 161         $bug->{$addr_type} = $s->resultset('Correspondent')->find_or_create({addr => lc($addrs[0]->address())});
 162         # insert the full name as well
 163         my $full_name = $addrs[0]->phrase();
 164         $full_name =~ s/^\"|\"$//g;
 165         $full_name =~ s/^\s+|\s+$//g;
 166         if (length $full_name) {
 167             $bug->{$addr_type}->
 168                 update_or_create_related('correspondent_full_names',
 169                                         {full_name=>$full_name,
 170                                          last_seen => 'NOW()'});
 171         }
 172     }
 173     my $b = $s->resultset('Bug')->update_or_create($bug) or
 174         die "Unable to update or create bug $bug->{id}";
 175      $s->txn_do(sub {
 176                    for my $ff (qw(found fixed)) {
 177                        my @elements = $s->resultset('BugVer')->search({bug => $data->{bug_num},
 178                                                                        found  => $ff eq 'found'?1:0,
 179                                                                       });
 180                        my %elements_to_delete = map {($elements[$_]->ver_string(),$elements[$_])} 0..$#elements;
 181                        my %elements_to_add;
 182                        my @elements_to_keep;
 183                        for my $version (@{$data->{"${ff}_versions"}}) {
 184                            if (exists $elements_to_delete{$version}) {
 185                                push @elements_to_keep,$version;
 186                            } else {
 187                                $elements_to_add{$version} = 1;
 188                            }
 189                        }
 190                        for my $version (@elements_to_keep) {
 191                            delete $elements_to_delete{$version};
 192                        }
 193                        for my $element (keys %elements_to_delete) {
 194                            $elements_to_delete{$element}->delete();
 195                        }
 196                        for my $element (keys %elements_to_add) {
 197                            # find source package and source version id
 198                            my $ne = $s->resultset('BugVer')->new_result({bug => $data->{bug_num},
 199                                                                          ver_string => $element,
 200                                                                          found => $ff eq 'found'?1:0,
 201                                                                         }
 202                                                                        );
 203                            if (my ($src_pkg,$src_ver) = $element =~ m{^([^\/]+)/(.+)$}) {
 204                                my $src_pkg_e = $s->resultset('SrcPkg')->single({pkg => $src_pkg});
 205                                if (defined $src_pkg_e) {
 206                                    $ne->src_pkg($src_pkg_e->id());
 207                                    my $src_ver_e = $s->resultset('SrcVer')->single({src_pkg => $src_pkg_e->id(),
 208                                                                                     ver => $src_ver
 209                                                                                    });
 210                                    $ne->src_ver($src_ver_e->id()) if defined $src_ver_e;
 211                                }
 212                            }
 213                            $ne->insert();
 214                        }
 215                    }
 216                });
 217     $s->txn_do(sub {
 218                    my $t = $s->resultset('BugTag')->search({bug => $data->{bug_num}});
 219                    $t->delete() if defined $t;
 220                    $s->populate(BugTag => [[qw(bug tag)], map {[$data->{bug_num}, $_->id()]} values %tags]);
 221                });
 222     # because these bugs reference other bugs which might not exist
 223     # yet, we can't handle them until we've loaded all bugs. queue
 224     # them up.
 225     for my $merge_block (qw(merged block)) {
 226         my $data_key = $merge_block;
 227         $data_key .= 'with' if $merge_block eq 'merged';
 228         if (@{$data->{$data_key}||[]}) {
 229             my $count = $s->resultset('Bug')->search({id => [@{$data->{$data_key}}]})->count();
 230             if ($count == @{$data->{$data_key}}) {
 231                 handle_load_bug_queue(db=>$s,
 232                                       queue => {$merge_block,
 233                                                {$data->{bug_num},[@{$data->{$data_key}}]}
 234                                                });
 235             } else {
 236                 $queue->{$merge_block}{$data->{bug_num}} = [@{$data->{$data_key}}];
 237             }
 238         }
 239     }
 240
 241     if (not $can_queue and keys %{$queue}) {
 242         handle_load_bug_queue(db => $s,queue => $queue);
 243     }
 244
 245     # still need to handle merges, versions, etc.
 246 }
 247
 248 =item handle_load_bug_queue
 249
 250      handle_load_bug_queue(db => $schema,queue => $queue);
 251
 252 Handles a queue of operations created by load bug. [These operations
 253 are used to handle cases where a bug referenced by a loaded bug may
 254 not exist yet. In cases where the bugs should exist, the queue is
 255 cleared automatically by load_bug if queue is undefined.
 256
 257 =cut
 258
 259 sub handle_load_bug_queue{
 260     my %param = validate_with(params => \@_,
 261                               spec => {db => {type => OBJECT,
 262                                              },
 263                                        queue => {type => HASHREF,
 264                                                 },
 265                                       });
 266     my $s = $param{db};
 267     my $queue = $param{queue};
 268     my %queue_types =
 269         (merged => {set => 'BugMerged',
 270                     columns => [qw(bug merged)],
 271                     bug => 'bug',
 272                    },
 273          blocks => {set => 'BugBlock',
 274                     columns => [qw(bug blocks)],
 275                     bug => 'bug',
 276                    },
 277         );
 278     for my $queue_type (keys %queue_types) {
 279         for my $bug (%{$queue->{$queue_type}}) {
 280             my $qt = $queue_types{$queue_type};
 281             $s->txn_do(sub {
 282                            $s->resultset($qt->{set})->search({$qt->{bug},$bug})->delete();
 283                            $s->populate($qt->{set},[[@{$qt->{columns}}],
 284                                                     map {[$bug,$_]} @{$queue->{$queue_type}{$bug}}]) if
 285                                @{$queue->{$queue_type}{$bug}//[]};
 286                        }
 287                       );
 288         }
 289     }
 290 }
 291
 292 =item load_bug_log -- load bug logs
 293
 294        load_bug_log(db  => $s,
 295                     bug => $bug);
 296
 297
 298 =over
 299
 300 =item db -- database
 301
 302 =item bug -- bug whose log should be loaded
 303
 304 =back
 305
 306 =cut
 307
 308 sub load_bug_log {
 309     my %param = validate_with(params => \@_,
 310                               spec => {db => {type => OBJECT,
 311                                              },
 312                                        bug => {type => SCALAR,
 313                                               },
 314                                        queue => {type => HASHREF,
 315                                                  optional => 1},
 316                                       });
 317     my $s = $param{db};
 318     my $msg_num=0;
 319     my %seen_msg_ids;
 320     my $log = Debbugs::Log->new(bug_num => $param{bug}) or
 321         die "Unable to open log for $param{bug} for reading: $!";
 322     while (my $record = $log->read_record()) {
 323         next unless $record->{type} eq 'incoming-recv';
 324         my ($msg_id) = $record->{text} =~ /^Message-Id:\s+<(.+)>/im;
 325         next if defined $msg_id and exists $seen_msg_ids{$msg_id};
 326         $seen_msg_ids{$msg_id} = 1 if defined $msg_id;
 327         next if defined $msg_id and $msg_id =~ /handler\..+\.ack(?:info)?\@/;
 328         my $message = parse($record->{text});
 329         # search for a message with this message id in the database
 330
 331         # check to see if the subject, to, and from match. if so, it's
 332         # probably the same message.
 333
 334         # if not, create a new message
 335
 336         # add correspondents if necessary
 337
 338         # link message to bugs if necessary
 339
 340     }
 341
 342 }
 343
 344 =back
 345
 346 =head2 Debinfo
 347
 348 Commands to handle src and package version loading from debinfo files
 349
 350 =over
 351
 352 =item load_debinfo
 353
 354      load_debinfo($schema,$binname, $binver, $binarch, $srcname, $srcver);
 355
 356
 357
 358 =cut
 359
 360 sub load_debinfo {
 361     my ($schema,$binname, $binver, $binarch, $srcname, $srcver) = @_;
 362     my $sp = $schema->resultset('SrcPkg')->find_or_create({pkg => $srcname});
 363     my $sv = $schema->resultset('SrcVer')->find_or_create({src_pkg=>$sp->id(),
 364                                                            ver => $srcver});
 365     my $arch = $schema->resultset('Arch')->find_or_create({arch => $binarch});
 366     my $bp = $schema->resultset('BinPkg')->find_or_create({pkg => $binname});
 367     $schema->resultset('BinVer')->find_or_create({bin_pkg_id => $bp->id(),
 368                                                   src_ver_id => $sv->id(),
 369                                                   arch_id    => $arch->id(),
 370                                                   ver        => $binver,
 371                                                  });
 372 }
 373
 374
 375 =back
 376
 377 =head Packages
 378
 379 =over
 380
 381 =item load_package
 382
 383      load_package($schema,$suite,$component,$arch,$pkg)
 384
 385 =cut
 386
 387 sub load_packages {
 388     my ($schema,$suite,$pkgs,$p) = @_;
 389     my $suite_id = $schema->resultset('Suite')->
 390         find_or_create({codename => $suite})->id;
 391     my %maint_cache;
 392     my %arch_cache;
 393     my %source_cache;
 394     my $src_max_last_modified = $schema->resultset('SrcAssociation')->
 395         search_rs({suite => $suite_id},
 396                  {order_by => {-desc => ['me.modified']},
 397                   rows => 1,
 398                   page => 1
 399                  }
 400                  )->single();
 401     my $bin_max_last_modified = $schema->resultset('BinAssociation')->
 402         search_rs({suite => $suite_id},
 403                  {order_by => {-desc => ['me.modified']},
 404                   rows => 1,
 405                   page => 1
 406                  }
 407                  )->single();
 408     print STDERR time." handling packages\n";
 409     for my $pkg_tuple (@{$pkgs}) {
 410         my ($arch,$component,$pkg) = @{$pkg_tuple};
 411         $p->update() if $p;
 412         if ($arch eq 'source') {
 413             my $source = $pkg->{Package};
 414             my $source_ver = $pkg->{Version};
 415             if (not exists $maint_cache{$pkg->{Maintainer}}) {
 416                 my @addrs = getparsedaddrs($pkg->{Maintainer} // '');
 417                 if (@addrs) {
 418                     my $mc = $schema->resultset('Correspondent')->
 419                         find_or_create({addr => lc($addrs[0]->address())},
 420                                       {key => 'correspondent_addr_idx'}
 421                                       );
 422                     my $full_name = $addrs[0]->phrase();
 423                     $full_name =~ s/^\"|\"$//g;
 424                     $full_name =~ s/^\s+|\s+$//g;
 425                     # $sv->discard_changes;
 426                     my $maint = $schema->resultset('Maintainer')->
 427                         find_or_create({name => $pkg->{Maintainer},
 428                                         correspondent => $mc->id},
 429                                       {key => 'maintainer_name_idx'},
 430                                       );
 431                     $mc->find_or_create_related('correspondent_full_names',
 432                                                {full_name => $full_name},
 433                                                {key => 'correspondent_full_name_correspondent_full_name_idx'}
 434                                                );
 435                     $mc->update;
 436                     $maint_cache{$pkg->{Maintainer}} = $maint;
 437                 }
 438             }
 439             if (not exists $source_cache{$source}{$source_ver}) {
 440                 my $sp = $schema->resultset('SrcPkg')->
 441                     find_or_create({pkg => $source});
 442                 my $sv = $sp->find_or_create_related('src_vers',
 443                                                     {ver => $source_ver});
 444                 $source_cache{$source}{$source_ver} = $sv;
 445                 if (exists $maint_cache{$pkg->{Maintainer}}) {
 446                     $source_cache{$source}{$source_ver}->
 447                         set_from_related('maintainer',
 448                                          $maint_cache{$pkg->{Maintainer}}
 449                                         );
 450                     $source_cache{$source}{$source_ver}->update;
 451                 }
 452             }
 453             $schema->resultset('SrcAssociation')->
 454                 update_or_create({suite => $suite_id,
 455                                   source => $source_cache{$source}{$source_ver}->id,
 456                                   modified => 'NOW()',
 457                                  },
 458                                 {key => 'src_associations_source_suite'}
 459                                 );
 460         } else {
 461             my $ar = $schema->resultset('Arch')->
 462                 find_or_create(arch => $arch);
 463             my $bp = $schema->resultset('BinPkg')->
 464                 find_or_create({pkg => $pkg->{Package}});
 465             my $source = $pkg->{Source} // $pkg->{Package};
 466             my $source_ver = $pkg->{Version};
 467             if ($source =~ /^\s*(\S+) \(([^\)]+)\)\s*$/) {
 468                 ($source,$source_ver) = ($1,$2);
 469             }
 470             if (not exists $source_cache{$source}{$source_ver}) {
 471                 my $sp = $schema->resultset('SrcPkg')->
 472                     find_or_create({pkg => $source});
 473                 my $sv = $sp->find_or_create_related('src_vers',
 474                                                     {ver => $source_ver});
 475                 $source_cache{$source}{$source_ver} = $sv;
 476             }
 477             my $bv = $bp->find_or_create_related('bin_vers',
 478                                                 {ver => $pkg->{Version},
 479                                                  src_ver => $source_cache{$source}{$source_ver}->id,
 480                                                  arch => $ar->id,
 481                                                 });
 482             $schema->resultset('BinAssociation')->
 483                 update_or_create({suite => $suite_id,
 484                                   bin => $bv->id,
 485                                   modified => 'NOW()',
 486                                  },
 487                                 {key => 'bin_associations_bin_suite'}
 488                                 );
 489         }
 490     }
 491     print STDERR time." deleting associations\n";
 492     # delete old binary associations in this suite which have not recently been
 493     # modified
 494     $schema->resultset('BinAssociation')->
 495         search_rs({suite => $suite_id,
 496                    modified => {'<',$bin_max_last_modified->modified()},
 497                   }) if defined
 498                       $bin_max_last_modified;
 499     $schema->resultset('SrcAssociation')->
 500         search_rs({suite => $suite_id,
 501                    modified => {'<',$src_max_last_modified->modified()},
 502                   }) if defined
 503                       $src_max_last_modified;
 504 }
 505
 506 =back
 507
 508 =cut
 509
 510 =head Suites
 511
 512 =over
 513
 514 =item load_suite
 515
 516      load_suite($schema,$codename,$suite,$version,$active);
 517
 518 =cut
 519
 520 sub load_suite {
 521     my ($schema,$codename,$suite,$version,$active) = @_;
 522     if (ref($codename)) {
 523         ($codename,$suite,$version) =
 524             @{$codename}{qw(Codename Suite Version)};
 525         $active = 1;
 526     }
 527     my $s = $schema->resultset('Suite')->find_or_create({codename => $codename});
 528     $s->suite_name($suite);
 529     $s->version($version);
 530     $s->active($active);
 531     $s->update();
 532     return $s;
 533
 534 }
 535
 536 =back
 537
 538 =cut
 539
 540 1;
 541
 542
 543 __END__