Debbugs/DB/Load.pm

   1 # This module is part of debbugs, and is released
   2 # under the terms of the GPL version 2, or any later version. See the
   3 # file README and COPYING for more information.
   4 # Copyright 2013 by Don Armstrong <don@donarmstrong.com>.
   5
   6 package Debbugs::DB::Load;
   7
   8 =head1 NAME
   9
  10 Debbugs::DB::Load -- Utility routines for loading the database
  11
  12 =head1 SYNOPSIS
  13
  14
  15 =head1 DESCRIPTION
  16
  17
  18 =head1 BUGS
  19
  20 None known.
  21
  22 =cut
  23
  24 use warnings;
  25 use strict;
  26 use vars qw($VERSION $DEBUG %EXPORT_TAGS @EXPORT_OK @EXPORT);
  27 use base qw(Exporter);
  28
  29 BEGIN{
  30      ($VERSION) = q$Revision$ =~ /^Revision:\s+([^\s+])/;
  31      $DEBUG = 0 unless defined $DEBUG;
  32
  33      @EXPORT = ();
  34      %EXPORT_TAGS = (load_bug    => [qw(load_bug handle_load_bug_queue load_bug_log)],
  35                      load_debinfo => [qw(load_debinfo)],
  36                      load_package => [qw(load_package)],
  37                     );
  38      @EXPORT_OK = ();
  39      Exporter::export_ok_tags(keys %EXPORT_TAGS);
  40      $EXPORT_TAGS{all} = [@EXPORT_OK];
  41 }
  42
  43 use Params::Validate qw(validate_with :types);
  44
  45 use Debbugs::Status qw(read_bug split_status_fields);
  46 use Debbugs::DB;
  47 use DateTime;
  48 use Debbugs::Common qw(make_list getparsedaddrs);
  49 use Debbugs::Config qw(:config);
  50 use Carp;
  51
  52 =head2 Bug loading
  53
  54 Routines to load bug; exported with :load_bug
  55
  56 =over
  57
  58 =item load_bug
  59
  60      load_bug(db => $schema,
  61               data => split_status_fields($data),
  62               tags => \%tags,
  63               queue => \%queue);
  64
  65 Loads a bug's metadata into the database. (Does not load any messages)
  66
  67 =over
  68
  69 =item db -- Debbugs::DB object
  70
  71 =item data -- Bug data (from read_bug) which has been split with split_status_fields
  72
  73 =item tags -- tag cache (hashref); optional
  74
  75 =item queue -- queue of operations to perform after bug is loaded; optional.
  76
  77 =back
  78
  79 =cut
  80
  81 sub load_bug {
  82     my %param = validate_with(params => \@_,
  83                               spec => {db => {type => OBJECT,
  84                                              },
  85                                        data => {type => HASHREF,
  86                                                 optional => 1,
  87                                                },
  88                                        bug => {type => SCALAR,
  89                                                optional => 1,
  90                                               },
  91                                        tags => {type => HASHREF,
  92                                                 default => sub {return {}},
  93                                                 optional => 1},
  94                                        severities => {type => HASHREF,
  95                                                       default => sub {return {}},
  96                                                       optional => 1,
  97                                                      },
  98                                        queue => {type => HASHREF,
  99                                                  optional => 1},
 100                                       });
 101     my $s = $param{db};
 102     if (not exists $param{data} and not exists $param{bug}) {
 103         croak "One of data or bug must be provided to load_bug";
 104     }
 105     if (not exists $param{data}) {
 106         $param{data} = read_bug(bug => $param{bug});
 107     }
 108     my $data = $param{data};
 109     my $tags = $param{tags};
 110     my $queue = $param{queue};
 111     my $severities = $param{severities};
 112     my $can_queue = 1;
 113     if (not defined $queue) {
 114         $can_queue = 0;
 115         $queue = {};
 116     }
 117     my %tags;
 118     my $s_data = split_status_fields($data);
 119     for my $tag (make_list($s_data->{keywords})) {
 120         next unless defined $tag and length $tag;
 121         # this allows for invalid tags. But we'll use this to try to
 122         # find those bugs and clean them up
 123         if (not exists $tags->{$tag}) {
 124             $tags->{$tag} = $s->resultset('Tag')->
 125             find_or_create({tag => $tag});
 126         }
 127         $tags{$tag} = $tags->{$tag};
 128     }
 129     my $severity = length($data->{severity}) ? $data->{severity} : $config{default_severity};
 130     if (exists $severities->{$severity}) {
 131         $severity = $severities->{$severity};
 132     } else {
 133         $severity = $s->resultset('Severity')->
 134             find_or_create({severity => $severity});
 135     }
 136     my $bug =
 137         {id => $data->{bug_num},
 138          creation => DateTime->from_epoch(epoch => $data->{date}),
 139          log_modified => DateTime->from_epoch(epoch => $data->{log_modified}),
 140          last_modified => DateTime->from_epoch(epoch => $data->{last_modified}),
 141          archived => $data->{archived},
 142          (defined $data->{unarchived} and length($data->{unarchived}))?(unarchived => DateTime->from_epoch(epoch => $data->{unarchived})):(),
 143          forwarded => $data->{forwarded} // '',
 144          summary => $data->{summary} // '',
 145          outlook => $data->{outlook} // '',
 146          subject => $data->{subject} // '',
 147          done_full => $data->{done} // '',
 148          severity => $severity,
 149          owner_full => $data->{owner} // '',
 150          submitter_full => $data->{originator} // '',
 151         };
 152     my %addr_map =
 153         (done => 'done',
 154          owner => 'owner',
 155          submitter => 'originator',
 156         );
 157     for my $addr_type (keys %addr_map) {
 158         my @addrs = getparsedaddrs($data->{$addr_map{$addr_type}} // '');
 159         next unless @addrs;
 160         $bug->{$addr_type} = $s->resultset('Correspondent')->find_or_create({addr => lc($addrs[0]->address())});
 161         # insert the full name as well
 162         my $full_name = $addrs[0]->phrase();
 163         $full_name =~ s/^\"|\"$//g;
 164         $full_name =~ s/^\s+|\s+$//g;
 165         if (length $full_name) {
 166             $bug->{$addr_type}->
 167                 update_or_create_related('correspondent_full_names',
 168                                         {full_name=>$full_name,
 169                                          last_seen => 'NOW()'});
 170         }
 171     }
 172     my $b = $s->resultset('Bug')->update_or_create($bug) or
 173         die "Unable to update or create bug $bug->{id}";
 174      $s->txn_do(sub {
 175                    for my $ff (qw(found fixed)) {
 176                        my @elements = $s->resultset('BugVer')->search({bug => $data->{bug_num},
 177                                                                        found  => $ff eq 'found'?1:0,
 178                                                                       });
 179                        my %elements_to_delete = map {($elements[$_]->ver_string(),$elements[$_])} 0..$#elements;
 180                        my %elements_to_add;
 181                        my @elements_to_keep;
 182                        for my $version (@{$data->{"${ff}_versions"}}) {
 183                            if (exists $elements_to_delete{$version}) {
 184                                push @elements_to_keep,$version;
 185                            } else {
 186                                $elements_to_add{$version} = 1;
 187                            }
 188                        }
 189                        for my $version (@elements_to_keep) {
 190                            delete $elements_to_delete{$version};
 191                        }
 192                        for my $element (keys %elements_to_delete) {
 193                            $elements_to_delete{$element}->delete();
 194                        }
 195                        for my $element (keys %elements_to_add) {
 196                            # find source package and source version id
 197                            my $ne = $s->resultset('BugVer')->new_result({bug => $data->{bug_num},
 198                                                                          ver_string => $element,
 199                                                                          found => $ff eq 'found'?1:0,
 200                                                                         }
 201                                                                        );
 202                            if (my ($src_pkg,$src_ver) = $element =~ m{^([^\/]+)/(.+)$}) {
 203                                my $src_pkg_e = $s->resultset('SrcPkg')->single({pkg => $src_pkg});
 204                                if (defined $src_pkg_e) {
 205                                    $ne->src_pkg($src_pkg_e->id());
 206                                    my $src_ver_e = $s->resultset('SrcVer')->single({src_pkg => $src_pkg_e->id(),
 207                                                                                     ver => $src_ver
 208                                                                                    });
 209                                    $ne->src_ver($src_ver_e->id()) if defined $src_ver_e;
 210                                }
 211                            }
 212                            $ne->insert();
 213                        }
 214                    }
 215                });
 216     $s->txn_do(sub {
 217                    my $t = $s->resultset('BugTag')->search({bug => $data->{bug_num}});
 218                    $t->delete() if defined $t;
 219                    $s->populate(BugTag => [[qw(bug tag)], map {[$data->{bug_num}, $_->id()]} values %tags]);
 220                });
 221     # because these bugs reference other bugs which might not exist
 222     # yet, we can't handle them until we've loaded all bugs. queue
 223     # them up.
 224     for my $merge_block (qw(merged block)) {
 225         my $data_key = $merge_block;
 226         $data_key .= 'with' if $merge_block eq 'merged';
 227         if (@{$data->{$data_key}||[]}) {
 228             my $count = $s->resultset('Bug')->search({id => [@{$data->{$data_key}}]})->count();
 229             if ($count == @{$data->{$data_key}}) {
 230                 handle_load_bug_queue(db=>$s,
 231                                       queue => {$merge_block,
 232                                                {$data->{bug_num},[@{$data->{$data_key}}]}
 233                                                });
 234             } else {
 235                 $queue->{$merge_block}{$data->{bug_num}} = [@{$data->{$data_key}}];
 236             }
 237         }
 238     }
 239
 240     if (not $can_queue and keys %{$queue}) {
 241         handle_load_bug_queue(db => $s,queue => $queue);
 242     }
 243
 244     # still need to handle merges, versions, etc.
 245 }
 246
 247 =item handle_load_bug_queue
 248
 249      handle_load_bug_queue(db => $schema,queue => $queue);
 250
 251 Handles a queue of operations created by load bug. [These operations
 252 are used to handle cases where a bug referenced by a loaded bug may
 253 not exist yet. In cases where the bugs should exist, the queue is
 254 cleared automatically by load_bug if queue is undefined.
 255
 256 =cut
 257
 258 sub handle_load_bug_queue{
 259     my %param = validate_with(params => \@_,
 260                               spec => {db => {type => OBJECT,
 261                                              },
 262                                        queue => {type => HASHREF,
 263                                                 },
 264                                       });
 265     my $s = $param{db};
 266     my $queue = $param{queue};
 267     my %queue_types =
 268         (merged => {set => 'BugMerged',
 269                     columns => [qw(bug merged)],
 270                     bug => 'bug',
 271                    },
 272          blocks => {set => 'BugBlock',
 273                     columns => [qw(bug blocks)],
 274                     bug => 'bug',
 275                    },
 276         );
 277     for my $queue_type (keys %queue_types) {
 278         for my $bug (%{$queue->{$queue_type}}) {
 279             my $qt = $queue_types{$queue_type};
 280             $s->txn_do(sub {
 281                            $s->resultset($qt->{set})->search({$qt->{bug},$bug})->delete();
 282                            $s->populate($qt->{set},[[@{$qt->{columns}}],
 283                                                     map {[$bug,$_]} @{$queue->{$queue_type}{$bug}}]) if
 284                                @{$queue->{$queue_type}{$bug}//[]};
 285                        }
 286                       );
 287         }
 288     }
 289 }
 290
 291 =item load_bug_log -- load bug logs
 292
 293        load_bug_log(db  => $s,
 294                     bug => $bug);
 295
 296
 297 =over
 298
 299 =item db -- database
 300
 301 =item bug -- bug whose log should be loaded
 302
 303 =back
 304
 305 =cut
 306
 307 sub load_bug_log {
 308     my %param = validate_with(params => \@_,
 309                               spec => {db => {type => OBJECT,
 310                                              },
 311                                        bug => {type => SCALAR,
 312                                               },
 313                                        queue => {type => HASHREF,
 314                                                  optional => 1},
 315                                       });
 316     my $s = $param{db};
 317     my $msg_num=0;
 318     my %seen_msg_ids;
 319     my $log = Debbugs::Log->new(bug_num => $param{bug}) or
 320         die "Unable to open log for $param{bug} for reading: $!";
 321     while (my $record = $log->read_record()) {
 322         next unless $record->{type} eq 'incoming-recv';
 323         my ($msg_id) = $record->{text} =~ /^Message-Id:\s+<(.+)>/im;
 324         next if defined $msg_id and exists $seen_msg_ids{$msg_id};
 325         $seen_msg_ids{$msg_id} = 1 if defined $msg_id;
 326         next if defined $msg_id and $msg_id =~ /handler\..+\.ack(?:info)?\@/;
 327         my $message = parse($record->{text});
 328         # search for a message with this message id in the database
 329
 330         # check to see if the subject, to, and from match. if so, it's
 331         # probably the same message.
 332
 333         # if not, create a new message
 334
 335         # add correspondents if necessary
 336
 337         # link message to bugs if necessary
 338
 339     }
 340
 341 }
 342
 343 =back
 344
 345 =head2 Debinfo
 346
 347 Commands to handle src and package version loading from debinfo files
 348
 349 =over
 350
 351 =item load_debinfo
 352
 353      load_debinfo($schema,$binname, $binver, $binarch, $srcname, $srcver);
 354
 355
 356
 357 =cut
 358
 359 sub load_debinfo {
 360     my ($schema,$binname, $binver, $binarch, $srcname, $srcver) = @_;
 361     my $sp = $schema->resultset('SrcPkg')->find_or_create({pkg => $srcname});
 362     my $sv = $schema->resultset('SrcVer')->find_or_create({src_pkg=>$sp->id(),
 363                                                            ver => $srcver});
 364     my $arch = $schema->resultset('Arch')->find_or_create({arch => $binarch});
 365     my $bp = $schema->resultset('BinPkg')->find_or_create({pkg => $binname});
 366     $schema->resultset('BinVer')->find_or_create({bin_pkg_id => $bp->id(),
 367                                                   src_ver_id => $sv->id(),
 368                                                   arch_id    => $arch->id(),
 369                                                   ver        => $binver,
 370                                                  });
 371 }
 372
 373
 374 =back
 375
 376 =head Packages
 377
 378 =over
 379
 380 =item load_package
 381
 382      load_package($schema,$suite,$component,$arch,$pkg)
 383
 384 =cut
 385
 386 sub load_package {
 387     my ($schema,$suite,$component,$arch,$pkg) = @_;
 388     if ($arch eq 'source') {
 389         my $sp = $schema->resultset('SrcPkg')->find_or_create({pkg => $pkg->{Package}});
 390         my $suite = $schema->resultset('Suite')->find_or_create({suite_name => $suite});
 391         my $sv = $schema->resultset('SrcVer')->find_or_create({src_pkg =>$sp->id,
 392                                                                ver => $pkg->{Version}});
 393         my @addrs = getparsedaddrs($pkg->{Maintainer} // '');
 394         if (@addrs) {
 395             my $mc = $schema->resultset('Correspondent')->
 396                 find_or_create({addr => lc($addrs[0]->address())});
 397             my $full_name = $addrs[0]->phrase();
 398             $full_name =~ s/^\"|\"$//g;
 399             $full_name =~ s/^\s+|\s+$//g;
 400             $sv->discard_changes;
 401             $sv->find_or_create_related('maintainer',
 402                                        {name => $full_name,
 403                                         correspondent => $mc->id},
 404                                        );
 405             $mc->update_or_create_related('correspondent_full_names',
 406                                          {full_name=>$full_name,
 407                                           last_seen => 'NOW()'});
 408         }
 409         # update the link for this source package
 410         $schema->
 411             txndo(sub {
 412                      # delete associations for this source package in this
 413                      # suite
 414                      $schema->resultset('SrcAssociations')->
 415                          search_rs({suite => $suite->id,})->
 416                          search_related_rs('src_pkg',
 417                                           {src_pkg => $sp->id})->delete;
 418                      $schema->resultset('SrcAssociations')->
 419                          create({suite => $suite->id,
 420                                  source => $sv->id,
 421                                 });
 422                  });
 423     } else {
 424         my $bp = $schema->resultset('BinPkg')->find_or_create({pkg => $pkg->{Package}});
 425         my $suite = $schema->resultset('Suite')->find_or_create({suite_name => $suite});
 426         my ($bv) = $bp->search_related('bin_vers',{ver => $pkg->{Version}});
 427         # if there isn't already a binary version for this package, we don't
 428         # know what source it belongs to, so we can't associate it with a
 429         # release
 430         return if (not defined $bv);
 431         $schema->
 432             txndo(sub {
 433                       $schema->resultset('BinAssociations')->
 434                           search_rs({suite => $suite->id,})->
 435                           search_related_rs('bin_pkg',
 436                                            {bin_pkg_id => $bp->id}
 437                                            )->delete;
 438                       $schema->resultset('BinAssociations')->
 439                           create({suite => $suite->id,
 440                                   bin => $bv->id
 441                                  });
 442                   });
 443     }
 444 }
 445
 446 =back
 447
 448 =cut
 449
 450 1;
 451
 452
 453 __END__