Debbugs/DB/Load.pm

   1 # This module is part of debbugs, and is released
   2 # under the terms of the GPL version 2, or any later version. See the
   3 # file README and COPYING for more information.
   4 # Copyright 2013 by Don Armstrong <don@donarmstrong.com>.
   5
   6 package Debbugs::DB::Load;
   7
   8 =head1 NAME
   9
  10 Debbugs::DB::Load -- Utility routines for loading the database
  11
  12 =head1 SYNOPSIS
  13
  14
  15 =head1 DESCRIPTION
  16
  17
  18 =head1 BUGS
  19
  20 None known.
  21
  22 =cut
  23
  24 use warnings;
  25 use strict;
  26 use vars qw($VERSION $DEBUG %EXPORT_TAGS @EXPORT_OK @EXPORT);
  27 use base qw(Exporter);
  28
  29 BEGIN{
  30      ($VERSION) = q$Revision$ =~ /^Revision:\s+([^\s+])/;
  31      $DEBUG = 0 unless defined $DEBUG;
  32
  33      @EXPORT = ();
  34      %EXPORT_TAGS = (load_bug    => [qw(load_bug handle_load_bug_queue)],
  35                     );
  36      @EXPORT_OK = ();
  37      Exporter::export_ok_tags(keys %EXPORT_TAGS);
  38      $EXPORT_TAGS{all} = [@EXPORT_OK];
  39 }
  40
  41 use Params::Validate qw(validate_with :types);
  42
  43 use Debbugs::Status qw(read_bug split_status_fields);
  44 use Debbugs::DB;
  45 use DateTime;
  46 use Debbugs::Common qw(make_list getparsedaddrs);
  47 use Debbugs::Config qw(:config);
  48 use Carp;
  49
  50 =head2 Bug loading
  51
  52 Routines to load bug; exported with :load_bug
  53
  54 =over
  55
  56 =item load_bug
  57
  58      load_bug(db => $schema,
  59               data => split_status_fields($data),
  60               tags => \%tags,
  61               queue => \%queue);
  62
  63 Loads a bug's metadata into the database. (Does not load any messages)
  64
  65 =over
  66
  67 =item db -- Debbugs::DB object
  68
  69 =item data -- Bug data (from read_bug) which has been split with split_status_fields
  70
  71 =item tags -- tag cache (hashref); optional
  72
  73 =item queue -- queue of operations to perform after bug is loaded; optional.
  74
  75 =back
  76
  77 =cut
  78
  79 sub load_bug {
  80     my %param = validate_with(params => \@_,
  81                               spec => {db => {type => OBJECT,
  82                                              },
  83                                        data => {type => HASHREF,
  84                                                 optional => 1,
  85                                                },
  86                                        bug => {type => SCALAR,
  87                                                optional => 1,
  88                                               },
  89                                        tags => {type => HASHREF,
  90                                                 default => sub {return {}},
  91                                                 optional => 1},
  92                                        severities => {type => HASHREF,
  93                                                       default => sub {return {}},
  94                                                       optional => 1,
  95                                                      },
  96                                        queue => {type => HASHREF,
  97                                                  optional => 1},
  98                                       });
  99     my $s = $param{db};
 100     if (not exists $param{data} and not exists $param{bug}) {
 101         croak "One of data or bug must be provided to load_bug";
 102     }
 103     if (not exists $param{data}) {
 104         $param{data} = read_bug(bug => $param{bug});
 105     }
 106     my $data = $param{data};
 107     my $tags = $param{tags};
 108     my $queue = $param{queue};
 109     my $severities = $param{severities};
 110     my $can_queue = 1;
 111     if (not defined $queue) {
 112         $can_queue = 0;
 113         $queue = {};
 114     }
 115     my %tags;
 116     my $s_data = split_status_fields($data);
 117     for my $tag (make_list($s_data->{keywords})) {
 118         next unless defined $tag and length $tag;
 119         # this allows for invalid tags. But we'll use this to try to
 120         # find those bugs and clean them up
 121         if (not exists $tags->{$tag}) {
 122             $tags->{$tag} = $s->resultset('Tag')->
 123             find_or_create({tag => $tag});
 124         }
 125         $tags{$tag} = $tags->{$tag};
 126     }
 127     my $severity = length($data->{severity}) ? $data->{severity} : $config{default_severity};
 128     if (exists $severities->{$severity}) {
 129         $severity = $severities->{$severity};
 130     } else {
 131         $severity = $s->resultset('Severity')->
 132             find_or_create({severity => $severity});
 133     }
 134     my $bug =
 135         {id => $data->{bug_num},
 136          creation => DateTime->from_epoch(epoch => $data->{date}),
 137          log_modified => DateTime->from_epoch(epoch => $data->{log_modified}),
 138          last_modified => DateTime->from_epoch(epoch => $data->{last_modified}),
 139          archived => $data->{archived},
 140          (defined $data->{unarchived} and length($data->{unarchived}))?(unarchived => DateTime->from_epoch(epoch => $data->{unarchived})):(),
 141          forwarded => $data->{forwarded} // '',
 142          summary => $data->{summary} // '',
 143          outlook => $data->{outlook} // '',
 144          subject => $data->{subject} // '',
 145          done_full => $data->{done} // '',
 146          severity => $severity,
 147          owner_full => $data->{owner} // '',
 148          submitter_full => $data->{originator} // '',
 149         };
 150     my %addr_map =
 151         (done => 'done',
 152          owner => 'owner',
 153          submitter => 'originator',
 154         );
 155     for my $addr_type (keys %addr_map) {
 156         my @addrs = getparsedaddrs($data->{$addr_map{$addr_type}} // '');
 157         next unless @addrs;
 158         $bug->{$addr_type} = $s->resultset('Correspondent')->find_or_create({addr => $addrs[0]->address()});
 159         # insert the full name as well
 160         my $full_name = $addrs[0]->phrase();
 161         $full_name =~ s/^\"|\"$//g;
 162         $full_name =~ s/^\s+|\s+$//g;
 163         if (length $full_name) {
 164             $bug->{$addr_type}->
 165                 update_or_create_related('correspondent_full_names',
 166                                         {full_name=>$full_name,
 167                                          last_seen => 'NOW()'});
 168         }
 169     }
 170     my $b = $s->resultset('Bug')->update_or_create($bug) or
 171         die "Unable to update or create bug $bug->{id}";
 172      $s->txn_do(sub {
 173                    for my $ff (qw(found fixed)) {
 174                        my @elements = $s->resultset('BugVer')->search({bug => $data->{bug_num},
 175                                                                        found  => $ff eq 'found'?1:0,
 176                                                                       });
 177                        my %elements_to_delete = map {($elements[$_]->ver_string(),$elements[$_])} 0..$#elements;
 178                        my %elements_to_add;
 179                        my @elements_to_keep;
 180                        for my $version (@{$data->{"${ff}_versions"}}) {
 181                            if (exists $elements_to_delete{$version}) {
 182                                push @elements_to_keep,$version;
 183                            } else {
 184                                $elements_to_add{$version} = 1;
 185                            }
 186                        }
 187                        for my $version (@elements_to_keep) {
 188                            delete $elements_to_delete{$version};
 189                        }
 190                        for my $element (keys %elements_to_delete) {
 191                            $elements_to_delete{$element}->delete();
 192                        }
 193                        for my $element (keys %elements_to_add) {
 194                            # find source package and source version id
 195                            my $ne = $s->resultset('BugVer')->new_result({bug => $data->{bug_num},
 196                                                                          ver_string => $element,
 197                                                                          found => $ff eq 'found'?1:0,
 198                                                                         }
 199                                                                        );
 200                            if (my ($src_pkg,$src_ver) = $element =~ m{^([^\/]+)/(.+)$}) {
 201                                my $src_pkg_e = $s->resultset('SrcPkg')->single({pkg => $src_pkg});
 202                                if (defined $src_pkg_e) {
 203                                    $ne->src_pkg($src_pkg_e->id());
 204                                    my $src_ver_e = $s->resultset('SrcVer')->single({src_pkg => $src_pkg_e->id(),
 205                                                                                     ver => $src_ver
 206                                                                                    });
 207                                    $ne->src_ver($src_ver_e->id()) if defined $src_ver_e;
 208                                }
 209                            }
 210                            $ne->insert();
 211                        }
 212                    }
 213                });
 214     $s->txn_do(sub {
 215                    my $t = $s->resultset('BugTag')->search({bug => $data->{bug_num}});
 216                    $t->delete() if defined $t;
 217                    $s->populate(BugTag => [[qw(bug tag)], map {[$data->{bug_num}, $_->id()]} values %tags]);
 218                });
 219     # because these bugs reference other bugs which might not exist
 220     # yet, we can't handle them until we've loaded all bugs. queue
 221     # them up.
 222     for my $merge_block (qw(merged block)) {
 223         my $data_key = $merge_block;
 224         $data_key .= 'with' if $merge_block eq 'merged';
 225         if (@{$data->{$data_key}||[]}) {
 226             my $count = $s->resultset('Bug')->search({id => [@{$data->{$data_key}}]})->count();
 227             if ($count == @{$data->{$data_key}}) {
 228                 handle_load_bug_queue(db=>$s,
 229                                       queue => {$merge_block,
 230                                                {$data->{bug_num},[@{$data->{$data_key}}]}
 231                                                });
 232             } else {
 233                 $queue->{$merge_block}{$data->{bug_num}} = [@{$data->{$data_key}}];
 234             }
 235         }
 236     }
 237
 238     if (not $can_queue and keys %{$queue}) {
 239         handle_load_bug_queue(db => $s,queue => $queue);
 240     }
 241
 242     # still need to handle merges, versions, etc.
 243 }
 244
 245 =item handle_load_bug_queue
 246
 247      handle_load_bug_queue(db => $schema,queue => $queue);
 248
 249 Handles a queue of operations created by load bug. [These operations
 250 are used to handle cases where a bug referenced by a loaded bug may
 251 not exist yet. In cases where the bugs should exist, the queue is
 252 cleared automatically by load_bug if queue is undefined.
 253
 254 =cut
 255
 256 sub handle_load_bug_queue{
 257     my %param = validate_with(params => \@_,
 258                               spec => {db => {type => OBJECT,
 259                                              },
 260                                        queue => {type => HASHREF,
 261                                                 },
 262                                       });
 263     my $s = $param{db};
 264     my $queue = $param{queue};
 265     my %queue_types =
 266         (merged => {set => 'BugMerged',
 267                     columns => [qw(bug merged)],
 268                     bug => 'bug',
 269                    },
 270          blocks => {set => 'BugBlock',
 271                     columns => [qw(bug blocks)],
 272                     bug => 'bug',
 273                    },
 274         );
 275     for my $queue_type (keys %queue_types) {
 276         for my $bug (%{$queue->{$queue_type}}) {
 277             my $qt = $queue_types{$queue_type};
 278             $s->txn_do(sub {
 279                            $s->resultset($qt->{set})->search({$qt->{bug},$bug})->delete();
 280                            $s->populate($qt->{set},[[@{$qt->{columns}}],
 281                                                     map {[$bug,$_]} @{$queue->{$queue_type}{$bug}}]) if
 282                                @{$queue->{$queue_type}{$bug}//[]};
 283                        }
 284                       );
 285         }
 286     }
 287 }
 288
 289 =back
 290
 291 =head2 Debinfo
 292
 293 Commands to handle src and package version loading from debinfo files
 294
 295 =item load_debinfo
 296
 297      load_debinfo($schema,$binname, $binver, $binarch, $srcname, $srcver);
 298
 299
 300
 301 =cut
 302
 303 sub load_debinfo {
 304     my ($schema,$binname, $binver, $binarch, $srcname, $srcver) = @_;
 305     my $sp = $schema->resultset('SrcPkg')->find_or_create({pkg => $srcname});
 306     my $sv = $schema->resultset('SrcVer')->find_or_create({src_pkg_id=>$sp->id(),
 307                                                            ver => $srcver});
 308     my $arch = $schema->resultset('Arch')->find_or_create({arch => $binarch});
 309     my $bp = $schema->resultset('BinPkg')->find_or_create({pkg => $binname});
 310     $schema->resultset('BinVer')->find_or_create({bin_pkg_id => $bp->id(),
 311                                                   src_ver_id => $sv->id(),
 312                                                   arch_id    => $arch->id(),
 313                                                   ver        => $binver,
 314                                                  });
 315 }
 316
 317 1;
 318
 319
 320 __END__