X-Git-Url: https://git.donarmstrong.com/?a=blobdiff_plain;f=Debbugs%2FDB%2FLoad.pm;h=03ab770636eb8ec3a1d554ad78815f1f17794d8f;hb=466f7faff129a5699c7674f59900a92aa256175d;hp=a85a4ec28eaaa97a3876c1c9258aebbe8d070a27;hpb=758f570dbc0f13d4dd3b656b24882154ef218e37;p=debbugs.git

diff --git a/Debbugs/DB/Load.pm b/Debbugs/DB/Load.pm
index a85a4ec..03ab770 100644
--- a/Debbugs/DB/Load.pm
+++ b/Debbugs/DB/Load.pm
@@ -23,6 +23,7 @@ None known.
 
 use warnings;
 use strict;
+use v5.10;
 use vars qw($VERSION $DEBUG %EXPORT_TAGS @EXPORT_OK @EXPORT);
 use base qw(Exporter);
 
@@ -42,7 +43,7 @@ BEGIN{
 }
 
 use Params::Validate qw(validate_with :types);
-use List::MoreUtils qw(natatime);
+use List::AllUtils qw(natatime);
 
 use Debbugs::Status qw(read_bug split_status_fields);
 use Debbugs::DB;
@@ -176,32 +177,45 @@ sub load_bug {
     my $b = $s->resultset('Bug')->update_or_create($bug) or
         die "Unable to update or create bug $bug->{id}";
     $s->txn_do(sub {
-		 $b->set_related_packages('binpackages',
-					  [grep {defined $_ and
-						   length $_ and $_ !~ /^src:/}
-					   make_list($data->{package})],
-					  $param{packages},
-					 );
-		 $b->set_related_packages('srcpackages',
-					  [grep {defined $_ and
-						   $_ =~ /^src:/}
-					   make_list($data->{package})],
-					  $param{packages},
-					 );
-		 $b->set_related_packages('affects_binpackages',
-					  [grep {defined $_ and
-						   length $_ and $_ !~ /^src:/}
-					   make_list($data->{affects})
-					  ],
-					  $param{packages},
-					 );
-		 $b->set_related_packages('affects_srcpackages',
-					  [grep {defined $_ and
-						   $_ =~ /^src:/}
-					   make_list($data->{affects})],
-					  $param{packages},
-					 );
-		 for my $ff (qw(found fixed)) {
+                   my @unknown_packages;
+                   my @unknown_affects_packages;
+                   push @unknown_packages,
+                       $b->set_related_packages('binpackages',
+                                                [grep {defined $_ and
+                                                           length $_ and $_ !~ /^src:/}
+                                                 make_list($data->{package})],
+                                                $param{packages},
+                                               );
+                   push @unknown_packages,
+                       $b->set_related_packages('srcpackages',
+                                                [map {s/src://;
+                                                      $_}
+                                                 grep {defined $_ and
+                                                           $_ =~ /^src:/}
+                                                 make_list($data->{package})],
+                                                $param{packages},
+                                               );
+                   push @unknown_affects_packages,
+                       $b->set_related_packages('affects_binpackages',
+                                                [grep {defined $_ and
+                                                           length $_ and $_ !~ /^src:/}
+                                                 make_list($data->{affects})
+                                                ],
+                                                $param{packages},
+                                               );
+                   push @unknown_affects_packages,
+                       $b->set_related_packages('affects_srcpackages',
+                                                [map {s/src://;
+                                                      $_}
+                                                 grep {defined $_ and
+                                                           $_ =~ /^src:/}
+                                                 make_list($data->{affects})],
+                                                $param{packages},
+                                               );
+                   $b->unknown_packages(join(', ',@unknown_packages));
+                   $b->unknown_affects(join(', ',@unknown_affects_packages));
+                   $b->update();
+                   for my $ff (qw(found fixed)) {
 		       my @elements = $s->resultset('BugVer')->search({bug => $data->{bug_num},
 								       found  => $ff eq 'found'?1:0,
 								      });
@@ -248,19 +262,22 @@ sub load_bug {
     # because these bugs reference other bugs which might not exist
     # yet, we can't handle them until we've loaded all bugs. queue
     # them up.
-    for my $merge_block (qw(merged block)) {
-        my $data_key = $merge_block;
-        $data_key .= 'with' if $merge_block eq 'merged';
-        if (@{$data->{$data_key}||[]}) {
-            my $count = $s->resultset('Bug')->search({id => [@{$data->{$data_key}}]})->count();
-            if ($count == @{$data->{$data_key}}) {
-                handle_load_bug_queue(db=>$s,
-                                      queue => {$merge_block,
-                                               {$data->{bug_num},[@{$data->{$data_key}}]}
-                                               });
-            } else {
-                $queue->{$merge_block}{$data->{bug_num}} = [@{$data->{$data_key}}];
-            }
+    for my $merge_block (qw(mergedwith blocks)) {
+        my $count = 0;
+        if (@{$data->{$merge_block}}) {
+            $count =
+                $s->resultset('Bug')->
+                search({id => [@{$data->{$merge_block}}]})->
+                count();
+        }
+        # if all of the bugs exist, immediately fix the merge/blocks
+        if ($count == @{$data->{$merge_block}}) {
+            handle_load_bug_queue(db=>$s,
+                                  queue => {$merge_block,
+                                           {$data->{bug_num},[@{$data->{$merge_block}}]}
+                                           });
+        } else {
+            $queue->{$merge_block}{$data->{bug_num}} = [@{$data->{$merge_block}}];
         }
     }
 
@@ -292,26 +309,33 @@ sub handle_load_bug_queue{
     my $s = $param{db};
     my $queue = $param{queue};
     my %queue_types =
-	(merged => {set => 'BugMerged',
-		    columns => [qw(bug merged)],
-		    bug => 'bug',
-		   },
+	(mergedwith => {set => 'BugMerged',
+                        columns => [qw(bug merged)],
+                        bug => 'bug',
+                       },
 	 blocks => {set => 'BugBlock',
-		    columns => [qw(bug blocks)],
-		    bug => 'bug',
-		   },
+                    columns => [qw(bug blocks)],
+                    bug => 'bug',
+                   },
 	);
     for my $queue_type (keys %queue_types) {
-	for my $bug (%{$queue->{$queue_type}}) {
-	    my $qt = $queue_types{$queue_type};
-	    $s->txn_do(sub {
-			   $s->resultset($qt->{set})->search({$qt->{bug},$bug})->delete();
-			   $s->populate($qt->{set},[[@{$qt->{columns}}],
-                                                    map {[$bug,$_]} @{$queue->{$queue_type}{$bug}}]) if
-			       @{$queue->{$queue_type}{$bug}//[]};
+        my $qt = $queue_types{$queue_type};
+        my @bugs = keys %{$queue->{$queue_type}};
+        next unless @bugs;
+        my @entries;
+        for my $bug (@bugs) {
+            push @entries,
+                map {[$bug,$_]}
+                @{$queue->{$queue_type}{$bug}};
+        }
+        $s->txn_do(sub {
+                       $s->resultset($qt->{set})->
+                           search({$qt->{bug}=>\@bugs})->delete();
+                       $s->resultset($qt->{set})->
+                           populate([[@{$qt->{columns}}],
+                                     @entries]) if @entries;
 		       }
-		      );
-	}
+                  );
     }
 }
 
@@ -353,17 +377,19 @@ sub load_bug_log {
         next if defined $msg_id and $msg_id =~ /handler\..+\.ack(?:info)?\@/;
         my $entity = parse_to_mime_entity($record);
         # search for a message with this message id in the database
-        $msg_id = $entity->head->get('Message-Id:');
+        $msg_id = $entity->head->get('Message-Id') //
+            $entity->head->get('Resent-Message-ID') //
+            '';
 	$msg_id =~ s/^\s*\<//;
 	$msg_id =~ s/>\s*$//;
 	# check to see if the subject, to, and from match. if so, it's
         # probably the same message.
-	my $subject = decode_rfc1522($entity->head->get('Subject:'));
-	$subject =~ s/\n(?:(\s)\s*|\s*$)/$1/g;
-	my $to = decode_rfc1522($entity->head->get('To:'));
-	$to =~ s/\n(?:(\s)\s*|\s*$)/$1/g;
-	my $from = decode_rfc1522($entity->head->get('From:'));
-	$from =~ s/\n(?:(\s)\s*|\s*$)/$1/g;
+	my $subject = decode_rfc1522($entity->head->get('Subject')//'');
+	$subject =~ s/\n(?:(\s)\s*|\s*$)//g;
+	my $to = decode_rfc1522($entity->head->get('To')//'');
+	$to =~ s/\n(?:(\s)\s*|\s*$)//g;
+	my $from = decode_rfc1522($entity->head->get('From')//'');
+	$from =~ s/\n(?:(\s)\s*|\s*$)//g;
 	my $m = $s->resultset('Message')->
 	    find({msgid => $msg_id,
 		  from_complete => $from,
@@ -379,24 +405,30 @@ sub load_bug_log {
 				subject => $subject
 			       });
 	    eval {
-		$m->sent_date(DateTime::Format::Mail->
-			      parse_datetime($entity->head->get('Date:',0)));
+		my $date = DateTime::Format::Mail->
+                    parse_datetime($entity->head->get('Date',0));
+                if (abs($date->offset) >= 60 * 60 * 12) {
+                    $date = $date->set_time_zone('UTC');
+                }
+                $m->sent_date($date);
 	    };
-	    my $spam = $entity->head->get('X-Spam-Status:',0);
+	    my $spam = $entity->head->get('X-Spam-Status',0)//'';
 	    if ($spam=~ /score=([\d\.]+)/) {
 		$m->spam_score($1);
 	    }
 	    my %corr;
 	    @{$corr{from}} = getparsedaddrs($from);
 	    @{$corr{to}} = getparsedaddrs($to);
-	    @{$corr{cc}} = getparsedaddrs($entity->head->get('Cc:'));
+	    @{$corr{cc}} = getparsedaddrs($entity->head->get('Cc'));
 	    # add correspondents if necessary
 	    my @cors;
 	    for my $type (keys %corr) {
 		for my $addr (@{$corr{$type}}) {
+                    my $cor = $s->resultset('Correspondent')->
+                        get_correspondent_id($addr);
+                    next unless defined $cor;
 		    push @cors,
-			{correspondent => $s->resultset('Correspondent')->
-			 get_correspondent_id($addr),
+			{correspondent => $cor,
 			 correspondent_type => $type,
 			};
 		}
@@ -404,12 +436,13 @@ sub load_bug_log {
 	    $m->update();
 	    $s->txn_do(sub {
 			   $m->message_correspondents()->delete();
-			   $m->add_to_message_correspondents(@cors);
+			   $m->add_to_message_correspondents(@cors) if
+                               @cors;
 		       }
 		      );
 	}
 	my $recv;
-	if ($entity->head->get('Received:',0)
+	if ($entity->head->get('Received',0)
 	    =~ /via spool by (\S+)/) {
 	    $recv = $s->resultset('Correspondent')->
 		get_correspondent_id($1);
@@ -441,17 +474,52 @@ Commands to handle src and package version loading from debinfo files
 =cut
 
 sub load_debinfo {
-    my ($schema,$binname, $binver, $binarch, $srcname, $srcver) = @_;
-    my $sp = $schema->resultset('SrcPkg')->find_or_create({pkg => $srcname});
-    my $sv = $schema->resultset('SrcVer')->find_or_create({src_pkg=>$sp->id(),
-                                                           ver => $srcver});
-    my $arch = $schema->resultset('Arch')->find_or_create({arch => $binarch});
-    my $bp = $schema->resultset('BinPkg')->find_or_create({pkg => $binname});
-    $schema->resultset('BinVer')->find_or_create({bin_pkg_id => $bp->id(),
-                                                  src_ver_id => $sv->id(),
-                                                  arch_id    => $arch->id(),
-                                                  ver        => $binver,
-                                                 });
+    my ($s,$binname, $binver, $binarch, $srcname, $srcver,$ct_date,$cache) = @_;
+    $cache //= {};
+    my $sp;
+    if (not defined $cache->{sp}{$srcname}) {
+        $cache->{sp}{$srcname} =
+            $s->resultset('SrcPkg')->find_or_create({pkg => $srcname});
+    }
+    $sp = $cache->{sp}{$srcname};
+    # update the creation date if the data we have is earlier
+    if (defined $ct_date and
+        (not defined $sp->creation or
+         $ct_date < $sp->creation)) {
+        $sp->creation($ct_date);
+        $sp->last_modified(DateTime->now);
+        $sp->update;
+    }
+    my $sv;
+    if (not defined $cache->{sv}{$srcname}{$srcver}) {
+        $cache->{sv}{$srcname}{$srcver} =
+            $s->resultset('SrcVer')->
+            find_or_create({src_pkg =>$sp->id(),
+                            ver => $srcver});
+    }
+    $sv = $cache->{sv}{$srcname}{$srcver};
+    if (defined $ct_date and
+        (not defined $sv->upload_date() or $ct_date < $sv->upload_date())) {
+        $sv->upload_date($ct_date);
+        $sv->update;
+    }
+    my $arch;
+    if (not defined $cache->{arch}{$binarch}) {
+        $cache->{arch}{$binarch} =
+            $s->resultset('Arch')->
+            find_or_create({arch => $binarch},
+                          )->id();
+    }
+    $arch = $cache->{arch}{$binarch};
+    my $bp;
+    if (not defined $cache->{bp}{$binname}) {
+        $cache->{bp}{$binname} =
+            $s->resultset('BinPkg')->
+            get_or_create_bin_pkg_id($binname);
+    }
+    $bp = $cache->{bp}{$binname};
+    $s->resultset('BinVer')->
+        get_bin_ver_id($bp,$binver,$arch,$sv->id());
 }
 
 
@@ -557,7 +625,7 @@ sub load_packages {
 	    sub {
 		for my $svm (@_) {
 		    my $s_id = $schema->resultset('SrcPkg')->
-			get_src_pkg_id($svm->[0]);
+			get_or_create_src_pkg_id($svm->[0]);
 		    my $sv_id = $schema->resultset('SrcVer')->
 			get_src_ver_id($s_id,$svm->[1],$maints->{$svm->[2]});
 		    $schema->resultset('SrcAssociation')->
@@ -624,11 +692,11 @@ sub load_packages {
 	sub {
 	    for my $bvm (@_) {
 		my $s_id = $schema->resultset('SrcPkg')->
-		    get_src_pkg_id($bvm->{source});
+		    get_or_create_src_pkg_id($bvm->{source});
 		my $sv_id = $schema->resultset('SrcVer')->
 		    get_src_ver_id($s_id,$bvm->{src_ver},$maints->{$bvm->{maint}});
 		my $b_id = $schema->resultset('BinPkg')->
-		    get_bin_pkg_id($bvm->{bin});
+		    get_or_create_bin_pkg_id($bvm->{bin});
 		my $bv_id = $schema->resultset('BinVer')->
 		    get_bin_ver_id($b_id,$bvm->{bin_ver},
 				   $archs->{$bvm->{arch}},$sv_id);