X-Git-Url: https://git.donarmstrong.com/?a=blobdiff_plain;f=Debbugs%2FDB%2FLoad.pm;h=a85a4ec28eaaa97a3876c1c9258aebbe8d070a27;hb=758f570dbc0f13d4dd3b656b24882154ef218e37;hp=d2774ab773678da03cba76957b08fe9afece5489;hpb=cd0cfe3e8040445f33e496eb9e807b3c96945187;p=debbugs.git

diff --git a/Debbugs/DB/Load.pm b/Debbugs/DB/Load.pm
index d2774ab..a85a4ec 100644
--- a/Debbugs/DB/Load.pm
+++ b/Debbugs/DB/Load.pm
@@ -32,6 +32,9 @@ BEGIN{
 
      @EXPORT = ();
      %EXPORT_TAGS = (load_bug    => [qw(load_bug handle_load_bug_queue load_bug_log)],
+		     load_debinfo => [qw(load_debinfo)],
+		     load_package => [qw(load_packages)],
+		     load_suite => [qw(load_suite)],
 		    );
      @EXPORT_OK = ();
      Exporter::export_ok_tags(keys %EXPORT_TAGS);
@@ -39,12 +42,15 @@ BEGIN{
 }
 
 use Params::Validate qw(validate_with :types);
+use List::MoreUtils qw(natatime);
 
 use Debbugs::Status qw(read_bug split_status_fields);
 use Debbugs::DB;
 use DateTime;
 use Debbugs::Common qw(make_list getparsedaddrs);
 use Debbugs::Config qw(:config);
+use Debbugs::MIME qw(parse_to_mime_entity decode_rfc1522);
+use DateTime::Format::Mail;
 use Carp;
 
 =head2 Bug loading
@@ -95,6 +101,10 @@ sub load_bug {
                                                      },
                                        queue => {type => HASHREF,
                                                  optional => 1},
+				       packages => {type => HASHREF,
+						    default => sub {return {}},
+						    optional => 1,
+						   },
                                       });
     my $s = $param{db};
     if (not exists $param{data} and not exists $param{bug}) {
@@ -113,8 +123,8 @@ sub load_bug {
         $queue = {};
     }
     my %tags;
-    my $s_data = split_status_fields($data);
-    for my $tag (make_list($s_data->{keywords})) {
+    $data = split_status_fields($data);
+    for my $tag (make_list($data->{keywords})) {
 	next unless defined $tag and length $tag;
 	# this allows for invalid tags. But we'll use this to try to
 	# find those bugs and clean them up
@@ -124,20 +134,23 @@ sub load_bug {
 	}
 	$tags{$tag} = $tags->{$tag};
     }
-    my $severity = length($data->{severity}) ? $data->{severity} : $config{default_severity};
-    if (exists $severities->{$severity}) {
-        $severity = $severities->{$severity};
-    } else {
-        $severity = $s->resultset('Severity')->
-            find_or_create({severity => $severity});
+    my $severity = length($data->{severity}) ? $data->{severity} :
+	$config{default_severity};
+    if (not exists $severities->{$severity}) {
+	$severities->{$severity} =
+	    $s->resultset('Severity')->
+            find_or_create({severity => $severity},
+			  );
     }
+    $severity = $severities->{$severity};
     my $bug =
         {id => $data->{bug_num},
          creation => DateTime->from_epoch(epoch => $data->{date}),
          log_modified => DateTime->from_epoch(epoch => $data->{log_modified}),
          last_modified => DateTime->from_epoch(epoch => $data->{last_modified}),
          archived => $data->{archived},
-         (defined $data->{unarchived} and length($data->{unarchived}))?(unarchived => DateTime->from_epoch(epoch => $data->{unarchived})):(),
+         (defined $data->{unarchived} and length($data->{unarchived}))?
+	 (unarchived => DateTime->from_epoch(epoch => $data->{unarchived})):(),
          forwarded => $data->{forwarded} // '',
          summary => $data->{summary} // '',
          outlook => $data->{outlook} // '',
@@ -153,28 +166,47 @@ sub load_bug {
          submitter => 'originator',
         );
     for my $addr_type (keys %addr_map) {
-        my @addrs = getparsedaddrs($data->{$addr_map{$addr_type}} // '');
-        next unless @addrs;
-        $bug->{$addr_type} = $s->resultset('Correspondent')->find_or_create({addr => $addrs[0]->address()});
-        # insert the full name as well
-        my $full_name = $addrs[0]->phrase();
-        $full_name =~ s/^\"|\"$//g;
-        $full_name =~ s/^\s+|\s+$//g;
-        if (length $full_name) {
-            $bug->{$addr_type}->
-                update_or_create_related('correspondent_full_names',
-                                        {full_name=>$full_name,
-                                         last_seen => 'NOW()'});
-        }
+	$bug->{$addr_type} = undef;
+	next unless defined $data->{$addr_map{$addr_type}} and
+	    length($data->{$addr_map{$addr_type}});
+        $bug->{$addr_type} =
+	    $s->resultset('Correspondent')->
+	    get_correspondent_id($data->{$addr_map{$addr_type}})
     }
     my $b = $s->resultset('Bug')->update_or_create($bug) or
         die "Unable to update or create bug $bug->{id}";
-     $s->txn_do(sub {
-		   for my $ff (qw(found fixed)) {
+    $s->txn_do(sub {
+		 $b->set_related_packages('binpackages',
+					  [grep {defined $_ and
+						   length $_ and $_ !~ /^src:/}
+					   make_list($data->{package})],
+					  $param{packages},
+					 );
+		 $b->set_related_packages('srcpackages',
+					  [grep {defined $_ and
+						   $_ =~ /^src:/}
+					   make_list($data->{package})],
+					  $param{packages},
+					 );
+		 $b->set_related_packages('affects_binpackages',
+					  [grep {defined $_ and
+						   length $_ and $_ !~ /^src:/}
+					   make_list($data->{affects})
+					  ],
+					  $param{packages},
+					 );
+		 $b->set_related_packages('affects_srcpackages',
+					  [grep {defined $_ and
+						   $_ =~ /^src:/}
+					   make_list($data->{affects})],
+					  $param{packages},
+					 );
+		 for my $ff (qw(found fixed)) {
 		       my @elements = $s->resultset('BugVer')->search({bug => $data->{bug_num},
 								       found  => $ff eq 'found'?1:0,
 								      });
-		       my %elements_to_delete = map {($elements[$_]->ver_string(),$elements[$_])} 0..$#elements;
+		       my %elements_to_delete = map {($elements[$_]->ver_string(),
+						      $elements[$_])} 0..$#elements;
 		       my %elements_to_add;
                        my @elements_to_keep;
 		       for my $version (@{$data->{"${ff}_versions"}}) {
@@ -211,11 +243,8 @@ sub load_bug {
 		       }
 		   }
 	       });
-    $s->txn_do(sub {
-		   my $t = $s->resultset('BugTag')->search({bug => $data->{bug_num}});
-                   $t->delete() if defined $t;
-		   $s->populate(BugTag => [[qw(bug tag)], map {[$data->{bug_num}, $_->id()]} values %tags]);
-	       });
+    ### set bug tags
+    $s->txn_do(sub {$b->set_tags([values %tags ] )});
     # because these bugs reference other bugs which might not exist
     # yet, we can't handle them until we've loaded all bugs. queue
     # them up.
@@ -322,18 +351,75 @@ sub load_bug_log {
         next if defined $msg_id and exists $seen_msg_ids{$msg_id};
         $seen_msg_ids{$msg_id} = 1 if defined $msg_id;
         next if defined $msg_id and $msg_id =~ /handler\..+\.ack(?:info)?\@/;
-        my $message = parse($record->{text});
+        my $entity = parse_to_mime_entity($record);
         # search for a message with this message id in the database
-        
-        # check to see if the subject, to, and from match. if so, it's
+        $msg_id = $entity->head->get('Message-Id:');
+	$msg_id =~ s/^\s*\<//;
+	$msg_id =~ s/>\s*$//;
+	# check to see if the subject, to, and from match. if so, it's
         # probably the same message.
-
-        # if not, create a new message
-
-        # add correspondents if necessary
-
+	my $subject = decode_rfc1522($entity->head->get('Subject:'));
+	$subject =~ s/\n(?:(\s)\s*|\s*$)/$1/g;
+	my $to = decode_rfc1522($entity->head->get('To:'));
+	$to =~ s/\n(?:(\s)\s*|\s*$)/$1/g;
+	my $from = decode_rfc1522($entity->head->get('From:'));
+	$from =~ s/\n(?:(\s)\s*|\s*$)/$1/g;
+	my $m = $s->resultset('Message')->
+	    find({msgid => $msg_id,
+		  from_complete => $from,
+		  to_complete => $to,
+		  subject => $subject
+		 });
+	if (not defined $m) {
+	    # if not, create a new message
+	    $m = $s->resultset('Message')->
+		find_or_create({msgid => $msg_id,
+				from_complete => $from,
+				to_complete => $to,
+				subject => $subject
+			       });
+	    eval {
+		$m->sent_date(DateTime::Format::Mail->
+			      parse_datetime($entity->head->get('Date:',0)));
+	    };
+	    my $spam = $entity->head->get('X-Spam-Status:',0);
+	    if ($spam=~ /score=([\d\.]+)/) {
+		$m->spam_score($1);
+	    }
+	    my %corr;
+	    @{$corr{from}} = getparsedaddrs($from);
+	    @{$corr{to}} = getparsedaddrs($to);
+	    @{$corr{cc}} = getparsedaddrs($entity->head->get('Cc:'));
+	    # add correspondents if necessary
+	    my @cors;
+	    for my $type (keys %corr) {
+		for my $addr (@{$corr{$type}}) {
+		    push @cors,
+			{correspondent => $s->resultset('Correspondent')->
+			 get_correspondent_id($addr),
+			 correspondent_type => $type,
+			};
+		}
+	    }
+	    $m->update();
+	    $s->txn_do(sub {
+			   $m->message_correspondents()->delete();
+			   $m->add_to_message_correspondents(@cors);
+		       }
+		      );
+	}
+	my $recv;
+	if ($entity->head->get('Received:',0)
+	    =~ /via spool by (\S+)/) {
+	    $recv = $s->resultset('Correspondent')->
+		get_correspondent_id($1);
+	    $m->add_to_message_correspondents({correspondent=>$recv,
+					       correspondent_type => 'recv'});
+	}
         # link message to bugs if necessary
-
+	$m->find_or_create_related('bug_messages',
+				  {bug=>$param{bug},
+				   message_number => $msg_num});
     }
 
 }
@@ -357,7 +443,7 @@ Commands to handle src and package version loading from debinfo files
 sub load_debinfo {
     my ($schema,$binname, $binver, $binarch, $srcname, $srcver) = @_;
     my $sp = $schema->resultset('SrcPkg')->find_or_create({pkg => $srcname});
-    my $sv = $schema->resultset('SrcVer')->find_or_create({src_pkg_id=>$sp->id(),
+    my $sv = $schema->resultset('SrcVer')->find_or_create({src_pkg=>$sp->id(),
                                                            ver => $srcver});
     my $arch = $schema->resultset('Arch')->find_or_create({arch => $binarch});
     my $bp = $schema->resultset('BinPkg')->find_or_create({pkg => $binname});
@@ -369,6 +455,240 @@ sub load_debinfo {
 }
 
 
+=back
+
+=head2 Packages
+
+=over
+
+=item load_package
+
+     load_package($schema,$suite,$component,$arch,$pkg)
+
+=cut
+
+sub load_packages {
+    my ($schema,$suite,$pkgs,$p) = @_;
+    my $suite_id = $schema->resultset('Suite')->
+	find_or_create({codename => $suite})->id;
+    my %maint_cache;
+    my %arch_cache;
+    my %source_cache;
+    my $src_max_last_modified = $schema->resultset('SrcAssociation')->
+	search_rs({suite => $suite_id},
+		 {order_by => {-desc => ['me.modified']},
+		  rows => 1,
+		  page => 1
+		 }
+		 )->single();
+    my $bin_max_last_modified = $schema->resultset('BinAssociation')->
+	search_rs({suite => $suite_id},
+		 {order_by => {-desc => ['me.modified']},
+		  rows => 1,
+		  page => 1
+		 }
+		 )->single();
+    my %maints;
+    my %sources;
+    my %bins;
+    for my $pkg_tuple (@{$pkgs}) {
+	my ($arch,$component,$pkg) = @{$pkg_tuple};
+	$maints{$pkg->{Maintainer}} = $pkg->{Maintainer};
+	if ($arch eq 'source') {
+	    my $source = $pkg->{Package};
+	    my $source_ver = $pkg->{Version};
+	    $sources{$source}{$source_ver} = $pkg->{Maintainer};
+	} else {
+	    my $source = $pkg->{Source} // $pkg->{Package};
+	    my $source_ver = $pkg->{Version};
+	    if ($source =~ /^\s*(\S+) \(([^\)]+)\)\s*$/) {
+		($source,$source_ver) = ($1,$2);
+	    }
+	    $sources{$source}{$source_ver} = $pkg->{Maintainer};
+	    $bins{$arch}{$pkg->{Package}} =
+	       {arch => $arch,
+		bin => $pkg->{Package},
+		bin_ver => $pkg->{Version},
+		src_ver => $source_ver,
+		source  => $source,
+		maint   => $pkg->{Maintainer},
+	       };
+	}
+    }
+    # Retrieve and Insert new maintainers
+    my $maints =
+	$schema->resultset('Maintainer')->
+	get_maintainers(keys %maints);
+    my $archs =
+	$schema->resultset('Arch')->
+	get_archs(keys %bins);
+    # We want all of the source package/versions which are in this suite to
+    # start with
+    my @sa_to_add;
+    my @sa_to_del;
+    my %included_sa;
+    # Calculate which source packages are no longer in this suite
+    for my $s ($schema->resultset('SrcPkg')->
+	       src_pkg_and_ver_in_suite($suite)) {
+	if (not exists $sources{$s->{pkg}} or
+	    not exists $sources{$s->{pkg}}{$s->{src_vers}{ver}}
+	   ) {
+	    push @sa_to_del,
+		$s->{src_associations}{id};
+	}
+	$included_sa{$s->{pkg}}{$s->{src_vers}} = 1;
+    }
+    # Calculate which source packages are newly in this suite
+    for my $s (keys %sources) {
+	for my $v (keys %{$sources{$s}}) {
+	    if (not exists $included_sa{$s} and
+		not $included_sa{$s}{$v}) {
+		push @sa_to_add,
+		    [$s,$v,$sources{$s}{$v}];
+	    } else {
+		$p->update() if defined $p;
+	    }
+	}
+    }
+    # add new source packages
+    my $it = natatime 100, @sa_to_add;
+    while (my @v = $it->()) {
+	$schema->txn_do(
+	    sub {
+		for my $svm (@_) {
+		    my $s_id = $schema->resultset('SrcPkg')->
+			get_src_pkg_id($svm->[0]);
+		    my $sv_id = $schema->resultset('SrcVer')->
+			get_src_ver_id($s_id,$svm->[1],$maints->{$svm->[2]});
+		    $schema->resultset('SrcAssociation')->
+			insert_suite_src_ver_association($suite_id,$sv_id);
+		}
+	    },
+			@v
+		       );
+	$p->update($p->last_update()+
+		   scalar @v) if defined $p;
+    }
+    # remove associations for packages not in this suite
+    if (@sa_to_del) {
+        $it = natatime 1000, @sa_to_del;
+        while (my @v = $it->()) {
+            $schema->
+                txn_do(sub {
+                           $schema->resultset('SrcAssociation')->
+                               search_rs({id => \@v})->
+                               delete();
+                       });
+        }
+    }
+    # update packages in this suite to have a modification time of now
+    $schema->resultset('SrcAssociation')->
+	search_rs({suite => $suite_id})->
+	update({modified => 'NOW()'});
+    ## Handle binary packages
+    my @bin_to_del;
+    my @bin_to_add;
+    my %included_bin;
+    # calculate which binary packages are no longer in this suite
+    for my $b ($schema->resultset('BinPkg')->
+	       bin_pkg_and_ver_in_suite($suite)) {
+	if (not exists $bins{$b->{arch}{arch}} or
+	    not exists $bins{$b->{arch}{arch}}{$b->{pkg}} or
+	    ($bins{$b->{arch}{arch}}{$b->{pkg}}{bin_ver} ne
+	     $b->{bin_vers}{ver}
+	    )
+	   ) {
+	    push @bin_to_del,
+		$b->{bin_associations}{id};
+	}
+	$included_bin{$b->{arch}{arch}}{$b->{pkg}} =
+	    $b->{bin_vers}{ver};
+    }
+    # calculate which binary packages are newly in this suite
+    for my $a (keys %bins) {
+	for my $pkg (keys %{$bins{$a}}) {
+	    if (not exists $included_bin{$a} or
+		not exists $included_bin{$a}{$pkg} or
+		$bins{$a}{$pkg}{bin_ver} ne
+		$included_bin{$a}{$pkg}) {
+		push @bin_to_add,
+		    $bins{$a}{$pkg};
+	    } else {
+		$p->update() if defined $p;
+	    }
+	}
+    }
+    $it = natatime 100, @bin_to_add;
+    while (my @v = $it->()) {
+	$schema->txn_do(
+	sub {
+	    for my $bvm (@_) {
+		my $s_id = $schema->resultset('SrcPkg')->
+		    get_src_pkg_id($bvm->{source});
+		my $sv_id = $schema->resultset('SrcVer')->
+		    get_src_ver_id($s_id,$bvm->{src_ver},$maints->{$bvm->{maint}});
+		my $b_id = $schema->resultset('BinPkg')->
+		    get_bin_pkg_id($bvm->{bin});
+		my $bv_id = $schema->resultset('BinVer')->
+		    get_bin_ver_id($b_id,$bvm->{bin_ver},
+				   $archs->{$bvm->{arch}},$sv_id);
+		$schema->resultset('BinAssociation')->
+		    insert_suite_bin_ver_association($suite_id,$bv_id);
+	    }
+	},
+			@v
+		       );
+	$p->update($p->last_update()+
+		   scalar @v) if defined $p;
+    }
+    if (@bin_to_del) {
+        $it = natatime 1000, @bin_to_del;
+        while (my @v = $it->()) {
+            $schema->
+                txn_do(sub {
+                           $schema->resultset('BinAssociation')->
+                               search_rs({id => \@v})->
+                               delete();
+                       });
+        }
+    }
+    $schema->resultset('BinAssociation')->
+	search_rs({suite => $suite_id})->
+	update({modified => 'NOW()'});
+
+}
+
+
+=back
+
+=cut
+
+=head2 Suites
+
+=over
+
+=item load_suite
+
+     load_suite($schema,$codename,$suite,$version,$active);
+
+=cut
+
+sub load_suite {
+    my ($schema,$codename,$suite,$version,$active) = @_;
+    if (ref($codename)) {
+	($codename,$suite,$version) =
+	    @{$codename}{qw(Codename Suite Version)};
+	$active = 1;
+    }
+    my $s = $schema->resultset('Suite')->find_or_create({codename => $codename});
+    $s->suite_name($suite);
+    $s->version($version);
+    $s->active($active);
+    $s->update();
+    return $s;
+
+}
+
 =back
 
 =cut
@@ -377,3 +697,7 @@ sub load_debinfo {
 
 
 __END__
+# Local Variables:
+# indent-tabs-mode: nil
+# cperl-indent-level: 4
+# End: