From 6ba69a1c420be91ba3489e678c2a99bab08975fa Mon Sep 17 00:00:00 2001
From: Don Armstrong <don@donarmstrong.com>
Date: Wed, 8 Mar 2017 20:43:45 -0800
Subject: [PATCH] implement load bug logs

---
 Debbugs/DB/Load.pm  | 75 +++++++++++++++++++++++++++++++++++++++------
 bin/debbugs-loadsql | 29 +++++++++++++++---
 2 files changed, 90 insertions(+), 14 deletions(-)

diff --git a/Debbugs/DB/Load.pm b/Debbugs/DB/Load.pm
index d3328ec..12acbcd 100644
--- a/Debbugs/DB/Load.pm
+++ b/Debbugs/DB/Load.pm
@@ -319,18 +319,75 @@ sub load_bug_log {
         next if defined $msg_id and exists $seen_msg_ids{$msg_id};
         $seen_msg_ids{$msg_id} = 1 if defined $msg_id;
         next if defined $msg_id and $msg_id =~ /handler\..+\.ack(?:info)?\@/;
-        my $message = parse($record->{text});
+        my $entity = parse_to_mime_entity($record);
         # search for a message with this message id in the database
-        
-        # check to see if the subject, to, and from match. if so, it's
+        $msg_id = $entity->head->get('Message-Id:');
+	$msg_id =~ s/^\s*\<//;
+	$msg_id =~ s/>\s*$//;
+	# check to see if the subject, to, and from match. if so, it's
         # probably the same message.
-
-        # if not, create a new message
-
-        # add correspondents if necessary
-
+	my $subject = decode_rfc1522($entity->head->get('Subject:'));
+	$subject =~ s/\n(?:(\s)\s*|\s*$)/$1/g;
+	my $to = decode_rfc1522($entity->head->get('To:'));
+	$to =~ s/\n(?:(\s)\s*|\s*$)/$1/g;
+	my $from = decode_rfc1522($entity->head->get('From:'));
+	$from =~ s/\n(?:(\s)\s*|\s*$)/$1/g;
+	my $m = $s->resultset('Message')->
+	    find({msgid => $msg_id,
+		  from_complete => $from,
+		  to_complete => $to,
+		  subject => $subject
+		 });
+	if (not defined $m) {
+	    # if not, create a new message
+	    $m = $s->resultset('Message')->
+		find_or_create({msgid => $msg_id,
+				from_complete => $from,
+				to_complete => $to,
+				subject => $subject
+			       });
+	    eval {
+		$m->sent_date(DateTime::Format::Mail->
+			      parse_datetime($entity->head->get('Date:',0)));
+	    };
+	    my $spam = $entity->head->get('X-Spam-Status:',0);
+	    if ($spam=~ /score=([\d\.]+)/) {
+		$m->spam_score($1);
+	    }
+	    my %corr;
+	    @{$corr{from}} = getparsedaddrs($from);
+	    @{$corr{to}} = getparsedaddrs($to);
+	    @{$corr{cc}} = getparsedaddrs($entity->head->get('Cc:'));
+	    # add correspondents if necessary
+	    my @cors;
+	    for my $type (keys %corr) {
+		for my $addr (@{$corr{$type}}) {
+		    push @cors,
+			{correspondent => $s->resultset('Correspondent')->
+			 get_correspondent_id($addr),
+			 correspondent_type => $type,
+			};
+		}
+	    }
+	    $m->update();
+	    $s->txn_do(sub {
+			   $m->message_correspondents()->delete();
+			   $m->add_to_message_correspondents(@cors);
+		       }
+		      );
+	}
+	my $recv;
+	if ($entity->head->get('Received:',0)
+	    =~ /via spool by (\S+)/) {
+	    $recv = $s->resultset('Correspondent')->
+		get_correspondent_id($1);
+	    $m->add_to_message_correspondents({correspondent=>$recv,
+					       correspondent_type => 'recv'});
+	}
         # link message to bugs if necessary
-
+	$m->find_or_create_related('bug_messages',
+				  {bug=>$param{bug},
+				   message_number => $msg_num});
     }
 
 }
diff --git a/bin/debbugs-loadsql b/bin/debbugs-loadsql
index 72e8c39..59a8502 100755
--- a/bin/debbugs-loadsql
+++ b/bin/debbugs-loadsql
@@ -440,25 +440,35 @@ sub add_configuration {
 
     # tags
     # add all tags
+    my %tags;
+    for my $tag (@{$config{tags}}) {
+	$tags{$tag} = 1;
+	$s->resultset('Tag')->find_or_create({tag => $tag});
+    }
     # mark obsolete tags
+    for my $tag ($s->resultset('Tag')->search_rs()->all()) {
+	next if exists $tags{$tag->tag};
+	$tag->obsolete(1);
+	$tag->update;
+    }
 
     # severities
     my %sev_names;
-    my $order = 0;
-    for my $sev_name (@{$config{severities}}) {
+    my $order = -1;
+    for my $sev_name (($config{default_severity},@{$config{severity_list}})) {
         # add all severitites
         my $sev = $s->resultset('Severity')->find_or_create({severity => $sev_name});
         # mark strong severities
         if (grep {$_ eq $sev_name} @{$config{strong_severities}}) {
             $sev->strong(1);
         }
-        $sev->order($order);
+        $sev->ordering($order);
         $sev->update();
         $order++;
         $sev_names{$sev_name} = 1;
     }
     # mark obsolete severities
-    for my $sev ($s->resultset('Severity')->find()) {
+    for my $sev ($s->resultset('Severity')->search_rs()->all()) {
         next if exists $sev_names{$sev->severity()};
         $sev->obsolete(1);
         $sev->update();
@@ -509,7 +519,16 @@ sub add_logs {
               $verbose,
               sub {
                   my $bug = shift;
-                  eval { 
+		  my $stat = stat(getbugcomponent($bug,'log',$initialdir));
+                  if (not defined $stat) {
+                      print STDERR "Unable to stat $bug $!\n";
+                      next;
+                  }
+                  if ($options{quick}) {
+                      my $rs = $s->resultset('Bug')->search({bug=>$bug})->single();
+                      next if defined $rs and $stat->mtime < $rs->last_modified()->epoch();
+                  }
+                  eval {
                       load_bug_log(db => $s,
                                    bug => $bug);
                   };
-- 
2.39.2