implement load bug logs

diff --git a/Debbugs/DB/Load.pm b/Debbugs/DB/Load.pm
index d3328ecadc165ed7d30ac975782e2e4f7ab72419..12acbcdbcc55c25d2174f4cd9a029195b393dc59 100644 (file)
--- a/Debbugs/DB/Load.pm
+++ b/Debbugs/DB/Load.pm
@@ -319,18 +319,75 @@ sub load_bug_log {
         next if defined $msg_id and exists $seen_msg_ids{$msg_id};
         $seen_msg_ids{$msg_id} = 1 if defined $msg_id;
         next if defined $msg_id and $msg_id =~ /handler\..+\.ack(?:info)?\@/;
-        my $message = parse($record->{text});
+        my $entity = parse_to_mime_entity($record);
         # search for a message with this message id in the database
-        
-        # check to see if the subject, to, and from match. if so, it's
+        $msg_id = $entity->head->get('Message-Id:');
+       $msg_id =~ s/^\s*\<//;
+       $msg_id =~ s/>\s*$//;
+       # check to see if the subject, to, and from match. if so, it's
         # probably the same message.
-
-        # if not, create a new message
-
-        # add correspondents if necessary
-
+       my $subject = decode_rfc1522($entity->head->get('Subject:'));
+       $subject =~ s/\n(?:(\s)\s*|\s*$)/$1/g;
+       my $to = decode_rfc1522($entity->head->get('To:'));
+       $to =~ s/\n(?:(\s)\s*|\s*$)/$1/g;
+       my $from = decode_rfc1522($entity->head->get('From:'));
+       $from =~ s/\n(?:(\s)\s*|\s*$)/$1/g;
+       my $m = $s->resultset('Message')->
+           find({msgid => $msg_id,
+                 from_complete => $from,
+                 to_complete => $to,
+                 subject => $subject
+                });
+       if (not defined $m) {
+           # if not, create a new message
+           $m = $s->resultset('Message')->
+               find_or_create({msgid => $msg_id,
+                               from_complete => $from,
+                               to_complete => $to,
+                               subject => $subject
+                              });
+           eval {
+               $m->sent_date(DateTime::Format::Mail->
+                             parse_datetime($entity->head->get('Date:',0)));
+           };
+           my $spam = $entity->head->get('X-Spam-Status:',0);
+           if ($spam=~ /score=([\d\.]+)/) {
+               $m->spam_score($1);
+           }
+           my %corr;
+           @{$corr{from}} = getparsedaddrs($from);
+           @{$corr{to}} = getparsedaddrs($to);
+           @{$corr{cc}} = getparsedaddrs($entity->head->get('Cc:'));
+           # add correspondents if necessary
+           my @cors;
+           for my $type (keys %corr) {
+               for my $addr (@{$corr{$type}}) {
+                   push @cors,
+                       {correspondent => $s->resultset('Correspondent')->
+                        get_correspondent_id($addr),
+                        correspondent_type => $type,
+                       };
+               }
+           }
+           $m->update();
+           $s->txn_do(sub {
+                          $m->message_correspondents()->delete();
+                          $m->add_to_message_correspondents(@cors);
+                      }
+                     );
+       }
+       my $recv;
+       if ($entity->head->get('Received:',0)
+           =~ /via spool by (\S+)/) {
+           $recv = $s->resultset('Correspondent')->
+               get_correspondent_id($1);
+           $m->add_to_message_correspondents({correspondent=>$recv,
+                                              correspondent_type => 'recv'});
+       }
         # link message to bugs if necessary
-
+       $m->find_or_create_related('bug_messages',
+                                 {bug=>$param{bug},
+                                  message_number => $msg_num});
     }
 
 }

diff --git a/bin/debbugs-loadsql b/bin/debbugs-loadsql
index 72e8c3913fc4593de335c11623a803e184656e61..59a8502d4bc14dc6d2cb261b66cbe43b78b24bcc 100755 (executable)
--- a/bin/debbugs-loadsql
+++ b/bin/debbugs-loadsql
@@ -440,25 +440,35 @@ sub add_configuration {
 
     # tags
     # add all tags
+    my %tags;
+    for my $tag (@{$config{tags}}) {
+       $tags{$tag} = 1;
+       $s->resultset('Tag')->find_or_create({tag => $tag});
+    }
     # mark obsolete tags
+    for my $tag ($s->resultset('Tag')->search_rs()->all()) {
+       next if exists $tags{$tag->tag};
+       $tag->obsolete(1);
+       $tag->update;
+    }
 
     # severities
     my %sev_names;
-    my $order = 0;
-    for my $sev_name (@{$config{severities}}) {
+    my $order = -1;
+    for my $sev_name (($config{default_severity},@{$config{severity_list}})) {
         # add all severitites
         my $sev = $s->resultset('Severity')->find_or_create({severity => $sev_name});
         # mark strong severities
         if (grep {$_ eq $sev_name} @{$config{strong_severities}}) {
             $sev->strong(1);
         }
-        $sev->order($order);
+        $sev->ordering($order);
         $sev->update();
         $order++;
         $sev_names{$sev_name} = 1;
     }
     # mark obsolete severities
-    for my $sev ($s->resultset('Severity')->find()) {
+    for my $sev ($s->resultset('Severity')->search_rs()->all()) {
         next if exists $sev_names{$sev->severity()};
         $sev->obsolete(1);
         $sev->update();
@@ -509,7 +519,16 @@ sub add_logs {
               $verbose,
               sub {
                   my $bug = shift;
-                  eval { 
+                 my $stat = stat(getbugcomponent($bug,'log',$initialdir));
+                  if (not defined $stat) {
+                      print STDERR "Unable to stat $bug $!\n";
+                      next;
+                  }
+                  if ($options{quick}) {
+                      my $rs = $s->resultset('Bug')->search({bug=>$bug})->single();
+                      next if defined $rs and $stat->mtime < $rs->last_modified()->epoch();
+                  }
+                  eval {
                       load_bug_log(db => $s,
                                    bug => $bug);
                   };