From: Don Armstrong Date: Fri, 22 Mar 2013 22:42:14 +0000 (-0700) Subject: Merge branch 'master' into database X-Git-Url: https://git.donarmstrong.com/?a=commitdiff_plain;h=3a699b3de4b043be2a611e300e21e36afcfae786;hp=7c72f7dc51eeaeb62016a03887ce8f6fdbeab2d4;p=debbugs.git Merge branch 'master' into database --- diff --git a/Debbugs/CGI/Bugreport.pm b/Debbugs/CGI/Bugreport.pm index 29602c5..d9a8744 100644 --- a/Debbugs/CGI/Bugreport.pm +++ b/Debbugs/CGI/Bugreport.pm @@ -40,7 +40,7 @@ use Debbugs::Common qw(globify_scalar english_join); use Debbugs::UTF8; use Debbugs::Config qw(:config); use POSIX qw(strftime); -use Encode qw(decode_utf8); +use Encode qw(decode_utf8 encode_utf8); BEGIN{ ($VERSION) = q$Revision: 494 $ =~ /^Revision:\s+([^\s+])/; @@ -113,10 +113,13 @@ sub display_entity { trim_headers => {type => BOOLEAN, default => 1, }, + avatars => {type => BOOLEAN, + default => 1, + }, } ); - $param{output} = globify_scalar($param{output}); + my $output = globify_scalar($param{output}); my $entity = $param{entity}; my $ref = $param{bug_num}; my $top = $param{outer}; @@ -135,25 +138,26 @@ sub display_entity { not $param{terse} and not exists $param{att}) { my $header = $entity->head; - print {$param{output}} "
\n"; + print {$output} "
\n"; if ($param{trim_headers}) { my @headers; foreach (qw(From To Cc Subject Date)) { my $head_field = $head->get($_); next unless defined $head_field and $head_field ne ''; - if ($_ eq 'From') { + chomp $head_field; + if ($_ eq 'From' and $param{avatars}) { my $libravatar_url = __libravatar_url(decode_rfc1522($head_field)); if (defined $libravatar_url and length $libravatar_url) { - push @headers,q(); + push @headers,q(\n); } } - push @headers, qq(

$_: ) . html_escape(decode_rfc1522($head_field))."

"; + push @headers, qq(
$_: ) . html_escape(decode_rfc1522($head_field))."
\n"; } - print {$param{output}} join(qq(), @headers); + print {$output} join(qq(), @headers); } else { - print {$param{output}} "
".html_escape(decode_rfc1522($entity->head->stringify))."
\n"; + print {$output} "
".html_escape(decode_rfc1522($entity->head->stringify))."
\n"; } - print {$param{output}} "
\n"; + print {$output} "
\n"; } if (not (($param{outer} and $type =~ m{^text(?:/plain)?(?:;|$)}) @@ -168,32 +172,31 @@ sub display_entity { my $body = $entity->stringify_body; # this attachment has its own content type, so we must not # try to convert it to UTF-8 or do anything funky. - my @layers = PerlIO::get_layers($param{output}); - binmode($param{output},':raw'); - print {$param{output}} "Content-Type: $type"; + binmode($output,':raw'); + print {$output} "Content-Type: $type"; my ($charset) = $head->get('Content-Type:') =~ m/charset\s*=\s*\"?([\w-]+)\"?/i; - print {$param{output}} qq(; charset="$charset") if defined $charset; - print {$param{output}} "\n"; + print {$output} qq(; charset="$charset") if defined $charset; + print {$output} "\n"; if ($filename ne '') { my $qf = $filename; $qf =~ s/"/\\"/g; $qf =~ s[.*/][]; - print {$param{output}} qq{Content-Disposition: inline; filename="$qf"\n}; + print {$output} qq{Content-Disposition: inline; filename="$qf"\n}; } - print {$param{output}} "\n"; + print {$output} "\n"; my $decoder = MIME::Decoder->new($head->mime_encoding); - $decoder->decode(IO::Scalar->new(\$body), $param{output}); - if (grep {/utf8/} @layers) { - binmode($param{output},':utf8'); - } - return; + $decoder->decode(IO::Scalar->new(\$body), $output); + # we don't reset the layers here, because it makes no + # sense to add anything to the output handle after this + # point. + return(1); } elsif (not exists $param{att}) { my @dlargs = (msg=>$xmessage, att=>$#$attachments); push @dlargs, (filename=>$filename) if $filename ne ''; my $printname = $filename; $printname = 'Message part ' . ($#$attachments + 1) if $filename eq ''; - print {$param{output}} '
[[ $part,
+                               bug_num => $ref,
+                               outer => 0,
+                               msg_num => $xmessage,
+                               output => $output,
+                               attachments => $attachments,
+                               terse => $param{terse},
+                               exists $param{msg}?(msg=>$param{msg}):(),
+                               exists $param{att}?(att=>$param{att}):(),
+                               exists $param{avatars}?(avatars=>$param{avatars}):(),
+                              );
+            if ($raw_output) {
+                return $raw_output;
+            }
+	    # print {$output} "\n";
 	}
     } elsif ($entity->parts) {
 	# We must be dealing with a nested message.
 	 if (not exists $param{att}) {
-	      print {$param{output}} "
\n";
+	      print {$output} "
\n";
 	 }
 	my @parts = $entity->parts;
 	foreach my $part (@parts) {
@@ -234,16 +242,17 @@ sub display_entity {
 			   bug_num => $ref,
 			   outer => 1,
 			   msg_num => $xmessage,
-			   output => $param{output},
+			   output => $output,
 			   attachments => $attachments,
 			   terse => $param{terse},
 			   exists $param{msg}?(msg=>$param{msg}):(),
 			   exists $param{att}?(att=>$param{att}):(),
+                           exists $param{avatars}?(avatars=>$param{avatars}):(),
 			  );
-	    # print {$param{output}} "\n";
+	    # print {$output} "\n";
 	}
 	 if (not exists $param{att}) {
-	      print {$param{output}} "
\n";
+	      print {$output} "
\n";
 	 }
     } elsif (not $param{terse}) {
 	 my $content_type = $entity->head->get('Content-Type:') || "text/html";
@@ -277,9 +286,10 @@ sub display_entity {
 		       {$1$2$3}gxm;
 	 }
 	 if (not exists $param{att}) {
-	      print {$param{output}} qq(
$body
\n);
+	      print {$output} qq(
$body
\n);
 	 }
     }
+    return 0;
 }
 
 
@@ -299,9 +309,8 @@ appropriate.
 sub handle_email_message{
      my ($email,%param) = @_;
 
-     # output needs to have the is_utf8 flag on to avoid double
-     # encoding
-     my $output = decode_utf8('');
+     my $output;
+     my $output_fh = globify_scalar(\$output);
      my $parser = MIME::Parser->new();
      # Because we are using memory, not tempfiles, there's no need to
      # clean up here like in Debbugs::MIME
@@ -309,19 +318,20 @@ sub handle_email_message{
      $parser->output_to_core(1);
      my $entity = $parser->parse_data( $email);
      my @attachments = ();
-     display_entity(entity  => $entity,
-		    bug_num => $param{ref},
-		    outer   => 1,
-		    msg_num => $param{msg_num},
-		    output => \$output,
-		    attachments => \@attachments,
-		    terse       => $param{terse},
-		    exists $param{msg}?(msg=>$param{msg}):(),
-		    exists $param{att}?(att=>$param{att}):(),
-		    exists $param{trim_headers}?(trim_headers=>$param{trim_headers}):(),
-		   );
-     return $output;
-
+     my $raw_output =
+         display_entity(entity  => $entity,
+                        bug_num => $param{ref},
+                        outer   => 1,
+                        msg_num => $param{msg_num},
+                        output => $output_fh,
+                        attachments => \@attachments,
+                        terse       => $param{terse},
+                        exists $param{msg}?(msg=>$param{msg}):(),
+                        exists $param{att}?(att=>$param{att}):(),
+                        exists $param{trim_headers}?(trim_headers=>$param{trim_headers}):(),
+                        exists $param{avatars}?(avatars=>$param{avatars}):(),
+                       );
+     return $raw_output?$output:decode_utf8($output);
 }
 
 =head2 handle_record
@@ -335,7 +345,7 @@ should be output to the browser.
 =cut
 
 sub handle_record{
-     my ($record,$bug_number,$msg_number,$seen_msg_ids) = @_;
+     my ($record,$bug_number,$msg_number,$seen_msg_ids,%param) = @_;
 
      # output needs to have the is_utf8 flag on to avoid double
      # encoding
@@ -398,6 +408,7 @@ sub handle_record{
 	  $output .= handle_email_message($record->{text},
 					  ref     => $bug_number,
 					  msg_num => $msg_number,
+                                          %param,
 					 );
      }
      elsif (/autocheck/) {
@@ -425,6 +436,7 @@ sub handle_record{
 	  $output .= handle_email_message($record->{text},
 					  ref     => $bug_number,
 					  msg_num => $msg_number,
+                                          %param,
 					 );
      }
      else {
diff --git a/Debbugs/Common.pm b/Debbugs/Common.pm
index 4595433..cf53b07 100644
--- a/Debbugs/Common.pm
+++ b/Debbugs/Common.pm
@@ -75,6 +75,7 @@ use Storable qw(dclone);
 use Params::Validate qw(validate_with :types);
 
 use Fcntl qw(:DEFAULT :flock);
+use Encode qw(is_utf8 decode_utf8);
 
 our $DEBUG_FH = \*STDERR if not defined $DEBUG_FH;
 
@@ -816,6 +817,10 @@ Will carp if given a scalar which isn't a scalarref or a glob (or
 globref), and return /dev/null. May return undef if IO::Scalar or
 IO::File fails. (Check $!)
 
+The scalar will fill with octets, not perl's internal encoding, so you
+must use decode_utf8() after on the scalar, and encode_utf8() on it
+before. This appears to be a bug in the underlying modules.
+
 =cut
 
 sub globify_scalar {
@@ -825,7 +830,11 @@ sub globify_scalar {
 	  if (defined ref($scalar)) {
 	       if (ref($scalar) eq 'SCALAR' and
 		   not UNIVERSAL::isa($scalar,'GLOB')) {
-		    open $handle, '>:scalar', $scalar;
+                   if (is_utf8(${$scalar})) {
+                       ${$scalar} = decode_utf8(${$scalar});
+                       carp(q(\$scalar must not be in perl's internal encoding));
+                   }
+		    open $handle, '>:scalar:utf8', $scalar;
 		    return $handle;
 	       }
 	       else {
diff --git a/Debbugs/Config.pm b/Debbugs/Config.pm
index 3a6e7cf..cb30eda 100644
--- a/Debbugs/Config.pm
+++ b/Debbugs/Config.pm
@@ -78,10 +78,11 @@ BEGIN {
 				],
 		     text     => [qw($gBadEmailPrefix $gHTMLTail $gHTMLExpireNote),
 				 ],
+                     cgi => [qw($gLibravatarUri $gLibravatarUriOptions)],
 		     config   => [qw(%config)],
 		    );
      @EXPORT_OK = ();
-     Exporter::export_ok_tags(qw(globals text config));
+     Exporter::export_ok_tags(keys %EXPORT_TAGS);
      $EXPORT_TAGS{all} = [@EXPORT_OK];
      $ENV{HOME} = '' if not defined $ENV{HOME};
 }
@@ -368,7 +369,7 @@ set_default(\%config,'bug_subscription_domain',$config{list_domain});
 
 =over
 
-=item libravatar_uri
+=item libravatar_uri $gLibravatarUri
 
 URI to a libravatar configuration. If empty or undefined, libravatar
 support will be disabled. Defaults to
@@ -379,7 +380,7 @@ and falls back to gravatar if necessary.
 
 set_default(\%config,'libravatar_uri',"http://cdn.libravatar.org/avatar/");
 
-=item libravatar_uri_options
+=item libravatar_uri_options $gLibravatarUriOptions
 
 Options to append to the md5_hex of the e-mail. This sets the default
 avatar used when an avatar isn't available. Currently defaults to
@@ -1088,7 +1089,7 @@ sub read_config{
 	  die "Error in configuration file: $@" if $@;
 	  # Now what we do is check out the contents of %EXPORT_TAGS to see exactly which variables
 	  # we want to glob in from the configuration file
-	  for my $variable (@{$EXPORT_TAGS{globals}}) {
+	  for my $variable (map {$_ =~ /^(?:config|all)$/ ? () : @{$EXPORT_TAGS{$_}}} keys %EXPORT_TAGS) {
 	       my ($hash_name,$glob_name,$glob_type) = __convert_name($variable);
 	       my $var_glob = $cpt->varglob($glob_name);
 	       my $value; #= $cpt->reval("return $variable");
diff --git a/Debbugs/Recipients.pm b/Debbugs/Recipients.pm
index 47c6485..ae739e9 100644
--- a/Debbugs/Recipients.pm
+++ b/Debbugs/Recipients.pm
@@ -245,7 +245,7 @@ Using the recipient hashref, determines the set of recipients.
 If you specify one of C, C, or C, you will receive only a
 LIST of recipients which the main should be Bcc'ed, Cc'ed, or To'ed
 respectively. By default, a LIST with keys bcc, cc, and to is returned
-with ARRAYREF values correponding to the users to whom a message
+with ARRAYREF values corresponding to the users to whom a message
 should be sent.
 
 =over
diff --git a/Debbugs/SOAP.pm b/Debbugs/SOAP.pm
index c1fc85f..f87581e 100644
--- a/Debbugs/SOAP.pm
+++ b/Debbugs/SOAP.pm
@@ -343,7 +343,7 @@ architectures are at which versions.
 
 =back
 
-This function correponds to L
+This function corresponds to L
 
 =cut
 
diff --git a/Debbugs/Status.pm b/Debbugs/Status.pm
index c66ab41..f44f9fc 100644
--- a/Debbugs/Status.pm
+++ b/Debbugs/Status.pm
@@ -70,6 +70,7 @@ BEGIN{
 				  qw(removefoundversions removefixedversions)
 				 ],
 		     hook     => [qw(bughook bughook_archive)],
+                     indexdb  => [qw(generate_index_db_line)],
 		     fields   => [qw(%fields)],
 		    );
      @EXPORT_OK = ();
@@ -207,6 +208,7 @@ sub read_bug{
 	 $log = $status;
 	 $log =~ s/\.summary$/.log/;
 	 ($location) = $status =~ m/(db-h|db|archive)/;
+         ($param{bug}) = $status =~ m/(\d+)\.summary$/;
     }
     if ($param{lock}) {
 	filelock("$config{spool_dir}/lock/$param{bug}",exists $param{locks}?$param{locks}:());
@@ -655,7 +657,7 @@ sub makestatus {
 
 Writes the bug status and summary files out.
 
-Skips writting out a status file if minversion is 2
+Skips writing out a status file if minversion is 2
 
 Does not call bughook if disablebughook is true.
 
@@ -773,7 +775,7 @@ exactly are removed. Otherwise, all versions matching the version
 number are removed.
 
 Currently $package and $isbinary are entirely ignored, but accepted
-for backwards compatibilty.
+for backwards compatibility.
 
 =cut
 
@@ -1599,6 +1601,39 @@ sub isstrongseverity {
     return grep { $_ eq $severity } @{$config{strong_severities}};
 }
 
+=head1 indexdb
+
+=head2 generate_index_db_line
+
+      	my $data = read_bug(bug => $bug,
+			    location => $initialdir);
+        # generate_index_db_line hasn't been written yet at all.
+        my $line = generate_index_db_line($data);
+
+Returns a line for a bug suitable to be written out to index.db.
+
+=cut
+
+sub generate_index_db_line {
+    my ($data,$bug) = @_;
+
+    # just in case someone has given us a split out data
+    $data = join_status_fields($data);
+
+    my $whendone = "open";
+    my $severity = $config{default_severity};
+    (my $pkglist = $data->{package}) =~ s/[,\s]+/,/g;
+    $pkglist =~ s/^,+//;
+    $pkglist =~ s/,+$//;
+    $whendone = "forwarded" if defined $data->{forwarded} and length $data->{forwarded};
+    $whendone = "done" if defined $data->{done} and length $data->{done};
+    $severity = $data->{severity} if length $data->{severity};
+    return sprintf "%s %d %d %s [%s] %s %s\n",
+        $pkglist, $data->{bug_num}//$bug, $data->{date}, $whendone,
+            $data->{originator}, $severity, $data->{keywords};
+}
+
+
 
 =head1 PRIVATE FUNCTIONS
 
@@ -1615,6 +1650,7 @@ sub update_realtime {
 	my $idx_new = IO::File->new($file.'.new','w')
 	     or die "Couldn't open ${file}.new: $!";
 
+        binmode($idx_old,':raw:utf8');
         binmode($idx_new,':raw:encoding(UTF-8)');
 	my $min_bug = min(keys %bugs);
 	my $line;
@@ -1680,19 +1716,7 @@ sub bughook {
 	     my $data = $bugs_temp{$bug};
 	     appendfile("$config{spool_dir}/debbugs.trace","$type $bug\n",makestatus($data, 1));
 
-	     my $whendone = "open";
-	     my $severity = $config{default_severity};
-	     (my $pkglist = $data->{package}) =~ s/[,\s]+/,/g;
-	     $pkglist =~ s/^,+//;
-	     $pkglist =~ s/,+$//;
-	     $whendone = "forwarded" if defined $data->{forwarded} and length $data->{forwarded};
-	     $whendone = "done" if defined $data->{done} and length $data->{done};
-	     $severity = $data->{severity} if length $data->{severity};
-
-	     my $k = sprintf "%s %d %d %s [%s] %s %s\n",
-		  $pkglist, $bug, $data->{date}, $whendone,
-		       $data->{originator}, $severity, $data->{keywords};
-	     $bugs{$bug} = $k;
+	     $bugs{$bug} = generate_index_db_line($data,$bug);
 	}
 	update_realtime("$config{spool_dir}/index.db.realtime", %bugs);
 
diff --git a/Debbugs/UTF8.pm b/Debbugs/UTF8.pm
index 74a4042..c90cedf 100644
--- a/Debbugs/UTF8.pm
+++ b/Debbugs/UTF8.pm
@@ -146,12 +146,13 @@ sub decode_utf8_safely{
 our %iconv_converters;
 
 sub convert_to_utf8 {
-    my ($data,$charset) = @_;
+    my ($data,$charset,$internal_call) = @_;
+    $internal_call //= 0;
     if (is_utf8($data)) {
         cluck("utf8 flag is set when calling convert_to_utf8");
         return $data;
     }
-    $charset = uc($charset);
+    $charset = uc($charset//'UTF-8');
     if ($charset eq 'RAW') {
         croak("Charset must not be raw when calling convert_to_utf8");
     }
@@ -161,6 +162,7 @@ sub convert_to_utf8 {
                 die "Unable to create converter for '$charset'";
         };
         if ($@) {
+            return undef if $internal_call;
             warn $@;
             # We weren't able to create the converter, so use Encode
             # instead
@@ -168,6 +170,7 @@ sub convert_to_utf8 {
         }
     }
     if (not defined $iconv_converters{$charset}) {
+        return undef if $internal_call;
         warn "The converter for $charset wasn't created properly somehow!";
         return __fallback_convert_to_utf8($data,$charset);
     }
@@ -178,7 +181,17 @@ sub convert_to_utf8 {
     if (not defined $retval or
         $retval < 0
        ) {
-        warn "failed to convert to utf8";
+        # try iso8559-1 first
+        if (not $internal_call) {
+            my $call_back_data = convert_to_utf8($data,'ISO8859-1',1);
+            # if there's an à (0xC3), it's probably something
+            # horrible, and we shouldn't try to convert it.
+            if (defined $call_back_data and $call_back_data !~ /\x{C3}/) {
+                warn "failed to convert to utf8 (charset: $charset, data: $data), but succeeded with ISO8859-1: ".encode_utf8($call_back_data);
+                return $call_back_data;
+            }
+        }
+        warn "failed to convert to utf8 (charset: $charset, data: $data)";
         # Fallback to encode, which will probably also fail.
         return __fallback_convert_to_utf8($data,$charset);
     }
@@ -199,7 +212,7 @@ sub __fallback_convert_to_utf8 {
      $charset //= 'utf8';
      my $result;
      eval {
-	 $result = decode($charset,$data);
+	 $result = decode($charset,$data,0);
      };
      if ($@) {
 	  warn "Unable to decode charset; '$charset' and '$data': $@";
diff --git a/bin/debbugs-rebuild-index.db b/bin/debbugs-rebuild-index.db
new file mode 100755
index 0000000..c44bc32
--- /dev/null
+++ b/bin/debbugs-rebuild-index.db
@@ -0,0 +1,157 @@
+#! /usr/bin/perl
+# debbugs-rebuild-index.db is part of debbugs, and is released
+# under the terms of the GPL version 2, or any later version, at your
+# option. See the file README and COPYING for more information.
+# Copyright 2012 by Don Armstrong .
+
+
+use warnings;
+use strict;
+
+use Getopt::Long qw(:config no_ignore_case);
+use Pod::Usage;
+
+=head1 NAME
+
+debbugs-rebuild-index.db -- rebuild Debbug's index.db
+
+=head1 SYNOPSIS
+
+debbugs-rebuild-index.db [options]
+
+ Options:
+  --spool-dir debbugs spool directory
+  --debug, -d debugging level (Default 0)
+  --help, -h display this help
+  --man, -m display manual
+
+=head1 OPTIONS
+
+=over
+
+=item B<--spool-dir>
+
+Debbugs spool directory; defaults to the value configured in the
+debbugs configuration file.
+
+=item B<--debug, -d>
+
+Debug verbosity.
+
+=item B<--help, -h>
+
+Display brief useage information.
+
+=item B<--man, -m>
+
+Display this manual.
+
+=back
+
+=head1 EXAMPLES
+
+Rebuild the index.db for db-h.
+
+ debbugs-rebuild-index.db;
+
+Rebuild the index.db for archive
+
+ debbugs-rebuild-index.db archive;
+
+
+=cut
+
+
+use vars qw($DEBUG);
+
+use Debbugs::Common qw(checkpid lockpid get_hashname getparsedaddrs getbugcomponent make_list);
+use Debbugs::Config qw(:config);
+use Debbugs::Status qw(read_bug split_status_fields generate_index_db_line);
+
+my %options = (debug           => 0,
+	       help            => 0,
+	       man             => 0,
+	       verbose         => 0,
+	       quiet           => 0,
+	       quick           => 0,
+	       service         => 'debbugs',
+	      );
+
+
+GetOptions(\%options,
+	   'quick|q',
+	   'service|s',
+	   'sysconfdir|c',
+	   'spool_dir|spool-dir=s',
+	   'debug|d+','help|h|?','man|m');
+
+pod2usage() if $options{help};
+pod2usage({verbose=>2}) if $options{man};
+
+$DEBUG = $options{debug};
+
+my @USAGE_ERRORS;
+$options{verbose} = $options{verbose} - $options{quiet};
+
+pod2usage(join("\n",@USAGE_ERRORS)) if @USAGE_ERRORS;
+
+if (exists $options{spool_dir} and defined $options{spool_dir}) {
+    $config{spool_dir} = $options{spool_dir};
+}
+chdir($config{spool_dir}) or die "chdir $config{spool_dir} failed: $!";
+
+my $verbose = $options{debug};
+
+my $initialdir = "db-h";
+
+if (defined $ARGV[0] and $ARGV[0] eq "archive") {
+    $initialdir = "archive";
+}
+
+if (not lockpid($config{spool_dir}.'/lock/debbugs-rebuild-index.db')) {
+     print STDERR "Another debbugs-rebuild-index.db is running; stopping\n";
+     exit 1;
+}
+
+my $fh_type = $initialdir;
+# if initaldir is db-h, the file is db.
+$fh_type = 'db' if $initialdir eq 'db-h';
+
+my $file = "index.${fh_type}.realtime";
+my $idx_rebuild = IO::File->new($file.'.rebuild','w')
+    or die "Couldn't open ${file}.rebuild: $!";
+
+
+my @dirs = ($initialdir);
+my $cnt = 0;
+my %bugs;
+while (my $dir = shift @dirs) {
+    printf "Doing dir %s ...\n", $dir if $verbose;
+
+    opendir(DIR, "$dir/.") or die "opendir $dir: $!";
+    my @subdirs = readdir(DIR);
+    closedir(DIR);
+
+    my @list = map { m/^(\d+)\.summary$/?($1):() } @subdirs;
+    push @dirs, map { m/^(\d+)$/ && -d "$dir/$1"?("$dir/$1"):() } @subdirs;
+
+    for my $bug (@list) {
+	print "Up to $cnt bugs...\n" if (++$cnt % 100 == 0 && $verbose);
+	my $stat = stat(getbugcomponent($bug,'summary',$initialdir));
+	if (not defined $stat) {
+	    print STDERR "Unable to stat $bug $!\n";
+	    next;
+	}
+	my $data = read_bug(bug => $bug,
+			    location => $initialdir);
+        my $line = generate_index_db_line($data);
+        $bugs{$bug} = $line;
+    }
+}
+binmode($idx_rebuild,':raw:encoding(UTF-8)');
+print {$idx_rebuild} $bugs{$_} foreach sort {$a <=> $b} keys %bugs;
+close($idx_rebuild);
+rename("$file.rebuild", $file);
+
+
+__END__
diff --git a/cgi/bugreport.cgi b/cgi/bugreport.cgi
index 70f4229..3899a9d 100755
--- a/cgi/bugreport.cgi
+++ b/cgi/bugreport.cgi
@@ -43,7 +43,7 @@ my %param = cgi_parameters(query => $q,
 			   single => [qw(bug msg att boring terse),
 				      qw(reverse mbox mime trim),
 				      qw(mboxstat mboxmaint archive),
-				      qw(repeatmerged)
+				      qw(repeatmerged avatars),
 				     ],
 			   default => {# msg       => '',
 				       boring    => 'no',
@@ -55,6 +55,7 @@ my %param = cgi_parameters(query => $q,
 				       mboxmaint => 'no',
 				       archive   => 'no',
 				       repeatmerged => 'yes',
+                                       avatars   => 'yes',
 				      },
 			  );
 # This is craptacular.
@@ -70,6 +71,7 @@ my $terse = $param{'terse'} eq 'yes';
 my $reverse = $param{'reverse'} eq 'yes';
 my $mbox = $param{'mbox'} eq 'yes';
 my $mime = $param{'mime'} eq 'yes';
+my $avatars = $param{avatars} eq 'yes';
 
 my %bugusertags;
 my %ut;
@@ -265,7 +267,11 @@ else {
 	       next;
 	  }
 	  $skip_next = 1 if $record->{type} eq 'html' and not $boring;
-	  push @log, handle_record($record,$ref,$msg_num,\%seen_msg_ids);
+	  push @log, handle_record($record,$ref,$msg_num,
+                                   \%seen_msg_ids,
+                                   trim_headers => $trim_headers,
+                                   avatars => $avatars,
+                                  );
      }
 }
 
diff --git a/debian/changelog b/debian/changelog
index 14a0d66..e2ab47d 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -17,6 +17,7 @@ debbugs (2.4.2~exp2) UNRELEASED; urgency=low
   * Fix failure to forcibly merge/merge when found/fixed is not qualified
     (closes: #670456). Thanks to Jonathan Nieder and Bernhard Schmidt.
   * Add libravatar support.
+  * Fix double encoding of attachments (closes: #703300)
 
   [Thanks to Arnout Engelen: ]
   * Add Homepage (closes: #670555).
@@ -52,9 +53,25 @@ debbugs (2.4.2~exp2) UNRELEASED; urgency=low
     a site admin aware of its existence (closes: #688345).
   * Make sure that mails to gSubscriptionDomain and gBugSubscriptionDomain
     are only sent out if the variables in config are defined and
-    have a lenght < 0 (closes: #688344).
+    have a length < 0 (closes: #688344).
   * Use locale independent date format for mail processing and service mail
     generation (closes: #688745).
+  * Fix spelling error in /debian/changelog.
+  * Fix lintian issue: description-synopsis-starts-with-article.
+  * Fix lintian issues: debian-rules-missing-recommended-target build-arch,
+    debian-rules-missing-recommended-target build-indep.
+  * Switch to debian/source/format: 3.0 (native).
+  * Fix lintian issue: debbugs source: duplicate-in-relation-field in
+    libdebbugs-perl depends: libtext-template-perl, libtext-template-perl.
+  * Fix lintian issues: debbugs source:
+    debian-rules-makemaker-prefix-is-deprecated line 44,
+    debian-rules-makemaker-prefix-is-deprecated line 56.
+  * Fix lintian issues: libdebbugs-perl: spelling-error-in-manpage for:
+    + usr/share/man/man3/Debbugs::Recipients.3pm.gz
+    + usr/share/man/man3/Debbugs::SOAP.3pm.gz
+    + usr/share/man/man3/Debbugs::Status.3pm.gz
+  * Raise Standards to 3.9.4 (after fixing several lintian issues).
+  * Pass param bug= to bugreport.html call in bugreport.cgi.
 
  -- Don Armstrong   Wed, 25 Aug 2010 01:57:38 -0700
 
diff --git a/debian/control b/debian/control
index f0f5d61..d8f19b5 100644
--- a/debian/control
+++ b/debian/control
@@ -3,7 +3,7 @@ Section: misc
 Priority: extra
 Maintainer: Debbugs developers 
 Uploaders: Colin Watson , Don Armstrong 
-Standards-Version: 3.8.2
+Standards-Version: 3.9.4
 Vcs-Browser: http://bugs.debian.org/debbugs-source/mainline
 Vcs-Git: http://bugs.debian.org/debbugs-source/debbugs.git
 Build-Depends: debhelper (>= 5)
@@ -25,7 +25,7 @@ Depends:
  libdebbugs-perl,
 Recommends: debbugs-web
 Suggests: spamassassin (>= 3.0), libcgi-alert-perl
-Description: The bug tracking system based on the active Debian BTS
+Description: bug tracking system based on the active Debian BTS
  Debian has a bug tracking system which files details of bugs reported by
  users and developers. Each bug is given a number, and is kept on file until
  it is marked as having been dealt with. The system is mainly controlled by
@@ -44,7 +44,6 @@ Depends:
  libio-stringy-perl, libmldbm-perl, liburi-perl, libsoap-lite-perl,
  libcgi-simple-perl, libparams-validate-perl, libtext-template-perl,
  libsafe-hole-perl, libmail-rfc822-address-perl, liblist-moreutils-perl,
- libtext-template-perl
 Section: perl
 Description: modules used by the active Debian BTS
  Debian has a bug tracking system which files details of bugs reported by
@@ -77,7 +76,7 @@ Depends:
  ${misc:Depends},
  libdebbugs-perl, debbugs-web, libconfig-simple-perl,
  libuser-perl, rsync, libhttp-server-simple-perl, libnet-server-perl
-Description: run and maintains a local mirror of the Debian BTS
+Description: Run and maintains a local mirror of the Debian BTS
  Debian has a bug tracking system which files details of bugs reported
  by users and developers. Each bug is given a number, and is kept on
  file until it is marked as having been dealt with. The system is
diff --git a/debian/rules b/debian/rules
index c348379..7bdd341 100755
--- a/debian/rules
+++ b/debian/rules
@@ -10,7 +10,13 @@ DEST_DIR := $(CURDIR)/debian/tmp
 PERL ?= /usr/bin/perl
 
 
-build: build-stamp
+build: build-arch build-indep
+
+build-arch:
+# nothing to do, as there aren't any architecture-dependent packages
+
+build-indep: build-stamp
+
 build-stamp:
 # Call the test suite
 	$(PERL) Makefile.PL INSTALLDIRS=vendor
@@ -35,7 +41,7 @@ install-stamp: build
 	dh_clean -k
 	dh_installdirs
 	$(MAKE) install_mostfiles DESTDIR=$(DEST_DIR)
-	$(MAKE) -f Makefile.perl install PREFIX=$(DEST_DIR)/usr
+	$(MAKE) -f Makefile.perl install DESTDIR=$(DEST_DIR)
 	touch $@
 
 binary-arch:
@@ -47,7 +53,7 @@ binary-indep: build install
 	dh_clean -k
 	dh_installdirs
 	$(MAKE) install_mostfiles DESTDIR=$(DEST_DIR)
-	$(MAKE) -f Makefile.perl install PREFIX=$(DEST_DIR)/usr
+	$(MAKE) -f Makefile.perl install DESTDIR=$(DEST_DIR)
 	dh_install --sourcedir=debian/tmp --fail-missing
 	dh_installdocs
 	dh_installchangelogs
diff --git a/debian/source/format b/debian/source/format
new file mode 100644
index 0000000..9f67427
--- /dev/null
+++ b/debian/source/format
@@ -0,0 +1 @@
+3.0 (native)
\ No newline at end of file
diff --git a/examples/debian/postpa/21bugclosers b/examples/debian/postpa/21bugclosers
index 382b779..dfcc44a 100755
--- a/examples/debian/postpa/21bugclosers
+++ b/examples/debian/postpa/21bugclosers
@@ -39,6 +39,7 @@ my %bugclosers = count_bugs(function => sub {
 
 open BUGCLOSERS, '> /org/bugs.debian.org/www/stats/bugclosers.txt.new'
     or die "can't open bugclosers.txt.new: $!";
+binmode(BUGCLOSERS,':raw:encoding(UTF-8)');
 for my $closer (sort { $bugclosers{$a} <=> $bugclosers{$b} } keys %bugclosers) {
     printf BUGCLOSERS "%4d %s\n", $bugclosers{$closer}, $closer
 	or die "can't write to bugclosers.txt.new: $!";
diff --git a/html/bugs.css b/html/bugs.css
index c251cea..9019b9d 100644
--- a/html/bugs.css
+++ b/html/bugs.css
@@ -99,7 +99,7 @@ a.submitter:hover, a.submitter:visited:hover {
 
 pre.message {
     font-family: monospace;
-    padding-top: 0;
+    padding-top: 8px;
     margin-top: 0;
     border-top: 0;
 }
@@ -133,6 +133,15 @@ div.headers {
     overflow: auto;
  }
 
+div.header {
+    font-family: sans-serif;
+    font-size: 95%;
+    color: #3c3c3c;
+    padding: 0px;
+    line-height: 120%;
+    margin: 0px;
+}
+
 div.headers p {
     font-family: sans-serif;
     font-size: 95%;
@@ -146,7 +155,7 @@ div.headers img {
     float: right;
 }
 
-span.header { 
+span.headerfield { 
   font-weight: bold
  };
 
diff --git a/scripts/spamscan-sa b/scripts/spamscan-sa
index 29a1fb6..3f6c26f 100755
--- a/scripts/spamscan-sa
+++ b/scripts/spamscan-sa
@@ -24,7 +24,8 @@ sub header_or_empty ($$) {
     my $value = $mail->get_header($hdr);
     if (defined $value) {
 	chomp $value;
-	$value =~ tr/\n/\\n/;
+        # replace newlines with '\n'
+	$value =~ s/\n/\\n/g;
 	return $value;
     }
     return '';
diff --git a/t/01_pod.t b/t/01_pod.t
index 8fb590f..2f1d2df 100644
--- a/t/01_pod.t
+++ b/t/01_pod.t
@@ -2,4 +2,6 @@
 use Test::More;
 eval "use Test::Pod 1.00";
 plan skip_all => "Test::Pod 1.00 required for testing POD" if $@;
-all_pod_files_ok(all_pod_files('.'));
+all_pod_files_ok(grep {$_ !~ /[~#]$/} all_pod_files((-e 'blib'?'blib':(qw(Debbugs Mail))),
+                                                     (qw(bin cgi scripts))
+                                                    ));
diff --git a/t/13_utf8_mail.t b/t/13_utf8_mail.t
index d5397dc..be6b6c6 100644
--- a/t/13_utf8_mail.t
+++ b/t/13_utf8_mail.t
@@ -1,7 +1,7 @@
 # -*- mode: cperl;-*-
 # $Id: 05_mail.t,v 1.1 2005/08/17 21:46:17 don Exp $
 
-use Test::More tests => 12;
+use Test::More tests => 13;
 
 use warnings;
 use strict;
@@ -24,7 +24,7 @@ use File::Basename qw(dirname basename);
 use lib qw(t/lib);
 use DebbugsTest qw(:all);
 use Data::Dumper;
-use Encode qw(decode encode decode_utf8);
+use Encode qw(decode encode decode_utf8 encode_utf8);
 
 # HTTP::Server:::Simple defines a SIG{CHLD} handler that breaks system; undef it here.
 $SIG{CHLD} = sub {};
@@ -53,7 +53,7 @@ END{
 
 send_message(to=>'submit@bugs.something',
 	     headers => [To   => 'submit@bugs.something',
-			 From => 'foöff@bugs.something',
+			 From => 'föoff@bugs.something',
 			 Subject => 'Submiting a bug',
 			],
 	     body => < [{Type=>"text/plain",Charset=>"utf-8",Data=><{subject} eq 'ütff8 title encoding test','bug 1 retitled');
 ok($status->{severity} eq 'wishlist','bug 1 wishlisted');
 ok(system('sh','-c','[ $(egrep "retitle.*encoding test" '.$spool_dir.'/db-h/01/1.log|grep -v "=C3=BCt=EF=AC=808"|wc -l) -eq 0 ]') == 0,
    'Control messages escaped properly');
-
+ok(system('sh','-c',encode_utf8('grep -q "föoff@bugs.something" '.$spool_dir.'/index.db'))==0,
+   'index.db not double escaped');
diff --git a/t/15_rebuild_indexdb.t b/t/15_rebuild_indexdb.t
new file mode 100644
index 0000000..a3036ec
--- /dev/null
+++ b/t/15_rebuild_indexdb.t
@@ -0,0 +1,128 @@
+# -*- mode: cperl;-*-
+
+use Test::More tests => 9;
+
+use warnings;
+use strict;
+
+use utf8;
+
+# Here, we're going to shoot messages through a set of things that can
+# happen.
+
+# First, we're going to send mesages to receive.
+# To do so, we'll first send a message to submit,
+# then send messages to the newly created bugnumber.
+
+use IO::File;
+use File::Temp qw(tempdir);
+use Cwd qw(getcwd);
+use Debbugs::MIME qw(create_mime_message);
+use File::Basename qw(dirname basename);
+# The test functions are placed here to make things easier
+use lib qw(t/lib);
+use DebbugsTest qw(:all);
+use Data::Dumper;
+use Encode qw(decode encode decode_utf8 encode_utf8);
+
+# HTTP::Server:::Simple defines a SIG{CHLD} handler that breaks system; undef it here.
+$SIG{CHLD} = sub {};
+my %config;
+eval {
+     %config = create_debbugs_configuration(debug => exists $ENV{DEBUG}?$ENV{DEBUG}:0);
+};
+if ($@) {
+     BAIL_OUT($@);
+}
+
+my $sendmail_dir = $config{sendmail_dir};
+my $spool_dir = $config{spool_dir};
+my $config_dir = $config{config_dir};
+
+END{
+     if ($ENV{DEBUG}) {
+	  diag("spool_dir:   $spool_dir\n");
+	  diag("config_dir:   $config_dir\n");
+	  diag("sendmail_dir: $sendmail_dir\n");
+     }
+}
+
+# We're going to use create mime message to create these messages, and
+# then just send them to receive.
+
+send_message(to=>'submit@bugs.something',
+	     headers => [To   => 'submit@bugs.something',
+			 From => 'föoff@bugs.something',
+			 Subject => 'Submiting a bug',
+			],
+	     body => < [{Type=>"text/plain",Charset=>"utf-8",Data=>< '1@bugs.something',
+	     headers => [To   => '1@bugs.something',
+			 From => 'föoff@bugs.something',
+			 Subject => 'Sending a message to a bug',
+			],
+	     body => < 'control@bugs.something',
+	     headers => [To   => 'control@bugs.something',
+			 From => 'föoff@bugs.something',
+			 Subject => 'Munging a bug',
+			],
+	     body => <
-
Send a report that this bug log contains spam.

+
Send a report that this bug log contains spam.

 

 {include(q(html/html_tail))}