Merge branch 'don/rebuild-index'

diff --git a/Debbugs/UTF8.pm b/Debbugs/UTF8.pm
index 7ab8f9cb619954a1e1fbe6f3056fdc3f99397d3a..c90cedf48109f0c6301b1e87ebdc964e28d1a170 100644 (file)
--- a/Debbugs/UTF8.pm
+++ b/Debbugs/UTF8.pm
@@ -146,7 +146,8 @@ sub decode_utf8_safely{
 our %iconv_converters;
 
 sub convert_to_utf8 {
-    my ($data,$charset) = @_;
+    my ($data,$charset,$internal_call) = @_;
+    $internal_call //= 0;
     if (is_utf8($data)) {
         cluck("utf8 flag is set when calling convert_to_utf8");
         return $data;
@@ -161,6 +162,7 @@ sub convert_to_utf8 {
                 die "Unable to create converter for '$charset'";
         };
         if ($@) {
+            return undef if $internal_call;
             warn $@;
             # We weren't able to create the converter, so use Encode
             # instead
@@ -168,6 +170,7 @@ sub convert_to_utf8 {
         }
     }
     if (not defined $iconv_converters{$charset}) {
+        return undef if $internal_call;
         warn "The converter for $charset wasn't created properly somehow!";
         return __fallback_convert_to_utf8($data,$charset);
     }
@@ -178,6 +181,16 @@ sub convert_to_utf8 {
     if (not defined $retval or
         $retval < 0
        ) {
+        # try iso8559-1 first
+        if (not $internal_call) {
+            my $call_back_data = convert_to_utf8($data,'ISO8859-1',1);
+            # if there's an à (0xC3), it's probably something
+            # horrible, and we shouldn't try to convert it.
+            if (defined $call_back_data and $call_back_data !~ /\x{C3}/) {
+                warn "failed to convert to utf8 (charset: $charset, data: $data), but succeeded with ISO8859-1: ".encode_utf8($call_back_data);
+                return $call_back_data;
+            }
+        }
         warn "failed to convert to utf8 (charset: $charset, data: $data)";
         # Fallback to encode, which will probably also fail.
         return __fallback_convert_to_utf8($data,$charset);
@@ -199,7 +212,7 @@ sub __fallback_convert_to_utf8 {
      $charset //= 'utf8';
      my $result;
      eval {
-        $result = decode($charset,$data);
+        $result = decode($charset,$data,0);
      };
      if ($@) {
          warn "Unable to decode charset; '$charset' and '$data': $@";

diff --git a/bin/debbugs-rebuild-index.db b/bin/debbugs-rebuild-index.db
index 49b925be63239f731ac4b636a6fc15492753cb0f..c44bc325cbfda479170b8c797b4daf546dad44e0 100755 (executable)
--- a/bin/debbugs-rebuild-index.db
+++ b/bin/debbugs-rebuild-index.db
@@ -34,7 +34,7 @@ debbugs-rebuild-index.db [options]
 Debbugs spool directory; defaults to the value configured in the
 debbugs configuration file.
 
-=item B<--debug, -d
+=item B<--debug, -d>
 
 Debug verbosity.
 
@@ -113,7 +113,11 @@ if (not lockpid($config{spool_dir}.'/lock/debbugs-rebuild-index.db')) {
      exit 1;
 }
 
-my $file = "index.${initialdir}.realtime";
+my $fh_type = $initialdir;
+# if initaldir is db-h, the file is db.
+$fh_type = 'db' if $initialdir eq 'db-h';
+
+my $file = "index.${fh_type}.realtime";
 my $idx_rebuild = IO::File->new($file.'.rebuild','w')
     or die "Couldn't open ${file}.rebuild: $!";
 
@@ -145,7 +149,7 @@ while (my $dir = shift @dirs) {
     }
 }
 binmode($idx_rebuild,':raw:encoding(UTF-8)');
-print {$idx_rebuild} $bugs{$_} foreach sort keys %bugs;
+print {$idx_rebuild} $bugs{$_} foreach sort {$a <=> $b} keys %bugs;
 close($idx_rebuild);
 rename("$file.rebuild", $file);