Because we now properly decode UTF8, in order to strip the BOM, we

[debbugs.git] / scripts / process

diff --git a/scripts/process b/scripts/process
index 0524c38c75de3f2039b621d5f929eeae5b7d82e9..d42a1f92b995c8a8c98a77969e62a436a595a348 100755 (executable)
--- a/scripts/process
+++ b/scripts/process
@@ -79,6 +79,7 @@ my $debugfh = IO::File->new('/dev/null','w') or
 if ($DEBUG > 0) {
     $debugfh = \*STDERR;
 }
+binmode($debugfh,':raw:encoding(UTF-8)');
 
 # these are the valid bug addresses
 my %baddress = (B => 'submit',
@@ -174,13 +175,14 @@ my %header;
 
 my @common_headers;
 for my $hdr (@headerlines) {
+    my $orig_hdr = $hdr;
     $hdr = decode_rfc1522($hdr);
     $_ = $hdr;
     s/\n\s/ /g;
     finish() if m/^x-loop: (\S+)$/i && $1 eq "$gMaintainerEmail";
     my $ins = !m/^subject:/i && !m/^reply-to:/i && !m/^return-path:/i
            && !m/^From / && !m/^X-Debbugs-/i;
-    $fwd .= $hdr."\n" if $ins;
+    $fwd .= encode_utf8($hdr)."\n" if $ins;
     # print {$debugfh} ">$_<\n";
     if (s/^(\S+):\s*//) {
        my $v = lc $1;
@@ -222,6 +224,7 @@ for my $phline (@bodylines)
     # Remove BOM markers from UTF-8 strings
     # Fixes #488554
     $phline =~ s/\xef\xbb\xbf//g;
+    $phline =~ s/\N{U+FEFF}//g;
     last if $phline !~ m/^([\w-]+):\s*(\S.*)/;
     my ($fn, $fv) = ($1, $2);
     $fv =~ s/\s*$//;
@@ -248,7 +251,9 @@ my $i = 0;
 ++$i while $msg[$i] =~ /./;
 $fwd .= join("\n",@msg[$i..$#msg]);
 
+binmode($debugfh,':raw');
 print {$debugfh} "***\n$fwd\n***\n";
+binmode($debugfh,':raw:encoding(UTF-8)');
 
 if (defined $header{'resent-from'} && !defined $header{'from'}) {
     $header{'from'} = $header{'resent-from'};
@@ -1203,7 +1208,7 @@ sub sendmessage {
     write_log_records(logfh => $logfh,
                      records => {text => stripbccs($msg),
                                  type => 'recips',
-                                 recips => [@{$recips}],
+                                 recips => [map {encode_utf8($_)} @{$recips}],
                                 },
                     );
     if (ref($bcc)) {