X-Git-Url: https://git.donarmstrong.com/?a=blobdiff_plain;f=scripts%2Fprocess;h=d3e9238d2acb002e896418f0cdc0844e1d02b3f9;hb=2c25f8427ad4939a1ee2926b8fe8a743b47fee8d;hp=75c487a1a05a534c5aa9d046e28709d4ab85b1e8;hpb=907fa43dbb9e516daba3ceded78dbafab29d09e5;p=debbugs.git diff --git a/scripts/process b/scripts/process index 75c487a..d3e9238 100755 --- a/scripts/process +++ b/scripts/process @@ -7,7 +7,6 @@ use warnings; use strict; -use locale; use POSIX qw(strftime locale_h); setlocale(LC_TIME, "C"); @@ -35,6 +34,7 @@ use Debbugs::Control qw(append_action_to_log); use Debbugs::Control::Service qw(valid_control control_line); use Debbugs::Recipients qw(determine_recipients); use Encode qw(encode_utf8 decode); +use List::AllUtils qw(first uniqnum); =head1 NAME @@ -79,6 +79,7 @@ my $debugfh = IO::File->new('/dev/null','w') or if ($DEBUG > 0) { $debugfh = \*STDERR; } +binmode($debugfh,':raw:encoding(UTF-8)'); # these are the valid bug addresses my %baddress = (B => 'submit', @@ -174,13 +175,16 @@ my %header; my @common_headers; for my $hdr (@headerlines) { + my $orig_hdr = $hdr; $hdr = decode_rfc1522($hdr); $_ = $hdr; s/\n\s/ /g; finish() if m/^x-loop: (\S+)$/i && $1 eq "$gMaintainerEmail"; - my $ins = !m/^subject:/i && !m/^reply-to:/i && !m/^return-path:/i - && !m/^From / && !m/^X-Debbugs-/i; - $fwd .= $hdr."\n" if $ins; + my $ins = !m/^(?:(?:subject|reply-to|return-path| + mail-followup-to| + references): + |From\s|X-Debbugs-)/xi; + $fwd .= encode_utf8($hdr)."\n" if $ins; # print {$debugfh} ">$_<\n"; if (s/^(\S+):\s*//) { my $v = lc $1; @@ -216,22 +220,32 @@ if (@bodylines and $bodylines[0] =~ /^-----BEGIN PGP SIGNED/) { #psuedoheaders my %pheader; my @control_bits; +my @usertag_bits; # extract pseudo-headers for my $phline (@bodylines) { # Remove BOM markers from UTF-8 strings # Fixes #488554 $phline =~ s/\xef\xbb\xbf//g; - last if $phline !~ m/^([\w-]+):\s*(\S.*)/; + $phline =~ s/\N{U+FEFF}//g; + last if $phline !~ m/^([\w-]+): # psuedoheader + (?:\s|\N{U+00A0})* # zero or more spaces, including + # non-breaking space + (\S.*)/x; # pseudoheader value my ($fn, $fv) = ($1, $2); $fv =~ s/\s*$//; - print {$debugfh} ">$fn|$fv|\n"; $fn = lc $fn; + # pluralize tag/usertag + $fn = $fn.'s' if $fn =~ /^(?:tag|usertag)$/; + print {$debugfh} ">$fn|$fv|\n"; if ($fn =~ /^control$/) { push @control_bits,$fv; + } elsif ($fn =~ /^(?:user|usertags)$/) { + $fv = lc $fv; + push @usertag_bits, [$fn, $fv]; } else { # Don't lc owner or forwarded - $fv = lc $fv unless $fn =~ /^(?:owner|forwarded|usertags|version|source-version)$/; + $fv = lc $fv unless $fn =~ /^(?:owner|forwarded|version|source-version|done)$/; $pheader{$fn} = $fv; } print {$debugfh} ">$fn~$fv<\n"; @@ -245,10 +259,12 @@ for my $key (grep /X-Debbugs-.*/i, keys %pheader) { # set $i to beginning of encoded body data, so we can dump it out # verbatim later my $i = 0; -++$i while $msg[$i] =~ /./; +++$i while $i <= $#msg and $msg[$i] =~ /./; $fwd .= join("\n",@msg[$i..$#msg]); +binmode($debugfh,':raw'); print {$debugfh} "***\n$fwd\n***\n"; +binmode($debugfh,':raw:encoding(UTF-8)'); if (defined $header{'resent-from'} && !defined $header{'from'}) { $header{'from'} = $header{'resent-from'}; @@ -275,8 +291,18 @@ if (!defined($header{'subject'})) my $ref=-1; # remove Re: from the subject line $subject =~ s/^Re:\s*//i; -# remove remaining mailing list name markers from the subject line -$subject =~ s/^\[.*\]\s*//i; +# remove remaining mailing list name markers from the subject line if +# this appears to be a message that has traversed a mailing list +if (exists $header{'list-id'} or exists $header{'list-subscribe'} or + (exists $header{'precedence'} and defined $header{'precedence'} and + $header{'precedence'} eq 'bulk') or + exists $header{'mailing-list'} or exists $header{'list-processor-version'} + ){ + # if a mailing list didn't match any of the above, it's probably + # so horribly configured that we wouldn't be able to figure it out + # anyway. + $subject =~ s/^\[.*\]\s*//i; +} $_= $subject."\n"; if (not defined $tryref and m/^Bug ?\#(\d+)\D/i) { $tryref = $1 if $1 > 0; @@ -291,7 +317,7 @@ if (defined $tryref) { $ref= $tryref; } else { &sendmessage(create_mime_message( - [From => "$gMaintainerEmail ($gProject $gBug Tracking System)", + [From => qq("$gProject $gBug Tracking System" <$gMaintainerEmail>), To => $replyto, Subject => "Unknown problem report $gBug#$tryref ($subject)", 'Message-ID' => "", @@ -368,7 +394,7 @@ if ($codeletter eq 'D' || $codeletter eq 'F') } $receivedat= "done\@$gEmailDomain"; $markaswhat= 'done'; - $set_done= $header{'from'}; + $set_done= $pheader{'done'} // $header{'from'}; if ( length( $gListDomain ) > 0 && length( $gDoneList ) > 0 ) { $generalcc= "$gDoneList\@$gListDomain"; push @generalcc, "$gDoneList\@$gListDomain"; @@ -382,7 +408,7 @@ if ($codeletter eq 'D' || $codeletter eq 'F') } if ($ref<0) { &sendmessage(create_mime_message( - [From => "$gMaintainerEmail ($gProject $gBug Tracking System)", + [From => qq("$gProject $gBug Tracking System" <$gMaintainerEmail>), To => $replyto, Subject => "Message with no $gBug number ignored by $receivedat ($subject)", 'Message-ID' => "", @@ -450,15 +476,13 @@ if ($codeletter eq 'D' || $codeletter eq 'F') # Add bug mailing list to $generalbcc as appropriate # This array is used to specify bcc in the cases where we're using create_mime_message. my @generalbcc = @generalcc; - my $generalbcc = $generalcc; if (defined $config{subscription_domain} and length $config{subscription_domain}) { - my @generalbcc = (@generalbcc, @addsrcaddrs); - my $generalbcc = join(', ', $generalbcc, @addsrcaddrs); + @generalbcc = (@generalbcc, @addsrcaddrs); } if (defined $config{bug_subscription_domain} and length $config{bug_subscription_domain}) { - my @generalbcc = (@generalbcc, "bugs=$ref\@$config{bug_subscription_domain}"); - my $generalbcc = join(', ', $generalbcc, "bugs=$ref\@$config{bug_subscription_domain}"); + @generalbcc = (@generalbcc, "bugs=$ref\@$config{bug_subscription_domain}"); } + my $generalbcc = join(', ', @generalbcc); $generalbcc =~ s/\s+\n\s+/ /g; $generalbcc =~ s/^\s+/ /; $generalbcc =~ s/\s+$//; if (length $generalbcc) {$generalbcc = "Bcc: $generalbcc\n"}; @@ -476,7 +500,7 @@ if ($codeletter eq 'D' || $codeletter eq 'F') &htmllog("Reply","sent",$replyto,"You have marked $gBug as forwarded."); &sendmessage(create_mime_message( [@common_headers, - From => "$gMaintainerEmail ($gProject $gBug Tracking System)", + From => qq("$gProject $gBug Tracking System" <$gMaintainerEmail>), To => "$replyto", Subject => "$gBug#$ref: marked as forwarded ($data->{subject})", "Message-ID" => "", @@ -488,6 +512,8 @@ if ($codeletter eq 'D' || $codeletter eq 'F') "X-$gProject-PR-Keywords" => $data->{keywords}, # Only have a X-$gProject-PR-Source when we know the source package (defined($source_package) and length($source_package))?("X-$gProject-PR-Source" => $source_package):(), + "Reply-To" => "$ref\@$gEmailDomain", + "Content-Type" => 'text/plain; charset="utf-8"', ],message_body_template('mail/process_mark_as_forwarded', {date => $header{date}, messageid => $header{'message-id'}, @@ -499,7 +525,7 @@ if ($codeletter eq 'D' || $codeletter eq 'F') &htmllog("Reply","sent",$replyto,"You have taken responsibility."); &sendmessage(create_mime_message( [@common_headers, - From => "$gMaintainerEmail ($gProject $gBug Tracking System)", + From => qq("$gProject $gBug Tracking System" <$gMaintainerEmail>), To => $replyto, Subject => "$gBug#$ref: marked as done ($data->{subject})", "Message-ID" => "", @@ -511,6 +537,8 @@ if ($codeletter eq 'D' || $codeletter eq 'F') "X-$gProject-PR-Keywords" => $data->{keywords}, # Only have a X-$gProject-PR-Source when we know the source package (defined($source_package) and length($source_package))?("X-$gProject-PR-Source" => $source_package):(), + "Reply-To" => "$ref\@$gEmailDomain", + "Content-Type" => 'text/plain; charset="utf-8"', ],message_body_template('mail/process_mark_as_done', {date => $header{date}, messageid => $header{'message-id'}, @@ -523,15 +551,15 @@ if ($codeletter eq 'D' || $codeletter eq 'F') "$gBug acknowledged by developer."); &sendmessage(create_mime_message( [@common_headers, - From => "$gMaintainerEmail ($gProject $gBug Tracking System)", + From => qq("$gProject $gBug Tracking System" <$gMaintainerEmail>), To => "$data->{originator}", Subject => "$gBug#$ref closed by $markedby ($header{'subject'})", "Message-ID" => "", (defined $data->{msgid})?("In-Reply-To" => $data->{msgid}):(), - References => join(' ',grep {defined $_} ($header{'message-id'},$data->{msgid})), + References => join(' ',grep {defined $_} ($header{'message-id'},$data->{msgid},'')), "X-$gProject-PR-Message" => "they-closed $ref", - "X-$gProject-PR-Package" => "$data->{package}", - "X-$gProject-PR-Keywords" => "$data->{keywords}", + (defined $data->{package})?("X-$gProject-PR-Package" => $data->{package}):(), + (defined $data->{keywords})?("X-$gProject-PR-Keywords" => $data->{keywords}):(), # Only have a X-$gProject-PR-Source when we know the source package (defined($source_package) and length($source_package))?("X-$gProject-PR-Source" => $source_package):(), "Reply-To" => "$ref\@$gEmailDomain", @@ -554,7 +582,7 @@ if ($ref<0) { # new bug report if ($codeletter eq 'U') { # -submitter &sendmessage(create_mime_message( [@common_headers, - From => "$gMaintainerEmail ($gProject $gBug Tracking System)", + From => qq("$gProject $gBug Tracking System" <$gMaintainerEmail>), To => $replyto, Subject => "Message with no $gBug number cannot be sent to submitter! ($subject)", 'Message-ID' => "", @@ -579,7 +607,8 @@ if ($ref<0) { # new bug report if (defined $pheader{source}) { # source packages are identified by the src: prefix - $data->{package} = 'src:'.$pheader{source}; + $data->{package} = $pheader{source}; + $data->{package} =~ s/(^|,\s*)/${1}src:/g; } elsif (defined $pheader{package}) { $data->{package} = $pheader{package}; if ($data->{package} =~ /^src:(.+)/) { @@ -593,7 +622,7 @@ if ($ref<0) { # new bug report ); &sendmessage(create_mime_message( [@common_headers, - From => "$gMaintainerEmail ($gProject $gBug Tracking System)", + From => qq("$gProject $gBug Tracking System" <$gMaintainerEmail>), To => $replyto, Subject => "Message with no Package: tag cannot be processed! ($subject)", "Message-ID" => "", @@ -671,32 +700,37 @@ if ($ref<0) { # new bug report $data->{msgid} = $header{'message-id'}; writebug($ref, $data); # Deal with usertags - if (exists $pheader{usertags}) { - my $user = $replyto; - $user = $pheader{user} if exists $pheader{user}; - $user =~ s/,.*//; - $user =~ s/^.*<(.*)>.*$/$1/; - $user =~ s/[(].*[)]//; - $user =~ s/^\s*(\S+)\s+.*$/$1/; - if ($user ne '' and Debbugs::User::is_valid_user($user)) { - $pheader{usertags} =~ s/(?:^\s+|\s+$)//g; - my %user_tags; - read_usertags(\%user_tags,$user); - for my $tag (split /[,\s]+/, $pheader{usertags}) { - if ($tag =~ /^[a-zA-Z0-9.+\@-]+/) { - my %bugs_with_tag; - @bugs_with_tag{@{$user_tags{$tag}||[]}} = (1) x @{$user_tags{$tag}||[]}; - $bugs_with_tag{$ref} = 1; - $user_tags{$tag} = [keys %bugs_with_tag]; - } - } - write_usertags(\%user_tags,$user); - } - else { - $brokenness .= fill_template('mail/invalid_user', - {user => $user} - ); - } + my $current_user = $replyto; + for my $field (@usertag_bits) { + my ($name, $value) = @$field; + if ($name eq 'user') { + my $user = $value; + $user =~ s/,.*//; + $user =~ s/^.*<(.*)>.*$/$1/; + $user =~ s/[(].*[)]//; + $user =~ s/^\s*(\S+)\s+.*$/$1/; + if ($user ne '' and Debbugs::User::is_valid_user($user)) { + $current_user = $user; + } else { + $brokenness .= fill_template('mail/invalid_user', + {user => $user} + ); + } + } + if ($name eq 'usertags'){ + my %user_tags; + read_usertags(\%user_tags, $current_user); + $value =~ s/(?:^\s+|\s+$)//g; + for my $tag (split /[,\s]+/, $value) { + if ($tag =~ /^[a-zA-Z0-9.+\@-]+/) { + my %bugs_with_tag; + @bugs_with_tag{@{$user_tags{$tag}||[]}} = (1) x @{$user_tags{$tag}||[]}; + $bugs_with_tag{$ref} = 1; + $user_tags{$tag} = [keys %bugs_with_tag]; + } + } + write_usertags(\%user_tags,$current_user); + } } overwritefile("db-h/$hash/$ref.report", map {"$_\n"} @msg); @@ -789,6 +823,12 @@ if (length($resentccval)) { $resentcc= "Resent-CC: $resentccval\n"; } +my $referencesval = join(' ',grep {defined $_} $header{'references'},$data->{msgid}); +my $references = ''; +if (!$newref && length($referencesval)) { + $references = "References: $referencesval\n"; +} + my $common_headers=''; { my @tmp = @common_headers; @@ -808,7 +848,7 @@ Resent-Sender: $gMaintainerEmail X-$gProject-PR-Message: report $ref X-$gProject-PR-Package: $data->{package} X-$gProject-PR-Keywords: $data->{keywords} -${source_pr_header} +${references}${source_pr_header} END chomp $enc_msg; $enc_msg = encode_utf8($enc_msg).$fwd."\n"; @@ -831,7 +871,7 @@ Resent-Sender: $gMaintainerEmail X-$gProject-PR-Message: $report_followup $ref X-$gProject-PR-Package: $data->{package} X-$gProject-PR-Keywords: $data->{keywords} -${source_pr_header} +${references}${source_pr_header} END chomp $enc_msg; $enc_msg = encode_utf8($enc_msg).$fwd."\n"; @@ -865,7 +905,7 @@ Resent-Sender: $gMaintainerEmail ${common_headers}X-$gProject-PR-Message: $report_followup $ref X-$gProject-PR-Package: $data->{package} X-$gProject-PR-Keywords: $data->{keywords} -${source_pr_header} +${references}${source_pr_header} END chomp $enc_msg; $enc_msg = encode_utf8($enc_msg).$fwd."\n"; @@ -959,7 +999,7 @@ if (not exists $header{'x-debbugs-no-ack'} and ); &sendmessage(create_mime_message( [@common_headers, - From => "$gMaintainerEmail ($gProject $gBug Tracking System)", + From => qq("$gProject $gBug Tracking System" <$gMaintainerEmail>), To => $replyto, Subject => $t_h{subject}, "Message-ID" => $t_h{messageid}, @@ -1002,7 +1042,7 @@ if (@control_bits) { request_subject => $header{subject}, request_nn => $nn, request_replyto => $replyto, - message => $msg, + message => [$msg], affected_bugs => \%bug_affected, affected_packages => \%affected_packages, recipients => \%recipients, @@ -1040,6 +1080,7 @@ if (@control_bits) { } } else { + print {$transcript} "Unknown command or malformed arguments to command.\n\n"; $errors++; if (++$unknowns >= 5) { print {$transcript} "Too many unknown commands, stopping here.\n\n"; @@ -1056,10 +1097,10 @@ if (@control_bits) { address_only => 1, cc => 1, ); - my $error_text = $errors > 0 ? " (with $errors errors)":''; + my $error_text = $errors > 0 ? " (with $errors error" . ($errors > 1 ? "s" : "") . ")" : ""; my $reply = create_mime_message(['X-Loop' => $gMaintainerEmail, - From => "$gMaintainerEmail ($gProject $gBug Tracking System)", + From => qq("$gProject $gBug Tracking System" <$gMaintainerEmail>), To => $replyto, @maintccs ? (Cc => join(', ',@maintccs)):(), Subject => "Processed${error_text}: $header{subject}", @@ -1194,7 +1235,7 @@ sub sendmessage { write_log_records(logfh => $logfh, records => {text => stripbccs($msg), type => 'recips', - recips => [@{$recips}], + recips => [map {encode_utf8($_)} @{$recips}], }, ); if (ref($bcc)) { @@ -1242,20 +1283,15 @@ sub fill_template{ my $variables = {config => \%config, defined($ref)?(ref => $ref):(), defined($data)?(data => $data):(), - refs => [map {exists $clonebugs{$_}?$clonebugs{$_}:$_} keys %bug_affected], + refs => [sort + uniqnum(defined($ref)?($ref):(), + map {exists $clonebugs{$_}?$clonebugs{$_}:$_} + keys %bug_affected)], %{$extra_var}, }; - my $hole_var = {'&bugurl' => - sub{"$_[0]: ". - 'http://'.$config{cgi_domain}.'/'. - Debbugs::CGI::bug_links(bug=>$_[0], - links_only => 1, - ); - } - }; return fill_in_template(template => $template, variables => $variables, - hole_var => $hole_var, + output_type => 'text', ); }