X-Git-Url: https://git.donarmstrong.com/?a=blobdiff_plain;f=scripts%2Fprocess;h=4c38000121c02e388828aee0374672c5d0abe7a5;hb=HEAD;hp=e313a9d48c298078f79c281837e6c84d1a3754cf;hpb=b7843cf92e2fa55b89e0993eb82c4453d3736d46;p=debbugs.git diff --git a/scripts/process b/scripts/process index e313a9d..8310998 100755 --- a/scripts/process +++ b/scripts/process @@ -7,7 +7,8 @@ use warnings; use strict; -use POSIX qw(strftime); +use POSIX qw(strftime locale_h); +setlocale(LC_TIME, "C"); use IO::File; @@ -29,10 +30,11 @@ use Debbugs::Text qw(:templates); use Debbugs::Config qw(:globals :config); -use Debbugs::Control qw(append_action_to_log); +use Debbugs::Control qw(append_action_to_log valid_usertag); use Debbugs::Control::Service qw(valid_control control_line); use Debbugs::Recipients qw(determine_recipients); use Encode qw(encode_utf8 decode); +use List::AllUtils qw(first uniqnum); =head1 NAME @@ -77,6 +79,7 @@ my $debugfh = IO::File->new('/dev/null','w') or if ($DEBUG > 0) { $debugfh = \*STDERR; } +binmode($debugfh,':raw:encoding(UTF-8)'); # these are the valid bug addresses my %baddress = (B => 'submit', @@ -172,13 +175,16 @@ my %header; my @common_headers; for my $hdr (@headerlines) { + my $orig_hdr = $hdr; $hdr = decode_rfc1522($hdr); $_ = $hdr; s/\n\s/ /g; finish() if m/^x-loop: (\S+)$/i && $1 eq "$gMaintainerEmail"; - my $ins = !m/^subject:/i && !m/^reply-to:/i && !m/^return-path:/i - && !m/^From / && !m/^X-Debbugs-/i; - $fwd .= $hdr."\n" if $ins; + my $ins = !m/^(?:(?:subject|reply-to|return-path| + mail-followup-to| + references): + |From\s|X-Debbugs-)/xi; + $fwd .= $orig_hdr."\n" if $ins; # print {$debugfh} ">$_<\n"; if (s/^(\S+):\s*//) { my $v = lc $1; @@ -186,7 +192,13 @@ for my $hdr (@headerlines) { push @common_headers, 'X-Loop',$_; } print {$debugfh} ">$v=$_<\n"; - $header{$v} = $_; + # Handle a comma which is escaped being passed through un-escaped. See + # https://bugs.debian.org/1041638 + if ($_ =~ m/,/ and not $orig_hdr =~ m/,/) { + $header{$v} = handle_escaped_commas($_,$orig_hdr); + } else { + $header{$v} = $_; + } } else { print {$debugfh} "!>$_<\n"; } @@ -214,22 +226,32 @@ if (@bodylines and $bodylines[0] =~ /^-----BEGIN PGP SIGNED/) { #psuedoheaders my %pheader; my @control_bits; +my @usertag_bits; # extract pseudo-headers for my $phline (@bodylines) { # Remove BOM markers from UTF-8 strings # Fixes #488554 $phline =~ s/\xef\xbb\xbf//g; - last if $phline !~ m/^([\w-]+):\s*(\S.*)/; + $phline =~ s/\N{U+FEFF}//g; + last if $phline !~ m/^([\w-]+): # psuedoheader + (?:\s|\N{U+00A0})* # zero or more spaces, including + # non-breaking space + (\S.*)/x; # pseudoheader value my ($fn, $fv) = ($1, $2); $fv =~ s/\s*$//; - print {$debugfh} ">$fn|$fv|\n"; $fn = lc $fn; + # pluralize tag/usertag + $fn = $fn.'s' if $fn =~ /^(?:tag|usertag)$/; + print {$debugfh} ">$fn|$fv|\n"; if ($fn =~ /^control$/) { push @control_bits,$fv; + } elsif ($fn =~ /^(?:user|usertags)$/) { + $fv = lc $fv; + push @usertag_bits, [$fn, $fv]; } else { # Don't lc owner or forwarded - $fv = lc $fv unless $fn =~ /^(?:owner|forwarded|usertags|version|source-version)$/; + $fv = lc $fv unless $fn =~ /^(?:owner|forwarded|version|source-version|done)$/; $pheader{$fn} = $fv; } print {$debugfh} ">$fn~$fv<\n"; @@ -243,10 +265,12 @@ for my $key (grep /X-Debbugs-.*/i, keys %pheader) { # set $i to beginning of encoded body data, so we can dump it out # verbatim later my $i = 0; -++$i while $msg[$i] =~ /./; +++$i while $i <= $#msg and $msg[$i] =~ /./; $fwd .= join("\n",@msg[$i..$#msg]); +binmode($debugfh,':raw'); print {$debugfh} "***\n$fwd\n***\n"; +binmode($debugfh,':raw:encoding(UTF-8)'); if (defined $header{'resent-from'} && !defined $header{'from'}) { $header{'from'} = $header{'resent-from'}; @@ -273,8 +297,18 @@ if (!defined($header{'subject'})) my $ref=-1; # remove Re: from the subject line $subject =~ s/^Re:\s*//i; -# remove remaining mailing list name markers from the subject line -$subject =~ s/^\[.*\]\s*//i; +# remove remaining mailing list name markers from the subject line if +# this appears to be a message that has traversed a mailing list +if (exists $header{'list-id'} or exists $header{'list-subscribe'} or + (exists $header{'precedence'} and defined $header{'precedence'} and + $header{'precedence'} eq 'bulk') or + exists $header{'mailing-list'} or exists $header{'list-processor-version'} + ){ + # if a mailing list didn't match any of the above, it's probably + # so horribly configured that we wouldn't be able to figure it out + # anyway. + $subject =~ s/^\[.*\]\s*//i; +} $_= $subject."\n"; if (not defined $tryref and m/^Bug ?\#(\d+)\D/i) { $tryref = $1 if $1 > 0; @@ -289,7 +323,7 @@ if (defined $tryref) { $ref= $tryref; } else { &sendmessage(create_mime_message( - [From => "$gMaintainerEmail ($gProject $gBug Tracking System)", + [From => qq("$gProject $gBug Tracking System" <$gMaintainerEmail>), To => $replyto, Subject => "Unknown problem report $gBug#$tryref ($subject)", 'Message-ID' => "", @@ -366,7 +400,7 @@ if ($codeletter eq 'D' || $codeletter eq 'F') } $receivedat= "done\@$gEmailDomain"; $markaswhat= 'done'; - $set_done= $header{'from'}; + $set_done= $pheader{'done'} // $header{'from'}; if ( length( $gListDomain ) > 0 && length( $gDoneList ) > 0 ) { $generalcc= "$gDoneList\@$gListDomain"; push @generalcc, "$gDoneList\@$gListDomain"; @@ -380,7 +414,7 @@ if ($codeletter eq 'D' || $codeletter eq 'F') } if ($ref<0) { &sendmessage(create_mime_message( - [From => "$gMaintainerEmail ($gProject $gBug Tracking System)", + [From => qq("$gProject $gBug Tracking System" <$gMaintainerEmail>), To => $replyto, Subject => "Message with no $gBug number ignored by $receivedat ($subject)", 'Message-ID' => "", @@ -447,8 +481,14 @@ if ($codeletter eq 'D' || $codeletter eq 'F') # Add bug mailing list to $generalbcc as appropriate # This array is used to specify bcc in the cases where we're using create_mime_message. - my @generalbcc = (@generalcc,@addsrcaddrs,"bugs=$ref\@$gListDomain"); - my $generalbcc = join(', ', $generalcc, @addsrcaddrs,"bugs=$ref\@$gListDomain"); + my @generalbcc = @generalcc; + if (defined $config{subscription_domain} and length $config{subscription_domain}) { + @generalbcc = (@generalbcc, @addsrcaddrs); + } + if (defined $config{bug_subscription_domain} and length $config{bug_subscription_domain}) { + @generalbcc = (@generalbcc, "bugs=$ref\@$config{bug_subscription_domain}"); + } + my $generalbcc = join(', ', @generalbcc); $generalbcc =~ s/\s+\n\s+/ /g; $generalbcc =~ s/^\s+/ /; $generalbcc =~ s/\s+$//; if (length $generalbcc) {$generalbcc = "Bcc: $generalbcc\n"}; @@ -466,7 +506,7 @@ if ($codeletter eq 'D' || $codeletter eq 'F') &htmllog("Reply","sent",$replyto,"You have marked $gBug as forwarded."); &sendmessage(create_mime_message( [@common_headers, - From => "$gMaintainerEmail ($gProject $gBug Tracking System)", + From => qq("$gProject $gBug Tracking System" <$gMaintainerEmail>), To => "$replyto", Subject => "$gBug#$ref: marked as forwarded ($data->{subject})", "Message-ID" => "", @@ -478,6 +518,8 @@ if ($codeletter eq 'D' || $codeletter eq 'F') "X-$gProject-PR-Keywords" => $data->{keywords}, # Only have a X-$gProject-PR-Source when we know the source package (defined($source_package) and length($source_package))?("X-$gProject-PR-Source" => $source_package):(), + "Reply-To" => "$ref\@$gEmailDomain", + "Content-Type" => 'text/plain; charset="utf-8"', ],message_body_template('mail/process_mark_as_forwarded', {date => $header{date}, messageid => $header{'message-id'}, @@ -489,7 +531,7 @@ if ($codeletter eq 'D' || $codeletter eq 'F') &htmllog("Reply","sent",$replyto,"You have taken responsibility."); &sendmessage(create_mime_message( [@common_headers, - From => "$gMaintainerEmail ($gProject $gBug Tracking System)", + From => qq("$gProject $gBug Tracking System" <$gMaintainerEmail>), To => $replyto, Subject => "$gBug#$ref: marked as done ($data->{subject})", "Message-ID" => "", @@ -501,6 +543,8 @@ if ($codeletter eq 'D' || $codeletter eq 'F') "X-$gProject-PR-Keywords" => $data->{keywords}, # Only have a X-$gProject-PR-Source when we know the source package (defined($source_package) and length($source_package))?("X-$gProject-PR-Source" => $source_package):(), + "Reply-To" => "$ref\@$gEmailDomain", + "Content-Type" => 'text/plain; charset="utf-8"', ],message_body_template('mail/process_mark_as_done', {date => $header{date}, messageid => $header{'message-id'}, @@ -513,15 +557,15 @@ if ($codeletter eq 'D' || $codeletter eq 'F') "$gBug acknowledged by developer."); &sendmessage(create_mime_message( [@common_headers, - From => "$gMaintainerEmail ($gProject $gBug Tracking System)", + From => qq("$gProject $gBug Tracking System" <$gMaintainerEmail>), To => "$data->{originator}", Subject => "$gBug#$ref closed by $markedby ($header{'subject'})", "Message-ID" => "", (defined $data->{msgid})?("In-Reply-To" => $data->{msgid}):(), - References => join(' ',grep {defined $_} ($header{'message-id'},$data->{msgid})), + References => join(' ',grep {defined $_} ($header{'message-id'},$data->{msgid},'')), "X-$gProject-PR-Message" => "they-closed $ref", - "X-$gProject-PR-Package" => "$data->{package}", - "X-$gProject-PR-Keywords" => "$data->{keywords}", + (defined $data->{package})?("X-$gProject-PR-Package" => $data->{package}):(), + (defined $data->{keywords})?("X-$gProject-PR-Keywords" => $data->{keywords}):(), # Only have a X-$gProject-PR-Source when we know the source package (defined($source_package) and length($source_package))?("X-$gProject-PR-Source" => $source_package):(), "Reply-To" => "$ref\@$gEmailDomain", @@ -544,7 +588,7 @@ if ($ref<0) { # new bug report if ($codeletter eq 'U') { # -submitter &sendmessage(create_mime_message( [@common_headers, - From => "$gMaintainerEmail ($gProject $gBug Tracking System)", + From => qq("$gProject $gBug Tracking System" <$gMaintainerEmail>), To => $replyto, Subject => "Message with no $gBug number cannot be sent to submitter! ($subject)", 'Message-ID' => "", @@ -569,7 +613,8 @@ if ($ref<0) { # new bug report if (defined $pheader{source}) { # source packages are identified by the src: prefix - $data->{package} = 'src:'.$pheader{source}; + $data->{package} = $pheader{source}; + $data->{package} =~ s/(^|,\s*)/${1}src:/g; } elsif (defined $pheader{package}) { $data->{package} = $pheader{package}; if ($data->{package} =~ /^src:(.+)/) { @@ -583,7 +628,7 @@ if ($ref<0) { # new bug report ); &sendmessage(create_mime_message( [@common_headers, - From => "$gMaintainerEmail ($gProject $gBug Tracking System)", + From => qq("$gProject $gBug Tracking System" <$gMaintainerEmail>), To => $replyto, Subject => "Message with no Package: tag cannot be processed! ($subject)", "Message-ID" => "", @@ -661,32 +706,38 @@ if ($ref<0) { # new bug report $data->{msgid} = $header{'message-id'}; writebug($ref, $data); # Deal with usertags - if (exists $pheader{usertags}) { - my $user = $replyto; - $user = $pheader{user} if exists $pheader{user}; - $user =~ s/,.*//; - $user =~ s/^.*<(.*)>.*$/$1/; - $user =~ s/[(].*[)]//; - $user =~ s/^\s*(\S+)\s+.*$/$1/; - if ($user ne '' and Debbugs::User::is_valid_user($user)) { - $pheader{usertags} =~ s/(?:^\s+|\s+$)//g; - my %user_tags; - read_usertags(\%user_tags,$user); - for my $tag (split /[,\s]+/, $pheader{usertags}) { - if ($tag =~ /^[a-zA-Z0-9.+\@-]+/) { - my %bugs_with_tag; - @bugs_with_tag{@{$user_tags{$tag}||[]}} = (1) x @{$user_tags{$tag}||[]}; - $bugs_with_tag{$ref} = 1; - $user_tags{$tag} = [keys %bugs_with_tag]; - } - } - write_usertags(\%user_tags,$user); - } - else { - $brokenness .= fill_template('mail/invalid_user', - {user => $user} - ); - } + my $current_user; + unshift @usertag_bits, ['user', $replyto]; + for my $field (@usertag_bits) { + my ($name, $value) = @$field; + if ($name eq 'user') { + my $user = $value; + $user =~ s/,.*//; + $user =~ s/^.*<(.*)>.*$/$1/; + $user =~ s/[(].*[)]//; + $user =~ s/^\s*(\S+)\s+.*$/$1/; + if ($user ne '' and Debbugs::User::is_valid_user($user)) { + $current_user = $user; + } else { + $brokenness .= fill_template('mail/invalid_user', + {user => $user} + ); + } + } + if ($name eq 'usertags' and defined $current_user){ + my %user_tags; + read_usertags(\%user_tags, $current_user); + $value =~ s/(?:^\s+|\s+$)//g; + for my $tag (split /[,\s]+/, $value) { + if (valid_usertag($tag)) { + my %bugs_with_tag; + @bugs_with_tag{@{$user_tags{$tag}||[]}} = (1) x @{$user_tags{$tag}||[]}; + $bugs_with_tag{$ref} = 1; + $user_tags{$tag} = [keys %bugs_with_tag]; + } + } + write_usertags(\%user_tags,$current_user); + } } overwritefile("db-h/$hash/$ref.report", map {"$_\n"} @msg); @@ -721,7 +772,9 @@ if (defined $gStrongList and isstrongseverity($data->{severity})) { } # Send mail to the per bug list subscription too -push @bccs, "bugs=$ref\@$gListDomain"; +if (defined $config{bug_subscription_domain} and length $config{bug_subscription_domain}) { + push @bccs, "bugs=$ref\@$config{bug_subscription_domain}"; +} if (defined $pheader{source}) { # Prefix source versions with the name of the source package. They @@ -777,6 +830,12 @@ if (length($resentccval)) { $resentcc= "Resent-CC: $resentccval\n"; } +my $referencesval = join(' ',grep {defined $_} $header{'references'},$data->{msgid}); +my $references = ''; +if (!$newref && length($referencesval)) { + $references = "References: $referencesval\n"; +} + my $common_headers=''; { my @tmp = @common_headers; @@ -796,7 +855,7 @@ Resent-Sender: $gMaintainerEmail X-$gProject-PR-Message: report $ref X-$gProject-PR-Package: $data->{package} X-$gProject-PR-Keywords: $data->{keywords} -${source_pr_header} +${references}${source_pr_header} END chomp $enc_msg; $enc_msg = encode_utf8($enc_msg).$fwd."\n"; @@ -819,7 +878,7 @@ Resent-Sender: $gMaintainerEmail X-$gProject-PR-Message: $report_followup $ref X-$gProject-PR-Package: $data->{package} X-$gProject-PR-Keywords: $data->{keywords} -${source_pr_header} +${references}${source_pr_header} END chomp $enc_msg; $enc_msg = encode_utf8($enc_msg).$fwd."\n"; @@ -853,7 +912,7 @@ Resent-Sender: $gMaintainerEmail ${common_headers}X-$gProject-PR-Message: $report_followup $ref X-$gProject-PR-Package: $data->{package} X-$gProject-PR-Keywords: $data->{keywords} -${source_pr_header} +${references}${source_pr_header} END chomp $enc_msg; $enc_msg = encode_utf8($enc_msg).$fwd."\n"; @@ -947,7 +1006,7 @@ if (not exists $header{'x-debbugs-no-ack'} and ); &sendmessage(create_mime_message( [@common_headers, - From => "$gMaintainerEmail ($gProject $gBug Tracking System)", + From => qq("$gProject $gBug Tracking System" <$gMaintainerEmail>), To => $replyto, Subject => $t_h{subject}, "Message-ID" => $t_h{messageid}, @@ -990,7 +1049,7 @@ if (@control_bits) { request_subject => $header{subject}, request_nn => $nn, request_replyto => $replyto, - message => $msg, + message => [$msg], affected_bugs => \%bug_affected, affected_packages => \%affected_packages, recipients => \%recipients, @@ -1028,6 +1087,7 @@ if (@control_bits) { } } else { + print {$transcript} "Unknown command or malformed arguments to command.\n\n"; $errors++; if (++$unknowns >= 5) { print {$transcript} "Too many unknown commands, stopping here.\n\n"; @@ -1044,10 +1104,10 @@ if (@control_bits) { address_only => 1, cc => 1, ); - my $error_text = $errors > 0 ? " (with $errors errors)":''; + my $error_text = $errors > 0 ? " (with $errors error" . ($errors > 1 ? "s" : "") . ")" : ""; my $reply = create_mime_message(['X-Loop' => $gMaintainerEmail, - From => "$gMaintainerEmail ($gProject $gBug Tracking System)", + From => qq("$gProject $gBug Tracking System" <$gMaintainerEmail>), To => $replyto, @maintccs ? (Cc => join(', ',@maintccs)):(), Subject => "Processed${error_text}: $header{subject}", @@ -1182,7 +1242,7 @@ sub sendmessage { write_log_records(logfh => $logfh, records => {text => stripbccs($msg), type => 'recips', - recips => [@{$recips}], + recips => [map {encode_utf8($_)} @{$recips}], }, ); if (ref($bcc)) { @@ -1230,20 +1290,15 @@ sub fill_template{ my $variables = {config => \%config, defined($ref)?(ref => $ref):(), defined($data)?(data => $data):(), - refs => [map {exists $clonebugs{$_}?$clonebugs{$_}:$_} keys %bug_affected], + refs => [sort + uniqnum(defined($ref)?($ref):(), + map {exists $clonebugs{$_}?$clonebugs{$_}:$_} + keys %bug_affected)], %{$extra_var}, }; - my $hole_var = {'&bugurl' => - sub{"$_[0]: ". - 'http://'.$config{cgi_domain}.'/'. - Debbugs::CGI::bug_links(bug=>$_[0], - links_only => 1, - ); - } - }; return fill_in_template(template => $template, variables => $variables, - hole_var => $hole_var, + output_type => 'text', ); } @@ -1260,15 +1315,15 @@ sub checkmaintainers { $p =~ /((?:src:)?[a-z0-9.+-]+)/; $p = $1; next unless defined $p; - if (defined $gSubscriptionDomain) { + if (defined $config{subscription_domain} and length $config{subscription_domain}) { my @source = binary_to_source(binary => $p, source_only => 1, ); if (@source) { push @addsrcaddrs, - map {"$_\@$gSubscriptionDomain"} @source; + map {"$_\@$config{subscription_domain}"} @source; } else { - push @addsrcaddrs, "$p\@$gSubscriptionDomain"; + push @addsrcaddrs, "$p\@$config{subscription_domain}"; } } # this is utter hackery until we switch to Debbugs::Recipients