]> git.donarmstrong.com Git - debbugs.git/blob - scripts/process
use slurp and io file to read the original report
[debbugs.git] / scripts / process
1 #!/usr/bin/perl
2 # $Id: process.in,v 1.109 2006/02/09 22:02:04 don Exp $
3 #
4 # Usage: process nn
5 # Temps:  incoming/Pnn
6
7 use warnings;
8 use strict;
9
10 use POSIX qw(strftime);
11
12 use IO::File;
13
14 use Getopt::Long;
15 use Pod::Usage;
16 use MIME::Parser;
17 use Debbugs::MIME qw(decode_rfc1522 create_mime_message getmailbody);
18 use Debbugs::Mail qw(send_mail_message encode_headers get_addresses);
19 use Debbugs::Packages qw(getpkgsrc binary_to_source);
20 use Debbugs::User qw(read_usertags write_usertags);
21 use Debbugs::Common qw(:lock get_hashname buglog package_maintainer overwritefile);
22 use Debbugs::Status qw(writebug isstrongseverity lockreadbugmerge lockreadbug new_bug read_bug splitpackages  :versions);
23
24 use Debbugs::CGI qw(html_escape bug_url);
25
26 use Debbugs::Log qw(:misc :write);
27
28 use Debbugs::Text qw(:templates);
29
30 use Debbugs::Config qw(:globals :config);
31
32 use Debbugs::Control qw(append_action_to_log);
33 use Encode qw(encode_utf8);
34
35 =head1 NAME
36
37 process - Handle e-mails emails sent to bugs
38
39 =head1 SYNOPSIS
40
41 process nn
42
43  Options:
44   --debug, -d debugging level (Default 0)
45
46 =head1 OPTIONS
47
48 =over
49
50 =item <--debug,-d>
51
52 Debugging level (default 0)
53
54 =back
55
56 =cut
57
58 use vars qw($DEBUG);
59
60 my %options = (debug           => 0,
61                help            => 0,
62                man             => 0,
63               );
64
65 GetOptions(\%options,
66            'debug|d+','help|h|?','man|m');
67
68 pod2usage() if $options{help};
69 pod2usage({verbose=>2}) if $options{man};
70
71
72 $DEBUG=$options{debug};
73 my $debugfh = IO::File->new('/dev/null','w') or
74     die "Unable to open /dev/null for writing; $!";
75 if ($DEBUG > 0) {
76     $debugfh = \*STDERR;
77 }
78
79 # these are the valid bug addresses
80 my %baddress = (B => 'submit',
81                 M => 'maintonly',
82                 Q => 'quiet',
83                 F => 'forwarded',
84                 D => 'done',
85                 S => 'submitter',
86                 L => 'list',
87                );
88 my $valid_codeletters = join('',keys %baddress);
89
90
91 chdir($config{spool_dir}) or die "Unable to chdir to spool ($config{spool_dir}): $!";
92
93 umask(002);
94
95 my $intdate = time or die "failed to get time: $!";
96
97 my ($nn) = @ARGV;
98 my ($codeletter,$tryref) =
99     $nn =~ m/^([$valid_codeletters])(\d*)\.\d+$/
100     or die "bad argument: $_";
101 $tryref = undef unless length ($tryref) and
102     $tryref > 0;
103
104 if (!rename("incoming/G$nn","incoming/P$nn"))  {
105     my $error = $!;
106     $error = '' if not defined $error;
107     # this is very fragile, but we should probably die here anyway
108     if ($error =~ m/no such file or directory/i) {
109         exit 0;
110     }
111     die "Unable to rename incoming/G$nn to lock: $error";
112 }
113
114 # die here to avoid continuously processing this mail
115 if (not exists $baddress{$codeletter}) {
116     die "bad codeletter $codeletter";
117 }
118
119 my $baddress = $baddress{$codeletter};
120 if ($baddress eq 'list') {
121     bug_list_forward($nn) if $codeletter eq 'L';
122 }
123
124 my $baddressroot= $baddress;
125 $baddress= "$tryref-$baddress" if defined $tryref;
126
127 my $msg;
128 my @msg;
129
130 {
131     my $log = IO::File->new("incoming/P$nn",'r') or
132         die "Unable to open 'incoming/P$nn' for reading; $!";
133     local $/;
134     $msg=<$log>;
135     @msg = split /\n/, $msg;
136     close($log);
137 }
138
139
140 my $tdate = strftime "%a, %d %h %Y %T +0000", gmtime;
141 my $fwd= "Received: via spool by $baddress\@$gEmailDomain id=$nn\n".
142     "          (code $codeletter".(defined($tryref)?" ref $tryref":'')."); $tdate\n";
143
144 # header and decoded body respectively
145 my (@headerlines, @bodylines);
146
147 # whether maintainer addresses have been checked
148 our $maintainerschecked = 0;
149 #maintainer address for this message
150 our @maintaddrs;
151 # other src addresses
152 our @addsrcaddrs;
153 our @resentccs;
154 our @bccs;
155
156 my $resentccexplain='';
157
158 # whether there's a new reference with this email
159 our $newref = 0;
160
161 our $brokenness = '';
162
163 my $parser_output = Debbugs::MIME::parse($msg);
164
165 @headerlines = @{$parser_output->{header}};
166 @bodylines = @{$parser_output->{body}};
167
168 my %header;
169
170 my @common_headers;
171 for my $hdr (@headerlines) {
172     $hdr = decode_rfc1522($hdr);
173     $_ = $hdr;
174     s/\n\s/ /g;
175     finish() if m/^x-loop: (\S+)$/i && $1 eq "$gMaintainerEmail";
176     my $ins = !m/^subject:/i && !m/^reply-to:/i && !m/^return-path:/i
177            && !m/^From / && !m/^X-Debbugs-/i;
178     $fwd .= $hdr."\n" if $ins;
179     # print {$debugfh} ">$_<\n";
180     if (s/^(\S+):\s*//) {
181         my $v = lc $1;
182         if ($v eq 'x-loop') {
183             push @common_headers, 'X-Loop',$_;
184         }
185         print {$debugfh} ">$v=$_<\n";
186         $header{$v} = $_;
187     } else {
188         print {$debugfh} "!>$_<\n";
189     }
190 }
191 $header{'message-id'} = '' if not defined $header{'message-id'};
192
193 push @common_headers, 'X-Loop',$gMaintainerEmail;
194
195 # remove blank lines
196 shift @bodylines while @bodylines and $bodylines[0] !~ /\S/;
197
198 # Strip off RFC2440-style PGP clearsigning.
199 if (@bodylines and $bodylines[0] =~ /^-----BEGIN PGP SIGNED/) {
200     shift @bodylines while @bodylines and length $bodylines[0];
201     shift @bodylines while @bodylines and $bodylines[0] !~ /\S/;
202     for my $findsig (0 .. $#bodylines) {
203         if ($bodylines[$findsig] =~ /^-----BEGIN PGP SIGNATURE/) {
204             $#bodylines = $findsig - 1;
205             last;
206         }
207     }
208     map { s/^- // } @bodylines;
209 }
210
211 #psuedoheaders
212 my %pheader;
213 # extract pseudo-headers
214 for my $phline (@bodylines)
215 {
216     # Remove BOM markers from UTF-8 strings
217     # Fixes #488554
218     $phline =~ s/\xef\xbb\xbf//g;
219     last if $phline !~ m/^([\w-]+):\s*(\S.*)/;
220     my ($fn, $fv) = ($1, $2);
221     $fv =~ s/\s*$//;
222     print {$debugfh} ">$fn|$fv|\n";
223     $fn = lc $fn;
224     # Don't lc owner or forwarded
225     $fv = lc $fv unless $fn =~ /^(?:owner|forwarded|usertags|version|source-version)$/;
226     $pheader{$fn} = $fv;
227     print {$debugfh} ">$fn~$fv<\n";
228 }
229
230 # Allow pseudo headers to set x-debbugs- stuff [#179340]
231 for my $key (grep /X-Debbugs-.*/i, keys %pheader) {
232      $header{$key} = $pheader{$key} if not exists $header{$key};
233 }
234
235 # set $i to beginning of encoded body data, so we can dump it out
236 # verbatim later
237 my $i = 0;
238 ++$i while $msg[$i] =~ /./;
239 $fwd .= join("\n",@msg[$i..$#msg]);
240
241 print {$debugfh} "***\n$fwd\n***\n";
242
243 if (defined $header{'resent-from'} && !defined $header{'from'}) {
244     $header{'from'} = $header{'resent-from'};
245 }
246 defined($header{'from'}) || die "no From header";
247
248 my $replyto = $header{'reply-to'};
249 $replyto = '' unless defined $replyto;
250 $replyto =~ s/^ +//;
251 $replyto =~ s/ +$//;
252 unless (length $replyto) {
253     $replyto = $header{'from'};
254 }
255
256 my $subject = '(no subject)';
257 if (!defined($header{'subject'})) 
258 {
259         $brokenness.= fill_template('mail/process_broken_subject');
260
261 } else { 
262     $subject= $header{'subject'}; 
263 }
264
265 my $ref=-1;
266 $subject =~ s/^Re:\s*//i; $_= $subject."\n";
267 if (not defined $tryref and m/^Bug ?\#(\d+)\D/i) {
268     $tryref = $1 if $1 > 0;
269 }
270 my $data;
271 if (defined $tryref) {
272      my $bfound;
273     ($bfound, $data)= &lockreadbugmerge($tryref);
274     if ($bfound and not $data->{archived}) {
275         $ref= $tryref; 
276     } else {
277         &sendmessage(create_mime_message(
278           [From          => "$gMaintainerEmail ($gProject $gBug Tracking System)",
279            To            => $replyto,
280            Subject       => "Unknown problem report $gBug#$tryref ($subject)",
281            'Message-ID'  => "<handler.x.$nn.unknown\@$gEmailDomain>",
282            'In-Reply-To' => $header{'message-id'},
283            References    => join(' ',grep {defined $_} $header{'message-id'},$data->{msgid}),
284            Precedence    => 'bulk',
285            "X-$gProject-PR-Message" => 'error',
286            @common_headers,
287           ],message_body_template('mail/process_unknown_bug_number',
288                                   {subject => $subject,
289                                    date    => $header{date},
290                                    baddress => $baddress,
291                                    tryref   => $tryref,
292                                    messageid => $header{'message-id'},
293                                   },
294                                  )),'');
295         appendlog($ref,$msg);
296         finish();
297     }
298 } else { 
299     &filelock('lock/-1'); 
300 }
301
302 # Attempt to determine which source package this is
303 my $source_pr_header = '';
304 my $source_package = '';
305 if (defined $pheader{source}) {
306      $source_package = $pheader{source};
307 }
308 elsif (defined $data->{package} or defined $pheader{package}) {
309      $source_package = binary_to_source(binary => $data->{package} // $pheader{package});
310 }
311 $source_pr_header = "X-$gProject-PR-Source: $source_package\n"
312      if defined $source_package and length $source_package;
313
314 # Done and Forwarded Bugs
315 if ($codeletter eq 'D' || $codeletter eq 'F') 
316 {
317     if ($replyto =~ m/$gBounceFroms/o ||
318         $header{'from'} =~ m/$gBounceFroms/o)
319     {
320          print STDERR "bounce detected !  Mwaap! Mwaap!";
321          exit 1;
322     }
323     my $markedby= $header{'from'} eq $replyto ? $replyto :
324                "$header{'from'} (reply to $replyto)";
325     my @generalcc;
326     my $receivedat;
327     my $markaswhat;
328     my $set_forwarded;
329     my $generalcc;
330     my $set_done;
331     if ($codeletter eq 'F') { # Forwarded
332         (appendlog($ref,$msg),finish()) if defined $data->{forwarded} and length($data->{forwarded});
333         $receivedat= "forwarded\@$gEmailDomain";
334         $markaswhat= 'forwarded';
335         $set_forwarded= $header{'to'};
336         # Dissallow forwarded being set to this bug tracking system
337         if (defined $set_forwarded and $set_forwarded =~ /\Q$gEmailDomain\E/) {
338              undef $set_forwarded;
339         }
340         if ( length( $gListDomain ) > 0 && length( $gForwardList ) > 0 ) {
341             push @generalcc, "$gForwardList\@$gListDomain";
342             $generalcc= "$gForwardList\@$gListDomain";
343         } else { 
344             $generalcc='';
345         }
346     } else { # Done
347         if (defined $data->{done} and length($data->{done}) and
348                 not defined $pheader{'source-version'} and
349                 not defined $pheader{'version'}) {
350             appendlog($ref,$msg);
351             finish();
352         }
353         $receivedat= "done\@$gEmailDomain";
354         $markaswhat= 'done';
355         $set_done= $header{'from'};
356         if ( length( $gListDomain ) > 0 && length( $gDoneList ) > 0 ) {
357             $generalcc= "$gDoneList\@$gListDomain";
358             push @generalcc, "$gDoneList\@$gListDomain";
359         } else { 
360             $generalcc=''; 
361         }
362     }
363     if (defined $gStrongList and isstrongseverity($data->{severity})) {
364         $generalcc = join ', ', $generalcc, "$gStrongList\@$gListDomain";
365         push @generalcc,"$gStrongList\@$gListDomain";
366     }
367     if ($ref<0) {
368         &sendmessage(create_mime_message(
369           [From          => "$gMaintainerEmail ($gProject $gBug Tracking System)",
370            To            => $replyto,
371            Subject       => "Message with no $gBug number ignored by $receivedat ($subject)",
372            'Message-ID'  => "<handler.x.$nn.warnignore\@$gEmailDomain>",
373            'In-Reply-To' => $header{'message-id'},
374            References    => join(' ',grep {defined $_} $header{'message-id'},$data->{msgid}),
375            Precedence    => 'bulk',
376            "X-$gProject-PR-Message" => 'error',
377            @common_headers,
378           ],message_body_template('mail/process_no_bug_number',
379                                   {subject => $subject,
380                                    date    => $header{date},
381                                    markaswhat => $markaswhat,
382                                    receivedat => $receivedat,
383                                    messageid => $header{'message-id'},
384                                   },
385                                  )),'');
386         appendlog($ref,$msg);
387         finish();
388     }
389
390     &checkmaintainers;
391
392     my @noticecc = grep($_ ne $replyto,@maintaddrs);
393     my $noticeccval.= join(', ', grep($_ ne $replyto,@maintaddrs));
394     $noticeccval =~ s/\s+\n\s+/ /g; 
395     $noticeccval =~ s/^\s+/ /; $noticeccval =~ s/\s+$//;
396
397     my @process= ($ref,split(/ /,$data->{mergedwith}));
398     my $orgref= $ref;
399
400     for $ref (@process) {
401         if ($ref != $orgref) {
402             &unfilelock;
403             $data = &lockreadbug($ref)
404                 || die "huh ? $ref from $orgref out of ".join(' ',@process);
405         }
406         $data->{done}= $set_done if defined($set_done);
407         $data->{forwarded}= $set_forwarded if defined($set_forwarded);
408         if ($codeletter eq 'D') {
409             $data->{keywords} = join ' ', grep $_ ne 'pending',
410                                      split ' ', $data->{keywords};
411             if (defined $pheader{'source-version'}) {
412                  if ($pheader{'source-version'} !~ m/^$config{package_version_re}$/) {
413                       $brokenness .= fill_template('mail/invalid_version',
414                                                    {version => $pheader{'source-version'}},
415                                                   );
416                  }
417                  else {
418                       addfixedversions($data, $pheader{source}, $pheader{'source-version'}, '');
419                  }
420             } elsif (defined $pheader{version}) {
421                  if ($pheader{version} !~ m/^$config{package_version_re}$/) {
422                       $brokenness .= fill_template('mail/invalid_version',
423                                                    {version => $pheader{version}},
424                                                   );
425                  }
426                  else {
427                       addfixedversions($data, $pheader{package}, $pheader{version}, '');
428                  }
429             }
430         }
431
432         # Add bug mailing list to $generalbcc as appropriate
433         # This array is used to specify bcc in the cases where we're using create_mime_message.
434         my @generalbcc = (@generalcc,@addsrcaddrs,"bugs=$ref\@$gListDomain");
435         my $generalbcc = join(', ', $generalcc, @addsrcaddrs,"bugs=$ref\@$gListDomain");
436         $generalbcc =~ s/\s+\n\s+/ /g;
437         $generalbcc =~ s/^\s+/ /; $generalbcc =~ s/\s+$//;
438         if (length $generalbcc) {$generalbcc = "Bcc: $generalbcc\n"};
439
440         writebug($ref, $data);
441
442         my $hash = get_hashname($ref);
443         my $orig_report_fh = IO::File->new("db-h/$hash/$ref.report") or
444             die "Unable to read original report: $!";
445         my $orig_report;
446         { local $/; $orig_report = <$orig_report_fh>;}
447         close($orig_report_fh) or
448             die "Unable to close original report filehandle: $!";
449         if ($codeletter eq 'F') {
450             &htmllog("Reply","sent",$replyto,"You have marked $gBug as forwarded.");
451             &sendmessage(create_mime_message(
452              [@common_headers,
453               From          => "$gMaintainerEmail ($gProject $gBug Tracking System)",
454               To            => "$replyto",
455               Subject       => "$gBug#$ref: marked as forwarded ($data->{subject})",
456               "Message-ID"  => "<header.$ref.$nn.ackfwdd\@$gEmailDomain>",
457               "In-Reply-To" => $header{'message-id'},
458               References    => join(' ',grep {defined $_} $header{'message-id'},$data->{msgid}),
459               Precedence    => 'bulk',
460               "X-$gProject-PR-Message"  => "forwarded $ref",
461               "X-$gProject-PR-Package"  => $data->{package},
462               "X-$gProject-PR-Keywords" => $data->{keywords},
463               # Only have a X-$gProject-PR-Source when we know the source package
464               (defined($source_package) and length($source_package))?("X-$gProject-PR-Source" => $source_package):(),
465              ],message_body_template('mail/process_mark_as_forwarded',
466                                      {date => $header{date},
467                                       messageid => $header{'message-id'},
468                                       data      => $data,
469                                      },
470                                     ),
471              [join("\n",@msg)]),'',[$replyto,@generalbcc,@noticecc],1);
472         } else {
473             &htmllog("Reply","sent",$replyto,"You have taken responsibility.");
474             &sendmessage(create_mime_message(
475              [@common_headers,
476               From          => "$gMaintainerEmail ($gProject $gBug Tracking System)",
477               To            => $replyto,
478               Subject       => "$gBug#$ref: marked as done ($data->{subject})",
479               "Message-ID"  => "<handler.$ref.$nn.ackdone\@$gEmailDomain>",
480               "In-Reply-To" => $header{'message-id'},
481               References    => join(' ',grep {defined $_} $header{'message-id'},$data->{msgid}),
482               Precedence    => 'bulk',
483               "X-$gProject-PR-Message"  => "closed $ref",
484               "X-$gProject-PR-Package"  => $data->{package},
485               "X-$gProject-PR-Keywords" => $data->{keywords},
486               # Only have a X-$gProject-PR-Source when we know the source package
487               (defined($source_package) and length($source_package))?("X-$gProject-PR-Source" => $source_package):(),
488              ],message_body_template('mail/process_mark_as_done',
489                                      {date => $header{date},
490                                       messageid => $header{'message-id'},
491                                       subject   => $header{subject},
492                                       data      => $data,
493                                      },
494                                     ),
495              [$orig_report,join("\n",@msg)]),'',[$replyto,@generalbcc,@noticecc],1);
496             &htmllog("Notification","sent",$data->{originator},
497                 "$gBug acknowledged by developer.");
498             &sendmessage(create_mime_message(
499              [@common_headers,
500               From          => "$gMaintainerEmail ($gProject $gBug Tracking System)",
501               To            => "$data->{originator}",
502               Subject       => "$gBug#$ref closed by $markedby ($header{'subject'})",
503               "Message-ID"  => "<handler.$ref.$nn.notifdone\@$gEmailDomain>",
504               (defined $data->{msgid})?("In-Reply-To" => $data->{msgid}):(),
505               References    => join(' ',grep {defined $_} ($header{'message-id'},$data->{msgid})),
506               "X-$gProject-PR-Message"  => "they-closed $ref",
507               "X-$gProject-PR-Package"  => "$data->{package}",
508               "X-$gProject-PR-Keywords" => "$data->{keywords}",
509               # Only have a X-$gProject-PR-Source when we know the source package
510               (defined($source_package) and length($source_package))?("X-$gProject-PR-Source" => $source_package):(),
511               "Reply-To"                => "$ref\@$gEmailDomain",
512               "Content-Type"            => 'text/plain; charset="utf-8"',
513              ],message_body_template('mail/process_your_bug_done',
514                                      {data      => $data,
515                                       markedby  => $markedby,
516                                       messageid => $header{'message-id'},
517                                       subject   => $header{subject},
518                                      },
519                                     ),
520              [join("\n",@msg),$orig_report]),'',undef,1);
521         }
522         appendlog($ref,$msg);
523     }
524     finish();
525 }
526
527 if ($ref<0) { # new bug report
528     if ($codeletter eq 'U') { # -submitter
529         &sendmessage(create_mime_message(
530           [@common_headers,
531            From          => "$gMaintainerEmail ($gProject $gBug Tracking System)",
532            To            => $replyto,
533            Subject       => "Message with no $gBug number cannot be sent to submitter! ($subject)",
534            'Message-ID'  => "<handler.x.$nn.nonumnosub\@$gEmailDomain>",
535            'In-Reply-To' => $header{'message-id'},
536            References    => join(' ',grep {defined $_} $header{'message-id'},$data->{msgid}),
537            Precedence    => 'bulk',
538            "X-$gProject-PR-Message" => 'error',
539           ],message_body_template('mail/process_no_bug_number',
540                                   {subject => $subject,
541                                    date    => $header{date},
542                                    markaswhat => 'submitter',
543                                    receivedat => "$baddress\@$gEmailDomain",
544                                    messageid => $header{'message-id'},
545                                   },
546                                  )),'');
547         appendlog($ref,$msg);
548         finish();
549     }
550
551     $data->{found_versions} = [];
552     $data->{fixed_versions} = [];
553
554     if (defined $pheader{source}) {
555         # source packages are identified by the src: prefix
556         $data->{package} = 'src:'.$pheader{source};
557     } elsif (defined $pheader{package}) {
558         $data->{package} = $pheader{package};
559         if ($data->{package} =~ /^src:(.+)/) {
560             $pheader{source} = $1;
561         }
562     } elsif (defined $config{default_package}) {
563         $data->{package} = $config{default_package},
564     }
565     else {
566         my $body = message_body_template('mail/process_no_package',
567                                         );
568         &sendmessage(create_mime_message(
569                        [@common_headers,
570                         From          => "$gMaintainerEmail ($gProject $gBug Tracking System)",
571                         To            => $replyto,
572                         Subject       => "Message with no Package: tag cannot be processed! ($subject)",
573                         "Message-ID"  => "<handler.x.$nn.nonumnosub\@$gEmailDomain>",
574                         "In-Reply-To" => $header{'message-id'},
575                         References    => join(' ',grep {defined $_} $header{'message-id'},$data->{msgid}),
576                         Precedence    => 'bulk',
577                         "X-$gProject-PR-Message" => 'error'
578                        ],
579            message_body_template('mail/process_no_package',
580                                  {date => $header{date},
581                                   subject => $subject,
582                                   messageid => $header{'message-id'},
583                                   baddress => $baddress,
584                                  },
585                                 ),[join("\n", @msg)]), '',undef,1);
586         appendlog($ref,$msg);
587         finish();
588     }
589
590     if (defined $config{default_package}) {
591          &checkmaintainers;
592          # if there are no maintainers for this package, assign it to the default package
593          if (not @maintaddrs) {
594               $data->{package} = $config{default_package};
595               $brokenness.= fill_template('mail/process_default_package_selected',
596                                           {old_package => $pheader{source} || $pheader{package} || 'No package',
597                                            new_package => $data->{package},
598                                           }
599                                          );
600               # force the maintainers to be rechecked
601               $maintainerschecked = 0;
602               &checkmaintainers;
603          }
604     }
605
606     $data->{keywords}= '';
607     if (defined($pheader{'keywords'})) {
608         $data->{keywords}= $pheader{'keywords'};
609     } elsif (defined($pheader{'tags'})) {
610         $data->{keywords}= $pheader{'tags'};
611     }
612     if (length($data->{keywords})) {
613         my @kws;
614         my %gkws = map { ($_, 1) } @gTags;
615         foreach my $kw (sort split(/[,\s]+/, lc($data->{keywords}))) {
616             push @kws, $kw if (defined $gkws{$kw});
617         }
618         $data->{keywords} = join(" ", @kws);
619     }
620     $data->{severity}= '';
621     if (defined($pheader{'severity'}) || defined($pheader{'priority'})) {
622         $data->{severity}= $pheader{'severity'};
623         $data->{severity}= $pheader{'priority'} unless ($data->{severity});
624         $data->{severity} =~ s/^\s*(.+)\s*$/$1/;
625
626         if (!grep($_ eq $data->{severity}, @gSeverityList, "$gDefaultSeverity")) {
627             $brokenness.= fill_template('mail/invalid_severity',
628                                         {severity=>$data->{severity}}
629                                        );
630             $data->{severity}= '';
631         }
632     }
633     if (defined($pheader{owner})) {
634         $data->{owner}= $pheader{owner};
635     }
636     if (defined($pheader{forwarded})) {
637         $data->{forwarded} = $pheader{forwarded};
638     }
639     $ref = new_bug();
640     $newref = $ref;
641     my $hash = get_hashname($ref);
642     $data->{originator} = $replyto;
643     $data->{date} = $intdate;
644     $data->{subject} = $subject;
645     $data->{msgid} = $header{'message-id'};
646     writebug($ref, $data);
647     # Deal with usertags
648     if (exists $pheader{usertags}) {
649          my $user = $replyto;
650          $user = $pheader{user} if exists $pheader{user};
651          $user =~ s/,.*//;
652          $user =~ s/^.*<(.*)>.*$/$1/;
653          $user =~ s/[(].*[)]//;
654          $user =~ s/^\s*(\S+)\s+.*$/$1/;
655          if ($user ne '' and Debbugs::User::is_valid_user($user)) {
656               $pheader{usertags} =~ s/(?:^\s+|\s+$)//g;
657               my %user_tags;
658               read_usertags(\%user_tags,$user);
659               for my $tag (split /[,\s]+/, $pheader{usertags}) {
660                    if ($tag =~ /^[a-zA-Z0-9.+\@-]+/) {
661                         my %bugs_with_tag; 
662                         @bugs_with_tag{@{$user_tags{$tag}||[]}} = (1) x @{$user_tags{$tag}||[]};
663                         $bugs_with_tag{$ref} = 1;
664                         $user_tags{$tag} = [keys %bugs_with_tag];
665                    }
666               }
667               write_usertags(\%user_tags,$user);
668          }
669          else {
670               $brokenness .= fill_template('mail/invalid_user',
671                                            {user => $user}
672                                           );
673          }
674     }
675     overwritefile("db-h/$hash/$ref.report",
676                   map {"$_\n"} @msg);
677 }
678
679 &checkmaintainers;
680
681 print {$debugfh} "maintainers >".join(' ',@maintaddrs)."<\n";
682
683 my $orgsender= defined($header{'sender'}) ? "Original-Sender: $header{'sender'}\n" : '';
684 my $newsubject= $subject;  $newsubject =~ s/^$gBug#$ref:*\s*//;
685
686 my $xcchdr= $header{ 'x-debbugs-cc' } || '';
687 if ($xcchdr =~ m/\S/) {
688     push(@resentccs,get_addresses($xcchdr));
689     $resentccexplain.= fill_template('mail/xdebbugscc',
690                                      {xcchdr => $xcchdr},
691                                     );
692 }
693
694 if (@maintaddrs && ($codeletter eq 'B' || $codeletter eq 'M')) {
695     push(@resentccs,@maintaddrs);
696     $resentccexplain.= fill_template('mail/maintainercc',
697                                      {maintaddrs => \@maintaddrs,
698                                      },
699                                     );
700 }
701
702 @bccs = @addsrcaddrs;
703 if (defined $gStrongList and isstrongseverity($data->{severity})) {
704     push @bccs, "$gStrongList\@$gListDomain";
705 }
706
707 # Send mail to the per bug list subscription too
708 push @bccs, "bugs=$ref\@$gListDomain";
709
710 if (defined $pheader{source}) {
711     # Prefix source versions with the name of the source package. They
712     # appear that way in version trees so that we can deal with binary
713     # packages moving from one source package to another.
714     if (defined $pheader{'source-version'}) {
715          if ($pheader{'source-version'} !~ m/^$config{package_version_re}$/) {
716               $brokenness .= fill_template('mail/invalid_version',
717                                            {version => $pheader{'source-version'}},
718                                           );
719          }
720          else {
721               addfoundversions($data, $pheader{source}, $pheader{'source-version'}, '');
722          }
723     } elsif (defined $pheader{version}) {
724          if ($pheader{version} !~ m/^$config{package_version_re}$/) {
725               $brokenness .= fill_template('mail/invalid_version',
726                                            {version => $pheader{version}},
727                                           );
728          }
729          else {
730               addfoundversions($data, $pheader{source}, $pheader{version}, '');
731          }
732     }
733     writebug($ref, $data);
734 } elsif (defined $pheader{package}) {
735     # TODO: could handle Source-Version: by looking up the source package?
736      if (defined $pheader{version}) {
737           if ($pheader{version} !~ m/^$config{package_version_re}$/) {
738                $brokenness .= fill_template('mail/invalid_version',
739                                             {version => $pheader{version}},
740                                            );
741           }
742           else {
743                addfoundversions($data, $pheader{package}, $pheader{version}, 'binary');
744           }
745      }
746      writebug($ref, $data);
747 }
748
749 my $veryquiet= $codeletter eq 'Q';
750 if ($codeletter eq 'M' && !@maintaddrs) {
751     $veryquiet= 1;
752     $brokenness.= fill_template('mail/invalid_maintainer',
753                                 {},
754                                );
755 }
756
757 my $resentccval.= join(', ',@resentccs);
758 $resentccval =~ s/\s+\n\s+/ /g; $resentccval =~ s/^\s+/ /; $resentccval =~ s/\s+$//;
759 my $resentcc = '';
760 if (length($resentccval)) { 
761     $resentcc= "Resent-CC: $resentccval\n"; 
762 }
763
764 my $common_headers='';
765 {
766     my @tmp = @common_headers;
767     while (my ($key,$value) = splice(@tmp, 0,2)) {
768         $common_headers .= qq($key: $value\n);
769     }
770 }
771 if ($codeletter eq 'U') { # sent to -submitter
772     &htmllog("Message", "sent on", $data->{originator}, "$gBug#$ref.");
773     my $enc_msg=<<END;
774 Subject: $gBug#$ref: $newsubject
775 Reply-To: $replyto, $ref-quiet\@$gEmailDomain
776 ${orgsender}Resent-To: $data->{originator}
777 ${resentcc}${common_headers}Resent-Date: $tdate
778 Resent-Message-ID: <handler.$ref.$nn\@$gEmailDomain>
779 Resent-Sender: $gMaintainerEmail
780 X-$gProject-PR-Message: report $ref
781 X-$gProject-PR-Package: $data->{package}
782 X-$gProject-PR-Keywords: $data->{keywords}
783 ${source_pr_header}
784 END
785     chomp $enc_msg;
786     $enc_msg = encode_utf8($enc_msg).$fwd."\n";
787     &sendmessage($enc_msg,[$data->{originator},@resentccs],[@bccs]);
788 } elsif ($codeletter eq 'B') { # Sent to submit
789     my $report_followup = $newref ? 'report' : 'followup';
790     &htmllog($newref ? "Report" : "Information", "forwarded",
791              join(', ',"$gSubmitList\@$gListDomain",@resentccs),
792              "<code>$gBug#$ref</code>".
793              (length($data->{package})? "; Package <code>".html_escape($data->{package})."</code>" : '').
794              ".");
795     my $enc_msg=<<END;
796 Subject: $gBug#$ref: $newsubject
797 Reply-To: $replyto, $ref\@$gEmailDomain
798 Resent-From: $header{'from'}
799 ${orgsender}Resent-To: $gSubmitList\@$gListDomain
800 ${resentcc}${common_headers}Resent-Date: $tdate
801 Resent-Message-ID: <handler.$ref.$nn\@$gEmailDomain>
802 Resent-Sender: $gMaintainerEmail
803 X-$gProject-PR-Message: $report_followup $ref
804 X-$gProject-PR-Package: $data->{package}
805 X-$gProject-PR-Keywords: $data->{keywords}
806 ${source_pr_header}
807 END
808     chomp $enc_msg;
809     $enc_msg = encode_utf8($enc_msg).$fwd."\n";
810     &sendmessage($enc_msg,["$gSubmitList\@$gListDomain",@resentccs],[@bccs]);
811 } elsif (@resentccs or @bccs) { # Quiet or Maintainer
812     # D and F done far earlier; B just done - so this must be M or Q
813     # We preserve whichever it was in the Reply-To (possibly adding
814     # the $gBug#).
815     my $report_followup = $newref ? 'report' : 'followup';
816     if (@resentccs) {
817         &htmllog($newref ? "Report" : "Information", "forwarded",
818                  $resentccval,
819                  "<code>$gBug#$ref</code>".
820                  (length($data->{package}) ? "; Package <code>".html_escape($data->{package})."</code>" : '').
821                  ".");
822     } else {
823         &htmllog($newref ? "Report" : "Information", "stored",
824                  "",
825                  "<code>$gBug#$ref</code>".
826                  (length($data->{package}) ? "; Package <code>".html_escape($data->{package})."</code>" : '').
827                  ".");
828     }
829     my $enc_msg=<<END;
830 Subject: $gBug#$ref: $newsubject
831 Reply-To: $replyto, $ref-$baddressroot\@$gEmailDomain
832 Resent-From: $header{'from'}
833 ${orgsender}Resent-To: $resentccval
834 Resent-Date: $tdate
835 Resent-Message-ID: <handler.$ref.$nn\@$gEmailDomain>
836 Resent-Sender: $gMaintainerEmail
837 ${common_headers}X-$gProject-PR-Message: $report_followup $ref
838 X-$gProject-PR-Package: $data->{package}
839 X-$gProject-PR-Keywords: $data->{keywords}
840 ${source_pr_header}
841 END
842     chomp $enc_msg;
843     $enc_msg = encode_utf8($enc_msg).$fwd."\n";
844     &sendmessage($enc_msg,[@resentccs],[@bccs]);
845 }
846
847 my $htmlbreak= length($brokenness) ? "<p>\n".html_escape($brokenness)."\n<p>\n" : '';
848 $htmlbreak =~ s/\n\n/\n<P>\n\n/g;
849 if (length($resentccval)) {
850     $htmlbreak = "  Copy sent to <code>".html_escape($resentccval)."</code>.".
851         $htmlbreak;
852 }
853
854 # Should we send an ack out?
855 if (not exists $header{'x-debbugs-no-ack'} and
856     ($newref or
857      ($codeletter ne 'U' and
858       (not defined $header{precedence} or
859        $header{'precedence'} !~ /\b(?:bulk|junk|list)\b/
860       )
861      )
862     )
863    ){
864
865      # figure out forward explanation
866      my $forwardexplain = '';
867      my $thanks = '';
868      my $extra_vars;
869      # will contain info and -info in moreinfo messages
870      my $info = '';
871      my $infod = '';
872      # temporary headers
873      my %t_h;
874      if ($newref) {
875           &htmllog("Acknowledgement","sent",$replyto,
876                    ($veryquiet ?
877                     "New $gBug report received and filed, but not forwarded." :
878                     "New $gBug report received and forwarded."). $htmlbreak);
879           $thanks = fill_template('mail/process_ack_thanks_new');
880      }
881      else {
882           &htmllog("Acknowledgement","sent",$replyto,
883                    ($veryquiet ? "Extra info received and filed, but not forwarded." :
884                     $codeletter eq 'M' ? "Extra info received and forwarded to maintainer." :
885                     "Extra info received and forwarded to list."). $htmlbreak);
886           $thanks = fill_template('mail/process_ack_thanks_additional');
887           $info = 'info';
888           $infod = '-info';
889      }
890      if ($veryquiet) {
891           $forwardexplain = fill_template('mail/forward_veryquiet',
892                                          );
893           # these are the headers that quiet messages override
894           $t_h{messageid}  = "<handler.$ref.$nn.ack${info}quiet\@$gEmailDomain>";
895           $t_h{pr_message} = "ack${infod}-quiet $ref";
896           $t_h{reply_to}   = "$ref-quiet\@$gEmailDomain";
897           $extra_vars->{refreplyto} = "$ref-quiet\@$gEmailDomain";
898           $t_h{subject}    = length($info)?
899                "$gBug#$ref: Info received and FILED only ($subject)":
900                "$gBug#$ref: Acknowledgement of QUIET report ($subject)";
901      }
902      elsif ($codeletter eq 'M') {
903           $forwardexplain = fill_template('mail/forward_maintonly',
904                                          );
905           # these are the headers that maintonly messages override
906           $t_h{messageid}  = "<handler.$ref.$nn.ack{$info}maintonly\@$gEmailDomain>";
907           $t_h{pr_message} = "ack${infod}-maintonly $ref";
908           $t_h{reply_to}   = "$ref-maintonly\@$gEmailDomain";
909           $extra_vars->{refreplyto} = "$ref-maintonly\@$gEmailDomain";
910           $t_h{subject}    = length($info)?
911                "$gBug#$ref: Info received for maintainer only ($subject)":
912                "$gBug#$ref: Acknowledgement of maintainer-only report ($subject)";
913      }
914      else {
915           $forwardexplain = fill_template('mail/forward_normal',
916                                          );
917           $t_h{messageid}  = "<handler.$ref.$nn.ack${info}\@$gEmailDomain>";
918           $t_h{pr_message} = "ack${infod} $ref";
919           $t_h{reply_to}   = "$ref\@$gEmailDomain";
920           $extra_vars->{refreplyto} = "$ref\@$gEmailDomain";
921           $t_h{subject}    = (defined $info and length($info))?
922                "$gBug#$ref: Info received ($subject)" :
923                "$gBug#$ref: Acknowledgement ($subject)";
924      }
925      my $body = message_body_template('mail/process_ack',
926                                       {forwardexplain  => $forwardexplain,
927                                        resentccexplain => $resentccexplain,
928                                        thanks          => $thanks,
929                                        %{$extra_vars}
930                                       }
931                                      );
932      &sendmessage(create_mime_message(
933                        [@common_headers,
934                         From          => "$gMaintainerEmail ($gProject $gBug Tracking System)",
935                         To            => $replyto,
936                         Subject       => $t_h{subject},
937                         "Message-ID"  => $t_h{messageid},
938                         "In-Reply-To" => $header{'message-id'},
939                         References    => $header{'message-id'},
940                         Precedence    => 'bulk',
941                         "X-$gProject-PR-Message"  => $t_h{pr_message} || "ack $ref",
942                         "X-$gProject-PR-Package"  => $data->{package},
943                         "X-$gProject-PR-Keywords" => $data->{keywords},
944                         # Only have a X-$gProject-PR-Source when we know the source package
945                         (defined($source_package) and length($source_package))?("X-$gProject-PR-Source" => $source_package):(),
946                         "Reply-To"                => $t_h{reply_to} || "$ref\@$gEmailDomain",
947                        ],$body,[]), '',undef,1);
948 }
949
950 appendlog($ref,$msg);
951 finish();
952
953 sub appendlog {
954     my ($ref,$msg) = @_;
955     my $log_location = buglog($ref);
956     die "Unable to find .log for $ref"
957         if not defined $log_location;
958     my $logfh = IO::File->new(">>$log_location") or
959         die "Unable to open $log_location for appending: $!";
960     write_log_records(logfh => $logfh,
961                       records => [{type => 'incoming-recv',
962                                    text => $msg,
963                                   }]);
964     close ($logfh) or die "Unable to close $log_location: $!";
965 }
966
967 sub finish {
968     my ($exit) = @_;
969     $exit ||= 0;
970     utime(time,time,"db");
971     # cleanups are run in an end block now.
972     #my ($u);
973     #while ($u= $cleanups[$#cleanups]) { &$u; }
974     unlink("incoming/P$nn") || die "unlinking incoming/P$nn: $!";
975     exit $exit;
976 }
977
978 die "wot no exit";
979
980 sub htmllog {
981     my ($whatobj,$whatverb,$where,$desc) = @_;
982     append_action_to_log(bug => $ref,
983                          action => "$whatobj $whatverb",
984                          requester => '',
985                          request_addr => $where,
986                          desc         => $desc,
987                          get_lock     => 0,
988                         );
989 }
990
991 sub stripbccs {
992     my $msg = shift;
993     my $ret = '';
994     my $bcc = 0;
995     while ($msg =~ s/(.*\n)//) {
996         local $_ = $1;
997         if (/^$/) {
998             $ret .= $_;
999             last;
1000         }
1001         if ($bcc) {
1002             # strip continuation lines too
1003             next if /^\s/;
1004             $bcc = 0;
1005         }
1006         if (/^Bcc:/i) {
1007             $bcc = 1;
1008         } else {
1009             $ret .= $_;
1010         }
1011     }
1012     return $ret . $msg;
1013 }
1014
1015 =head2 send_message
1016
1017      send_message($the_message,\@recipients,\@bcc,$do_not_encode)
1018
1019 The first argument is the scalar message, the second argument is the
1020 arrayref of recipients, the third is the arrayref of Bcc:'ed
1021 recipients.
1022
1023 The final argument turns off header encoding and the addition of the
1024 X-Loop header if true, defaults to false.
1025
1026 =cut
1027
1028
1029 sub sendmessage {
1030     my ($msg,$recips,$bcc,$no_encode) = @_;
1031     if (not defined $recips or (!ref($recips) && $recips eq '')
1032         or @$recips == 0) {
1033         $recips = ['-t'];
1034     }
1035     # This is suboptimal. The right solution is to send headers
1036     # separately from the rest of the message and encode them rather
1037     # than doing this.
1038     $msg = "X-Loop: $gMaintainerEmail\n" . $msg unless $no_encode;
1039     # The original message received is written out in appendlog, so
1040     # before writing out the other messages we've sent out, we need to
1041     # RFC1522 encode the header.
1042     $msg = encode_headers($msg) unless $no_encode;
1043
1044     my $hash = get_hashname($ref);
1045     #save email to the log
1046     my $logfh = IO::File->new(">>db-h/${hash}/${ref}.log") or
1047         die "opening db-h/$hash/${ref}.log: $!";
1048     write_log_records(logfh => $logfh,
1049                       records => {text => stripbccs($msg),
1050                                   type => 'recips',
1051                                   recips => [@{$recips}],
1052                                  },
1053                      );
1054     if (ref($bcc)) {
1055         shift @$recips if $recips->[0] eq '-t';
1056         push @$recips, @$bcc;
1057     }
1058
1059     send_mail_message(message        => $msg,
1060                       # Because we encode the headers above, we do not want to encode them here
1061                       encode_headers => 0,
1062                       recipients     => $recips);
1063 }
1064
1065 =head2 message_body_template
1066
1067      message_body_template('mail/ack',{ref=>'foo'});
1068
1069 Creates a message body using a template
1070
1071 =cut
1072
1073 sub message_body_template{
1074      my ($template,$extra_var) = @_;
1075      $extra_var ||={};
1076      my $body = fill_template($template,$extra_var);
1077      return fill_template('mail/message_body',
1078                           {%{$extra_var},
1079                            body => $body,
1080                           },
1081                          );
1082 }
1083
1084 =head2 fill_template
1085
1086      fill_template('mail/foo',{foo=>'bar'});
1087
1088 Calls fill_in_template with a default set of variables and any extras
1089 added in.
1090
1091 =cut
1092
1093 sub fill_template{
1094      my ($template,$extra_var) = @_;
1095      $extra_var ||={};
1096      my $variables = {config => \%config,
1097                       defined($ref)?(ref    => $ref):(),
1098                       defined($data)?(data  => $data):(),
1099                       %{$extra_var},
1100                      };
1101      my $hole_var = {'&bugurl' =>
1102                      sub{"$_[0]: ".
1103                               'http://'.$config{cgi_domain}.'/'.
1104                                    Debbugs::CGI::bug_links(bug=>$_[0],
1105                                                            links_only => 1,
1106                                                           );
1107                     }
1108                     };
1109      return fill_in_template(template => $template,
1110                              variables => $variables,
1111                              hole_var  => $hole_var,
1112                             );
1113 }
1114
1115
1116 # this shole routine is *bad*; will be changed to use
1117 # Debbugs::Recipients and stuff therin in short order.
1118 sub checkmaintainers {
1119     return if $maintainerschecked++;
1120     return if !length($data->{package});
1121
1122     my $anymaintfound=0; my $anymaintnotfound=0;
1123     for my $p (splitpackages($data->{package})) {
1124         $p =~ y/A-Z/a-z/;
1125         $p =~ /((?:src:)?[a-z0-9.+-]+)/;
1126         $p = $1;
1127         next unless defined $p;
1128         if (defined $gSubscriptionDomain) {
1129             my @source = binary_to_source(binary => $p,
1130                                           source_only => 1,
1131                                          );
1132             if (@source) {
1133                 push @addsrcaddrs,
1134                     map {"$_\@$gSubscriptionDomain"} @source;
1135             } else {
1136                 push @addsrcaddrs, "$p\@$gSubscriptionDomain";
1137             }
1138         }
1139         # this is utter hackery until we switch to Debbugs::Recipients
1140         my @maints = package_maintainer(binary => $p);
1141         if (@maints) {
1142             print {$debugfh} "maintainer add >$p|".join(',',@maints)."<\n";
1143             my %temp;
1144             @temp{@maintaddrs} = @maintaddrs;
1145             push(@maintaddrs,
1146                  grep {$_ ne $replyto and
1147                            not exists $temp{$_}} @maints);
1148             $anymaintfound++;
1149         } else {
1150             print {$debugfh} "maintainer none >$p<\n";
1151             push(@maintaddrs,$gUnknownMaintainerEmail) unless $anymaintnotfound;
1152             $anymaintnotfound++;
1153             last;
1154         }
1155     }
1156
1157     if (defined $data->{owner} and length $data->{owner}) {
1158         print {$debugfh} "owner add >$data->{package}|$data->{owner}<\n";
1159         my $addmaint = $data->{owner};
1160         push(@maintaddrs, $addmaint) unless
1161             $addmaint eq $replyto or grep($_ eq $addmaint, @maintaddrs);
1162     }
1163 }
1164
1165 =head2 bug_list_forward
1166
1167      bug_list_forward($spool_filename) if $codeletter eq 'L';
1168
1169
1170 Given the spool file, will forward a bug to the per bug mailing list
1171 subscription system.
1172
1173 =cut
1174
1175 sub bug_list_forward{
1176      my ($bug_fn) = @_;
1177      # Read the bug information and package information for passing to
1178      # the mailing list
1179      my $bug_fh = IO::File->new("incoming/P$bug_fn",'r') or
1180           die "Unable to open incoming/P$bug_fn $!";
1181
1182      if (not defined $config{bug_subscription_domain} or not
1183          length $config{bug_subscription_domain}) {
1184           unlink("incoming/P$bug_fn") or
1185                die "unlinking incoming/P$bug_fn: $!";
1186           exit 0;
1187      }
1188
1189      my ($bug_number) = $bug_fn =~ /^L(\d+)\./;
1190      my $data = read_bug(bug => $bug_number);
1191
1192      local $/ = undef;
1193      my $bug_message = <$bug_fh>;
1194      my ($bug_address) = $bug_message =~ /^Received: \(at ([^\)]+)\) by/;
1195      my ($envelope_from) = $bug_message =~ s/\nFrom\s+([^\s]+)[^\n]+\n/\n/;
1196      if (not defined $envelope_from) {
1197           # Try to use the From: header or something to set it 
1198           ($envelope_from) = $bug_message =~ /\nFrom:\s+(.+?)\n/;
1199           # Kludgy, and should really be using a full scale header
1200           # parser to do this.
1201           $envelope_from =~ s/^.+?<([^>]+)>.+$/$1/;
1202      }
1203      my ($header,$body) = split /\n\n/, $bug_message, 2;
1204      # Add X-$gProject-PR-Message: list bug_number, package name, and bug title headers
1205      $header .= qq(\nX-$gProject-PR-Message: list $bug_number\n).
1206           qq(X-$gProject-PR-Package: $data->{package}\n).
1207                qq(X-$gProject-PR-Title: $data->{subject})
1208                if defined $data;
1209      print STDERR "Tried to loop me with $envelope_from\n"
1210           and exit 1 if $envelope_from =~ /\Q$gListDomain\E|\Q$gEmailDomain\E/;
1211      print {$debugfh} $envelope_from,qq(\n);
1212      # If we don't have a bug address, something has gone horribly wrong.
1213      print STDERR "Doesn't match: $bug_address\n" and exit 1 unless defined $bug_address;
1214      $bug_address =~ s/\@.+//;
1215      print {$debugfh} "Sending message to bugs=$bug_address\@$config{bug_subscription_domain}\n";
1216      print {$debugfh} $header.qq(\n\n).$body;
1217      send_mail_message(message        => $header.qq(\n\n).$body,
1218                        recipients     => ["bugs=$bug_address\@$config{bug_subscription_domain}"],
1219                        envelope_from  => $envelope_from,
1220                        encode_headers => 0,
1221                       );
1222      unlink("incoming/P$bug_fn") || die "unlinking incoming/P$bug_fn: $!";
1223      exit 0;
1224 }