X-Git-Url: https://git.donarmstrong.com/?a=blobdiff_plain;f=bin%2Fdebbugs-spam;h=ba026da4547af6a1ac3b79d07df1e9af9c5b940f;hb=64267ddd252ea24e5aeb4bb999a22a1b1f0f2d85;hp=371fc572d8d3c99807d06b1f3e02f8ea8d6a8e23;hpb=d423d5f7230c43996c17f74e739c18e63f6ac2d1;p=debbugs.git diff --git a/bin/debbugs-spam b/bin/debbugs-spam index 371fc57..ba026da 100755 --- a/bin/debbugs-spam +++ b/bin/debbugs-spam @@ -62,6 +62,12 @@ ham. Output the score of all of the messages in a bug +=over + +=item B<--skip-seen> Skip messages which have previously been classified + +=back + =item B Mark messages as spam if there is a regex match to subject or message @@ -72,11 +78,25 @@ id Mark messages as ham if there is a regex match to subject or message id +=item B + +Learn from messages which are ham/spam + =back =head1 EXAMPLES +Start spamd: + + /usr/sbin/spamd --socketpath=/home/debbugs/spamd_socket \ + --nouser-config --cf='include /home/debbugs/.spamassassin/user_prefs' \ + --cf='allow_user_rules 1' --allow-tell; + +Then score bugs: + + debbugs-spam --spamc-opts '-U' --spamc-opts '/home/debbugs/spamd_socket' \ + score 859123; =cut @@ -112,15 +132,20 @@ handle_main_arguments(\%options, my %subcommands = ('auto-scan' => {function => \&auto_spamscan, - arguments => {'ham_threshold=s' => -5, + arguments => {'ham_threshold|ham-threshold=s' => 0, }, + defaults => {ham_threshold => -5}, }, 'score' => {function => \&score_bug, + arguments => {'skip_seen|skip-seen!' => 0 + }, }, 'mark-spam' => {function => \&mark_spam, }, 'mark-ham' => {function => \&mark_ham, }, + 'learn' => {fuction => \&learn, + }, 'help' => {function => sub {pod2usage({verbose => 2});}} ); @@ -151,7 +176,10 @@ if ($subcommand ne 'help') { chdir($config{spool_dir}) or die "chdir $config{spool_dir} failed: $!"; } my $opts = - handle_subcommand_arguments(\@ARGV,$subcommands{$subcommand}{arguments}); + handle_subcommand_arguments(\@ARGV, + $subcommands{$subcommand}{arguments}, + $subcommands{$subcommand}{defaults}, + ); $subcommands{$subcommand}{function}->(\%options,$opts,\%config,\@ARGV); @@ -180,11 +208,49 @@ sub mark_it { if ($mid =~ /\Q$regex\E/) { $is_match = 1; } - if ($spam_ham eq 'spam') { - $spam->add_spam($mid); + if ($is_match) { + print STDERR "it's a match" if $DEBUG; + if ($spam_ham eq 'spam') { + $spam->add_spam($mid); + } else { + $spam->add_ham($mid); + } + } + }, + $bug_num + ); + $spam->save(); + } +} + +sub learn { + my ($options,$opts,$config,$argv) = @_; + for my $bug_num (@{$argv}) { + my $spam = Debbugs::Log::Spam->new(bug_num => $bug_num) or + die "Unable to open bug log spam for $bug_num"; + foreachmsg(sub { + my ($bn,$rec,$mid) = @_; + my $score; + if ($spam->is_spam($mid)) { + $score //= + spam_score($rec,$options->{spamc}, + [@{$options->{spamc_opts}}, + '-L','spam' + ] + ); + print STDERR "learning spam" if $DEBUG; + } elsif ($spam->is_ham($mid)) { + $score //= + spam_score($rec,$options->{spamc}, + [@{$options->{spamc_opts}}, + '-L','ham' + ] + ); + print STDERR "learning ham" if $DEBUG; } else { - $spam->add_ham($mid); + print STDERR "not learning" if $DEBUG; } + print STDERR " from $mid" if $DEBUG; }, $bug_num ); @@ -197,8 +263,10 @@ sub score_bug { for my $bug_num (@{$argv}) { my @bug_score = spam_score_bug($bug_num, - $opts->{spamc}, - $opts->{spamc_opts}); + $options->{spamc}, + $options->{spamc_opts}, + $opts->{skip_seen}, + ); print "$_->{score} $_->{message_id} $_->{subject}\n" foreach @bug_score; } @@ -228,7 +296,7 @@ sub auto_spamscan { if ($is_spam) { print STDERR "it's spam ($score)\n" if $DEBUG; $spam->add_spam($mid); - } elsif ($score < $options->{ham_threshold}) { + } elsif ($score < $opts->{ham_threshold}) { print STDERR "it's really ham ($score)\n" if $DEBUG; $spam->add_ham($mid); } @@ -243,17 +311,31 @@ sub auto_spamscan { } sub spam_score_bug { - my ($bug,$spamc,$spamc_opts) = @_; + my ($bug,$spamc,$spamc_opts,$skip_seen) = @_; + my $spam; + if ($skip_seen) { + $spam = Debbugs::Log::Spam->new(bug_num => $bug) or + die "Unable to open bug log spam for $bug"; + } my @records; foreachmsg(sub { my ($bn,$rec,$mid) = @_; - my $score = + my $score; + if ($skip_seen) { + if ($spam->is_spam($mid)) { + $score = 999; + } elsif ($spam->is_ham($mid)) { + $score = -999; + } + } + $score //= spam_score($rec,$spamc,$spamc_opts); + my ($subject) = $rec->{text} =~ /^Subject: *(.+)$/mi; push @records, {message_id => $mid, score => $score, - subject => ($rec->{text} =~ /^Subject: *(.+)/i)[0], + subject => $subject, }; }, $bug @@ -278,25 +360,25 @@ sub spam_score { if (not $childpid) { die "Unable to fork spamc"; } - print {$spamc} $record->{text}; - close($spamc) or die "Unable to close spamc: $!"; + print {$spamc_in} $record->{text}; + close($spamc_in) or die "Unable to close spamc_in: $!"; waitpid($childpid,0); - if ($DEBUG) { - print STDERR "[$?;".($? >> 8)."] "; - print STDERR map {s/\n//; $_ } <$spamc_out>; - print STDERR " "; - } - close($spamc_out); - $SIG{"PIPE"} = $old_sig; if ($? >> 8) { $is_spam = 1; } my ($first_line,@report) = <$spamc_out>; + if ($DEBUG) { + print STDERR "[$?;".($? >> 8)."] "; + print STDERR $first_line,@report; + print STDERR " "; + } if (defined $first_line) { chomp $first_line; ($score,$threshold) = $first_line =~ m{^(-?[\d\.]+)/(-?[\d\.]+)$}; $report = join('',@report); } + close($spamc_out); + $SIG{"PIPE"} = $old_sig; }; if ($@) { carp "processing of message failed [$@]\n"; @@ -316,17 +398,18 @@ sub foreachmsg { my ($msg_id) = record_regex($record, qr/^Message-Id:\s+<(.+)>/mi); next unless defined $msg_id; + print STDERR "examining $msg_id: " if $DEBUG; if ($msg_id =~ /$config{email_domain}$/) { - print STDERR "skipping $msg_id\n" if $DEBUG; + print STDERR "skipping\n" if $DEBUG; next; } - print STDERR "examining $msg_id: " if $DEBUG; if ($seen_msgids{$msg_id}) { print STDERR "already seen\n" if $DEBUG; next; } $seen_msgids{$msg_id}=1; $sub->($bug_num,$record,$msg_id); + print STDERR "\n" if $DEBUG; } }