#!/usr/local/bin/perl -w # # # Original Version # ---------------- # http://www.ayradyss.org/programs/source/smreject.html # # # use diagnostics; #use strict; use Sys::Hostname; ## # smreject.pl - Processes rejected message entries from the sendmail logfile. ## # # usage: smreject [-h] [-d] [-t top#] [-j] [-m addr] [-s] [-u] [files] # # -t n list top n hosts or addresses for each ruleset. # n = 0 for no detail by host or email address. # # -j use the "junk file" (access.db source) to provide further information. # # -m addr mail the output to the given address # # -d enable debug mode # With this you can see which ruleset is recognized and what arguments are # extracted from the log. Especially useful for reporting bugs: # Send me an excerpt from the log and I'll fix smreject. # # -h enable HTML-output # # -s print date stamps -- not recommended # # -u keep a tally of unknown rejection reasons # # Ver 1.5.3 by Igor S. Livshits # Fixed minor cosmetic issues # Repaired some real brain-dead typographical errors # # Ver 1.5.2 by Scott Weikart # Increased debugging for bad entries # # Ver 1.5.1 by Igor S. Livshits # RSS added # Support for LOCALUSER checks added (check_local) # Support for "Message-Id must resolve" added (check_local) # Minor code clean up # Expanded findinjunk() # # Ver 1.5.0 by Igor S. Livshits # Use Getopt::Long instead of getopts (mostly for future flexibility) # New usage: smreject [-h] [-d] [-t top#] [-j] [-m addr] [-s] [files] # New option to print date ranges [-s] # Killed the log file option [-f]; all non-option strings are now file names to process # Also accept multiple files so you can parse several logs as one # # Ver 1.4.3 by Ralf Hildebrandt # DUL added # # Ver 1.4.2 by Ralf Hildebrandt # Corrected error that left old data in $from when a new ruleset was found # re-initialized variables to "" instead of 0. # # Ver 1.4.1 by Ralf Hildebrandt # ORBS added # # Ver 1.4 by Ralf Hildebrandt # major overhaul by Brian J. 'Big Stains' Coan # September 24th, 1998 # # $Log: smreject.pl,v $ # Revision 1.4 1998/09/23 21:59:01 brian # incorporated my changes into the new 1.3.2 release, which i got from # Ralf after i sent him my diffs. changes still relevant are: # * "-u" option, for spitting out unknown syslog entries we are unable # to analyse. # * allow mailto to contain '!' character. use hostname() # and report which host this is from. # * use a hash %junk instead of the array @junk. # * filter out comment lines from the spammers db. # * extract the arg2. more precisely match on arg1 and from fields. # seems to me check_rcpt should use arg1 for relevant arg also, not # the from arg. # * when the ruleset is check_relay, use arg2 as the # alternate key instead of the relay. # * my findinjunk() is more powerful. # # Revision 1.2 1998/09/22 18:47:01 brian # lots of work to adapt to our system. properly scan our spammers database. # added an option "-u" to spit out any ruleset= lines that the program can't # analise. load spammers database into a hash, not an array. also parse # out the syslog's "arg2" entry, maybe this didn't even exist when Ralf # wrote the program. he didn't seem to know about check_relay and # check_rcpt rulesets. overhauled findinjunk() routine so it would look # at matchine the sender's domain if given an email address; it would # look at higher levels of a domain; and it would look at higer levels # of an IP number. added all our sendmail.cf error strings - it would be # better to not have to hard-code these, but i'm not anxious to overhaul # that much of the program. # # Ver 1.3.3 Ralf Hildebrandt (11/09/1998) # * Igor S. Livshits supplied code that makes the use # of an external grep unnecessary. Neat! # * Typo in Claus's name corrected. # # Ver 1.3.2 Ralf Hildebrandt (10/09/1998) # * Igor S. Livshits complained that Claus Assmann's rulesets # are not being recognized. I'm starting to fix that step by # step! # * smreject doesn't die but barfs when it encounters a new # ruleset. Neat! # # Ver 1.3.1 Ralf Hildebrandt (31/08/1998) # * Rejections based upon relay were not recognized. # # Ver 1.3.0 Ralf Hildebrandt (12/08/1998) # * Ruleset check_mail was not recognized. # * changed the continuing on last ruleset part a little: # (gethostbyaddr lines were not parsed correctly). # # Ver 1.2.5 Ralf Hildebrandt (16/07/1998) # * Some variables were not initialized - fixed # * changed the continuing on last ruleset part a little. # # Ver 1.2.4 Ralf Hildebrandt (13/07/1998) # * Rejections due to the RBL were recognized & counted, but not displayed :( # # Ver 1.2.3 Ralf Hildebrandt (26/06/1998) # * Found a error in the F.A. that parses the logfile. Lines containing # "lost connection" were not properly recognized and made smreject choke. # * Added a lot of errors that may not have been properly recognized. # # Ver 1.2.2 Ralf Hildebrandt (22/06/1998) # * Made the matching operators in findinjunk() case-insensitive # # Ver 1.2.1 Ralf Hildebrandt (14/06/1998) # * Tweaked the output a bit. # * Still have find out what to do with rejection of a message # due to multiple rulesets. At the moment the rejection is only counted # once (good!), but only the last ruleset is evaluated (bad!) # # Ver 1.2 Ralf Hildebrandt # # * Have completely rewritten the following (30/05/1998): # - Logfileparser (some sort of F.A. now) # - Output routine (made that a function, gives us the From: now) # - Regexp'es for matching the lines of the logfile (now match correctly) # - other_reasons() renamed to find_reason() and cleaned/sped up. # # Ver 1.13 Ralf Hildebrandt # * Sendmail 8.9 has come... # * Added support for all new rulesets. # * Wrote the findinjunk() routine -- it replaces findhostname(), # findemail() and findIP() (which was not used anyway...) # * Fixed the usage line. # * Made use of junk file (/etc/mail/access) default. # Important: Only entries NOT ending with OK or RELAY are junk entries! # * Changed the routines for counting junk file entries # by splitting them into two counters, one found_in_junk # and one find_reason (e.g. RBL, no DNS etc.) # * Added debug mode # # Ver 1.12a (unofficial, Anne Bennett ) # Make "junk" optional; turn on iff opt_j. (Oct 22, 1997) # Initialize variables to avoid perl complaints if # certain counters are never incremented. (Oct 22, 1997) # Comment out never-used assignments. (Oct 22, 1997) # Highlight "configuration section". (Oct 22, 1997) # Initialize $top if not set with "-t". (Oct 22, 1997) # Added an option to change the logfile used. (Oct 22, 1997) # Print nothing if no rejects were found. (Oct 22, 1997) # Added option to mail output. (Oct 23, 1997) # # Ver 1.12 by Ted George (tgeorge@kcnet.com) # Count for junk file rejections included check_mail # errors. (Sep 24, 1997) # # Ver 1.11 Check_mail rejections from domain names were # not properly matched. (Sep 18, 1997) # # Ver 1.1 Cleaned up output slightly. Minor fixes including # the unknown category for entries that don't match # any items in junk file. (Sep 10, 1997) # # Ver 1.0 Original release (Sep 5, 1997) # # ======================================================= # change these locations to match your system # # location of sendmail log file @logfiles = ("/var/log/maillog"); #$logfile = "/var/log/syslog"; # location of the plaintext (!) access file (not the access(.db|.dbm|.hash) !) $junk = "/etc/mail/access"; #$junk = "/etc/mail/junk"; #$junk = "/etc/apc/sendmail.data/spammers"; # mail program which accepts "-s subject", address on command line, # and message on standard input. $mailprog = "mailx"; # $mailprog = "mutt"; # $mailprog = "Mail"; # path, used to call the $mailprog # $ENV{PATH} = "/users2/local/bin:/usr/local/bin:/usr/bin:/bin"; # ======================================================= # initialize some variables $check_received = 0; $check_from = 0; $check_message_id = 0; $check_rcpt = 0; $check_mail = 0; $check_relay = 0; $found_in_junk = 0; $other_reasons = 0; $mail_errors = 0; # parse command-line options use Getopt::Long; $top = 10; GetOptions("j:s" => \$dojunk, "m=s" => \$mailto, "t=i" => \$top, "d" => \$debug, "h" => \$html, "u" => \$unknown, "s" => \$printdates, ); $dojunk= 1 if (defined($dojunk) && !$dojunk); @logfiles= @ARGV if @ARGV; print "DEBUG!\n" if ($debug); $host = hostname(); if ($mailto) { # Sanitize "mailto" to satisfy "-T": address should contain # nothing but "word" characters (alphabetics, numerics, and # underscores), a hyphen, an at sign, a comma, or a dot. if ($mailto =~ /^([-\@!\w,.]+)$/) { $mailto = $1; } else { die "Bad characters in mail address: $mailto"; } $real_stdout = select(); open(MAILIT, "| $mailprog -s 'Sendmail Rejected Messages for $host' $mailto") or die("Cannot fork $mailprog"); select(MAILIT); } if ($dojunk) { %junk = (); # clear the hash before use open(JUNK, $junk) || die "Cannot access specified junk file [$junk]."; while(){ next if m/^\#/; # skip comment lines next if (/\b(OK|RELAY|ALLOWED)$/);# skip lines which allow access warn "bad entry on line $. of $junk:\n $_", next unless m/^(\S+)\s+(\S+)/; $junk{lc($1)} = $2; $junkcnt++ if $debug; } close(JUNK); # now works with custom error messages; Thanks to Jan Krueger # Code perl-ified; Thanks to Igor S. Livshits print "Loaded $junkcnt entries from $junk\n" if $debug; } $id = 0; $old_id = 1; $arg1 = ''; $relay = ''; $reject = ''; $next = 0; $earliest = 0; # the earliest file modification time (now) $latest = 50000; # the latest file modification time (100ya) $mostrecent = 1; # the first files is always most recent foreach $logfile (@logfiles) { stat($logfile); unless (-T _) { # skip non-text files print "Skipping improper file reference <$logfile>.\n" if $debug; next; } if (-M _ < $latest) { # found a later file than previous youngest $latest= -M _; $mostrecent= 1; print "New latest file is $latest days old.\n" if $debug; } else { # do not collect last date candidates $mostrecent= 0; # for this older file } if (-M _ > $earliest) { # found an earlier file than previous youngest $earliest= -M _; $first= ""; # we likely found an earlier file since last print "New earliest file is $earliest days old.\n" if $debug; } unless (open(INPUT, "$logfile")) { print "Can't open logfile [$logfile]...skipping.\n"; next; } while () { $first = $_ if ($printdates && !$first); $last = $_ if ($printdates && $mostrecent); $id = $1 if (/sendmail\[\d+\]: (\D{3}\d{5})/); if (/ruleset=/) { print "\nRuleset found!\n" if ($debug); $old_id = $id; # Changed this! Now we match everything! (I hope...) $ruleset = $1 if (/ruleset=(.+), arg1=/); if (/relay=/) { $arg1 = $1 if (/arg1=(.+), relay=/); } else { $arg1 = $1 if (/arg1=(.+),/); } if (/arg2=([^,]*)/) { $arg2 = $1; $next = 0; } else { $arg2 = ""; $next = 1; } $relay = $1 if (/relay=(.+), reject=/); $reject = $1 if (/reject=(.+)/); if ($debug) { print "Ruleset:\t",$ruleset,"\n", "Arg1:\t\t",$arg1,"\n", "Arg2:\t\t",$arg2,"\n", "Relay:\t\t",$relay,"\n", "Reject:\t\t",$reject,"\n"; } } elsif (($old_id eq $id) && $next) { print "...continuing with previous ruleset!\n" if ($debug); $next = 0; $from = $1 if ( /from=(.*?), size=/ ); $relay = $1 if ( /relay=(.+ \[.*\])/ ); if ($debug) { print "From:\t\t",$from,"\n"; print "Relay:\t\t",$relay,"\n"; } $next = 1 if ( /lost input channel/); $next = 1 if ( /gethostbyaddr/ ); # Just in case we found "lost input channel" or "gethostbyaddr" instead of "from=" } else { $ruleset = ""; $old_id = ""; $relay = ""; $arg1 = ""; $arg2 = ""; $next = ""; $from = ""; next; } # Now we have gathered all the data we need. # if (!$next) { $$ruleset++; # I wouldn't have believed that this works... if ($ruleset =~ /check_(received|message_id|from|mail|relay)/) { # added mail 12/08/98 # added relay 10/09/98 # Look for $arg1 in junk $relevant = $arg1; } elsif ($ruleset =~ /check_rcpt/) { # Look for $from in junk $relevant = $from; } else { print "Unknown ruleset $ruleset found.\n"; } if ($ruleset =~ m/check_relay/) { $reason = find_reason($relevant, $arg2, $reject); } else { $reason = find_reason($relevant, $relay, $reject); } if ($unknown and $reason =~ /^unknown$/) { print "Unprogrammed Unknown Rejections:\n" if $mail_errors == 1; print; } $name = "\$count_" . $ruleset; if ($ruleset =~ /check_(rcpt|from|mail|relay)/) { $$name{$relevant}++; # For rulesets check_rcpt, check_mail and check_from we only want # the $relevant to be displayed; that is sufficient! # added mail 12/08/98 } else { $$name{$relevant . "\n\tFrom: " . $from}++; # For the other rulesets we want the $from also. } $name = "\$count_relay_" . $ruleset; $$name{$relay}++; } } } close(INPUT); $totreject = $check_from + $check_rcpt + $check_received + $check_message_id + $check_relay + $check_mail; print "\nTotal rejections: $totreject\n\n" if $debug; if ( $totreject > 0 ) { if ($printdates) { $from = "unknown"; $to = "unknown"; $from = $1 if $first =~ /^(.{7}\d\d:\d\d:\d\d )/; $to = $1 if $last =~ /^(.{7}\d\d:\d\d:\d\d )/; } if ($html) { print "\nSendmail Rejected Messages for $host"; print ": $from - $to" if $printdates; print "\n"; } print "\n"; print "

" if ($html); print "Sendmail Rejected Messages for $host"; print " $from - $to" if $printdates; print "

" if ($html); print "\n"; if ($html) { print "

\n"; } else { print "-----------------------------------------------------------------\n"; } print "

" if ($html); print "$totreject rejections processed"; print "

" if ($html); print "\n"; if (($dojunk) && ($found_in_junk > 0)) { if ($html) { print "\n\n"; } else { print "\n*** $found_in_junk Rejections due to $junk data file entry ***\n\n"; } foreach (keys %junk) { &prjunk_entry($_); } print "
$found_in_junk Rejections due to $junk data file entry

\n" if ($html); } if ($other_reasons > 0) { if ($html) { print "\n\n" if ($html); } else { print "\n*** $other_reasons Rejections due to these reasons ***\n\n"; } &prjunk_entry("Sender banned"); &prjunk_entry("Domain banned"); &prjunk_entry("Network banned"); &prjunk_entry("Invalid Account Specified"); &prjunk_entry("Poorly Specified Recipient"); &prjunk_entry("DNS failure. Host must resolve"); &prjunk_entry("Host must be in Domain format"); &prjunk_entry("Relaying Denied"); &prjunk_entry("Bad Message-Id"); &prjunk_entry("Message-Id must resolve"); &prjunk_entry("DNS failure. Client must resolve"); &prjunk_entry("Client must resolve"); &prjunk_entry("RBL Entry"); &prjunk_entry("ORBS Entry"); &prjunk_entry("DUL Entry"); &prjunk_entry("RSS Entry"); &prjunk_entry("Sender domain must resolve"); &prjunk_entry("Sender domain must exist"); &prjunk_entry("Sender not valid"); &prjunk_entry("List:; syntax illegal for recipient addresses"); &prjunk_entry("User address required"); &prjunk_entry("Colon illegal in host name part"); &prjunk_entry("Invalid host name"); &prjunk_entry("User address required"); &prjunk_entry("User has moved"); &prjunk_entry("Real domain name required"); &prjunk_entry("Domain name required"); &prjunk_entry("Relaying denied"); &prjunk_entry("unknown"); print "
$other_reasons Rejections due to other reasons

\n" if ($html); } print_stats("check_message_id"); print_stats("check_from"); print_stats("check_received"); print_stats("check_rcpt"); print_stats("check_mail"); print_stats("check_relay"); print "\n" if ($html); if ($mailto) { select($real_stdout); close(MAILIT); } } sub sortbyvalues { my(@data); my(@datakeys); while (($key, $value) = each(%tmp)) { if ($html) { $key =~ s/\/\>\;/g; push(@data, sprintf("%6d%-s\n", $value, $key)); } else { push(@data, sprintf("%6d %-s\n", $value, $key)); } push(@datakeys, sprintf("%6d", $value)); } @data = @data[reverse sort { $datakeys[$a] <=> $datakeys[$b]; } $[..$#data]; $#data = $top - 1 if ($#data >= $top); return @data; } sub findinjunk { my($keys) = shift; my($key); foreach $key (split(/[\[\]<> ]/, $keys)) { next unless $key; $key = lc($key); return $key if exists($junk{$key}); if ($key=~ /.*@(.*)$/) { $key = $1; return $key if exists($junk{$key}); } if ($key=~ /.*\.([0-9]+)$/) { while ($key=~ /(.*)\.[^.]*$/) { $key = $1; return $key if exists($junk{$key}); } } else { while ($key=~ /[^.]*\.(.*)$/) { $key = $1; return $key if exists($junk{$key}); } } } } sub prjunk_entry { my($name) = shift; if ($count_junk{$name}) { if ($html) { printf("%6d%-s\n", $count_junk{$name}, $name) ; } else { printf("%6d %-s\n", $count_junk{$name}, $name); } } } sub print_stats { my($ruleset) = shift; my($name); local %tmp; if ($$ruleset > 0) { if ($html) { print "

$$ruleset rejections by ruleset $ruleset

\n"; } else { if ($ruleset =~ /check_(rcpt|from|relay)/) { print "\n*** $$ruleset rejections by ruleset $ruleset ***\n" ; } else { print "\n*** $$ruleset rejections by ruleset $ruleset ***\n" ; } } if ($top > 0) { if ($html) { if ($ruleset =~ /check_(rcpt|from|mail)/) { print "\n\n"; } elsif ($ruleset !~ /check_relay/) { print "
Top $top $ruleset rejections by relevant argument
\n\n"; } } else { if ($ruleset =~ /check_(rcpt|from|mail)/) { print "\nTop $top $ruleset rejections by relevant argument:\n\n"; } elsif ($ruleset !~ /check_relay/) { print "\nTop $top $ruleset rejections by relevant argument and From: :\n\n"; } } $name = "\$count_" . $ruleset; %tmp = %$name; if ($ruleset !~ /check_relay/) { print &sortbyvalues(); print "
Top $top $ruleset rejections by relevant argument and From:

\n" if ($html); } if ($html) { print "\n\n"; } else { print "\nTop $top $ruleset rejections by relay host:\n\n"; } $name = "\$count_relay_" . $ruleset; %tmp = %$name; print &sortbyvalues(); print "
Top $top $ruleset rejections by relay host

\n" if ($html); } print "\n"; } } sub find_reason { my($key) = shift; my($key2) = shift; my($reject) = shift; my($junk_entry); $junk_entry = "Invalid Account Specified" if ($reject =~ /Invalid Account Specified/); $junk_entry = "Poorly Specified Recipient" if ($reject =~ /user address required/); $junk_entry = "DNS failure. Host must resolve" if ($reject =~ /unresolvable/); $junk_entry = "Host must be in Domain format" if ($reject =~ / invalid host /); $junk_entry = "Relaying Denied" if ($reject =~ /Relaying Denied/); $junk_entry = "Relaying Denied" if ($reject =~ /Change your SMTP setting to your local SMTP server/); $junk_entry = "Bad Message-Id" if ($reject =~ /Bad Message-Id/); $junk_entry = "Message-Id must resolve" if ($reject =~ /Message-Id must resolve/); $junk_entry = "DNS failure. Client must resolve" if ($reject =~ /DNS failure/); $junk_entry = "Client must resolve" if ($reject =~ /Client must resolve/); $junk_entry = "RBL Entry" if ($reject =~ /maps.vix.com/); $junk_entry = "ORBS Entry" if ($reject =~ /www.orbs.org/); $junk_entry = "DUL Entry" if ($reject =~ /www.orca.bc.ca/); $junk_entry = "RSS Entry" if ($reject =~ /www.mail-abuse.org\/rss\//); $junk_entry = "Sender domain must resolve" if ($reject =~ /Sender domain must resolve/); $junk_entry = "Sender domain must exist" if ($reject =~ /Sender domain must exist/); $junk_entry = "Sender not valid" if ($reject =~ /Sender not valid/); $junk_entry = "List:; syntax illegal for recipient addresses" if ($reject =~ /recipient addresses/); $junk_entry = "User address required" if ($reject =~ /User address required/); $junk_entry = "Colon illegal in host name part" if ($reject =~ /Colon illegal in host name part/); $junk_entry = "Invalid host name" if ($reject =~ /Invalid host name/); $junk_entry = "User address required" if ($reject =~ /User address required/); $junk_entry = "User has moved" if ($reject =~ /User has moved/); $junk_entry = "Real domain name required" if ($reject =~ /Real domain name required/); $junk_entry = "Domain name required" if ($reject =~ /Domain name required/); $junk_entry = "Relaying denied" if ($reject =~ /Relaying denied/); if ($junk_entry) { $other_reasons++; } else { $junk_entry = &findinjunk($key); $junk_entry = &findinjunk($key2) unless $junk_entry; if ($junk_entry) { $found_in_junk++; } else { $junk_entry = "unknown"; # these were probably in spammers earlier in the day $junk_entry = "Domain banned" if ($reject =~ /\sdomain\s/); $junk_entry = "Host banned" if ($reject =~ /\syou\s/); $junk_entry = "Sender banned" if ($reject =~ /\syou\s/ and $key =~ /\@/); $junk_entry = "Network banned" if ($reject =~ /network/ and $key =~ /^([0-9]+\.?){1,4}$/); unless ($junk_entry) { $junk_entry = "unknown"; $other_reasons++; $mail_errors++; } if ($debug) { print "Key: ",$key,"\n"; print "Reject: ", $reject,"\n"; } } } $count_junk{$junk_entry}++; if ($debug) { print "Reason:\t\t", $junk_entry, " (", $count_junk{$junk_entry}, ")\n\n"; } $junk_entry; }