lisp/mh-e/mh-junk.el

   1 ;;; mh-junk.el --- MH-E interface to anti-spam measures
   2
   3 ;; Copyright (C) 2003, 2004, 2005, 2006, 2007 Free Software Foundation, Inc.
   4
   5 ;; Author: Satyaki Das <satyaki@theforce.stanford.edu>,
   6 ;;         Bill Wohler <wohler@newt.com>
   7 ;; Maintainer: Bill Wohler <wohler@newt.com>
   8 ;; Keywords:  mail, spam
   9
  10 ;; This file is part of GNU Emacs.
  11
  12 ;; GNU Emacs is free software; you can redistribute it and/or modify
  13 ;; it under the terms of the GNU General Public License as published by
  14 ;; the Free Software Foundation; either version 3, or (at your option)
  15 ;; any later version.
  16
  17 ;; GNU Emacs is distributed in the hope that it will be useful,
  18 ;; but WITHOUT ANY WARRANTY; without even the implied warranty of
  19 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  20 ;; GNU General Public License for more details.
  21
  22 ;; You should have received a copy of the GNU General Public License
  23 ;; along with GNU Emacs; see the file COPYING.  If not, write to the
  24 ;; Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
  25 ;; Boston, MA 02110-1301, USA.
  26
  27 ;;; Commentary:
  28
  29 ;; Spam handling in MH-E.
  30
  31 ;;; Change Log:
  32
  33 ;;; Code:
  34
  35 (require 'mh-e)
  36 (require 'mh-scan)
  37 (mh-require-cl)
  38
  39 ;;;###mh-autoload
  40 (defun mh-junk-blacklist (range)
  41   "Blacklist RANGE as spam.
  42
  43 This command trains the spam program in use (see the option
  44 `mh-junk-program') with the content of RANGE and then handles the
  45 message(s) as specified by the option `mh-junk-disposition'.
  46
  47 Check the documentation of `mh-interactive-range' to see how RANGE is
  48 read in interactive use.
  49
  50 For more information about using your particular spam fighting
  51 program, see:
  52
  53   - `mh-spamassassin-blacklist'
  54   - `mh-bogofilter-blacklist'
  55   - `mh-spamprobe-blacklist'"
  56   (interactive (list (mh-interactive-range "Blacklist")))
  57   (let ((blacklist-func (nth 1 (assoc mh-junk-choice mh-junk-function-alist))))
  58     (unless blacklist-func
  59       (error "Customize `mh-junk-program' appropriately"))
  60     (let ((dest (cond ((null mh-junk-disposition) nil)
  61                       ((equal mh-junk-disposition "") "+")
  62                       ((eq (aref mh-junk-disposition 0) ?+)
  63                        mh-junk-disposition)
  64                       ((eq (aref mh-junk-disposition 0) ?@)
  65                        (concat mh-current-folder "/"
  66                                (substring mh-junk-disposition 1)))
  67                       (t (concat "+" mh-junk-disposition)))))
  68       (mh-iterate-on-range msg range
  69         (message "Blacklisting message %d..." msg)
  70         (funcall (symbol-function blacklist-func) msg)
  71         (message "Blacklisting message %d...done" msg)
  72         (if (not (memq msg mh-seen-list))
  73             (setq mh-seen-list (cons msg mh-seen-list)))
  74         (if dest
  75             (mh-refile-a-msg nil (intern dest))
  76           (mh-delete-a-msg nil)))
  77       (mh-next-msg))))
  78
  79 ;;;###mh-autoload
  80 (defun mh-junk-whitelist (range)
  81   "Whitelist RANGE as ham.
  82
  83 This command reclassifies the RANGE as ham if it were incorrectly
  84 classified as spam (see the option `mh-junk-program'). It then
  85 refiles the message into the \"+inbox\" folder.
  86
  87 Check the documentation of `mh-interactive-range' to see how
  88 RANGE is read in interactive use."
  89   (interactive (list (mh-interactive-range "Whitelist")))
  90   (let ((whitelist-func (nth 2 (assoc mh-junk-choice mh-junk-function-alist))))
  91     (unless whitelist-func
  92       (error "Customize `mh-junk-program' appropriately"))
  93     (mh-iterate-on-range msg range
  94       (message "Whitelisting message %d..." msg)
  95       (funcall (symbol-function whitelist-func) msg)
  96       (message "Whitelisting message %d...done" msg)
  97       (mh-refile-a-msg nil (intern mh-inbox)))
  98     (mh-next-msg)))
  99
 100 \f
 101
 102 ;; Spamassassin Interface
 103
 104 (defvar mh-spamassassin-executable (executable-find "spamassassin"))
 105 (defvar mh-sa-learn-executable (executable-find "sa-learn"))
 106
 107 ;;;###mh-autoload
 108 (defun mh-spamassassin-blacklist (msg)
 109   "Blacklist MSG with SpamAssassin.
 110
 111 SpamAssassin is one of the more popular spam filtering programs.
 112 Get it from your local distribution or from
 113 http://spamassassin.org/.
 114
 115 To use SpamAssassin, add the following recipes to
 116 \".procmailrc\":
 117
 118     PATH=$PATH:/usr/bin/mh
 119     MAILDIR=$HOME/`mhparam Path`
 120
 121     # Fight spam with SpamAssassin.
 122     :0fw
 123     | spamc
 124
 125     # Anything with a spam level of 10 or more is junked immediately.
 126     :0:
 127     * ^X-Spam-Level: ..........
 128     /dev/null
 129
 130     :0:
 131     * ^X-Spam-Status: Yes
 132     spam/.
 133
 134 If you don't use \"spamc\", use \"spamassassin -P -a\".
 135
 136 Note that one of the recipes above throws away messages with a
 137 score greater than or equal to 10. Here's how you can determine a
 138 value that works best for you.
 139
 140 First, run \"spamassassin -t\" on every mail message in your
 141 archive and use Gnumeric to verify that the average plus the
 142 standard deviation of good mail is under 5, the SpamAssassin
 143 default for \"spam\".
 144
 145 Using Gnumeric, sort the messages by score and view the messages
 146 with the highest score. Determine the score which encompasses all
 147 of your interesting messages and add a couple of points to be
 148 conservative. Add that many dots to the \"X-Spam-Level:\" header
 149 field above to send messages with that score down the drain.
 150
 151 In the example above, messages with a score of 5-9 are set aside
 152 in the \"+spam\" folder for later review. The major weakness of
 153 rules-based filters is a plethora of false positives so it is
 154 worthwhile to check.
 155
 156 If SpamAssassin classifies a message incorrectly, or is unsure,
 157 you can use the MH-E commands \\[mh-junk-blacklist] and
 158 \\[mh-junk-whitelist].
 159
 160 The command \\[mh-junk-blacklist] adds a \"blacklist_from\" entry
 161 to \"~/spamassassin/user_prefs\", deletes the message, and sends
 162 the message to the Razor, so that others might not see this spam.
 163 If the \"sa-learn\" command is available, the message is also
 164 recategorized as spam.
 165
 166 The command \\[mh-junk-whitelist] adds a \"whitelist_from\" rule
 167 to the \"~/.spamassassin/user_prefs\" file. If the \"sa-learn\"
 168 command is available, the message is also recategorized as ham.
 169
 170 Over time, you'll observe that the same host or domain occurs
 171 repeatedly in the \"blacklist_from\" entries, so you might think
 172 that you could avoid future spam by blacklisting all mail from a
 173 particular domain. The utility function
 174 `mh-spamassassin-identify-spammers' helps you do precisely that.
 175 This function displays a frequency count of the hosts and domains
 176 in the \"blacklist_from\" entries from the last blank line in
 177 \"~/.spamassassin/user_prefs\" to the end of the file. This
 178 information can be used so that you can replace multiple
 179 \"blacklist_from\" entries with a single wildcard entry such as:
 180
 181     blacklist_from *@*amazingoffersdirect2u.com
 182
 183 In versions of SpamAssassin (2.50 and on) that support a Bayesian
 184 classifier, \\[mh-junk-blacklist] uses the program \"sa-learn\"
 185 to recategorize the message as spam. Neither MH-E, nor
 186 SpamAssassin, rebuilds the database after adding words, so you
 187 will need to run \"sa-learn --rebuild\" periodically. This can be
 188 done by adding the following to your crontab:
 189
 190     0 * * * *   sa-learn --rebuild > /dev/null 2>&1"
 191   (unless mh-spamassassin-executable
 192     (error "Unable to find the spamassassin executable"))
 193   (let ((current-folder mh-current-folder)
 194         (msg-file (mh-msg-filename msg mh-current-folder))
 195         (sender))
 196     (message "Reporting message %d..." msg)
 197     (mh-truncate-log-buffer)
 198     ;; Put call-process output in log buffer if we are saving it
 199     ;; (this happens if mh-junk-background is t).
 200     (with-current-buffer mh-log-buffer
 201       (call-process mh-spamassassin-executable msg-file mh-junk-background nil
 202                     ;;"--report" "--remove-from-whitelist"
 203                     "-r" "-R")          ; spamassassin V2.20
 204     (when mh-sa-learn-executable
 205       (message "Recategorizing message %d as spam..." msg)
 206       (mh-truncate-log-buffer)
 207       (call-process mh-sa-learn-executable msg-file mh-junk-background nil
 208                     "--single" "--spam" "--local" "--no-rebuild")))
 209     (message "Blacklisting sender of message %d..." msg)
 210     (with-current-buffer (get-buffer-create mh-temp-buffer)
 211       (erase-buffer)
 212       (call-process (expand-file-name mh-scan-prog mh-progs)
 213                     nil t nil
 214                     (format "%d" msg) current-folder
 215                     "-format" "%<(mymbox{from})%|%(addr{from})%>")
 216       (goto-char (point-min))
 217       (if (search-forward-regexp "^\\(.+\\)$" nil t)
 218           (progn
 219             (setq sender (match-string 0))
 220             (mh-spamassassin-add-rule "blacklist_from" sender)
 221             (message "Blacklisting sender of message %d...done" msg))
 222         (message "Blacklisting sender of message %d...not done (from my address)" msg)))))
 223
 224 ;;;###mh-autoload
 225 (defun mh-spamassassin-whitelist (msg)
 226   "Whitelist MSG with SpamAssassin.
 227
 228 The \\[mh-junk-whitelist] command adds a \"whitelist_from\" rule to
 229 the \"~/.spamassassin/user_prefs\" file. If the \"sa-learn\" command
 230 is available, the message is also recategorized as ham.
 231
 232 See `mh-spamassassin-blacklist' for more information."
 233   (unless mh-spamassassin-executable
 234     (error "Unable to find the spamassassin executable"))
 235   (let ((msg-file (mh-msg-filename msg mh-current-folder))
 236         (show-buffer (get-buffer mh-show-buffer))
 237         from)
 238     (with-current-buffer (get-buffer-create mh-temp-buffer)
 239       (erase-buffer)
 240       (message "Removing spamassassin markup from message %d..." msg)
 241       (call-process mh-spamassassin-executable msg-file t nil
 242                     ;; "--remove-markup"
 243                     "-d")               ; spamassassin V2.20
 244       (if show-buffer
 245           (kill-buffer show-buffer))
 246       (write-file msg-file)
 247       (when mh-sa-learn-executable
 248         (message "Recategorizing message %d as ham..." msg)
 249         (mh-truncate-log-buffer)
 250         ;; Put call-process output in log buffer if we are saving it
 251         ;; (this happens if mh-junk-background is t).
 252         (with-current-buffer mh-log-buffer
 253           (call-process mh-sa-learn-executable msg-file mh-junk-background nil
 254                         "--single" "--ham" "--local" "--no-rebuild")))
 255       (message "Whitelisting sender of message %d..." msg)
 256       (setq from
 257             (car (mh-funcall-if-exists
 258                   ietf-drums-parse-address (mh-get-header-field "From:"))))
 259       (kill-buffer nil)
 260       (unless (or (null from) (equal from ""))
 261         (mh-spamassassin-add-rule "whitelist_from" from))
 262       (message "Whitelisting sender of message %d...done" msg))))
 263
 264 (defun mh-spamassassin-add-rule (rule body)
 265   "Add a new rule to \"~/.spamassassin/user_prefs\".
 266 The name of the rule is RULE and its body is BODY."
 267   (save-window-excursion
 268     (let* ((line (format "%s\t%s\n" rule body))
 269            (case-fold-search t)
 270            (file (expand-file-name "~/.spamassassin/user_prefs"))
 271            (buffer-exists (find-buffer-visiting file)))
 272       (find-file file)
 273       (if (not (search-forward (format "\n%s" line) nil t))
 274           (progn
 275             (goto-char (point-max))
 276             (insert (if (bolp) "" "\n") line)
 277             (save-buffer)))
 278       (if (not buffer-exists)
 279           (kill-buffer nil)))))
 280
 281 ;;;###mh-autoload
 282 (defun mh-spamassassin-identify-spammers ()
 283   "Identify spammers who are repeat offenders.
 284
 285 This function displays a frequency count of the hosts and domains
 286 in the \"blacklist_from\" entries from the last blank line in
 287 \"~/.spamassassin/user_prefs\" to the end of the file. This
 288 information can be used so that you can replace multiple
 289 \"blacklist_from\" entries with a single wildcard entry such as:
 290
 291     blacklist_from *@*amazingoffersdirect2u.com"
 292   (interactive)
 293   (let* ((file (expand-file-name "~/.spamassassin/user_prefs"))
 294          (domains (make-hash-table :test 'equal)))
 295     (find-file file)
 296     ;; Only consider entries between last blank line and end of file.
 297     (goto-char (1- (point-max)))
 298     (search-backward-regexp "^$")
 299     ;; Perform frequency count.
 300     (save-excursion
 301       (while (search-forward-regexp "^blacklist_from\\s-*\\(.*\\)@\\(.*\\)$"
 302                                     nil t)
 303         (let ((host (match-string 2))
 304               value)
 305           ;; Remove top-level-domain from hostname.
 306           (setq host (cdr (reverse (split-string host "\\."))))
 307           ;; Add counts for each host and domain part.
 308           (while host
 309             (setq value (gethash (car host) domains))
 310             (setf (gethash (car host) domains) (1+ (if (not value) 0 value)))
 311             (setq host (cdr host))))))
 312
 313     ;; Output
 314     (delete-other-windows)
 315     (pop-to-buffer (get-buffer-create "*MH-E Spammer Frequencies*"))
 316     (erase-buffer)
 317     (maphash '(lambda (key value) ""
 318                 (if (> value 2)
 319                     (insert (format "%s %s\n" key value))))
 320              domains)
 321     (sort-numeric-fields 2 (point-min) (point-max))
 322     (reverse-region (point-min) (point-max))
 323     (goto-char (point-min))))
 324
 325 \f
 326
 327 ;; Bogofilter Interface
 328
 329 (defvar mh-bogofilter-executable (executable-find "bogofilter"))
 330
 331 ;;;###mh-autoload
 332 (defun mh-bogofilter-blacklist (msg)
 333   "Blacklist MSG with bogofilter.
 334
 335 Bogofilter is a Bayesian spam filtering program. Get it from your
 336 local distribution or from http://bogofilter.sourceforge.net/.
 337
 338 Bogofilter is taught by running:
 339
 340     bogofilter -n < good-message
 341
 342 on every good message, and
 343
 344     bogofilter -s < spam-message
 345
 346 on every spam message. This is called a full training; three other
 347 training methods are described in the FAQ that is distributed with
 348 bogofilter. Note that most Bayesian filters need 1000 to 5000 of each
 349 type of message to start doing a good job.
 350
 351 To use bogofilter, add the following recipes to \".procmailrc\":
 352
 353     PATH=$PATH:/usr/bin/mh
 354     MAILDIR=$HOME/`mhparam Path`
 355
 356     # Fight spam with bogofilter.
 357     :0fw
 358     | bogofilter -3 -e -p
 359
 360     :0:
 361     * ^X-Bogosity: Yes, tests=bogofilter
 362     spam/.
 363
 364     :0:
 365     * ^X-Bogosity: Unsure, tests=bogofilter
 366     spam/unsure/.
 367
 368 If bogofilter classifies a message incorrectly, or is unsure, you can
 369 use the MH-E commands \\[mh-junk-blacklist] and \\[mh-junk-whitelist]
 370 to update bogofilter's training.
 371
 372 The \"Bogofilter FAQ\" suggests that you run the following
 373 occasionally to shrink the database:
 374
 375     bogoutil -d wordlist.db | bogoutil -l wordlist.db.new
 376     mv wordlist.db wordlist.db.prv
 377     mv wordlist.db.new wordlist.db
 378
 379 The \"Bogofilter tuning HOWTO\" describes how you can fine-tune Bogofilter."
 380   (unless mh-bogofilter-executable
 381     (error "Unable to find the bogofilter executable"))
 382   (let ((msg-file (mh-msg-filename msg mh-current-folder)))
 383     (mh-truncate-log-buffer)
 384     ;; Put call-process output in log buffer if we are saving it
 385     ;; (this happens if mh-junk-background is t).
 386     (with-current-buffer mh-log-buffer
 387       (call-process mh-bogofilter-executable msg-file mh-junk-background
 388                     nil "-s"))))
 389
 390 ;;;###mh-autoload
 391 (defun mh-bogofilter-whitelist (msg)
 392   "Whitelist MSG with bogofilter.
 393
 394 See `mh-bogofilter-blacklist' for more information."
 395   (unless mh-bogofilter-executable
 396     (error "Unable to find the bogofilter executable"))
 397   (let ((msg-file (mh-msg-filename msg mh-current-folder)))
 398     (mh-truncate-log-buffer)
 399     ;; Put call-process output in log buffer if we are saving it
 400     ;; (this happens if mh-junk-background is t).
 401     (with-current-buffer mh-log-buffer
 402       (call-process mh-bogofilter-executable msg-file mh-junk-background
 403                     nil "-n"))))
 404
 405 \f
 406
 407 ;; Spamprobe Interface
 408
 409 (defvar mh-spamprobe-executable (executable-find "spamprobe"))
 410
 411 ;;;###mh-autoload
 412 (defun mh-spamprobe-blacklist (msg)
 413   "Blacklist MSG with SpamProbe.
 414
 415 SpamProbe is a Bayesian spam filtering program. Get it from your local
 416 distribution or from http://spamprobe.sourceforge.net.
 417
 418 To use SpamProbe, add the following recipes to \".procmailrc\":
 419
 420     PATH=$PATH:/usr/bin/mh
 421     MAILDIR=$HOME/`mhparam Path`
 422
 423     # Fight spam with SpamProbe.
 424     :0
 425     SCORE=| spamprobe receive
 426
 427     :0 wf
 428     | formail -I \"X-SpamProbe: $SCORE\"
 429
 430     :0:
 431     *^X-SpamProbe: SPAM
 432     spam/.
 433
 434 If SpamProbe classifies a message incorrectly, you can use the
 435 MH-E commands \\[mh-junk-blacklist] and \\[mh-junk-whitelist] to
 436 update SpamProbe's training."
 437   (unless mh-spamprobe-executable
 438     (error "Unable to find the spamprobe executable"))
 439   (let ((msg-file (mh-msg-filename msg mh-current-folder)))
 440     (mh-truncate-log-buffer)
 441     ;; Put call-process output in log buffer if we are saving it
 442     ;; (this happens if mh-junk-background is t).
 443     (with-current-buffer mh-log-buffer
 444       (call-process mh-spamprobe-executable msg-file mh-junk-background
 445                     nil "spam"))))
 446
 447 ;;;###mh-autoload
 448 (defun mh-spamprobe-whitelist (msg)
 449   "Whitelist MSG with SpamProbe.
 450
 451 See `mh-spamprobe-blacklist' for more information."
 452   (unless mh-spamprobe-executable
 453     (error "Unable to find the spamprobe executable"))
 454   (let ((msg-file (mh-msg-filename msg mh-current-folder)))
 455     (mh-truncate-log-buffer)
 456     ;; Put call-process output in log buffer if we are saving it
 457     ;; (this happens if mh-junk-background is t).
 458     (with-current-buffer mh-log-buffer
 459       (call-process mh-spamprobe-executable msg-file mh-junk-background
 460                     nil "good"))))
 461
 462 (provide 'mh-junk)
 463
 464 ;; Local Variables:
 465 ;; indent-tabs-mode: nil
 466 ;; sentence-end-double-space: nil
 467 ;; End:
 468
 469 ;; arch-tag: 603335f1-77ff-4306-8828-5d3dad51abe1
 470 ;;; mh-junk.el ends here