lisp/mh-e/mh-junk.el

   1 ;;; mh-junk.el --- MH-E interface to anti-spam measures
   2
   3 ;; Copyright (C) 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011
   4 ;;   Free Software Foundation, Inc.
   5
   6 ;; Author: Satyaki Das <satyaki@theforce.stanford.edu>,
   7 ;;         Bill Wohler <wohler@newt.com>
   8 ;; Maintainer: Bill Wohler <wohler@newt.com>
   9 ;; Keywords:  mail, spam
  10
  11 ;; This file is part of GNU Emacs.
  12
  13 ;; GNU Emacs is free software: you can redistribute it and/or modify
  14 ;; it under the terms of the GNU General Public License as published by
  15 ;; the Free Software Foundation, either version 3 of the License, or
  16 ;; (at your option) any later version.
  17
  18 ;; GNU Emacs is distributed in the hope that it will be useful,
  19 ;; but WITHOUT ANY WARRANTY; without even the implied warranty of
  20 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  21 ;; GNU General Public License for more details.
  22
  23 ;; You should have received a copy of the GNU General Public License
  24 ;; along with GNU Emacs.  If not, see <http://www.gnu.org/licenses/>.
  25
  26 ;;; Commentary:
  27
  28 ;; Spam handling in MH-E.
  29
  30 ;;; Change Log:
  31
  32 ;;; Code:
  33
  34 (require 'mh-e)
  35 (require 'mh-scan)
  36 (mh-require-cl)
  37
  38 ;;;###mh-autoload
  39 (defun mh-junk-blacklist (range)
  40   "Blacklist RANGE as spam.
  41
  42 This command trains the spam program in use (see the option
  43 `mh-junk-program') with the content of RANGE and then handles the
  44 message(s) as specified by the option `mh-junk-disposition'.
  45
  46 Check the documentation of `mh-interactive-range' to see how RANGE is
  47 read in interactive use.
  48
  49 For more information about using your particular spam fighting
  50 program, see:
  51
  52   - `mh-spamassassin-blacklist'
  53   - `mh-bogofilter-blacklist'
  54   - `mh-spamprobe-blacklist'"
  55   (interactive (list (mh-interactive-range "Blacklist")))
  56   (let ((blacklist-func (nth 1 (assoc mh-junk-choice mh-junk-function-alist))))
  57     (unless blacklist-func
  58       (error "Customize `mh-junk-program' appropriately"))
  59     (let ((dest (cond ((null mh-junk-disposition) nil)
  60                       ((equal mh-junk-disposition "") "+")
  61                       ((eq (aref mh-junk-disposition 0) ?+)
  62                        mh-junk-disposition)
  63                       ((eq (aref mh-junk-disposition 0) ?@)
  64                        (concat mh-current-folder "/"
  65                                (substring mh-junk-disposition 1)))
  66                       (t (concat "+" mh-junk-disposition)))))
  67       (mh-iterate-on-range msg range
  68         (message "Blacklisting message %d..." msg)
  69         (funcall (symbol-function blacklist-func) msg)
  70         (message "Blacklisting message %d...done" msg)
  71         (if (not (memq msg mh-seen-list))
  72             (setq mh-seen-list (cons msg mh-seen-list)))
  73         (if dest
  74             (mh-refile-a-msg nil (intern dest))
  75           (mh-delete-a-msg nil)))
  76       (mh-next-msg))))
  77
  78 ;;;###mh-autoload
  79 (defun mh-junk-whitelist (range)
  80   "Whitelist RANGE as ham.
  81
  82 This command reclassifies the RANGE as ham if it were incorrectly
  83 classified as spam (see the option `mh-junk-program'). It then
  84 refiles the message into the \"+inbox\" folder.
  85
  86 Check the documentation of `mh-interactive-range' to see how
  87 RANGE is read in interactive use."
  88   (interactive (list (mh-interactive-range "Whitelist")))
  89   (let ((whitelist-func (nth 2 (assoc mh-junk-choice mh-junk-function-alist))))
  90     (unless whitelist-func
  91       (error "Customize `mh-junk-program' appropriately"))
  92     (mh-iterate-on-range msg range
  93       (message "Whitelisting message %d..." msg)
  94       (funcall (symbol-function whitelist-func) msg)
  95       (message "Whitelisting message %d...done" msg)
  96       (mh-refile-a-msg nil (intern mh-inbox)))
  97     (mh-next-msg)))
  98
  99 \f
 100
 101 ;; Spamassassin Interface
 102
 103 (defvar mh-spamassassin-executable (executable-find "spamassassin"))
 104 (defvar mh-sa-learn-executable (executable-find "sa-learn"))
 105
 106 ;;;###mh-autoload
 107 (defun mh-spamassassin-blacklist (msg)
 108   "Blacklist MSG with SpamAssassin.
 109
 110 SpamAssassin is one of the more popular spam filtering programs.
 111 Get it from your local distribution or from the SpamAssassin web
 112 site at URL `http://spamassassin.org/'.
 113
 114 To use SpamAssassin, add the following recipes to
 115 \".procmailrc\":
 116
 117     PATH=$PATH:/usr/bin/mh
 118     MAILDIR=$HOME/`mhparam Path`
 119
 120     # Fight spam with SpamAssassin.
 121     :0fw
 122     | spamc
 123
 124     # Anything with a spam level of 10 or more is junked immediately.
 125     :0:
 126     * ^X-Spam-Level: ..........
 127     /dev/null
 128
 129     :0:
 130     * ^X-Spam-Status: Yes
 131     spam/.
 132
 133 If you don't use \"spamc\", use \"spamassassin -P -a\".
 134
 135 Note that one of the recipes above throws away messages with a
 136 score greater than or equal to 10. Here's how you can determine a
 137 value that works best for you.
 138
 139 First, run \"spamassassin -t\" on every mail message in your
 140 archive and use Gnumeric to verify that the average plus the
 141 standard deviation of good mail is under 5, the SpamAssassin
 142 default for \"spam\".
 143
 144 Using Gnumeric, sort the messages by score and view the messages
 145 with the highest score. Determine the score which encompasses all
 146 of your interesting messages and add a couple of points to be
 147 conservative. Add that many dots to the \"X-Spam-Level:\" header
 148 field above to send messages with that score down the drain.
 149
 150 In the example above, messages with a score of 5-9 are set aside
 151 in the \"+spam\" folder for later review. The major weakness of
 152 rules-based filters is a plethora of false positives so it is
 153 worthwhile to check.
 154
 155 If SpamAssassin classifies a message incorrectly, or is unsure,
 156 you can use the MH-E commands \\[mh-junk-blacklist] and
 157 \\[mh-junk-whitelist].
 158
 159 The command \\[mh-junk-blacklist] adds a \"blacklist_from\" entry
 160 to \"~/spamassassin/user_prefs\", deletes the message, and sends
 161 the message to the Razor, so that others might not see this spam.
 162 If the \"sa-learn\" command is available, the message is also
 163 recategorized as spam.
 164
 165 The command \\[mh-junk-whitelist] adds a \"whitelist_from\" rule
 166 to the \"~/.spamassassin/user_prefs\" file. If the \"sa-learn\"
 167 command is available, the message is also recategorized as ham.
 168
 169 Over time, you'll observe that the same host or domain occurs
 170 repeatedly in the \"blacklist_from\" entries, so you might think
 171 that you could avoid future spam by blacklisting all mail from a
 172 particular domain. The utility function
 173 `mh-spamassassin-identify-spammers' helps you do precisely that.
 174 This function displays a frequency count of the hosts and domains
 175 in the \"blacklist_from\" entries from the last blank line in
 176 \"~/.spamassassin/user_prefs\" to the end of the file. This
 177 information can be used so that you can replace multiple
 178 \"blacklist_from\" entries with a single wildcard entry such as:
 179
 180     blacklist_from *@*amazingoffersdirect2u.com
 181
 182 In versions of SpamAssassin (2.50 and on) that support a Bayesian
 183 classifier, \\[mh-junk-blacklist] uses the program \"sa-learn\"
 184 to recategorize the message as spam. Neither MH-E, nor
 185 SpamAssassin, rebuilds the database after adding words, so you
 186 will need to run \"sa-learn --rebuild\" periodically. This can be
 187 done by adding the following to your crontab:
 188
 189     0 * * * *   sa-learn --rebuild > /dev/null 2>&1"
 190   (unless mh-spamassassin-executable
 191     (error "Unable to find the spamassassin executable"))
 192   (let ((current-folder mh-current-folder)
 193         (msg-file (mh-msg-filename msg mh-current-folder))
 194         (sender))
 195     (message "Reporting message %d..." msg)
 196     (mh-truncate-log-buffer)
 197     ;; Put call-process output in log buffer if we are saving it
 198     ;; (this happens if mh-junk-background is t).
 199     (with-current-buffer mh-log-buffer
 200       (call-process mh-spamassassin-executable msg-file mh-junk-background nil
 201                     ;;"--report" "--remove-from-whitelist"
 202                     "-r" "-R")          ; spamassassin V2.20
 203     (when mh-sa-learn-executable
 204       (message "Recategorizing message %d as spam..." msg)
 205       (mh-truncate-log-buffer)
 206       (call-process mh-sa-learn-executable msg-file mh-junk-background nil
 207                     "--single" "--spam" "--local" "--no-rebuild")))
 208     (message "Blacklisting sender of message %d..." msg)
 209     (with-current-buffer (get-buffer-create mh-temp-buffer)
 210       (erase-buffer)
 211       (call-process (expand-file-name mh-scan-prog mh-progs)
 212                     nil t nil
 213                     (format "%d" msg) current-folder
 214                     "-format" "%<(mymbox{from})%|%(addr{from})%>")
 215       (goto-char (point-min))
 216       (if (search-forward-regexp "^\\(.+\\)$" nil t)
 217           (progn
 218             (setq sender (match-string 0))
 219             (mh-spamassassin-add-rule "blacklist_from" sender)
 220             (message "Blacklisting sender of message %d...done" msg))
 221         (message "Blacklisting sender of message %d...not done (from my address)" msg)))))
 222
 223 ;;;###mh-autoload
 224 (defun mh-spamassassin-whitelist (msg)
 225   "Whitelist MSG with SpamAssassin.
 226
 227 The \\[mh-junk-whitelist] command adds a \"whitelist_from\" rule to
 228 the \"~/.spamassassin/user_prefs\" file. If the \"sa-learn\" command
 229 is available, the message is also recategorized as ham.
 230
 231 See `mh-spamassassin-blacklist' for more information."
 232   (unless mh-spamassassin-executable
 233     (error "Unable to find the spamassassin executable"))
 234   (let ((msg-file (mh-msg-filename msg mh-current-folder))
 235         (show-buffer (get-buffer mh-show-buffer))
 236         from)
 237     (with-current-buffer (get-buffer-create mh-temp-buffer)
 238       (erase-buffer)
 239       (message "Removing spamassassin markup from message %d..." msg)
 240       (call-process mh-spamassassin-executable msg-file t nil
 241                     ;; "--remove-markup"
 242                     "-d")               ; spamassassin V2.20
 243       (if show-buffer
 244           (kill-buffer show-buffer))
 245       (write-file msg-file)
 246       (when mh-sa-learn-executable
 247         (message "Recategorizing message %d as ham..." msg)
 248         (mh-truncate-log-buffer)
 249         ;; Put call-process output in log buffer if we are saving it
 250         ;; (this happens if mh-junk-background is t).
 251         (with-current-buffer mh-log-buffer
 252           (call-process mh-sa-learn-executable msg-file mh-junk-background nil
 253                         "--single" "--ham" "--local" "--no-rebuild")))
 254       (message "Whitelisting sender of message %d..." msg)
 255       (setq from
 256             (car (mh-funcall-if-exists
 257                   ietf-drums-parse-address (mh-get-header-field "From:"))))
 258       (kill-buffer nil)
 259       (unless (or (null from) (equal from ""))
 260         (mh-spamassassin-add-rule "whitelist_from" from))
 261       (message "Whitelisting sender of message %d...done" msg))))
 262
 263 (defun mh-spamassassin-add-rule (rule body)
 264   "Add a new rule to \"~/.spamassassin/user_prefs\".
 265 The name of the rule is RULE and its body is BODY."
 266   (save-window-excursion
 267     (let* ((line (format "%s\t%s\n" rule body))
 268            (case-fold-search t)
 269            (file (expand-file-name "~/.spamassassin/user_prefs"))
 270            (buffer-exists (find-buffer-visiting file)))
 271       (find-file file)
 272       (if (not (search-forward (format "\n%s" line) nil t))
 273           (progn
 274             (goto-char (point-max))
 275             (insert (if (bolp) "" "\n") line)
 276             (save-buffer)))
 277       (if (not buffer-exists)
 278           (kill-buffer nil)))))
 279
 280 ;;;###mh-autoload
 281 (defun mh-spamassassin-identify-spammers ()
 282   "Identify spammers who are repeat offenders.
 283
 284 This function displays a frequency count of the hosts and domains
 285 in the \"blacklist_from\" entries from the last blank line in
 286 \"~/.spamassassin/user_prefs\" to the end of the file. This
 287 information can be used so that you can replace multiple
 288 \"blacklist_from\" entries with a single wildcard entry such as:
 289
 290     blacklist_from *@*amazingoffersdirect2u.com"
 291   (interactive)
 292   (let* ((file (expand-file-name "~/.spamassassin/user_prefs"))
 293          (domains (make-hash-table :test 'equal)))
 294     (find-file file)
 295     ;; Only consider entries between last blank line and end of file.
 296     (goto-char (1- (point-max)))
 297     (search-backward-regexp "^$")
 298     ;; Perform frequency count.
 299     (save-excursion
 300       (while (search-forward-regexp "^blacklist_from\\s-*\\(.*\\)@\\(.*\\)$"
 301                                     nil t)
 302         (let ((host (match-string 2))
 303               value)
 304           ;; Remove top-level-domain from hostname.
 305           (setq host (cdr (reverse (split-string host "\\."))))
 306           ;; Add counts for each host and domain part.
 307           (while host
 308             (setq value (gethash (car host) domains))
 309             (setf (gethash (car host) domains) (1+ (if (not value) 0 value)))
 310             (setq host (cdr host))))))
 311
 312     ;; Output
 313     (delete-other-windows)
 314     (pop-to-buffer (get-buffer-create "*MH-E Spammer Frequencies*"))
 315     (erase-buffer)
 316     (maphash '(lambda (key value) ""
 317                 (if (> value 2)
 318                     (insert (format "%s %s\n" key value))))
 319              domains)
 320     (sort-numeric-fields 2 (point-min) (point-max))
 321     (reverse-region (point-min) (point-max))
 322     (goto-char (point-min))))
 323
 324 \f
 325
 326 ;; Bogofilter Interface
 327
 328 (defvar mh-bogofilter-executable (executable-find "bogofilter"))
 329
 330 ;;;###mh-autoload
 331 (defun mh-bogofilter-blacklist (msg)
 332   "Blacklist MSG with bogofilter.
 333
 334 Bogofilter is a Bayesian spam filtering program. Get it from your
 335 local distribution or from the bogofilter web site at URL
 336 `http://bogofilter.sourceforge.net/'.
 337
 338 Bogofilter is taught by running:
 339
 340     bogofilter -n < good-message
 341
 342 on every good message, and
 343
 344     bogofilter -s < spam-message
 345
 346 on every spam message. This is called a full training; three other
 347 training methods are described in the FAQ that is distributed with
 348 bogofilter. Note that most Bayesian filters need 1000 to 5000 of each
 349 type of message to start doing a good job.
 350
 351 To use bogofilter, add the following recipes to \".procmailrc\":
 352
 353     PATH=$PATH:/usr/bin/mh
 354     MAILDIR=$HOME/`mhparam Path`
 355
 356     # Fight spam with bogofilter.
 357     :0fw
 358     | bogofilter -3 -e -p
 359
 360     :0:
 361     * ^X-Bogosity: Yes, tests=bogofilter
 362     spam/.
 363
 364     :0:
 365     * ^X-Bogosity: Unsure, tests=bogofilter
 366     spam/unsure/.
 367
 368 If bogofilter classifies a message incorrectly, or is unsure, you can
 369 use the MH-E commands \\[mh-junk-blacklist] and \\[mh-junk-whitelist]
 370 to update bogofilter's training.
 371
 372 The \"Bogofilter FAQ\" suggests that you run the following
 373 occasionally to shrink the database:
 374
 375     bogoutil -d wordlist.db | bogoutil -l wordlist.db.new
 376     mv wordlist.db wordlist.db.prv
 377     mv wordlist.db.new wordlist.db
 378
 379 The \"Bogofilter tuning HOWTO\" describes how you can fine-tune Bogofilter."
 380   (unless mh-bogofilter-executable
 381     (error "Unable to find the bogofilter executable"))
 382   (let ((msg-file (mh-msg-filename msg mh-current-folder)))
 383     (mh-truncate-log-buffer)
 384     ;; Put call-process output in log buffer if we are saving it
 385     ;; (this happens if mh-junk-background is t).
 386     (with-current-buffer mh-log-buffer
 387       (call-process mh-bogofilter-executable msg-file mh-junk-background
 388                     nil "-s"))))
 389
 390 ;;;###mh-autoload
 391 (defun mh-bogofilter-whitelist (msg)
 392   "Whitelist MSG with bogofilter.
 393
 394 See `mh-bogofilter-blacklist' for more information."
 395   (unless mh-bogofilter-executable
 396     (error "Unable to find the bogofilter executable"))
 397   (let ((msg-file (mh-msg-filename msg mh-current-folder)))
 398     (mh-truncate-log-buffer)
 399     ;; Put call-process output in log buffer if we are saving it
 400     ;; (this happens if mh-junk-background is t).
 401     (with-current-buffer mh-log-buffer
 402       (call-process mh-bogofilter-executable msg-file mh-junk-background
 403                     nil "-n"))))
 404
 405 \f
 406
 407 ;; Spamprobe Interface
 408
 409 (defvar mh-spamprobe-executable (executable-find "spamprobe"))
 410
 411 ;;;###mh-autoload
 412 (defun mh-spamprobe-blacklist (msg)
 413   "Blacklist MSG with SpamProbe.
 414
 415 SpamProbe is a Bayesian spam filtering program. Get it from your
 416 local distribution or from the SpamProbe web site at URL
 417 `http://spamprobe.sourceforge.net'.
 418
 419 To use SpamProbe, add the following recipes to \".procmailrc\":
 420
 421     PATH=$PATH:/usr/bin/mh
 422     MAILDIR=$HOME/`mhparam Path`
 423
 424     # Fight spam with SpamProbe.
 425     :0
 426     SCORE=| spamprobe receive
 427
 428     :0 wf
 429     | formail -I \"X-SpamProbe: $SCORE\"
 430
 431     :0:
 432     *^X-SpamProbe: SPAM
 433     spam/.
 434
 435 If SpamProbe classifies a message incorrectly, you can use the
 436 MH-E commands \\[mh-junk-blacklist] and \\[mh-junk-whitelist] to
 437 update SpamProbe's training."
 438   (unless mh-spamprobe-executable
 439     (error "Unable to find the spamprobe executable"))
 440   (let ((msg-file (mh-msg-filename msg mh-current-folder)))
 441     (mh-truncate-log-buffer)
 442     ;; Put call-process output in log buffer if we are saving it
 443     ;; (this happens if mh-junk-background is t).
 444     (with-current-buffer mh-log-buffer
 445       (call-process mh-spamprobe-executable msg-file mh-junk-background
 446                     nil "spam"))))
 447
 448 ;;;###mh-autoload
 449 (defun mh-spamprobe-whitelist (msg)
 450   "Whitelist MSG with SpamProbe.
 451
 452 See `mh-spamprobe-blacklist' for more information."
 453   (unless mh-spamprobe-executable
 454     (error "Unable to find the spamprobe executable"))
 455   (let ((msg-file (mh-msg-filename msg mh-current-folder)))
 456     (mh-truncate-log-buffer)
 457     ;; Put call-process output in log buffer if we are saving it
 458     ;; (this happens if mh-junk-background is t).
 459     (with-current-buffer mh-log-buffer
 460       (call-process mh-spamprobe-executable msg-file mh-junk-background
 461                     nil "good"))))
 462
 463 (provide 'mh-junk)
 464
 465 ;; Local Variables:
 466 ;; indent-tabs-mode: nil
 467 ;; sentence-end-double-space: nil
 468 ;; End:
 469
 470 ;; arch-tag: 603335f1-77ff-4306-8828-5d3dad51abe1
 471 ;;; mh-junk.el ends here