lisp/mh-e/mh-junk.el

   1 ;;; mh-junk.el --- MH-E interface to anti-spam measures
   2
   3 ;; Copyright (C) 2003, 2004, 2005, 2006, 2007, 2008 Free Software Foundation, Inc.
   4
   5 ;; Author: Satyaki Das <satyaki@theforce.stanford.edu>,
   6 ;;         Bill Wohler <wohler@newt.com>
   7 ;; Maintainer: Bill Wohler <wohler@newt.com>
   8 ;; Keywords:  mail, spam
   9
  10 ;; This file is part of GNU Emacs.
  11
  12 ;; GNU Emacs is free software: you can redistribute it and/or modify
  13 ;; it under the terms of the GNU General Public License as published by
  14 ;; the Free Software Foundation, either version 3 of the License, or
  15 ;; (at your option) any later version.
  16
  17 ;; GNU Emacs is distributed in the hope that it will be useful,
  18 ;; but WITHOUT ANY WARRANTY; without even the implied warranty of
  19 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  20 ;; GNU General Public License for more details.
  21
  22 ;; You should have received a copy of the GNU General Public License
  23 ;; along with GNU Emacs.  If not, see <http://www.gnu.org/licenses/>.
  24
  25 ;;; Commentary:
  26
  27 ;; Spam handling in MH-E.
  28
  29 ;;; Change Log:
  30
  31 ;;; Code:
  32
  33 (require 'mh-e)
  34 (require 'mh-scan)
  35 (mh-require-cl)
  36
  37 ;;;###mh-autoload
  38 (defun mh-junk-blacklist (range)
  39   "Blacklist RANGE as spam.
  40
  41 This command trains the spam program in use (see the option
  42 `mh-junk-program') with the content of RANGE and then handles the
  43 message(s) as specified by the option `mh-junk-disposition'.
  44
  45 Check the documentation of `mh-interactive-range' to see how RANGE is
  46 read in interactive use.
  47
  48 For more information about using your particular spam fighting
  49 program, see:
  50
  51   - `mh-spamassassin-blacklist'
  52   - `mh-bogofilter-blacklist'
  53   - `mh-spamprobe-blacklist'"
  54   (interactive (list (mh-interactive-range "Blacklist")))
  55   (let ((blacklist-func (nth 1 (assoc mh-junk-choice mh-junk-function-alist))))
  56     (unless blacklist-func
  57       (error "Customize `mh-junk-program' appropriately"))
  58     (let ((dest (cond ((null mh-junk-disposition) nil)
  59                       ((equal mh-junk-disposition "") "+")
  60                       ((eq (aref mh-junk-disposition 0) ?+)
  61                        mh-junk-disposition)
  62                       ((eq (aref mh-junk-disposition 0) ?@)
  63                        (concat mh-current-folder "/"
  64                                (substring mh-junk-disposition 1)))
  65                       (t (concat "+" mh-junk-disposition)))))
  66       (mh-iterate-on-range msg range
  67         (message "Blacklisting message %d..." msg)
  68         (funcall (symbol-function blacklist-func) msg)
  69         (message "Blacklisting message %d...done" msg)
  70         (if (not (memq msg mh-seen-list))
  71             (setq mh-seen-list (cons msg mh-seen-list)))
  72         (if dest
  73             (mh-refile-a-msg nil (intern dest))
  74           (mh-delete-a-msg nil)))
  75       (mh-next-msg))))
  76
  77 ;;;###mh-autoload
  78 (defun mh-junk-whitelist (range)
  79   "Whitelist RANGE as ham.
  80
  81 This command reclassifies the RANGE as ham if it were incorrectly
  82 classified as spam (see the option `mh-junk-program'). It then
  83 refiles the message into the \"+inbox\" folder.
  84
  85 Check the documentation of `mh-interactive-range' to see how
  86 RANGE is read in interactive use."
  87   (interactive (list (mh-interactive-range "Whitelist")))
  88   (let ((whitelist-func (nth 2 (assoc mh-junk-choice mh-junk-function-alist))))
  89     (unless whitelist-func
  90       (error "Customize `mh-junk-program' appropriately"))
  91     (mh-iterate-on-range msg range
  92       (message "Whitelisting message %d..." msg)
  93       (funcall (symbol-function whitelist-func) msg)
  94       (message "Whitelisting message %d...done" msg)
  95       (mh-refile-a-msg nil (intern mh-inbox)))
  96     (mh-next-msg)))
  97
  98 \f
  99
 100 ;; Spamassassin Interface
 101
 102 (defvar mh-spamassassin-executable (executable-find "spamassassin"))
 103 (defvar mh-sa-learn-executable (executable-find "sa-learn"))
 104
 105 ;;;###mh-autoload
 106 (defun mh-spamassassin-blacklist (msg)
 107   "Blacklist MSG with SpamAssassin.
 108
 109 SpamAssassin is one of the more popular spam filtering programs.
 110 Get it from your local distribution or from
 111 http://spamassassin.org/.
 112
 113 To use SpamAssassin, add the following recipes to
 114 \".procmailrc\":
 115
 116     PATH=$PATH:/usr/bin/mh
 117     MAILDIR=$HOME/`mhparam Path`
 118
 119     # Fight spam with SpamAssassin.
 120     :0fw
 121     | spamc
 122
 123     # Anything with a spam level of 10 or more is junked immediately.
 124     :0:
 125     * ^X-Spam-Level: ..........
 126     /dev/null
 127
 128     :0:
 129     * ^X-Spam-Status: Yes
 130     spam/.
 131
 132 If you don't use \"spamc\", use \"spamassassin -P -a\".
 133
 134 Note that one of the recipes above throws away messages with a
 135 score greater than or equal to 10. Here's how you can determine a
 136 value that works best for you.
 137
 138 First, run \"spamassassin -t\" on every mail message in your
 139 archive and use Gnumeric to verify that the average plus the
 140 standard deviation of good mail is under 5, the SpamAssassin
 141 default for \"spam\".
 142
 143 Using Gnumeric, sort the messages by score and view the messages
 144 with the highest score. Determine the score which encompasses all
 145 of your interesting messages and add a couple of points to be
 146 conservative. Add that many dots to the \"X-Spam-Level:\" header
 147 field above to send messages with that score down the drain.
 148
 149 In the example above, messages with a score of 5-9 are set aside
 150 in the \"+spam\" folder for later review. The major weakness of
 151 rules-based filters is a plethora of false positives so it is
 152 worthwhile to check.
 153
 154 If SpamAssassin classifies a message incorrectly, or is unsure,
 155 you can use the MH-E commands \\[mh-junk-blacklist] and
 156 \\[mh-junk-whitelist].
 157
 158 The command \\[mh-junk-blacklist] adds a \"blacklist_from\" entry
 159 to \"~/spamassassin/user_prefs\", deletes the message, and sends
 160 the message to the Razor, so that others might not see this spam.
 161 If the \"sa-learn\" command is available, the message is also
 162 recategorized as spam.
 163
 164 The command \\[mh-junk-whitelist] adds a \"whitelist_from\" rule
 165 to the \"~/.spamassassin/user_prefs\" file. If the \"sa-learn\"
 166 command is available, the message is also recategorized as ham.
 167
 168 Over time, you'll observe that the same host or domain occurs
 169 repeatedly in the \"blacklist_from\" entries, so you might think
 170 that you could avoid future spam by blacklisting all mail from a
 171 particular domain. The utility function
 172 `mh-spamassassin-identify-spammers' helps you do precisely that.
 173 This function displays a frequency count of the hosts and domains
 174 in the \"blacklist_from\" entries from the last blank line in
 175 \"~/.spamassassin/user_prefs\" to the end of the file. This
 176 information can be used so that you can replace multiple
 177 \"blacklist_from\" entries with a single wildcard entry such as:
 178
 179     blacklist_from *@*amazingoffersdirect2u.com
 180
 181 In versions of SpamAssassin (2.50 and on) that support a Bayesian
 182 classifier, \\[mh-junk-blacklist] uses the program \"sa-learn\"
 183 to recategorize the message as spam. Neither MH-E, nor
 184 SpamAssassin, rebuilds the database after adding words, so you
 185 will need to run \"sa-learn --rebuild\" periodically. This can be
 186 done by adding the following to your crontab:
 187
 188     0 * * * *   sa-learn --rebuild > /dev/null 2>&1"
 189   (unless mh-spamassassin-executable
 190     (error "Unable to find the spamassassin executable"))
 191   (let ((current-folder mh-current-folder)
 192         (msg-file (mh-msg-filename msg mh-current-folder))
 193         (sender))
 194     (message "Reporting message %d..." msg)
 195     (mh-truncate-log-buffer)
 196     ;; Put call-process output in log buffer if we are saving it
 197     ;; (this happens if mh-junk-background is t).
 198     (with-current-buffer mh-log-buffer
 199       (call-process mh-spamassassin-executable msg-file mh-junk-background nil
 200                     ;;"--report" "--remove-from-whitelist"
 201                     "-r" "-R")          ; spamassassin V2.20
 202     (when mh-sa-learn-executable
 203       (message "Recategorizing message %d as spam..." msg)
 204       (mh-truncate-log-buffer)
 205       (call-process mh-sa-learn-executable msg-file mh-junk-background nil
 206                     "--single" "--spam" "--local" "--no-rebuild")))
 207     (message "Blacklisting sender of message %d..." msg)
 208     (with-current-buffer (get-buffer-create mh-temp-buffer)
 209       (erase-buffer)
 210       (call-process (expand-file-name mh-scan-prog mh-progs)
 211                     nil t nil
 212                     (format "%d" msg) current-folder
 213                     "-format" "%<(mymbox{from})%|%(addr{from})%>")
 214       (goto-char (point-min))
 215       (if (search-forward-regexp "^\\(.+\\)$" nil t)
 216           (progn
 217             (setq sender (match-string 0))
 218             (mh-spamassassin-add-rule "blacklist_from" sender)
 219             (message "Blacklisting sender of message %d...done" msg))
 220         (message "Blacklisting sender of message %d...not done (from my address)" msg)))))
 221
 222 ;;;###mh-autoload
 223 (defun mh-spamassassin-whitelist (msg)
 224   "Whitelist MSG with SpamAssassin.
 225
 226 The \\[mh-junk-whitelist] command adds a \"whitelist_from\" rule to
 227 the \"~/.spamassassin/user_prefs\" file. If the \"sa-learn\" command
 228 is available, the message is also recategorized as ham.
 229
 230 See `mh-spamassassin-blacklist' for more information."
 231   (unless mh-spamassassin-executable
 232     (error "Unable to find the spamassassin executable"))
 233   (let ((msg-file (mh-msg-filename msg mh-current-folder))
 234         (show-buffer (get-buffer mh-show-buffer))
 235         from)
 236     (with-current-buffer (get-buffer-create mh-temp-buffer)
 237       (erase-buffer)
 238       (message "Removing spamassassin markup from message %d..." msg)
 239       (call-process mh-spamassassin-executable msg-file t nil
 240                     ;; "--remove-markup"
 241                     "-d")               ; spamassassin V2.20
 242       (if show-buffer
 243           (kill-buffer show-buffer))
 244       (write-file msg-file)
 245       (when mh-sa-learn-executable
 246         (message "Recategorizing message %d as ham..." msg)
 247         (mh-truncate-log-buffer)
 248         ;; Put call-process output in log buffer if we are saving it
 249         ;; (this happens if mh-junk-background is t).
 250         (with-current-buffer mh-log-buffer
 251           (call-process mh-sa-learn-executable msg-file mh-junk-background nil
 252                         "--single" "--ham" "--local" "--no-rebuild")))
 253       (message "Whitelisting sender of message %d..." msg)
 254       (setq from
 255             (car (mh-funcall-if-exists
 256                   ietf-drums-parse-address (mh-get-header-field "From:"))))
 257       (kill-buffer nil)
 258       (unless (or (null from) (equal from ""))
 259         (mh-spamassassin-add-rule "whitelist_from" from))
 260       (message "Whitelisting sender of message %d...done" msg))))
 261
 262 (defun mh-spamassassin-add-rule (rule body)
 263   "Add a new rule to \"~/.spamassassin/user_prefs\".
 264 The name of the rule is RULE and its body is BODY."
 265   (save-window-excursion
 266     (let* ((line (format "%s\t%s\n" rule body))
 267            (case-fold-search t)
 268            (file (expand-file-name "~/.spamassassin/user_prefs"))
 269            (buffer-exists (find-buffer-visiting file)))
 270       (find-file file)
 271       (if (not (search-forward (format "\n%s" line) nil t))
 272           (progn
 273             (goto-char (point-max))
 274             (insert (if (bolp) "" "\n") line)
 275             (save-buffer)))
 276       (if (not buffer-exists)
 277           (kill-buffer nil)))))
 278
 279 ;;;###mh-autoload
 280 (defun mh-spamassassin-identify-spammers ()
 281   "Identify spammers who are repeat offenders.
 282
 283 This function displays a frequency count of the hosts and domains
 284 in the \"blacklist_from\" entries from the last blank line in
 285 \"~/.spamassassin/user_prefs\" to the end of the file. This
 286 information can be used so that you can replace multiple
 287 \"blacklist_from\" entries with a single wildcard entry such as:
 288
 289     blacklist_from *@*amazingoffersdirect2u.com"
 290   (interactive)
 291   (let* ((file (expand-file-name "~/.spamassassin/user_prefs"))
 292          (domains (make-hash-table :test 'equal)))
 293     (find-file file)
 294     ;; Only consider entries between last blank line and end of file.
 295     (goto-char (1- (point-max)))
 296     (search-backward-regexp "^$")
 297     ;; Perform frequency count.
 298     (save-excursion
 299       (while (search-forward-regexp "^blacklist_from\\s-*\\(.*\\)@\\(.*\\)$"
 300                                     nil t)
 301         (let ((host (match-string 2))
 302               value)
 303           ;; Remove top-level-domain from hostname.
 304           (setq host (cdr (reverse (split-string host "\\."))))
 305           ;; Add counts for each host and domain part.
 306           (while host
 307             (setq value (gethash (car host) domains))
 308             (setf (gethash (car host) domains) (1+ (if (not value) 0 value)))
 309             (setq host (cdr host))))))
 310
 311     ;; Output
 312     (delete-other-windows)
 313     (pop-to-buffer (get-buffer-create "*MH-E Spammer Frequencies*"))
 314     (erase-buffer)
 315     (maphash '(lambda (key value) ""
 316                 (if (> value 2)
 317                     (insert (format "%s %s\n" key value))))
 318              domains)
 319     (sort-numeric-fields 2 (point-min) (point-max))
 320     (reverse-region (point-min) (point-max))
 321     (goto-char (point-min))))
 322
 323 \f
 324
 325 ;; Bogofilter Interface
 326
 327 (defvar mh-bogofilter-executable (executable-find "bogofilter"))
 328
 329 ;;;###mh-autoload
 330 (defun mh-bogofilter-blacklist (msg)
 331   "Blacklist MSG with bogofilter.
 332
 333 Bogofilter is a Bayesian spam filtering program. Get it from your
 334 local distribution or from http://bogofilter.sourceforge.net/.
 335
 336 Bogofilter is taught by running:
 337
 338     bogofilter -n < good-message
 339
 340 on every good message, and
 341
 342     bogofilter -s < spam-message
 343
 344 on every spam message. This is called a full training; three other
 345 training methods are described in the FAQ that is distributed with
 346 bogofilter. Note that most Bayesian filters need 1000 to 5000 of each
 347 type of message to start doing a good job.
 348
 349 To use bogofilter, add the following recipes to \".procmailrc\":
 350
 351     PATH=$PATH:/usr/bin/mh
 352     MAILDIR=$HOME/`mhparam Path`
 353
 354     # Fight spam with bogofilter.
 355     :0fw
 356     | bogofilter -3 -e -p
 357
 358     :0:
 359     * ^X-Bogosity: Yes, tests=bogofilter
 360     spam/.
 361
 362     :0:
 363     * ^X-Bogosity: Unsure, tests=bogofilter
 364     spam/unsure/.
 365
 366 If bogofilter classifies a message incorrectly, or is unsure, you can
 367 use the MH-E commands \\[mh-junk-blacklist] and \\[mh-junk-whitelist]
 368 to update bogofilter's training.
 369
 370 The \"Bogofilter FAQ\" suggests that you run the following
 371 occasionally to shrink the database:
 372
 373     bogoutil -d wordlist.db | bogoutil -l wordlist.db.new
 374     mv wordlist.db wordlist.db.prv
 375     mv wordlist.db.new wordlist.db
 376
 377 The \"Bogofilter tuning HOWTO\" describes how you can fine-tune Bogofilter."
 378   (unless mh-bogofilter-executable
 379     (error "Unable to find the bogofilter executable"))
 380   (let ((msg-file (mh-msg-filename msg mh-current-folder)))
 381     (mh-truncate-log-buffer)
 382     ;; Put call-process output in log buffer if we are saving it
 383     ;; (this happens if mh-junk-background is t).
 384     (with-current-buffer mh-log-buffer
 385       (call-process mh-bogofilter-executable msg-file mh-junk-background
 386                     nil "-s"))))
 387
 388 ;;;###mh-autoload
 389 (defun mh-bogofilter-whitelist (msg)
 390   "Whitelist MSG with bogofilter.
 391
 392 See `mh-bogofilter-blacklist' for more information."
 393   (unless mh-bogofilter-executable
 394     (error "Unable to find the bogofilter executable"))
 395   (let ((msg-file (mh-msg-filename msg mh-current-folder)))
 396     (mh-truncate-log-buffer)
 397     ;; Put call-process output in log buffer if we are saving it
 398     ;; (this happens if mh-junk-background is t).
 399     (with-current-buffer mh-log-buffer
 400       (call-process mh-bogofilter-executable msg-file mh-junk-background
 401                     nil "-n"))))
 402
 403 \f
 404
 405 ;; Spamprobe Interface
 406
 407 (defvar mh-spamprobe-executable (executable-find "spamprobe"))
 408
 409 ;;;###mh-autoload
 410 (defun mh-spamprobe-blacklist (msg)
 411   "Blacklist MSG with SpamProbe.
 412
 413 SpamProbe is a Bayesian spam filtering program. Get it from your local
 414 distribution or from http://spamprobe.sourceforge.net.
 415
 416 To use SpamProbe, add the following recipes to \".procmailrc\":
 417
 418     PATH=$PATH:/usr/bin/mh
 419     MAILDIR=$HOME/`mhparam Path`
 420
 421     # Fight spam with SpamProbe.
 422     :0
 423     SCORE=| spamprobe receive
 424
 425     :0 wf
 426     | formail -I \"X-SpamProbe: $SCORE\"
 427
 428     :0:
 429     *^X-SpamProbe: SPAM
 430     spam/.
 431
 432 If SpamProbe classifies a message incorrectly, you can use the
 433 MH-E commands \\[mh-junk-blacklist] and \\[mh-junk-whitelist] to
 434 update SpamProbe's training."
 435   (unless mh-spamprobe-executable
 436     (error "Unable to find the spamprobe executable"))
 437   (let ((msg-file (mh-msg-filename msg mh-current-folder)))
 438     (mh-truncate-log-buffer)
 439     ;; Put call-process output in log buffer if we are saving it
 440     ;; (this happens if mh-junk-background is t).
 441     (with-current-buffer mh-log-buffer
 442       (call-process mh-spamprobe-executable msg-file mh-junk-background
 443                     nil "spam"))))
 444
 445 ;;;###mh-autoload
 446 (defun mh-spamprobe-whitelist (msg)
 447   "Whitelist MSG with SpamProbe.
 448
 449 See `mh-spamprobe-blacklist' for more information."
 450   (unless mh-spamprobe-executable
 451     (error "Unable to find the spamprobe executable"))
 452   (let ((msg-file (mh-msg-filename msg mh-current-folder)))
 453     (mh-truncate-log-buffer)
 454     ;; Put call-process output in log buffer if we are saving it
 455     ;; (this happens if mh-junk-background is t).
 456     (with-current-buffer mh-log-buffer
 457       (call-process mh-spamprobe-executable msg-file mh-junk-background
 458                     nil "good"))))
 459
 460 (provide 'mh-junk)
 461
 462 ;; Local Variables:
 463 ;; indent-tabs-mode: nil
 464 ;; sentence-end-double-space: nil
 465 ;; End:
 466
 467 ;; arch-tag: 603335f1-77ff-4306-8828-5d3dad51abe1
 468 ;;; mh-junk.el ends here