lisp/mh-e/mh-junk.el

   1 ;;; mh-junk.el --- MH-E interface to anti-spam measures
   2
   3 ;; Copyright (C) 2003-2011  Free Software Foundation, Inc.
   4
   5 ;; Author: Satyaki Das <satyaki@theforce.stanford.edu>,
   6 ;;         Bill Wohler <wohler@newt.com>
   7 ;; Maintainer: Bill Wohler <wohler@newt.com>
   8 ;; Keywords:  mail, spam
   9
  10 ;; This file is part of GNU Emacs.
  11
  12 ;; GNU Emacs is free software: you can redistribute it and/or modify
  13 ;; it under the terms of the GNU General Public License as published by
  14 ;; the Free Software Foundation, either version 3 of the License, or
  15 ;; (at your option) any later version.
  16
  17 ;; GNU Emacs is distributed in the hope that it will be useful,
  18 ;; but WITHOUT ANY WARRANTY; without even the implied warranty of
  19 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  20 ;; GNU General Public License for more details.
  21
  22 ;; You should have received a copy of the GNU General Public License
  23 ;; along with GNU Emacs.  If not, see <http://www.gnu.org/licenses/>.
  24
  25 ;;; Commentary:
  26
  27 ;; Spam handling in MH-E.
  28
  29 ;;; Change Log:
  30
  31 ;;; Code:
  32
  33 (require 'mh-e)
  34 (require 'mh-scan)
  35 (mh-require-cl)
  36
  37 ;;;###mh-autoload
  38 (defun mh-junk-blacklist (range)
  39   "Blacklist RANGE as spam.
  40
  41 This command trains the spam program in use (see the option
  42 `mh-junk-program') with the content of RANGE and then handles the
  43 message(s) as specified by the option `mh-junk-disposition'.
  44
  45 Check the documentation of `mh-interactive-range' to see how RANGE is
  46 read in interactive use.
  47
  48 For more information about using your particular spam fighting
  49 program, see:
  50
  51   - `mh-spamassassin-blacklist'
  52   - `mh-bogofilter-blacklist'
  53   - `mh-spamprobe-blacklist'"
  54   (interactive (list (mh-interactive-range "Blacklist")))
  55   (let ((blacklist-func (nth 1 (assoc mh-junk-choice mh-junk-function-alist))))
  56     (unless blacklist-func
  57       (error "Customize `mh-junk-program' appropriately"))
  58     (let ((dest (cond ((null mh-junk-disposition) nil)
  59                       ((equal mh-junk-disposition "") "+")
  60                       ((eq (aref mh-junk-disposition 0) ?+)
  61                        mh-junk-disposition)
  62                       ((eq (aref mh-junk-disposition 0) ?@)
  63                        (concat mh-current-folder "/"
  64                                (substring mh-junk-disposition 1)))
  65                       (t (concat "+" mh-junk-disposition)))))
  66       (mh-iterate-on-range msg range
  67         (message "Blacklisting message %d..." msg)
  68         (funcall (symbol-function blacklist-func) msg)
  69         (message "Blacklisting message %d...done" msg)
  70         (if (not (memq msg mh-seen-list))
  71             (setq mh-seen-list (cons msg mh-seen-list)))
  72         (if dest
  73             (mh-refile-a-msg nil (intern dest))
  74           (mh-delete-a-msg nil)))
  75       (mh-next-msg))))
  76
  77 ;;;###mh-autoload
  78 (defun mh-junk-whitelist (range)
  79   "Whitelist RANGE as ham.
  80
  81 This command reclassifies the RANGE as ham if it were incorrectly
  82 classified as spam (see the option `mh-junk-program'). It then
  83 refiles the message into the \"+inbox\" folder.
  84
  85 Check the documentation of `mh-interactive-range' to see how
  86 RANGE is read in interactive use."
  87   (interactive (list (mh-interactive-range "Whitelist")))
  88   (let ((whitelist-func (nth 2 (assoc mh-junk-choice mh-junk-function-alist))))
  89     (unless whitelist-func
  90       (error "Customize `mh-junk-program' appropriately"))
  91     (mh-iterate-on-range msg range
  92       (message "Whitelisting message %d..." msg)
  93       (funcall (symbol-function whitelist-func) msg)
  94       (message "Whitelisting message %d...done" msg)
  95       (mh-refile-a-msg nil (intern mh-inbox)))
  96     (mh-next-msg)))
  97
  98 \f
  99
 100 ;; Spamassassin Interface
 101
 102 (defvar mh-spamassassin-executable (executable-find "spamassassin"))
 103 (defvar mh-sa-learn-executable (executable-find "sa-learn"))
 104
 105 ;;;###mh-autoload
 106 (defun mh-spamassassin-blacklist (msg)
 107   "Blacklist MSG with SpamAssassin.
 108
 109 SpamAssassin is one of the more popular spam filtering programs.
 110 Get it from your local distribution or from the SpamAssassin web
 111 site at URL `http://spamassassin.org/'.
 112
 113 To use SpamAssassin, add the following recipes to
 114 \".procmailrc\":
 115
 116     PATH=$PATH:/usr/bin/mh
 117     MAILDIR=$HOME/`mhparam Path`
 118
 119     # Fight spam with SpamAssassin.
 120     :0fw
 121     | spamc
 122
 123     # Anything with a spam level of 10 or more is junked immediately.
 124     :0:
 125     * ^X-Spam-Level: ..........
 126     /dev/null
 127
 128     :0:
 129     * ^X-Spam-Status: Yes
 130     spam/.
 131
 132 If you don't use \"spamc\", use \"spamassassin -P -a\".
 133
 134 Note that one of the recipes above throws away messages with a
 135 score greater than or equal to 10. Here's how you can determine a
 136 value that works best for you.
 137
 138 First, run \"spamassassin -t\" on every mail message in your
 139 archive and use Gnumeric to verify that the average plus the
 140 standard deviation of good mail is under 5, the SpamAssassin
 141 default for \"spam\".
 142
 143 Using Gnumeric, sort the messages by score and view the messages
 144 with the highest score. Determine the score which encompasses all
 145 of your interesting messages and add a couple of points to be
 146 conservative. Add that many dots to the \"X-Spam-Level:\" header
 147 field above to send messages with that score down the drain.
 148
 149 In the example above, messages with a score of 5-9 are set aside
 150 in the \"+spam\" folder for later review. The major weakness of
 151 rules-based filters is a plethora of false positives so it is
 152 worthwhile to check.
 153
 154 If SpamAssassin classifies a message incorrectly, or is unsure,
 155 you can use the MH-E commands \\[mh-junk-blacklist] and
 156 \\[mh-junk-whitelist].
 157
 158 The command \\[mh-junk-blacklist] adds a \"blacklist_from\" entry
 159 to \"~/spamassassin/user_prefs\", deletes the message, and sends
 160 the message to the Razor, so that others might not see this spam.
 161 If the \"sa-learn\" command is available, the message is also
 162 recategorized as spam.
 163
 164 The command \\[mh-junk-whitelist] adds a \"whitelist_from\" rule
 165 to the \"~/.spamassassin/user_prefs\" file. If the \"sa-learn\"
 166 command is available, the message is also recategorized as ham.
 167
 168 Over time, you'll observe that the same host or domain occurs
 169 repeatedly in the \"blacklist_from\" entries, so you might think
 170 that you could avoid future spam by blacklisting all mail from a
 171 particular domain. The utility function
 172 `mh-spamassassin-identify-spammers' helps you do precisely that.
 173 This function displays a frequency count of the hosts and domains
 174 in the \"blacklist_from\" entries from the last blank line in
 175 \"~/.spamassassin/user_prefs\" to the end of the file. This
 176 information can be used so that you can replace multiple
 177 \"blacklist_from\" entries with a single wildcard entry such as:
 178
 179     blacklist_from *@*amazingoffersdirect2u.com
 180
 181 In versions of SpamAssassin (2.50 and on) that support a Bayesian
 182 classifier, \\[mh-junk-blacklist] uses the program \"sa-learn\"
 183 to recategorize the message as spam. Neither MH-E, nor
 184 SpamAssassin, rebuilds the database after adding words, so you
 185 will need to run \"sa-learn --rebuild\" periodically. This can be
 186 done by adding the following to your crontab:
 187
 188     0 * * * *   sa-learn --rebuild > /dev/null 2>&1"
 189   (unless mh-spamassassin-executable
 190     (error "Unable to find the spamassassin executable"))
 191   (let ((current-folder mh-current-folder)
 192         (msg-file (mh-msg-filename msg mh-current-folder))
 193         (sender))
 194     (message "Reporting message %d..." msg)
 195     (mh-truncate-log-buffer)
 196     ;; Put call-process output in log buffer if we are saving it
 197     ;; (this happens if mh-junk-background is t).
 198     (with-current-buffer mh-log-buffer
 199       (call-process mh-spamassassin-executable msg-file mh-junk-background nil
 200                     ;;"--report" "--remove-from-whitelist"
 201                     "-r" "-R")          ; spamassassin V2.20
 202     (when mh-sa-learn-executable
 203       (message "Recategorizing message %d as spam..." msg)
 204       (mh-truncate-log-buffer)
 205       (call-process mh-sa-learn-executable msg-file mh-junk-background nil
 206                     "--single" "--spam" "--local" "--no-rebuild")))
 207     (message "Blacklisting sender of message %d..." msg)
 208     (with-current-buffer (get-buffer-create mh-temp-buffer)
 209       (erase-buffer)
 210       (call-process (expand-file-name mh-scan-prog mh-progs)
 211                     nil t nil
 212                     (format "%d" msg) current-folder
 213                     "-format" "%<(mymbox{from})%|%(addr{from})%>")
 214       (goto-char (point-min))
 215       (if (search-forward-regexp "^\\(.+\\)$" nil t)
 216           (progn
 217             (setq sender (match-string 0))
 218             (mh-spamassassin-add-rule "blacklist_from" sender)
 219             (message "Blacklisting sender of message %d...done" msg))
 220         (message "Blacklisting sender of message %d...not done (from my address)" msg)))))
 221
 222 ;;;###mh-autoload
 223 (defun mh-spamassassin-whitelist (msg)
 224   "Whitelist MSG with SpamAssassin.
 225
 226 The \\[mh-junk-whitelist] command adds a \"whitelist_from\" rule to
 227 the \"~/.spamassassin/user_prefs\" file. If the \"sa-learn\" command
 228 is available, the message is also recategorized as ham.
 229
 230 See `mh-spamassassin-blacklist' for more information."
 231   (unless mh-spamassassin-executable
 232     (error "Unable to find the spamassassin executable"))
 233   (let ((msg-file (mh-msg-filename msg mh-current-folder))
 234         (show-buffer (get-buffer mh-show-buffer))
 235         from)
 236     (with-current-buffer (get-buffer-create mh-temp-buffer)
 237       (erase-buffer)
 238       (message "Removing spamassassin markup from message %d..." msg)
 239       (call-process mh-spamassassin-executable msg-file t nil
 240                     ;; "--remove-markup"
 241                     "-d")               ; spamassassin V2.20
 242       (if show-buffer
 243           (kill-buffer show-buffer))
 244       (write-file msg-file)
 245       (when mh-sa-learn-executable
 246         (message "Recategorizing message %d as ham..." msg)
 247         (mh-truncate-log-buffer)
 248         ;; Put call-process output in log buffer if we are saving it
 249         ;; (this happens if mh-junk-background is t).
 250         (with-current-buffer mh-log-buffer
 251           (call-process mh-sa-learn-executable msg-file mh-junk-background nil
 252                         "--single" "--ham" "--local" "--no-rebuild")))
 253       (message "Whitelisting sender of message %d..." msg)
 254       (setq from
 255             (car (mh-funcall-if-exists
 256                   ietf-drums-parse-address (mh-get-header-field "From:"))))
 257       (kill-buffer nil)
 258       (unless (or (null from) (equal from ""))
 259         (mh-spamassassin-add-rule "whitelist_from" from))
 260       (message "Whitelisting sender of message %d...done" msg))))
 261
 262 (defun mh-spamassassin-add-rule (rule body)
 263   "Add a new rule to \"~/.spamassassin/user_prefs\".
 264 The name of the rule is RULE and its body is BODY."
 265   (save-window-excursion
 266     (let* ((line (format "%s\t%s\n" rule body))
 267            (case-fold-search t)
 268            (file (expand-file-name "~/.spamassassin/user_prefs"))
 269            (buffer-exists (find-buffer-visiting file)))
 270       (find-file file)
 271       (if (not (search-forward (format "\n%s" line) nil t))
 272           (progn
 273             (goto-char (point-max))
 274             (insert (if (bolp) "" "\n") line)
 275             (save-buffer)))
 276       (if (not buffer-exists)
 277           (kill-buffer nil)))))
 278
 279 ;;;###mh-autoload
 280 (defun mh-spamassassin-identify-spammers ()
 281   "Identify spammers who are repeat offenders.
 282
 283 This function displays a frequency count of the hosts and domains
 284 in the \"blacklist_from\" entries from the last blank line in
 285 \"~/.spamassassin/user_prefs\" to the end of the file. This
 286 information can be used so that you can replace multiple
 287 \"blacklist_from\" entries with a single wildcard entry such as:
 288
 289     blacklist_from *@*amazingoffersdirect2u.com"
 290   (interactive)
 291   (let* ((file (expand-file-name "~/.spamassassin/user_prefs"))
 292          (domains (make-hash-table :test 'equal)))
 293     (find-file file)
 294     ;; Only consider entries between last blank line and end of file.
 295     (goto-char (1- (point-max)))
 296     (search-backward-regexp "^$")
 297     ;; Perform frequency count.
 298     (save-excursion
 299       (while (search-forward-regexp "^blacklist_from\\s-*\\(.*\\)@\\(.*\\)$"
 300                                     nil t)
 301         (let ((host (match-string 2))
 302               value)
 303           ;; Remove top-level-domain from hostname.
 304           (setq host (cdr (reverse (split-string host "\\."))))
 305           ;; Add counts for each host and domain part.
 306           (while host
 307             (setq value (gethash (car host) domains))
 308             (setf (gethash (car host) domains) (1+ (if (not value) 0 value)))
 309             (setq host (cdr host))))))
 310
 311     ;; Output
 312     (delete-other-windows)
 313     (pop-to-buffer (get-buffer-create "*MH-E Spammer Frequencies*"))
 314     (erase-buffer)
 315     (maphash '(lambda (key value) ""
 316                 (if (> value 2)
 317                     (insert (format "%s %s\n" key value))))
 318              domains)
 319     (sort-numeric-fields 2 (point-min) (point-max))
 320     (reverse-region (point-min) (point-max))
 321     (goto-char (point-min))))
 322
 323 \f
 324
 325 ;; Bogofilter Interface
 326
 327 (defvar mh-bogofilter-executable (executable-find "bogofilter"))
 328
 329 ;;;###mh-autoload
 330 (defun mh-bogofilter-blacklist (msg)
 331   "Blacklist MSG with bogofilter.
 332
 333 Bogofilter is a Bayesian spam filtering program. Get it from your
 334 local distribution or from the bogofilter web site at URL
 335 `http://bogofilter.sourceforge.net/'.
 336
 337 Bogofilter is taught by running:
 338
 339     bogofilter -n < good-message
 340
 341 on every good message, and
 342
 343     bogofilter -s < spam-message
 344
 345 on every spam message. This is called a full training; three other
 346 training methods are described in the FAQ that is distributed with
 347 bogofilter. Note that most Bayesian filters need 1000 to 5000 of each
 348 type of message to start doing a good job.
 349
 350 To use bogofilter, add the following recipes to \".procmailrc\":
 351
 352     PATH=$PATH:/usr/bin/mh
 353     MAILDIR=$HOME/`mhparam Path`
 354
 355     # Fight spam with bogofilter.
 356     :0fw
 357     | bogofilter -3 -e -p
 358
 359     :0:
 360     * ^X-Bogosity: Yes, tests=bogofilter
 361     spam/.
 362
 363     :0:
 364     * ^X-Bogosity: Unsure, tests=bogofilter
 365     spam/unsure/.
 366
 367 If bogofilter classifies a message incorrectly, or is unsure, you can
 368 use the MH-E commands \\[mh-junk-blacklist] and \\[mh-junk-whitelist]
 369 to update bogofilter's training.
 370
 371 The \"Bogofilter FAQ\" suggests that you run the following
 372 occasionally to shrink the database:
 373
 374     bogoutil -d wordlist.db | bogoutil -l wordlist.db.new
 375     mv wordlist.db wordlist.db.prv
 376     mv wordlist.db.new wordlist.db
 377
 378 The \"Bogofilter tuning HOWTO\" describes how you can fine-tune Bogofilter."
 379   (unless mh-bogofilter-executable
 380     (error "Unable to find the bogofilter executable"))
 381   (let ((msg-file (mh-msg-filename msg mh-current-folder)))
 382     (mh-truncate-log-buffer)
 383     ;; Put call-process output in log buffer if we are saving it
 384     ;; (this happens if mh-junk-background is t).
 385     (with-current-buffer mh-log-buffer
 386       (call-process mh-bogofilter-executable msg-file mh-junk-background
 387                     nil "-s"))))
 388
 389 ;;;###mh-autoload
 390 (defun mh-bogofilter-whitelist (msg)
 391   "Whitelist MSG with bogofilter.
 392
 393 See `mh-bogofilter-blacklist' for more information."
 394   (unless mh-bogofilter-executable
 395     (error "Unable to find the bogofilter executable"))
 396   (let ((msg-file (mh-msg-filename msg mh-current-folder)))
 397     (mh-truncate-log-buffer)
 398     ;; Put call-process output in log buffer if we are saving it
 399     ;; (this happens if mh-junk-background is t).
 400     (with-current-buffer mh-log-buffer
 401       (call-process mh-bogofilter-executable msg-file mh-junk-background
 402                     nil "-n"))))
 403
 404 \f
 405
 406 ;; Spamprobe Interface
 407
 408 (defvar mh-spamprobe-executable (executable-find "spamprobe"))
 409
 410 ;;;###mh-autoload
 411 (defun mh-spamprobe-blacklist (msg)
 412   "Blacklist MSG with SpamProbe.
 413
 414 SpamProbe is a Bayesian spam filtering program. Get it from your
 415 local distribution or from the SpamProbe web site at URL
 416 `http://spamprobe.sourceforge.net'.
 417
 418 To use SpamProbe, add the following recipes to \".procmailrc\":
 419
 420     PATH=$PATH:/usr/bin/mh
 421     MAILDIR=$HOME/`mhparam Path`
 422
 423     # Fight spam with SpamProbe.
 424     :0
 425     SCORE=| spamprobe receive
 426
 427     :0 wf
 428     | formail -I \"X-SpamProbe: $SCORE\"
 429
 430     :0:
 431     *^X-SpamProbe: SPAM
 432     spam/.
 433
 434 If SpamProbe classifies a message incorrectly, you can use the
 435 MH-E commands \\[mh-junk-blacklist] and \\[mh-junk-whitelist] to
 436 update SpamProbe's training."
 437   (unless mh-spamprobe-executable
 438     (error "Unable to find the spamprobe executable"))
 439   (let ((msg-file (mh-msg-filename msg mh-current-folder)))
 440     (mh-truncate-log-buffer)
 441     ;; Put call-process output in log buffer if we are saving it
 442     ;; (this happens if mh-junk-background is t).
 443     (with-current-buffer mh-log-buffer
 444       (call-process mh-spamprobe-executable msg-file mh-junk-background
 445                     nil "spam"))))
 446
 447 ;;;###mh-autoload
 448 (defun mh-spamprobe-whitelist (msg)
 449   "Whitelist MSG with SpamProbe.
 450
 451 See `mh-spamprobe-blacklist' for more information."
 452   (unless mh-spamprobe-executable
 453     (error "Unable to find the spamprobe executable"))
 454   (let ((msg-file (mh-msg-filename msg mh-current-folder)))
 455     (mh-truncate-log-buffer)
 456     ;; Put call-process output in log buffer if we are saving it
 457     ;; (this happens if mh-junk-background is t).
 458     (with-current-buffer mh-log-buffer
 459       (call-process mh-spamprobe-executable msg-file mh-junk-background
 460                     nil "good"))))
 461
 462 (provide 'mh-junk)
 463
 464 ;; Local Variables:
 465 ;; indent-tabs-mode: nil
 466 ;; sentence-end-double-space: nil
 467 ;; End:
 468
 469 ;;; mh-junk.el ends here