Merge from emacs--rel--22
[emacs.git] / lisp / gnus / spam-stat.el
blob5b57f376cf897482c43f3d3e7f423c5a097834a8
1 ;;; spam-stat.el --- detecting spam based on statistics
3 ;; Copyright (C) 2002, 2003, 2004, 2005, 2006, 2007 Free Software Foundation, Inc.
5 ;; Author: Alex Schroeder <alex@gnu.org>
6 ;; Keywords: network
7 ;; URL: http://www.emacswiki.org/cgi-bin/wiki.pl?SpamStat
9 ;; This file is part of GNU Emacs.
11 ;; This is free software; you can redistribute it and/or modify it
12 ;; under the terms of the GNU General Public License as published by
13 ;; the Free Software Foundation; either version 3, or (at your option)
14 ;; any later version.
16 ;; This is distributed in the hope that it will be useful, but WITHOUT
17 ;; ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
18 ;; or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public
19 ;; License for more details.
21 ;; You should have received a copy of the GNU General Public License
22 ;; along with GNU Emacs; see the file COPYING. If not, write to the
23 ;; Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
24 ;; Boston, MA 02110-1301, USA.
26 ;;; Commentary:
28 ;; This implements spam analysis according to Paul Graham in "A Plan
29 ;; for Spam". The basis for all this is a statistical distribution of
30 ;; words for your spam and non-spam mails. We need this information
31 ;; in a hash-table so that the analysis can use the information when
32 ;; looking at your mails. Therefore, before you begin, you need tons
33 ;; of mails (Graham uses 4000 non-spam and 4000 spam mails for his
34 ;; experiments).
36 ;; The main interface to using spam-stat, are the following functions:
38 ;; `spam-stat-buffer-is-spam' -- called in a buffer, that buffer is
39 ;; considered to be a new spam mail; use this for new mail that has
40 ;; not been processed before
42 ;; `spam-stat-buffer-is-non-spam' -- called in a buffer, that buffer
43 ;; is considered to be a new non-spam mail; use this for new mail that
44 ;; has not been processed before
46 ;; `spam-stat-buffer-change-to-spam' -- called in a buffer, that
47 ;; buffer is no longer considered to be normal mail but spam; use this
48 ;; to change the status of a mail that has already been processed as
49 ;; non-spam
51 ;; `spam-stat-buffer-change-to-non-spam' -- called in a buffer, that
52 ;; buffer is no longer considered to be spam but normal mail; use this
53 ;; to change the status of a mail that has already been processed as
54 ;; spam
56 ;; `spam-stat-save' -- save the hash table to the file; the filename
57 ;; used is stored in the variable `spam-stat-file'
59 ;; `spam-stat-load' -- load the hash table from a file; the filename
60 ;; used is stored in the variable `spam-stat-file'
62 ;; `spam-stat-score-word' -- return the spam score for a word
64 ;; `spam-stat-score-buffer' -- return the spam score for a buffer
66 ;; `spam-stat-split-fancy' -- for fancy mail splitting; add
67 ;; the rule (: spam-stat-split-fancy) to `nnmail-split-fancy'
69 ;; This requires the following in your ~/.gnus file:
71 ;; (require 'spam-stat)
72 ;; (spam-stat-load)
74 ;;; Testing:
76 ;; Typical test will involve calls to the following functions:
78 ;; Reset: (spam-stat-reset)
79 ;; Learn spam: (spam-stat-process-spam-directory "~/Mail/mail/spam")
80 ;; Learn non-spam: (spam-stat-process-non-spam-directory "~/Mail/mail/misc")
81 ;; Save table: (spam-stat-save)
82 ;; File size: (nth 7 (file-attributes spam-stat-file))
83 ;; Number of words: (hash-table-count spam-stat)
84 ;; Test spam: (spam-stat-test-directory "~/Mail/mail/spam")
85 ;; Test non-spam: (spam-stat-test-directory "~/Mail/mail/misc")
86 ;; Reduce table size: (spam-stat-reduce-size)
87 ;; Save table: (spam-stat-save)
88 ;; File size: (nth 7 (file-attributes spam-stat-file))
89 ;; Number of words: (hash-table-count spam-stat)
90 ;; Test spam: (spam-stat-test-directory "~/Mail/mail/spam")
91 ;; Test non-spam: (spam-stat-test-directory "~/Mail/mail/misc")
93 ;;; Dictionary Creation:
95 ;; Typically, you will filter away mailing lists etc. using specific
96 ;; rules in `nnmail-split-fancy'. Somewhere among these rules, you
97 ;; will filter spam. Here is how you would create your dictionary:
99 ;; Reset: (spam-stat-reset)
100 ;; Learn spam: (spam-stat-process-spam-directory "~/Mail/mail/spam")
101 ;; Learn non-spam: (spam-stat-process-non-spam-directory "~/Mail/mail/misc")
102 ;; Repeat for any other non-spam group you need...
103 ;; Reduce table size: (spam-stat-reduce-size)
104 ;; Save table: (spam-stat-save)
106 ;;; Todo:
108 ;; Speed it up. Integrate with Gnus such that it uses spam and expiry
109 ;; marks to call the appropriate functions when leaving the summary
110 ;; buffer and saves the hash table when leaving Gnus. More testing:
111 ;; More mails, disabling SpamAssassin, double checking algorithm, find
112 ;; improved algorithm.
114 ;;; Thanks:
116 ;; Ted Zlatanov <tzz@lifelogs.com>
117 ;; Jesper Harder <harder@myrealbox.com>
118 ;; Dan Schmidt <dfan@dfan.org>
119 ;; Lasse Rasinen <lrasinen@iki.fi>
120 ;; Milan Zamazal <pdm@zamazal.org>
124 ;;; Code:
125 (require 'mail-parse)
127 (defvar gnus-original-article-buffer)
129 (defgroup spam-stat nil
130 "Statistical spam detection for Emacs.
131 Use the functions to build a dictionary of words and their statistical
132 distribution in spam and non-spam mails. Then use a function to determine
133 whether a buffer contains spam or not."
134 :version "22.1"
135 :group 'gnus)
137 (defcustom spam-stat-file "~/.spam-stat.el"
138 "File used to save and load the dictionary.
139 See `spam-stat-to-hash-table' for the format of the file."
140 :type 'file
141 :group 'spam-stat)
143 (defcustom spam-stat-install-hooks t
144 "Whether spam-stat should install its hooks in Gnus.
145 This is set to nil if you use spam-stat through spam.el."
146 :type 'boolean
147 :group 'spam-stat)
149 (defcustom spam-stat-unknown-word-score 0.2
150 "The score to use for unknown words.
151 Also used for words that don't appear often enough."
152 :type 'number
153 :group 'spam-stat)
155 (defcustom spam-stat-max-word-length 15
156 "Only words shorter than this will be considered."
157 :type 'integer
158 :group 'spam-stat)
160 (defcustom spam-stat-max-buffer-length 10240
161 "Only the beginning of buffers will be analyzed.
162 This variable says how many characters this will be."
163 :type 'integer
164 :group 'spam-stat)
166 (defcustom spam-stat-split-fancy-spam-group "mail.spam"
167 "Name of the group where spam should be stored.
168 If `spam-stat-split-fancy' is used in fancy splitting rules. Has
169 no effect when spam-stat is invoked through spam.el."
170 :type 'string
171 :group 'spam-stat)
173 (defcustom spam-stat-split-fancy-spam-threshold 0.9
174 "Spam score threshold in spam-stat-split-fancy."
175 :type 'number
176 :group 'spam-stat)
178 (defcustom spam-stat-washing-hook nil
179 "Hook applied to each message before analysis."
180 :type 'hook
181 :group 'spam-stat)
183 (defcustom spam-stat-score-buffer-user-functions nil
184 "List of additional scoring functions.
185 Called one by one on the buffer.
187 If all of these functions return non-nil answers, these numerical
188 answers are added to the computed spam stat score on the buffer. If
189 you defun such functions, make sure they don't return the buffer in a
190 narrowed state or such: use, for example, `save-excursion'. Each of
191 your functions is also passed the initial spam-stat score which might
192 aid in your scoring.
194 Also be careful when defining such functions. If they take a long
195 time, they will slow down your mail splitting. Thus, if the buffer is
196 large, don't forget to use smaller regions, by wrapping your work in,
197 say, `with-spam-stat-max-buffer-size'."
198 :type '(repeat sexp)
199 :group 'spam-stat)
201 (defcustom spam-stat-process-directory-age 90
202 "Max. age of files to be processed in directory, in days.
203 When using `spam-stat-process-spam-directory' or
204 `spam-stat-process-non-spam-directory', only files that have
205 been touched in this many days will be considered. Without
206 this filter, re-training spam-stat with several thousand messages
207 will start to take a very long time."
208 :type 'number
209 :group 'spam-stat)
211 (defvar spam-stat-last-saved-at nil
212 "Time stamp of last change of spam-stat-file on this run")
214 (defvar spam-stat-syntax-table
215 (let ((table (copy-syntax-table text-mode-syntax-table)))
216 (modify-syntax-entry ?- "w" table)
217 (modify-syntax-entry ?_ "w" table)
218 (modify-syntax-entry ?. "w" table)
219 (modify-syntax-entry ?! "w" table)
220 (modify-syntax-entry ?? "w" table)
221 (modify-syntax-entry ?+ "w" table)
222 table)
223 "Syntax table used when processing mails for statistical analysis.
224 The important part is which characters are word constituents.")
226 (defvar spam-stat-dirty nil
227 "Whether the spam-stat database needs saving.")
229 (defvar spam-stat-buffer nil
230 "Buffer to use for scoring while splitting.
231 This is set by hooking into Gnus.")
233 (defvar spam-stat-buffer-name " *spam stat buffer*"
234 "Name of the `spam-stat-buffer'.")
236 (defvar spam-stat-coding-system
237 (if (mm-coding-system-p 'emacs-mule) 'emacs-mule 'raw-text)
238 "Coding system used for `spam-stat-file'.")
240 ;; Hooking into Gnus
242 (defun spam-stat-store-current-buffer ()
243 "Store a copy of the current buffer in `spam-stat-buffer'."
244 (let ((buf (current-buffer)))
245 (with-current-buffer (get-buffer-create spam-stat-buffer-name)
246 (erase-buffer)
247 (insert-buffer-substring buf)
248 (setq spam-stat-buffer (current-buffer)))))
250 (defun spam-stat-store-gnus-article-buffer ()
251 "Store a copy of the current article in `spam-stat-buffer'.
252 This uses `gnus-article-buffer'."
253 (with-current-buffer gnus-original-article-buffer
254 (spam-stat-store-current-buffer)))
256 ;; Data -- not using defstruct in order to save space and time
258 (defvar spam-stat (make-hash-table :test 'equal)
259 "Hash table used to store the statistics.
260 Use `spam-stat-load' to load the file.
261 Every word is used as a key in this table. The value is a vector.
262 Use `spam-stat-ngood', `spam-stat-nbad', `spam-stat-good',
263 `spam-stat-bad', and `spam-stat-score' to access this vector.")
265 (defvar spam-stat-ngood 0
266 "The number of good mails in the dictionary.")
268 (defvar spam-stat-nbad 0
269 "The number of bad mails in the dictionary.")
271 (defvar spam-stat-error-holder nil
272 "A holder for condition-case errors while scoring buffers.")
274 (defsubst spam-stat-good (entry)
275 "Return the number of times this word belongs to good mails."
276 (aref entry 0))
278 (defsubst spam-stat-bad (entry)
279 "Return the number of times this word belongs to bad mails."
280 (aref entry 1))
282 (defsubst spam-stat-score (entry)
283 "Set the score of this word."
284 (if entry
285 (aref entry 2)
286 spam-stat-unknown-word-score))
288 (defsubst spam-stat-set-good (entry value)
289 "Set the number of times this word belongs to good mails."
290 (aset entry 0 value))
292 (defsubst spam-stat-set-bad (entry value)
293 "Set the number of times this word belongs to bad mails."
294 (aset entry 1 value))
296 (defsubst spam-stat-set-score (entry value)
297 "Set the score of this word."
298 (aset entry 2 value))
300 (defsubst spam-stat-make-entry (good bad)
301 "Return a vector with the given properties."
302 (let ((entry (vector good bad nil)))
303 (spam-stat-set-score entry (spam-stat-compute-score entry))
304 entry))
306 ;; Computing
308 (defun spam-stat-compute-score (entry)
309 "Compute the score of this word. 1.0 means spam."
310 ;; promote all numbers to floats for the divisions
311 (let* ((g (* 2.0 (spam-stat-good entry)))
312 (b (float (spam-stat-bad entry))))
313 (cond ((< (+ g b) 5)
315 ((= 0 spam-stat-ngood)
316 .99)
317 ((= 0 spam-stat-nbad)
318 .01)
320 (max .01
321 (min .99 (/ (/ b spam-stat-nbad)
322 (+ (/ g spam-stat-ngood)
323 (/ b spam-stat-nbad)))))))))
325 ;; Parsing
327 (defmacro with-spam-stat-max-buffer-size (&rest body)
328 "Narrow the buffer down to the first 4k characters, then evaluate BODY."
329 `(save-restriction
330 (when (> (- (point-max)
331 (point-min))
332 spam-stat-max-buffer-length)
333 (narrow-to-region (point-min)
334 (+ (point-min) spam-stat-max-buffer-length)))
335 ,@body))
337 (defun spam-stat-buffer-words ()
338 "Return a hash table of words and number of occurrences in the buffer."
339 (run-hooks 'spam-stat-washing-hook)
340 (with-spam-stat-max-buffer-size
341 (with-syntax-table spam-stat-syntax-table
342 (goto-char (point-min))
343 (let ((result (make-hash-table :test 'equal))
344 word count)
345 (while (re-search-forward "\\w+" nil t)
346 (setq word (match-string-no-properties 0)
347 count (1+ (gethash word result 0)))
348 (when (< (length word) spam-stat-max-word-length)
349 (puthash word count result)))
350 result))))
352 (defun spam-stat-buffer-is-spam ()
353 "Consider current buffer to be a new spam mail."
354 (setq spam-stat-nbad (1+ spam-stat-nbad))
355 (maphash
356 (lambda (word count)
357 (let ((entry (gethash word spam-stat)))
358 (if entry
359 (spam-stat-set-bad entry (+ count (spam-stat-bad entry)))
360 (setq entry (spam-stat-make-entry 0 count)))
361 (spam-stat-set-score entry (spam-stat-compute-score entry))
362 (puthash word entry spam-stat)))
363 (spam-stat-buffer-words))
364 (setq spam-stat-dirty t))
366 (defun spam-stat-buffer-is-non-spam ()
367 "Consider current buffer to be a new non-spam mail."
368 (setq spam-stat-ngood (1+ spam-stat-ngood))
369 (maphash
370 (lambda (word count)
371 (let ((entry (gethash word spam-stat)))
372 (if entry
373 (spam-stat-set-good entry (+ count (spam-stat-good entry)))
374 (setq entry (spam-stat-make-entry count 0)))
375 (spam-stat-set-score entry (spam-stat-compute-score entry))
376 (puthash word entry spam-stat)))
377 (spam-stat-buffer-words))
378 (setq spam-stat-dirty t))
380 (defun spam-stat-buffer-change-to-spam ()
381 "Consider current buffer no longer normal mail but spam."
382 (setq spam-stat-nbad (1+ spam-stat-nbad)
383 spam-stat-ngood (1- spam-stat-ngood))
384 (maphash
385 (lambda (word count)
386 (let ((entry (gethash word spam-stat)))
387 (if (not entry)
388 (gnus-message 8 "This buffer has unknown words in it")
389 (spam-stat-set-good entry (- (spam-stat-good entry) count))
390 (spam-stat-set-bad entry (+ (spam-stat-bad entry) count))
391 (spam-stat-set-score entry (spam-stat-compute-score entry))
392 (puthash word entry spam-stat))))
393 (spam-stat-buffer-words))
394 (setq spam-stat-dirty t))
396 (defun spam-stat-buffer-change-to-non-spam ()
397 "Consider current buffer no longer spam but normal mail."
398 (setq spam-stat-nbad (1- spam-stat-nbad)
399 spam-stat-ngood (1+ spam-stat-ngood))
400 (maphash
401 (lambda (word count)
402 (let ((entry (gethash word spam-stat)))
403 (if (not entry)
404 (gnus-message 8 "This buffer has unknown words in it")
405 (spam-stat-set-good entry (+ (spam-stat-good entry) count))
406 (spam-stat-set-bad entry (- (spam-stat-bad entry) count))
407 (spam-stat-set-score entry (spam-stat-compute-score entry))
408 (puthash word entry spam-stat))))
409 (spam-stat-buffer-words))
410 (setq spam-stat-dirty t))
412 ;; Saving and Loading
414 (defun spam-stat-save (&optional force)
415 "Save the `spam-stat' hash table as lisp file.
416 With a prefix argument save unconditionally."
417 (interactive "P")
418 (when (or force spam-stat-dirty)
419 (let ((coding-system-for-write spam-stat-coding-system))
420 (with-temp-file spam-stat-file
421 (let ((standard-output (current-buffer))
422 (font-lock-maximum-size 0))
423 (insert (format ";-*- coding: %s; -*-\n" spam-stat-coding-system))
424 (insert (format "(setq spam-stat-ngood %d spam-stat-nbad %d
425 spam-stat (spam-stat-to-hash-table '(" spam-stat-ngood spam-stat-nbad))
426 (maphash (lambda (word entry)
427 (prin1 (list word
428 (spam-stat-good entry)
429 (spam-stat-bad entry))))
430 spam-stat)
431 (insert ")))"))))
432 (message "Saved %s." spam-stat-file)
433 (setq spam-stat-dirty nil
434 spam-stat-last-saved-at (nth 5 (file-attributes spam-stat-file)))))
436 (defun spam-stat-load ()
437 "Read the `spam-stat' hash table from disk."
438 ;; TODO: maybe we should warn the user if spam-stat-dirty is t?
439 (let ((coding-system-for-read spam-stat-coding-system))
440 (cond (spam-stat-dirty (message "Spam stat not loaded: spam-stat-dirty t"))
441 ((or (not (boundp 'spam-stat-last-saved-at))
442 (null spam-stat-last-saved-at)
443 (not (equal spam-stat-last-saved-at
444 (nth 5 (file-attributes spam-stat-file)))))
445 (progn
446 (load-file spam-stat-file)
447 (setq spam-stat-dirty nil
448 spam-stat-last-saved-at
449 (nth 5 (file-attributes spam-stat-file)))))
450 (t (message "Spam stat file not loaded: no change in disk..")))))
452 (defun spam-stat-to-hash-table (entries)
453 "Turn list ENTRIES into a hash table and store as `spam-stat'.
454 Every element in ENTRIES has the form \(WORD GOOD BAD) where WORD is
455 the word string, NGOOD is the number of good mails it has appeared in,
456 NBAD is the number of bad mails it has appeared in, GOOD is the number
457 of times it appeared in good mails, and BAD is the number of times it
458 has appeared in bad mails."
459 (let ((table (make-hash-table :size (length entries)
460 :test 'equal)))
461 (mapc (lambda (l)
462 (puthash (car l)
463 (spam-stat-make-entry (nth 1 l) (nth 2 l))
464 table))
465 entries)
466 table))
468 (defun spam-stat-reset ()
469 "Reset `spam-stat' to an empty hash-table.
470 This deletes all the statistics."
471 (interactive)
472 (setq spam-stat (make-hash-table :test 'equal)
473 spam-stat-ngood 0
474 spam-stat-nbad 0)
475 (setq spam-stat-dirty t))
477 ;; Scoring buffers
479 (defvar spam-stat-score-data nil
480 "Raw data used in the last run of `spam-stat-score-buffer'.")
482 (defsubst spam-stat-score-word (word)
483 "Return score for WORD.
484 The default score for unknown words is stored in
485 `spam-stat-unknown-word-score'."
486 (spam-stat-score (gethash word spam-stat)))
488 (defun spam-stat-buffer-words-with-scores ()
489 "Process current buffer, return the 15 most conspicuous words.
490 These are the words whose spam-stat differs the most from 0.5.
491 The list returned contains elements of the form \(WORD SCORE DIFF),
492 where DIFF is the difference between SCORE and 0.5."
493 (let (result word score)
494 (maphash (lambda (word ignore)
495 (setq score (spam-stat-score-word word)
496 result (cons (list word score (abs (- score 0.5)))
497 result)))
498 (spam-stat-buffer-words))
499 (setq result (sort result (lambda (a b) (< (nth 2 b) (nth 2 a)))))
500 (setcdr (nthcdr 14 result) nil)
501 result))
503 (defun spam-stat-score-buffer ()
504 "Return a score describing the spam-probability for this buffer.
505 Add user supplied modifications if supplied."
506 (interactive) ; helps in debugging.
507 (setq spam-stat-score-data (spam-stat-buffer-words-with-scores))
508 (let* ((probs (mapcar 'cadr spam-stat-score-data))
509 (prod (apply #'* probs))
510 (score0
511 (/ prod (+ prod (apply #'* (mapcar #'(lambda (x) (- 1 x))
512 probs)))))
513 (score1s
514 (condition-case
515 spam-stat-error-holder
516 (spam-stat-score-buffer-user score0)
517 (error nil)))
518 (ans
519 (if score1s (+ score0 score1s) score0)))
520 (when (interactive-p)
521 (message "%S" ans))
522 ans))
524 (defun spam-stat-score-buffer-user (&rest args)
525 (let* ((scores
526 (mapcar
527 (lambda (fn)
528 (apply fn args))
529 spam-stat-score-buffer-user-functions)))
530 (if (memq nil scores) nil
531 (apply #'+ scores))))
533 (defun spam-stat-split-fancy ()
534 "Return the name of the spam group if the current mail is spam.
535 Use this function on `nnmail-split-fancy'. If you are interested in
536 the raw data used for the last run of `spam-stat-score-buffer',
537 check the variable `spam-stat-score-data'."
538 (condition-case spam-stat-error-holder
539 (progn
540 (set-buffer spam-stat-buffer)
541 (goto-char (point-min))
542 (when (> (spam-stat-score-buffer) spam-stat-split-fancy-spam-threshold)
543 (when (boundp 'nnmail-split-trace)
544 (mapc (lambda (entry)
545 (push entry nnmail-split-trace))
546 spam-stat-score-data))
547 spam-stat-split-fancy-spam-group))
548 (error (message "Error in spam-stat-split-fancy: %S" spam-stat-error-holder)
549 nil)))
551 ;; Testing
553 (defun spam-stat-strip-xref ()
554 "Strip the the Xref header."
555 (save-restriction
556 (mail-narrow-to-head)
557 (when (re-search-forward "^Xref:.*\n" nil t)
558 (delete-region (match-beginning 0) (match-end 0)))))
560 (defun spam-stat-process-directory (dir func)
561 "Process all the regular files in directory DIR using function FUNC."
562 (let* ((files (directory-files dir t "^[^.]"))
563 (max (/ (length files) 100.0))
564 (count 0))
565 (with-temp-buffer
566 (dolist (f files)
567 (when (and (file-readable-p f)
568 (file-regular-p f)
569 (> (nth 7 (file-attributes f)) 0)
570 (< (time-to-number-of-days (time-since (nth 5 (file-attributes f))))
571 spam-stat-process-directory-age))
572 (setq count (1+ count))
573 (message "Reading %s: %.2f%%" dir (/ count max))
574 (insert-file-contents-literally f)
575 (spam-stat-strip-xref)
576 (funcall func)
577 (erase-buffer))))))
579 (defun spam-stat-process-spam-directory (dir)
580 "Process all the regular files in directory DIR as spam."
581 (interactive "D")
582 (spam-stat-process-directory dir 'spam-stat-buffer-is-spam))
584 (defun spam-stat-process-non-spam-directory (dir)
585 "Process all the regular files in directory DIR as non-spam."
586 (interactive "D")
587 (spam-stat-process-directory dir 'spam-stat-buffer-is-non-spam))
589 (defun spam-stat-count ()
590 "Return size of `spam-stat'."
591 (interactive)
592 (hash-table-count spam-stat))
594 (defun spam-stat-test-directory (dir &optional verbose)
595 "Test all the regular files in directory DIR for spam.
596 If the result is 1.0, then all files are considered spam.
597 If the result is 0.0, non of the files is considered spam.
598 You can use this to determine error rates.
600 If VERBOSE is non-nil display names of files detected as spam or
601 non-spam in a temporary buffer. If it is the symbol `ham',
602 display non-spam files; otherwise display spam files."
603 (interactive "DDirectory: ")
604 (let* ((files (directory-files dir t "^[^.]"))
605 display-files
606 buffer-score
607 (total (length files))
608 (score 0.0); float
609 (max (/ total 100.0)); float
610 (count 0))
611 (with-temp-buffer
612 (dolist (f files)
613 (when (and (file-readable-p f)
614 (file-regular-p f)
615 (> (nth 7 (file-attributes f)) 0))
616 (setq count (1+ count))
617 (message "Reading %.2f%%, score %.2f"
618 (/ count max) (/ score count))
619 (insert-file-contents-literally f)
620 (setq buffer-score (spam-stat-score-buffer))
621 (when (> buffer-score 0.9)
622 (setq score (1+ score)))
623 (when verbose
624 (if (> buffer-score 0.9)
625 (unless (eq verbose 'ham) (push f display-files))
626 (when (eq verbose 'ham) (push f display-files))))
627 (erase-buffer))))
628 (when display-files
629 (with-output-to-temp-buffer "*spam-stat results*"
630 (dolist (file display-files)
631 (princ file)
632 (terpri))))
633 (message "Final score: %d / %d = %f" score total (/ score total))))
635 ;; Shrinking the dictionary
637 (defun spam-stat-reduce-size (&optional count)
638 "Reduce the size of `spam-stat'.
639 This removes all words that occur less than COUNT from the dictionary.
640 COUNT defaults to 5"
641 (interactive)
642 (setq count (or count 5))
643 (maphash (lambda (key entry)
644 (when (< (+ (spam-stat-good entry)
645 (spam-stat-bad entry))
646 count)
647 (remhash key spam-stat)))
648 spam-stat)
649 (setq spam-stat-dirty t))
651 (defun spam-stat-install-hooks-function ()
652 "Install the spam-stat function hooks."
653 (interactive)
654 (add-hook 'nnmail-prepare-incoming-message-hook
655 'spam-stat-store-current-buffer)
656 (add-hook 'gnus-select-article-hook
657 'spam-stat-store-gnus-article-buffer))
659 (when spam-stat-install-hooks
660 (spam-stat-install-hooks-function))
662 (defun spam-stat-unload-hook ()
663 "Uninstall the spam-stat function hooks."
664 (interactive)
665 (remove-hook 'nnmail-prepare-incoming-message-hook
666 'spam-stat-store-current-buffer)
667 (remove-hook 'gnus-select-article-hook
668 'spam-stat-store-gnus-article-buffer))
670 (add-hook 'spam-stat-unload-hook 'spam-stat-unload-hook)
672 (provide 'spam-stat)
674 ;;; arch-tag: ff1d2200-8ddb-42fb-bb7b-1b5e20448554
675 ;;; spam-stat.el ends here