Snapshot 44
[MacVim.git] / src / spell.c
blob512068121eaa50a24672c49f1a603f0f8911218e
1 /* vi:set ts=8 sts=4 sw=4:
3 * VIM - Vi IMproved by Bram Moolenaar
5 * Do ":help uganda" in Vim to read copying and usage conditions.
6 * Do ":help credits" in Vim to see a list of people who contributed.
7 * See README.txt for an overview of the Vim source code.
8 */
11 * spell.c: code for spell checking
13 * The spell checking mechanism uses a tree (aka trie). Each node in the tree
14 * has a list of bytes that can appear (siblings). For each byte there is a
15 * pointer to the node with the byte that follows in the word (child).
17 * A NUL byte is used where the word may end. The bytes are sorted, so that
18 * binary searching can be used and the NUL bytes are at the start. The
19 * number of possible bytes is stored before the list of bytes.
21 * The tree uses two arrays: "byts" stores the characters, "idxs" stores
22 * either the next index or flags. The tree starts at index 0. For example,
23 * to lookup "vi" this sequence is followed:
24 * i = 0
25 * len = byts[i]
26 * n = where "v" appears in byts[i + 1] to byts[i + len]
27 * i = idxs[n]
28 * len = byts[i]
29 * n = where "i" appears in byts[i + 1] to byts[i + len]
30 * i = idxs[n]
31 * len = byts[i]
32 * find that byts[i + 1] is 0, idxs[i + 1] has flags for "vi".
34 * There are two word trees: one with case-folded words and one with words in
35 * original case. The second one is only used for keep-case words and is
36 * usually small.
38 * There is one additional tree for when not all prefixes are applied when
39 * generating the .spl file. This tree stores all the possible prefixes, as
40 * if they were words. At each word (prefix) end the prefix nr is stored, the
41 * following word must support this prefix nr. And the condition nr is
42 * stored, used to lookup the condition that the word must match with.
44 * Thanks to Olaf Seibert for providing an example implementation of this tree
45 * and the compression mechanism.
46 * LZ trie ideas:
47 * http://www.irb.hr/hr/home/ristov/papers/RistovLZtrieRevision1.pdf
48 * More papers: http://www-igm.univ-mlv.fr/~laporte/publi_en.html
50 * Matching involves checking the caps type: Onecap ALLCAP KeepCap.
52 * Why doesn't Vim use aspell/ispell/myspell/etc.?
53 * See ":help develop-spell".
56 /* Use SPELL_PRINTTREE for debugging: dump the word tree after adding a word.
57 * Only use it for small word lists! */
58 #if 0
59 # define SPELL_PRINTTREE
60 #endif
62 /* Use DEBUG_TRIEWALK to print the changes made in suggest_trie_walk() for a
63 * specific word. */
64 #if 0
65 # define DEBUG_TRIEWALK
66 #endif
69 * Use this to adjust the score after finding suggestions, based on the
70 * suggested word sounding like the bad word. This is much faster than doing
71 * it for every possible suggestion.
72 * Disadvantage: When "the" is typed as "hte" it sounds quite different ("@"
73 * vs "ht") and goes down in the list.
74 * Used when 'spellsuggest' is set to "best".
76 #define RESCORE(word_score, sound_score) ((3 * word_score + sound_score) / 4)
79 * Do the opposite: based on a maximum end score and a known sound score,
80 * compute the maximum word score that can be used.
82 #define MAXSCORE(word_score, sound_score) ((4 * word_score - sound_score) / 3)
85 * Vim spell file format: <HEADER>
86 * <SECTIONS>
87 * <LWORDTREE>
88 * <KWORDTREE>
89 * <PREFIXTREE>
91 * <HEADER>: <fileID> <versionnr>
93 * <fileID> 8 bytes "VIMspell"
94 * <versionnr> 1 byte VIMSPELLVERSION
97 * Sections make it possible to add information to the .spl file without
98 * making it incompatible with previous versions. There are two kinds of
99 * sections:
100 * 1. Not essential for correct spell checking. E.g. for making suggestions.
101 * These are skipped when not supported.
102 * 2. Optional information, but essential for spell checking when present.
103 * E.g. conditions for affixes. When this section is present but not
104 * supported an error message is given.
106 * <SECTIONS>: <section> ... <sectionend>
108 * <section>: <sectionID> <sectionflags> <sectionlen> (section contents)
110 * <sectionID> 1 byte number from 0 to 254 identifying the section
112 * <sectionflags> 1 byte SNF_REQUIRED: this section is required for correct
113 * spell checking
115 * <sectionlen> 4 bytes length of section contents, MSB first
117 * <sectionend> 1 byte SN_END
120 * sectionID == SN_INFO: <infotext>
121 * <infotext> N bytes free format text with spell file info (version,
122 * website, etc)
124 * sectionID == SN_REGION: <regionname> ...
125 * <regionname> 2 bytes Up to 8 region names: ca, au, etc. Lower case.
126 * First <regionname> is region 1.
128 * sectionID == SN_CHARFLAGS: <charflagslen> <charflags>
129 * <folcharslen> <folchars>
130 * <charflagslen> 1 byte Number of bytes in <charflags> (should be 128).
131 * <charflags> N bytes List of flags (first one is for character 128):
132 * 0x01 word character CF_WORD
133 * 0x02 upper-case character CF_UPPER
134 * <folcharslen> 2 bytes Number of bytes in <folchars>.
135 * <folchars> N bytes Folded characters, first one is for character 128.
137 * sectionID == SN_MIDWORD: <midword>
138 * <midword> N bytes Characters that are word characters only when used
139 * in the middle of a word.
141 * sectionID == SN_PREFCOND: <prefcondcnt> <prefcond> ...
142 * <prefcondcnt> 2 bytes Number of <prefcond> items following.
143 * <prefcond> : <condlen> <condstr>
144 * <condlen> 1 byte Length of <condstr>.
145 * <condstr> N bytes Condition for the prefix.
147 * sectionID == SN_REP: <repcount> <rep> ...
148 * <repcount> 2 bytes number of <rep> items, MSB first.
149 * <rep> : <repfromlen> <repfrom> <reptolen> <repto>
150 * <repfromlen> 1 byte length of <repfrom>
151 * <repfrom> N bytes "from" part of replacement
152 * <reptolen> 1 byte length of <repto>
153 * <repto> N bytes "to" part of replacement
155 * sectionID == SN_REPSAL: <repcount> <rep> ...
156 * just like SN_REP but for soundfolded words
158 * sectionID == SN_SAL: <salflags> <salcount> <sal> ...
159 * <salflags> 1 byte flags for soundsalike conversion:
160 * SAL_F0LLOWUP
161 * SAL_COLLAPSE
162 * SAL_REM_ACCENTS
163 * <salcount> 2 bytes number of <sal> items following
164 * <sal> : <salfromlen> <salfrom> <saltolen> <salto>
165 * <salfromlen> 1 byte length of <salfrom>
166 * <salfrom> N bytes "from" part of soundsalike
167 * <saltolen> 1 byte length of <salto>
168 * <salto> N bytes "to" part of soundsalike
170 * sectionID == SN_SOFO: <sofofromlen> <sofofrom> <sofotolen> <sofoto>
171 * <sofofromlen> 2 bytes length of <sofofrom>
172 * <sofofrom> N bytes "from" part of soundfold
173 * <sofotolen> 2 bytes length of <sofoto>
174 * <sofoto> N bytes "to" part of soundfold
176 * sectionID == SN_SUGFILE: <timestamp>
177 * <timestamp> 8 bytes time in seconds that must match with .sug file
179 * sectionID == SN_NOSPLITSUGS: nothing
181 * sectionID == SN_WORDS: <word> ...
182 * <word> N bytes NUL terminated common word
184 * sectionID == SN_MAP: <mapstr>
185 * <mapstr> N bytes String with sequences of similar characters,
186 * separated by slashes.
188 * sectionID == SN_COMPOUND: <compmax> <compminlen> <compsylmax> <compoptions>
189 * <comppatcount> <comppattern> ... <compflags>
190 * <compmax> 1 byte Maximum nr of words in compound word.
191 * <compminlen> 1 byte Minimal word length for compounding.
192 * <compsylmax> 1 byte Maximum nr of syllables in compound word.
193 * <compoptions> 2 bytes COMP_ flags.
194 * <comppatcount> 2 bytes number of <comppattern> following
195 * <compflags> N bytes Flags from COMPOUNDRULE items, separated by
196 * slashes.
198 * <comppattern>: <comppatlen> <comppattext>
199 * <comppatlen> 1 byte length of <comppattext>
200 * <comppattext> N bytes end or begin chars from CHECKCOMPOUNDPATTERN
202 * sectionID == SN_NOBREAK: (empty, its presence is what matters)
204 * sectionID == SN_SYLLABLE: <syllable>
205 * <syllable> N bytes String from SYLLABLE item.
207 * <LWORDTREE>: <wordtree>
209 * <KWORDTREE>: <wordtree>
211 * <PREFIXTREE>: <wordtree>
214 * <wordtree>: <nodecount> <nodedata> ...
216 * <nodecount> 4 bytes Number of nodes following. MSB first.
218 * <nodedata>: <siblingcount> <sibling> ...
220 * <siblingcount> 1 byte Number of siblings in this node. The siblings
221 * follow in sorted order.
223 * <sibling>: <byte> [ <nodeidx> <xbyte>
224 * | <flags> [<flags2>] [<region>] [<affixID>]
225 * | [<pflags>] <affixID> <prefcondnr> ]
227 * <byte> 1 byte Byte value of the sibling. Special cases:
228 * BY_NOFLAGS: End of word without flags and for all
229 * regions.
230 * For PREFIXTREE <affixID> and
231 * <prefcondnr> follow.
232 * BY_FLAGS: End of word, <flags> follow.
233 * For PREFIXTREE <pflags>, <affixID>
234 * and <prefcondnr> follow.
235 * BY_FLAGS2: End of word, <flags> and <flags2>
236 * follow. Not used in PREFIXTREE.
237 * BY_INDEX: Child of sibling is shared, <nodeidx>
238 * and <xbyte> follow.
240 * <nodeidx> 3 bytes Index of child for this sibling, MSB first.
242 * <xbyte> 1 byte byte value of the sibling.
244 * <flags> 1 byte bitmask of:
245 * WF_ALLCAP word must have only capitals
246 * WF_ONECAP first char of word must be capital
247 * WF_KEEPCAP keep-case word
248 * WF_FIXCAP keep-case word, all caps not allowed
249 * WF_RARE rare word
250 * WF_BANNED bad word
251 * WF_REGION <region> follows
252 * WF_AFX <affixID> follows
254 * <flags2> 1 byte Bitmask of:
255 * WF_HAS_AFF >> 8 word includes affix
256 * WF_NEEDCOMP >> 8 word only valid in compound
257 * WF_NOSUGGEST >> 8 word not used for suggestions
258 * WF_COMPROOT >> 8 word already a compound
259 * WF_NOCOMPBEF >> 8 no compounding before this word
260 * WF_NOCOMPAFT >> 8 no compounding after this word
262 * <pflags> 1 byte bitmask of:
263 * WFP_RARE rare prefix
264 * WFP_NC non-combining prefix
265 * WFP_UP letter after prefix made upper case
267 * <region> 1 byte Bitmask for regions in which word is valid. When
268 * omitted it's valid in all regions.
269 * Lowest bit is for region 1.
271 * <affixID> 1 byte ID of affix that can be used with this word. In
272 * PREFIXTREE used for the required prefix ID.
274 * <prefcondnr> 2 bytes Prefix condition number, index in <prefcond> list
275 * from HEADER.
277 * All text characters are in 'encoding', but stored as single bytes.
281 * Vim .sug file format: <SUGHEADER>
282 * <SUGWORDTREE>
283 * <SUGTABLE>
285 * <SUGHEADER>: <fileID> <versionnr> <timestamp>
287 * <fileID> 6 bytes "VIMsug"
288 * <versionnr> 1 byte VIMSUGVERSION
289 * <timestamp> 8 bytes timestamp that must match with .spl file
292 * <SUGWORDTREE>: <wordtree> (see above, no flags or region used)
295 * <SUGTABLE>: <sugwcount> <sugline> ...
297 * <sugwcount> 4 bytes number of <sugline> following
299 * <sugline>: <sugnr> ... NUL
301 * <sugnr>: X bytes word number that results in this soundfolded word,
302 * stored as an offset to the previous number in as
303 * few bytes as possible, see offset2bytes())
306 #if defined(MSDOS) || defined(WIN16) || defined(WIN32) || defined(_WIN64)
307 # include "vimio.h" /* for lseek(), must be before vim.h */
308 #endif
310 #include "vim.h"
312 #if defined(FEAT_SPELL) || defined(PROTO)
314 #ifndef UNIX /* it's in os_unix.h for Unix */
315 # include <time.h> /* for time_t */
316 #endif
318 #define MAXWLEN 250 /* Assume max. word len is this many bytes.
319 Some places assume a word length fits in a
320 byte, thus it can't be above 255. */
322 /* Type used for indexes in the word tree need to be at least 4 bytes. If int
323 * is 8 bytes we could use something smaller, but what? */
324 #if SIZEOF_INT > 3
325 typedef int idx_T;
326 #else
327 typedef long idx_T;
328 #endif
330 /* Flags used for a word. Only the lowest byte can be used, the region byte
331 * comes above it. */
332 #define WF_REGION 0x01 /* region byte follows */
333 #define WF_ONECAP 0x02 /* word with one capital (or all capitals) */
334 #define WF_ALLCAP 0x04 /* word must be all capitals */
335 #define WF_RARE 0x08 /* rare word */
336 #define WF_BANNED 0x10 /* bad word */
337 #define WF_AFX 0x20 /* affix ID follows */
338 #define WF_FIXCAP 0x40 /* keep-case word, allcap not allowed */
339 #define WF_KEEPCAP 0x80 /* keep-case word */
341 /* for <flags2>, shifted up one byte to be used in wn_flags */
342 #define WF_HAS_AFF 0x0100 /* word includes affix */
343 #define WF_NEEDCOMP 0x0200 /* word only valid in compound */
344 #define WF_NOSUGGEST 0x0400 /* word not to be suggested */
345 #define WF_COMPROOT 0x0800 /* already compounded word, COMPOUNDROOT */
346 #define WF_NOCOMPBEF 0x1000 /* no compounding before this word */
347 #define WF_NOCOMPAFT 0x2000 /* no compounding after this word */
349 /* only used for su_badflags */
350 #define WF_MIXCAP 0x20 /* mix of upper and lower case: macaRONI */
352 #define WF_CAPMASK (WF_ONECAP | WF_ALLCAP | WF_KEEPCAP | WF_FIXCAP)
354 /* flags for <pflags> */
355 #define WFP_RARE 0x01 /* rare prefix */
356 #define WFP_NC 0x02 /* prefix is not combining */
357 #define WFP_UP 0x04 /* to-upper prefix */
358 #define WFP_COMPPERMIT 0x08 /* prefix with COMPOUNDPERMITFLAG */
359 #define WFP_COMPFORBID 0x10 /* prefix with COMPOUNDFORBIDFLAG */
361 /* Flags for postponed prefixes in "sl_pidxs". Must be above affixID (one
362 * byte) and prefcondnr (two bytes). */
363 #define WF_RAREPFX (WFP_RARE << 24) /* rare postponed prefix */
364 #define WF_PFX_NC (WFP_NC << 24) /* non-combining postponed prefix */
365 #define WF_PFX_UP (WFP_UP << 24) /* to-upper postponed prefix */
366 #define WF_PFX_COMPPERMIT (WFP_COMPPERMIT << 24) /* postponed prefix with
367 * COMPOUNDPERMITFLAG */
368 #define WF_PFX_COMPFORBID (WFP_COMPFORBID << 24) /* postponed prefix with
369 * COMPOUNDFORBIDFLAG */
372 /* flags for <compoptions> */
373 #define COMP_CHECKDUP 1 /* CHECKCOMPOUNDDUP */
374 #define COMP_CHECKREP 2 /* CHECKCOMPOUNDREP */
375 #define COMP_CHECKCASE 4 /* CHECKCOMPOUNDCASE */
376 #define COMP_CHECKTRIPLE 8 /* CHECKCOMPOUNDTRIPLE */
378 /* Special byte values for <byte>. Some are only used in the tree for
379 * postponed prefixes, some only in the other trees. This is a bit messy... */
380 #define BY_NOFLAGS 0 /* end of word without flags or region; for
381 * postponed prefix: no <pflags> */
382 #define BY_INDEX 1 /* child is shared, index follows */
383 #define BY_FLAGS 2 /* end of word, <flags> byte follows; for
384 * postponed prefix: <pflags> follows */
385 #define BY_FLAGS2 3 /* end of word, <flags> and <flags2> bytes
386 * follow; never used in prefix tree */
387 #define BY_SPECIAL BY_FLAGS2 /* highest special byte value */
389 /* Info from "REP", "REPSAL" and "SAL" entries in ".aff" file used in si_rep,
390 * si_repsal, sl_rep, and si_sal. Not for sl_sal!
391 * One replacement: from "ft_from" to "ft_to". */
392 typedef struct fromto_S
394 char_u *ft_from;
395 char_u *ft_to;
396 } fromto_T;
398 /* Info from "SAL" entries in ".aff" file used in sl_sal.
399 * The info is split for quick processing by spell_soundfold().
400 * Note that "sm_oneof" and "sm_rules" point into sm_lead. */
401 typedef struct salitem_S
403 char_u *sm_lead; /* leading letters */
404 int sm_leadlen; /* length of "sm_lead" */
405 char_u *sm_oneof; /* letters from () or NULL */
406 char_u *sm_rules; /* rules like ^, $, priority */
407 char_u *sm_to; /* replacement. */
408 #ifdef FEAT_MBYTE
409 int *sm_lead_w; /* wide character copy of "sm_lead" */
410 int *sm_oneof_w; /* wide character copy of "sm_oneof" */
411 int *sm_to_w; /* wide character copy of "sm_to" */
412 #endif
413 } salitem_T;
415 #ifdef FEAT_MBYTE
416 typedef int salfirst_T;
417 #else
418 typedef short salfirst_T;
419 #endif
421 /* Values for SP_*ERROR are negative, positive values are used by
422 * read_cnt_string(). */
423 #define SP_TRUNCERROR -1 /* spell file truncated error */
424 #define SP_FORMERROR -2 /* format error in spell file */
425 #define SP_OTHERERROR -3 /* other error while reading spell file */
428 * Structure used to store words and other info for one language, loaded from
429 * a .spl file.
430 * The main access is through the tree in "sl_fbyts/sl_fidxs", storing the
431 * case-folded words. "sl_kbyts/sl_kidxs" is for keep-case words.
433 * The "byts" array stores the possible bytes in each tree node, preceded by
434 * the number of possible bytes, sorted on byte value:
435 * <len> <byte1> <byte2> ...
436 * The "idxs" array stores the index of the child node corresponding to the
437 * byte in "byts".
438 * Exception: when the byte is zero, the word may end here and "idxs" holds
439 * the flags, region mask and affixID for the word. There may be several
440 * zeros in sequence for alternative flag/region/affixID combinations.
442 typedef struct slang_S slang_T;
443 struct slang_S
445 slang_T *sl_next; /* next language */
446 char_u *sl_name; /* language name "en", "en.rare", "nl", etc. */
447 char_u *sl_fname; /* name of .spl file */
448 int sl_add; /* TRUE if it's a .add file. */
450 char_u *sl_fbyts; /* case-folded word bytes */
451 idx_T *sl_fidxs; /* case-folded word indexes */
452 char_u *sl_kbyts; /* keep-case word bytes */
453 idx_T *sl_kidxs; /* keep-case word indexes */
454 char_u *sl_pbyts; /* prefix tree word bytes */
455 idx_T *sl_pidxs; /* prefix tree word indexes */
457 char_u *sl_info; /* infotext string or NULL */
459 char_u sl_regions[17]; /* table with up to 8 region names plus NUL */
461 char_u *sl_midword; /* MIDWORD string or NULL */
463 hashtab_T sl_wordcount; /* hashtable with word count, wordcount_T */
465 int sl_compmax; /* COMPOUNDWORDMAX (default: MAXWLEN) */
466 int sl_compminlen; /* COMPOUNDMIN (default: 0) */
467 int sl_compsylmax; /* COMPOUNDSYLMAX (default: MAXWLEN) */
468 int sl_compoptions; /* COMP_* flags */
469 garray_T sl_comppat; /* CHECKCOMPOUNDPATTERN items */
470 regprog_T *sl_compprog; /* COMPOUNDRULE turned into a regexp progrm
471 * (NULL when no compounding) */
472 char_u *sl_comprules; /* all COMPOUNDRULE concatenated (or NULL) */
473 char_u *sl_compstartflags; /* flags for first compound word */
474 char_u *sl_compallflags; /* all flags for compound words */
475 char_u sl_nobreak; /* When TRUE: no spaces between words */
476 char_u *sl_syllable; /* SYLLABLE repeatable chars or NULL */
477 garray_T sl_syl_items; /* syllable items */
479 int sl_prefixcnt; /* number of items in "sl_prefprog" */
480 regprog_T **sl_prefprog; /* table with regprogs for prefixes */
482 garray_T sl_rep; /* list of fromto_T entries from REP lines */
483 short sl_rep_first[256]; /* indexes where byte first appears, -1 if
484 there is none */
485 garray_T sl_sal; /* list of salitem_T entries from SAL lines */
486 salfirst_T sl_sal_first[256]; /* indexes where byte first appears, -1 if
487 there is none */
488 int sl_followup; /* SAL followup */
489 int sl_collapse; /* SAL collapse_result */
490 int sl_rem_accents; /* SAL remove_accents */
491 int sl_sofo; /* SOFOFROM and SOFOTO instead of SAL items:
492 * "sl_sal_first" maps chars, when has_mbyte
493 * "sl_sal" is a list of wide char lists. */
494 garray_T sl_repsal; /* list of fromto_T entries from REPSAL lines */
495 short sl_repsal_first[256]; /* sl_rep_first for REPSAL lines */
496 int sl_nosplitsugs; /* don't suggest splitting a word */
498 /* Info from the .sug file. Loaded on demand. */
499 time_t sl_sugtime; /* timestamp for .sug file */
500 char_u *sl_sbyts; /* soundfolded word bytes */
501 idx_T *sl_sidxs; /* soundfolded word indexes */
502 buf_T *sl_sugbuf; /* buffer with word number table */
503 int sl_sugloaded; /* TRUE when .sug file was loaded or failed to
504 load */
506 int sl_has_map; /* TRUE if there is a MAP line */
507 #ifdef FEAT_MBYTE
508 hashtab_T sl_map_hash; /* MAP for multi-byte chars */
509 int sl_map_array[256]; /* MAP for first 256 chars */
510 #else
511 char_u sl_map_array[256]; /* MAP for first 256 chars */
512 #endif
513 hashtab_T sl_sounddone; /* table with soundfolded words that have
514 handled, see add_sound_suggest() */
517 /* First language that is loaded, start of the linked list of loaded
518 * languages. */
519 static slang_T *first_lang = NULL;
521 /* Flags used in .spl file for soundsalike flags. */
522 #define SAL_F0LLOWUP 1
523 #define SAL_COLLAPSE 2
524 #define SAL_REM_ACCENTS 4
527 * Structure used in "b_langp", filled from 'spelllang'.
529 typedef struct langp_S
531 slang_T *lp_slang; /* info for this language */
532 slang_T *lp_sallang; /* language used for sound folding or NULL */
533 slang_T *lp_replang; /* language used for REP items or NULL */
534 int lp_region; /* bitmask for region or REGION_ALL */
535 } langp_T;
537 #define LANGP_ENTRY(ga, i) (((langp_T *)(ga).ga_data) + (i))
539 #define REGION_ALL 0xff /* word valid in all regions */
541 #define VIMSPELLMAGIC "VIMspell" /* string at start of Vim spell file */
542 #define VIMSPELLMAGICL 8
543 #define VIMSPELLVERSION 50
545 #define VIMSUGMAGIC "VIMsug" /* string at start of Vim .sug file */
546 #define VIMSUGMAGICL 6
547 #define VIMSUGVERSION 1
549 /* Section IDs. Only renumber them when VIMSPELLVERSION changes! */
550 #define SN_REGION 0 /* <regionname> section */
551 #define SN_CHARFLAGS 1 /* charflags section */
552 #define SN_MIDWORD 2 /* <midword> section */
553 #define SN_PREFCOND 3 /* <prefcond> section */
554 #define SN_REP 4 /* REP items section */
555 #define SN_SAL 5 /* SAL items section */
556 #define SN_SOFO 6 /* soundfolding section */
557 #define SN_MAP 7 /* MAP items section */
558 #define SN_COMPOUND 8 /* compound words section */
559 #define SN_SYLLABLE 9 /* syllable section */
560 #define SN_NOBREAK 10 /* NOBREAK section */
561 #define SN_SUGFILE 11 /* timestamp for .sug file */
562 #define SN_REPSAL 12 /* REPSAL items section */
563 #define SN_WORDS 13 /* common words */
564 #define SN_NOSPLITSUGS 14 /* don't split word for suggestions */
565 #define SN_INFO 15 /* info section */
566 #define SN_END 255 /* end of sections */
568 #define SNF_REQUIRED 1 /* <sectionflags>: required section */
570 /* Result values. Lower number is accepted over higher one. */
571 #define SP_BANNED -1
572 #define SP_OK 0
573 #define SP_RARE 1
574 #define SP_LOCAL 2
575 #define SP_BAD 3
577 /* file used for "zG" and "zW" */
578 static char_u *int_wordlist = NULL;
580 typedef struct wordcount_S
582 short_u wc_count; /* nr of times word was seen */
583 char_u wc_word[1]; /* word, actually longer */
584 } wordcount_T;
586 static wordcount_T dumwc;
587 #define WC_KEY_OFF (unsigned)(dumwc.wc_word - (char_u *)&dumwc)
588 #define HI2WC(hi) ((wordcount_T *)((hi)->hi_key - WC_KEY_OFF))
589 #define MAXWORDCOUNT 0xffff
592 * Information used when looking for suggestions.
594 typedef struct suginfo_S
596 garray_T su_ga; /* suggestions, contains "suggest_T" */
597 int su_maxcount; /* max. number of suggestions displayed */
598 int su_maxscore; /* maximum score for adding to su_ga */
599 int su_sfmaxscore; /* idem, for when doing soundfold words */
600 garray_T su_sga; /* like su_ga, sound-folded scoring */
601 char_u *su_badptr; /* start of bad word in line */
602 int su_badlen; /* length of detected bad word in line */
603 int su_badflags; /* caps flags for bad word */
604 char_u su_badword[MAXWLEN]; /* bad word truncated at su_badlen */
605 char_u su_fbadword[MAXWLEN]; /* su_badword case-folded */
606 char_u su_sal_badword[MAXWLEN]; /* su_badword soundfolded */
607 hashtab_T su_banned; /* table with banned words */
608 slang_T *su_sallang; /* default language for sound folding */
609 } suginfo_T;
611 /* One word suggestion. Used in "si_ga". */
612 typedef struct suggest_S
614 char_u *st_word; /* suggested word, allocated string */
615 int st_wordlen; /* STRLEN(st_word) */
616 int st_orglen; /* length of replaced text */
617 int st_score; /* lower is better */
618 int st_altscore; /* used when st_score compares equal */
619 int st_salscore; /* st_score is for soundalike */
620 int st_had_bonus; /* bonus already included in score */
621 slang_T *st_slang; /* language used for sound folding */
622 } suggest_T;
624 #define SUG(ga, i) (((suggest_T *)(ga).ga_data)[i])
626 /* TRUE if a word appears in the list of banned words. */
627 #define WAS_BANNED(su, word) (!HASHITEM_EMPTY(hash_find(&su->su_banned, word)))
629 /* Number of suggestions kept when cleaning up. We need to keep more than
630 * what is displayed, because when rescore_suggestions() is called the score
631 * may change and wrong suggestions may be removed later. */
632 #define SUG_CLEAN_COUNT(su) ((su)->su_maxcount < 130 ? 150 : (su)->su_maxcount + 20)
634 /* Threshold for sorting and cleaning up suggestions. Don't want to keep lots
635 * of suggestions that are not going to be displayed. */
636 #define SUG_MAX_COUNT(su) (SUG_CLEAN_COUNT(su) + 50)
638 /* score for various changes */
639 #define SCORE_SPLIT 149 /* split bad word */
640 #define SCORE_SPLIT_NO 249 /* split bad word with NOSPLITSUGS */
641 #define SCORE_ICASE 52 /* slightly different case */
642 #define SCORE_REGION 200 /* word is for different region */
643 #define SCORE_RARE 180 /* rare word */
644 #define SCORE_SWAP 75 /* swap two characters */
645 #define SCORE_SWAP3 110 /* swap two characters in three */
646 #define SCORE_REP 65 /* REP replacement */
647 #define SCORE_SUBST 93 /* substitute a character */
648 #define SCORE_SIMILAR 33 /* substitute a similar character */
649 #define SCORE_SUBCOMP 33 /* substitute a composing character */
650 #define SCORE_DEL 94 /* delete a character */
651 #define SCORE_DELDUP 66 /* delete a duplicated character */
652 #define SCORE_DELCOMP 28 /* delete a composing character */
653 #define SCORE_INS 96 /* insert a character */
654 #define SCORE_INSDUP 67 /* insert a duplicate character */
655 #define SCORE_INSCOMP 30 /* insert a composing character */
656 #define SCORE_NONWORD 103 /* change non-word to word char */
658 #define SCORE_FILE 30 /* suggestion from a file */
659 #define SCORE_MAXINIT 350 /* Initial maximum score: higher == slower.
660 * 350 allows for about three changes. */
662 #define SCORE_COMMON1 30 /* subtracted for words seen before */
663 #define SCORE_COMMON2 40 /* subtracted for words often seen */
664 #define SCORE_COMMON3 50 /* subtracted for words very often seen */
665 #define SCORE_THRES2 10 /* word count threshold for COMMON2 */
666 #define SCORE_THRES3 100 /* word count threshold for COMMON3 */
668 /* When trying changed soundfold words it becomes slow when trying more than
669 * two changes. With less then two changes it's slightly faster but we miss a
670 * few good suggestions. In rare cases we need to try three of four changes.
672 #define SCORE_SFMAX1 200 /* maximum score for first try */
673 #define SCORE_SFMAX2 300 /* maximum score for second try */
674 #define SCORE_SFMAX3 400 /* maximum score for third try */
676 #define SCORE_BIG SCORE_INS * 3 /* big difference */
677 #define SCORE_MAXMAX 999999 /* accept any score */
678 #define SCORE_LIMITMAX 350 /* for spell_edit_score_limit() */
680 /* for spell_edit_score_limit() we need to know the minimum value of
681 * SCORE_ICASE, SCORE_SWAP, SCORE_DEL, SCORE_SIMILAR and SCORE_INS */
682 #define SCORE_EDIT_MIN SCORE_SIMILAR
685 * Structure to store info for word matching.
687 typedef struct matchinf_S
689 langp_T *mi_lp; /* info for language and region */
691 /* pointers to original text to be checked */
692 char_u *mi_word; /* start of word being checked */
693 char_u *mi_end; /* end of matching word so far */
694 char_u *mi_fend; /* next char to be added to mi_fword */
695 char_u *mi_cend; /* char after what was used for
696 mi_capflags */
698 /* case-folded text */
699 char_u mi_fword[MAXWLEN + 1]; /* mi_word case-folded */
700 int mi_fwordlen; /* nr of valid bytes in mi_fword */
702 /* for when checking word after a prefix */
703 int mi_prefarridx; /* index in sl_pidxs with list of
704 affixID/condition */
705 int mi_prefcnt; /* number of entries at mi_prefarridx */
706 int mi_prefixlen; /* byte length of prefix */
707 #ifdef FEAT_MBYTE
708 int mi_cprefixlen; /* byte length of prefix in original
709 case */
710 #else
711 # define mi_cprefixlen mi_prefixlen /* it's the same value */
712 #endif
714 /* for when checking a compound word */
715 int mi_compoff; /* start of following word offset */
716 char_u mi_compflags[MAXWLEN]; /* flags for compound words used */
717 int mi_complen; /* nr of compound words used */
718 int mi_compextra; /* nr of COMPOUNDROOT words */
720 /* others */
721 int mi_result; /* result so far: SP_BAD, SP_OK, etc. */
722 int mi_capflags; /* WF_ONECAP WF_ALLCAP WF_KEEPCAP */
723 buf_T *mi_buf; /* buffer being checked */
725 /* for NOBREAK */
726 int mi_result2; /* "mi_resul" without following word */
727 char_u *mi_end2; /* "mi_end" without following word */
728 } matchinf_T;
731 * The tables used for recognizing word characters according to spelling.
732 * These are only used for the first 256 characters of 'encoding'.
734 typedef struct spelltab_S
736 char_u st_isw[256]; /* flags: is word char */
737 char_u st_isu[256]; /* flags: is uppercase char */
738 char_u st_fold[256]; /* chars: folded case */
739 char_u st_upper[256]; /* chars: upper case */
740 } spelltab_T;
742 static spelltab_T spelltab;
743 static int did_set_spelltab;
745 #define CF_WORD 0x01
746 #define CF_UPPER 0x02
748 static void clear_spell_chartab __ARGS((spelltab_T *sp));
749 static int set_spell_finish __ARGS((spelltab_T *new_st));
750 static int spell_iswordp __ARGS((char_u *p, buf_T *buf));
751 static int spell_iswordp_nmw __ARGS((char_u *p));
752 #ifdef FEAT_MBYTE
753 static int spell_mb_isword_class __ARGS((int cl));
754 static int spell_iswordp_w __ARGS((int *p, buf_T *buf));
755 #endif
756 static int write_spell_prefcond __ARGS((FILE *fd, garray_T *gap));
759 * For finding suggestions: At each node in the tree these states are tried:
761 typedef enum
763 STATE_START = 0, /* At start of node check for NUL bytes (goodword
764 * ends); if badword ends there is a match, otherwise
765 * try splitting word. */
766 STATE_NOPREFIX, /* try without prefix */
767 STATE_SPLITUNDO, /* Undo splitting. */
768 STATE_ENDNUL, /* Past NUL bytes at start of the node. */
769 STATE_PLAIN, /* Use each byte of the node. */
770 STATE_DEL, /* Delete a byte from the bad word. */
771 STATE_INS_PREP, /* Prepare for inserting bytes. */
772 STATE_INS, /* Insert a byte in the bad word. */
773 STATE_SWAP, /* Swap two bytes. */
774 STATE_UNSWAP, /* Undo swap two characters. */
775 STATE_SWAP3, /* Swap two characters over three. */
776 STATE_UNSWAP3, /* Undo Swap two characters over three. */
777 STATE_UNROT3L, /* Undo rotate three characters left */
778 STATE_UNROT3R, /* Undo rotate three characters right */
779 STATE_REP_INI, /* Prepare for using REP items. */
780 STATE_REP, /* Use matching REP items from the .aff file. */
781 STATE_REP_UNDO, /* Undo a REP item replacement. */
782 STATE_FINAL /* End of this node. */
783 } state_T;
786 * Struct to keep the state at each level in suggest_try_change().
788 typedef struct trystate_S
790 state_T ts_state; /* state at this level, STATE_ */
791 int ts_score; /* score */
792 idx_T ts_arridx; /* index in tree array, start of node */
793 short ts_curi; /* index in list of child nodes */
794 char_u ts_fidx; /* index in fword[], case-folded bad word */
795 char_u ts_fidxtry; /* ts_fidx at which bytes may be changed */
796 char_u ts_twordlen; /* valid length of tword[] */
797 char_u ts_prefixdepth; /* stack depth for end of prefix or
798 * PFD_PREFIXTREE or PFD_NOPREFIX */
799 char_u ts_flags; /* TSF_ flags */
800 #ifdef FEAT_MBYTE
801 char_u ts_tcharlen; /* number of bytes in tword character */
802 char_u ts_tcharidx; /* current byte index in tword character */
803 char_u ts_isdiff; /* DIFF_ values */
804 char_u ts_fcharstart; /* index in fword where badword char started */
805 #endif
806 char_u ts_prewordlen; /* length of word in "preword[]" */
807 char_u ts_splitoff; /* index in "tword" after last split */
808 char_u ts_splitfidx; /* "ts_fidx" at word split */
809 char_u ts_complen; /* nr of compound words used */
810 char_u ts_compsplit; /* index for "compflags" where word was spit */
811 char_u ts_save_badflags; /* su_badflags saved here */
812 char_u ts_delidx; /* index in fword for char that was deleted,
813 valid when "ts_flags" has TSF_DIDDEL */
814 } trystate_T;
816 /* values for ts_isdiff */
817 #define DIFF_NONE 0 /* no different byte (yet) */
818 #define DIFF_YES 1 /* different byte found */
819 #define DIFF_INSERT 2 /* inserting character */
821 /* values for ts_flags */
822 #define TSF_PREFIXOK 1 /* already checked that prefix is OK */
823 #define TSF_DIDSPLIT 2 /* tried split at this point */
824 #define TSF_DIDDEL 4 /* did a delete, "ts_delidx" has index */
826 /* special values ts_prefixdepth */
827 #define PFD_NOPREFIX 0xff /* not using prefixes */
828 #define PFD_PREFIXTREE 0xfe /* walking through the prefix tree */
829 #define PFD_NOTSPECIAL 0xfd /* highest value that's not special */
831 /* mode values for find_word */
832 #define FIND_FOLDWORD 0 /* find word case-folded */
833 #define FIND_KEEPWORD 1 /* find keep-case word */
834 #define FIND_PREFIX 2 /* find word after prefix */
835 #define FIND_COMPOUND 3 /* find case-folded compound word */
836 #define FIND_KEEPCOMPOUND 4 /* find keep-case compound word */
838 static slang_T *slang_alloc __ARGS((char_u *lang));
839 static void slang_free __ARGS((slang_T *lp));
840 static void slang_clear __ARGS((slang_T *lp));
841 static void slang_clear_sug __ARGS((slang_T *lp));
842 static void find_word __ARGS((matchinf_T *mip, int mode));
843 static int match_checkcompoundpattern __ARGS((char_u *ptr, int wlen, garray_T *gap));
844 static int can_compound __ARGS((slang_T *slang, char_u *word, char_u *flags));
845 static int can_be_compound __ARGS((trystate_T *sp, slang_T *slang, char_u *compflags, int flag));
846 static int match_compoundrule __ARGS((slang_T *slang, char_u *compflags));
847 static int valid_word_prefix __ARGS((int totprefcnt, int arridx, int flags, char_u *word, slang_T *slang, int cond_req));
848 static void find_prefix __ARGS((matchinf_T *mip, int mode));
849 static int fold_more __ARGS((matchinf_T *mip));
850 static int spell_valid_case __ARGS((int wordflags, int treeflags));
851 static int no_spell_checking __ARGS((win_T *wp));
852 static void spell_load_lang __ARGS((char_u *lang));
853 static char_u *spell_enc __ARGS((void));
854 static void int_wordlist_spl __ARGS((char_u *fname));
855 static void spell_load_cb __ARGS((char_u *fname, void *cookie));
856 static slang_T *spell_load_file __ARGS((char_u *fname, char_u *lang, slang_T *old_lp, int silent));
857 static int get2c __ARGS((FILE *fd));
858 static int get3c __ARGS((FILE *fd));
859 static int get4c __ARGS((FILE *fd));
860 static time_t get8c __ARGS((FILE *fd));
861 static char_u *read_cnt_string __ARGS((FILE *fd, int cnt_bytes, int *lenp));
862 static char_u *read_string __ARGS((FILE *fd, int cnt));
863 static int read_region_section __ARGS((FILE *fd, slang_T *slang, int len));
864 static int read_charflags_section __ARGS((FILE *fd));
865 static int read_prefcond_section __ARGS((FILE *fd, slang_T *lp));
866 static int read_rep_section __ARGS((FILE *fd, garray_T *gap, short *first));
867 static int read_sal_section __ARGS((FILE *fd, slang_T *slang));
868 static int read_words_section __ARGS((FILE *fd, slang_T *lp, int len));
869 static void count_common_word __ARGS((slang_T *lp, char_u *word, int len, int count));
870 static int score_wordcount_adj __ARGS((slang_T *slang, int score, char_u *word, int split));
871 static int read_sofo_section __ARGS((FILE *fd, slang_T *slang));
872 static int read_compound __ARGS((FILE *fd, slang_T *slang, int len));
873 static int byte_in_str __ARGS((char_u *str, int byte));
874 static int init_syl_tab __ARGS((slang_T *slang));
875 static int count_syllables __ARGS((slang_T *slang, char_u *word));
876 static int set_sofo __ARGS((slang_T *lp, char_u *from, char_u *to));
877 static void set_sal_first __ARGS((slang_T *lp));
878 #ifdef FEAT_MBYTE
879 static int *mb_str2wide __ARGS((char_u *s));
880 #endif
881 static int spell_read_tree __ARGS((FILE *fd, char_u **bytsp, idx_T **idxsp, int prefixtree, int prefixcnt));
882 static idx_T read_tree_node __ARGS((FILE *fd, char_u *byts, idx_T *idxs, int maxidx, int startidx, int prefixtree, int maxprefcondnr));
883 static void clear_midword __ARGS((buf_T *buf));
884 static void use_midword __ARGS((slang_T *lp, buf_T *buf));
885 static int find_region __ARGS((char_u *rp, char_u *region));
886 static int captype __ARGS((char_u *word, char_u *end));
887 static int badword_captype __ARGS((char_u *word, char_u *end));
888 static void spell_reload_one __ARGS((char_u *fname, int added_word));
889 static void set_spell_charflags __ARGS((char_u *flags, int cnt, char_u *upp));
890 static int set_spell_chartab __ARGS((char_u *fol, char_u *low, char_u *upp));
891 static int spell_casefold __ARGS((char_u *p, int len, char_u *buf, int buflen));
892 static int check_need_cap __ARGS((linenr_T lnum, colnr_T col));
893 static void spell_find_suggest __ARGS((char_u *badptr, int badlen, suginfo_T *su, int maxcount, int banbadword, int need_cap, int interactive));
894 #ifdef FEAT_EVAL
895 static void spell_suggest_expr __ARGS((suginfo_T *su, char_u *expr));
896 #endif
897 static void spell_suggest_file __ARGS((suginfo_T *su, char_u *fname));
898 static void spell_suggest_intern __ARGS((suginfo_T *su, int interactive));
899 static void suggest_load_files __ARGS((void));
900 static void tree_count_words __ARGS((char_u *byts, idx_T *idxs));
901 static void spell_find_cleanup __ARGS((suginfo_T *su));
902 static void onecap_copy __ARGS((char_u *word, char_u *wcopy, int upper));
903 static void allcap_copy __ARGS((char_u *word, char_u *wcopy));
904 static void suggest_try_special __ARGS((suginfo_T *su));
905 static void suggest_try_change __ARGS((suginfo_T *su));
906 static void suggest_trie_walk __ARGS((suginfo_T *su, langp_T *lp, char_u *fword, int soundfold));
907 static void go_deeper __ARGS((trystate_T *stack, int depth, int score_add));
908 #ifdef FEAT_MBYTE
909 static int nofold_len __ARGS((char_u *fword, int flen, char_u *word));
910 #endif
911 static void find_keepcap_word __ARGS((slang_T *slang, char_u *fword, char_u *kword));
912 static void score_comp_sal __ARGS((suginfo_T *su));
913 static void score_combine __ARGS((suginfo_T *su));
914 static int stp_sal_score __ARGS((suggest_T *stp, suginfo_T *su, slang_T *slang, char_u *badsound));
915 static void suggest_try_soundalike_prep __ARGS((void));
916 static void suggest_try_soundalike __ARGS((suginfo_T *su));
917 static void suggest_try_soundalike_finish __ARGS((void));
918 static void add_sound_suggest __ARGS((suginfo_T *su, char_u *goodword, int score, langp_T *lp));
919 static int soundfold_find __ARGS((slang_T *slang, char_u *word));
920 static void make_case_word __ARGS((char_u *fword, char_u *cword, int flags));
921 static void set_map_str __ARGS((slang_T *lp, char_u *map));
922 static int similar_chars __ARGS((slang_T *slang, int c1, int c2));
923 static void add_suggestion __ARGS((suginfo_T *su, garray_T *gap, char_u *goodword, int badlen, int score, int altscore, int had_bonus, slang_T *slang, int maxsf));
924 static void check_suggestions __ARGS((suginfo_T *su, garray_T *gap));
925 static void add_banned __ARGS((suginfo_T *su, char_u *word));
926 static void rescore_suggestions __ARGS((suginfo_T *su));
927 static void rescore_one __ARGS((suginfo_T *su, suggest_T *stp));
928 static int cleanup_suggestions __ARGS((garray_T *gap, int maxscore, int keep));
929 static void spell_soundfold __ARGS((slang_T *slang, char_u *inword, int folded, char_u *res));
930 static void spell_soundfold_sofo __ARGS((slang_T *slang, char_u *inword, char_u *res));
931 static void spell_soundfold_sal __ARGS((slang_T *slang, char_u *inword, char_u *res));
932 #ifdef FEAT_MBYTE
933 static void spell_soundfold_wsal __ARGS((slang_T *slang, char_u *inword, char_u *res));
934 #endif
935 static int soundalike_score __ARGS((char_u *goodsound, char_u *badsound));
936 static int spell_edit_score __ARGS((slang_T *slang, char_u *badword, char_u *goodword));
937 static int spell_edit_score_limit __ARGS((slang_T *slang, char_u *badword, char_u *goodword, int limit));
938 #ifdef FEAT_MBYTE
939 static int spell_edit_score_limit_w __ARGS((slang_T *slang, char_u *badword, char_u *goodword, int limit));
940 #endif
941 static void dump_word __ARGS((slang_T *slang, char_u *word, char_u *pat, int *dir, int round, int flags, linenr_T lnum));
942 static linenr_T dump_prefixes __ARGS((slang_T *slang, char_u *word, char_u *pat, int *dir, int round, int flags, linenr_T startlnum));
943 static buf_T *open_spellbuf __ARGS((void));
944 static void close_spellbuf __ARGS((buf_T *buf));
947 * Use our own character-case definitions, because the current locale may
948 * differ from what the .spl file uses.
949 * These must not be called with negative number!
951 #ifndef FEAT_MBYTE
952 /* Non-multi-byte implementation. */
953 # define SPELL_TOFOLD(c) ((c) < 256 ? spelltab.st_fold[c] : (c))
954 # define SPELL_TOUPPER(c) ((c) < 256 ? spelltab.st_upper[c] : (c))
955 # define SPELL_ISUPPER(c) ((c) < 256 ? spelltab.st_isu[c] : FALSE)
956 #else
957 # if defined(HAVE_WCHAR_H)
958 # include <wchar.h> /* for towupper() and towlower() */
959 # endif
960 /* Multi-byte implementation. For Unicode we can call utf_*(), but don't do
961 * that for ASCII, because we don't want to use 'casemap' here. Otherwise use
962 * the "w" library function for characters above 255 if available. */
963 # ifdef HAVE_TOWLOWER
964 # define SPELL_TOFOLD(c) (enc_utf8 && (c) >= 128 ? utf_fold(c) \
965 : (c) < 256 ? spelltab.st_fold[c] : towlower(c))
966 # else
967 # define SPELL_TOFOLD(c) (enc_utf8 && (c) >= 128 ? utf_fold(c) \
968 : (c) < 256 ? spelltab.st_fold[c] : (c))
969 # endif
971 # ifdef HAVE_TOWUPPER
972 # define SPELL_TOUPPER(c) (enc_utf8 && (c) >= 128 ? utf_toupper(c) \
973 : (c) < 256 ? spelltab.st_upper[c] : towupper(c))
974 # else
975 # define SPELL_TOUPPER(c) (enc_utf8 && (c) >= 128 ? utf_toupper(c) \
976 : (c) < 256 ? spelltab.st_upper[c] : (c))
977 # endif
979 # ifdef HAVE_ISWUPPER
980 # define SPELL_ISUPPER(c) (enc_utf8 && (c) >= 128 ? utf_isupper(c) \
981 : (c) < 256 ? spelltab.st_isu[c] : iswupper(c))
982 # else
983 # define SPELL_ISUPPER(c) (enc_utf8 && (c) >= 128 ? utf_isupper(c) \
984 : (c) < 256 ? spelltab.st_isu[c] : (FALSE))
985 # endif
986 #endif
989 static char *e_format = N_("E759: Format error in spell file");
990 static char *e_spell_trunc = N_("E758: Truncated spell file");
991 static char *e_afftrailing = N_("Trailing text in %s line %d: %s");
992 static char *e_affname = N_("Affix name too long in %s line %d: %s");
993 static char *e_affform = N_("E761: Format error in affix file FOL, LOW or UPP");
994 static char *e_affrange = N_("E762: Character in FOL, LOW or UPP is out of range");
995 static char *msg_compressing = N_("Compressing word tree...");
997 /* Remember what "z?" replaced. */
998 static char_u *repl_from = NULL;
999 static char_u *repl_to = NULL;
1002 * Main spell-checking function.
1003 * "ptr" points to a character that could be the start of a word.
1004 * "*attrp" is set to the highlight index for a badly spelled word. For a
1005 * non-word or when it's OK it remains unchanged.
1006 * This must only be called when 'spelllang' is not empty.
1008 * "capcol" is used to check for a Capitalised word after the end of a
1009 * sentence. If it's zero then perform the check. Return the column where to
1010 * check next, or -1 when no sentence end was found. If it's NULL then don't
1011 * worry.
1013 * Returns the length of the word in bytes, also when it's OK, so that the
1014 * caller can skip over the word.
1017 spell_check(wp, ptr, attrp, capcol, docount)
1018 win_T *wp; /* current window */
1019 char_u *ptr;
1020 hlf_T *attrp;
1021 int *capcol; /* column to check for Capital */
1022 int docount; /* count good words */
1024 matchinf_T mi; /* Most things are put in "mi" so that it can
1025 be passed to functions quickly. */
1026 int nrlen = 0; /* found a number first */
1027 int c;
1028 int wrongcaplen = 0;
1029 int lpi;
1030 int count_word = docount;
1032 /* A word never starts at a space or a control character. Return quickly
1033 * then, skipping over the character. */
1034 if (*ptr <= ' ')
1035 return 1;
1037 /* Return here when loading language files failed. */
1038 if (wp->w_buffer->b_langp.ga_len == 0)
1039 return 1;
1041 vim_memset(&mi, 0, sizeof(matchinf_T));
1043 /* A number is always OK. Also skip hexadecimal numbers 0xFF99 and
1044 * 0X99FF. But always do check spelling to find "3GPP" and "11
1045 * julifeest". */
1046 if (*ptr >= '0' && *ptr <= '9')
1048 if (*ptr == '0' && (ptr[1] == 'x' || ptr[1] == 'X'))
1049 mi.mi_end = skiphex(ptr + 2);
1050 else
1051 mi.mi_end = skipdigits(ptr);
1052 nrlen = (int)(mi.mi_end - ptr);
1055 /* Find the normal end of the word (until the next non-word character). */
1056 mi.mi_word = ptr;
1057 mi.mi_fend = ptr;
1058 if (spell_iswordp(mi.mi_fend, wp->w_buffer))
1062 mb_ptr_adv(mi.mi_fend);
1063 } while (*mi.mi_fend != NUL && spell_iswordp(mi.mi_fend, wp->w_buffer));
1065 if (capcol != NULL && *capcol == 0 && wp->w_buffer->b_cap_prog != NULL)
1067 /* Check word starting with capital letter. */
1068 c = PTR2CHAR(ptr);
1069 if (!SPELL_ISUPPER(c))
1070 wrongcaplen = (int)(mi.mi_fend - ptr);
1073 if (capcol != NULL)
1074 *capcol = -1;
1076 /* We always use the characters up to the next non-word character,
1077 * also for bad words. */
1078 mi.mi_end = mi.mi_fend;
1080 /* Check caps type later. */
1081 mi.mi_buf = wp->w_buffer;
1083 /* case-fold the word with one non-word character, so that we can check
1084 * for the word end. */
1085 if (*mi.mi_fend != NUL)
1086 mb_ptr_adv(mi.mi_fend);
1088 (void)spell_casefold(ptr, (int)(mi.mi_fend - ptr), mi.mi_fword,
1089 MAXWLEN + 1);
1090 mi.mi_fwordlen = (int)STRLEN(mi.mi_fword);
1092 /* The word is bad unless we recognize it. */
1093 mi.mi_result = SP_BAD;
1094 mi.mi_result2 = SP_BAD;
1097 * Loop over the languages specified in 'spelllang'.
1098 * We check them all, because a word may be matched longer in another
1099 * language.
1101 for (lpi = 0; lpi < wp->w_buffer->b_langp.ga_len; ++lpi)
1103 mi.mi_lp = LANGP_ENTRY(wp->w_buffer->b_langp, lpi);
1105 /* If reloading fails the language is still in the list but everything
1106 * has been cleared. */
1107 if (mi.mi_lp->lp_slang->sl_fidxs == NULL)
1108 continue;
1110 /* Check for a matching word in case-folded words. */
1111 find_word(&mi, FIND_FOLDWORD);
1113 /* Check for a matching word in keep-case words. */
1114 find_word(&mi, FIND_KEEPWORD);
1116 /* Check for matching prefixes. */
1117 find_prefix(&mi, FIND_FOLDWORD);
1119 /* For a NOBREAK language, may want to use a word without a following
1120 * word as a backup. */
1121 if (mi.mi_lp->lp_slang->sl_nobreak && mi.mi_result == SP_BAD
1122 && mi.mi_result2 != SP_BAD)
1124 mi.mi_result = mi.mi_result2;
1125 mi.mi_end = mi.mi_end2;
1128 /* Count the word in the first language where it's found to be OK. */
1129 if (count_word && mi.mi_result == SP_OK)
1131 count_common_word(mi.mi_lp->lp_slang, ptr,
1132 (int)(mi.mi_end - ptr), 1);
1133 count_word = FALSE;
1137 if (mi.mi_result != SP_OK)
1139 /* If we found a number skip over it. Allows for "42nd". Do flag
1140 * rare and local words, e.g., "3GPP". */
1141 if (nrlen > 0)
1143 if (mi.mi_result == SP_BAD || mi.mi_result == SP_BANNED)
1144 return nrlen;
1147 /* When we are at a non-word character there is no error, just
1148 * skip over the character (try looking for a word after it). */
1149 else if (!spell_iswordp_nmw(ptr))
1151 if (capcol != NULL && wp->w_buffer->b_cap_prog != NULL)
1153 regmatch_T regmatch;
1155 /* Check for end of sentence. */
1156 regmatch.regprog = wp->w_buffer->b_cap_prog;
1157 regmatch.rm_ic = FALSE;
1158 if (vim_regexec(&regmatch, ptr, 0))
1159 *capcol = (int)(regmatch.endp[0] - ptr);
1162 #ifdef FEAT_MBYTE
1163 if (has_mbyte)
1164 return (*mb_ptr2len)(ptr);
1165 #endif
1166 return 1;
1168 else if (mi.mi_end == ptr)
1169 /* Always include at least one character. Required for when there
1170 * is a mixup in "midword". */
1171 mb_ptr_adv(mi.mi_end);
1172 else if (mi.mi_result == SP_BAD
1173 && LANGP_ENTRY(wp->w_buffer->b_langp, 0)->lp_slang->sl_nobreak)
1175 char_u *p, *fp;
1176 int save_result = mi.mi_result;
1178 /* First language in 'spelllang' is NOBREAK. Find first position
1179 * at which any word would be valid. */
1180 mi.mi_lp = LANGP_ENTRY(wp->w_buffer->b_langp, 0);
1181 if (mi.mi_lp->lp_slang->sl_fidxs != NULL)
1183 p = mi.mi_word;
1184 fp = mi.mi_fword;
1185 for (;;)
1187 mb_ptr_adv(p);
1188 mb_ptr_adv(fp);
1189 if (p >= mi.mi_end)
1190 break;
1191 mi.mi_compoff = (int)(fp - mi.mi_fword);
1192 find_word(&mi, FIND_COMPOUND);
1193 if (mi.mi_result != SP_BAD)
1195 mi.mi_end = p;
1196 break;
1199 mi.mi_result = save_result;
1203 if (mi.mi_result == SP_BAD || mi.mi_result == SP_BANNED)
1204 *attrp = HLF_SPB;
1205 else if (mi.mi_result == SP_RARE)
1206 *attrp = HLF_SPR;
1207 else
1208 *attrp = HLF_SPL;
1211 if (wrongcaplen > 0 && (mi.mi_result == SP_OK || mi.mi_result == SP_RARE))
1213 /* Report SpellCap only when the word isn't badly spelled. */
1214 *attrp = HLF_SPC;
1215 return wrongcaplen;
1218 return (int)(mi.mi_end - ptr);
1222 * Check if the word at "mip->mi_word" is in the tree.
1223 * When "mode" is FIND_FOLDWORD check in fold-case word tree.
1224 * When "mode" is FIND_KEEPWORD check in keep-case word tree.
1225 * When "mode" is FIND_PREFIX check for word after prefix in fold-case word
1226 * tree.
1228 * For a match mip->mi_result is updated.
1230 static void
1231 find_word(mip, mode)
1232 matchinf_T *mip;
1233 int mode;
1235 idx_T arridx = 0;
1236 int endlen[MAXWLEN]; /* length at possible word endings */
1237 idx_T endidx[MAXWLEN]; /* possible word endings */
1238 int endidxcnt = 0;
1239 int len;
1240 int wlen = 0;
1241 int flen;
1242 int c;
1243 char_u *ptr;
1244 idx_T lo, hi, m;
1245 #ifdef FEAT_MBYTE
1246 char_u *s;
1247 #endif
1248 char_u *p;
1249 int res = SP_BAD;
1250 slang_T *slang = mip->mi_lp->lp_slang;
1251 unsigned flags;
1252 char_u *byts;
1253 idx_T *idxs;
1254 int word_ends;
1255 int prefix_found;
1256 int nobreak_result;
1258 if (mode == FIND_KEEPWORD || mode == FIND_KEEPCOMPOUND)
1260 /* Check for word with matching case in keep-case tree. */
1261 ptr = mip->mi_word;
1262 flen = 9999; /* no case folding, always enough bytes */
1263 byts = slang->sl_kbyts;
1264 idxs = slang->sl_kidxs;
1266 if (mode == FIND_KEEPCOMPOUND)
1267 /* Skip over the previously found word(s). */
1268 wlen += mip->mi_compoff;
1270 else
1272 /* Check for case-folded in case-folded tree. */
1273 ptr = mip->mi_fword;
1274 flen = mip->mi_fwordlen; /* available case-folded bytes */
1275 byts = slang->sl_fbyts;
1276 idxs = slang->sl_fidxs;
1278 if (mode == FIND_PREFIX)
1280 /* Skip over the prefix. */
1281 wlen = mip->mi_prefixlen;
1282 flen -= mip->mi_prefixlen;
1284 else if (mode == FIND_COMPOUND)
1286 /* Skip over the previously found word(s). */
1287 wlen = mip->mi_compoff;
1288 flen -= mip->mi_compoff;
1293 if (byts == NULL)
1294 return; /* array is empty */
1297 * Repeat advancing in the tree until:
1298 * - there is a byte that doesn't match,
1299 * - we reach the end of the tree,
1300 * - or we reach the end of the line.
1302 for (;;)
1304 if (flen <= 0 && *mip->mi_fend != NUL)
1305 flen = fold_more(mip);
1307 len = byts[arridx++];
1309 /* If the first possible byte is a zero the word could end here.
1310 * Remember this index, we first check for the longest word. */
1311 if (byts[arridx] == 0)
1313 if (endidxcnt == MAXWLEN)
1315 /* Must be a corrupted spell file. */
1316 EMSG(_(e_format));
1317 return;
1319 endlen[endidxcnt] = wlen;
1320 endidx[endidxcnt++] = arridx++;
1321 --len;
1323 /* Skip over the zeros, there can be several flag/region
1324 * combinations. */
1325 while (len > 0 && byts[arridx] == 0)
1327 ++arridx;
1328 --len;
1330 if (len == 0)
1331 break; /* no children, word must end here */
1334 /* Stop looking at end of the line. */
1335 if (ptr[wlen] == NUL)
1336 break;
1338 /* Perform a binary search in the list of accepted bytes. */
1339 c = ptr[wlen];
1340 if (c == TAB) /* <Tab> is handled like <Space> */
1341 c = ' ';
1342 lo = arridx;
1343 hi = arridx + len - 1;
1344 while (lo < hi)
1346 m = (lo + hi) / 2;
1347 if (byts[m] > c)
1348 hi = m - 1;
1349 else if (byts[m] < c)
1350 lo = m + 1;
1351 else
1353 lo = hi = m;
1354 break;
1358 /* Stop if there is no matching byte. */
1359 if (hi < lo || byts[lo] != c)
1360 break;
1362 /* Continue at the child (if there is one). */
1363 arridx = idxs[lo];
1364 ++wlen;
1365 --flen;
1367 /* One space in the good word may stand for several spaces in the
1368 * checked word. */
1369 if (c == ' ')
1371 for (;;)
1373 if (flen <= 0 && *mip->mi_fend != NUL)
1374 flen = fold_more(mip);
1375 if (ptr[wlen] != ' ' && ptr[wlen] != TAB)
1376 break;
1377 ++wlen;
1378 --flen;
1384 * Verify that one of the possible endings is valid. Try the longest
1385 * first.
1387 while (endidxcnt > 0)
1389 --endidxcnt;
1390 arridx = endidx[endidxcnt];
1391 wlen = endlen[endidxcnt];
1393 #ifdef FEAT_MBYTE
1394 if ((*mb_head_off)(ptr, ptr + wlen) > 0)
1395 continue; /* not at first byte of character */
1396 #endif
1397 if (spell_iswordp(ptr + wlen, mip->mi_buf))
1399 if (slang->sl_compprog == NULL && !slang->sl_nobreak)
1400 continue; /* next char is a word character */
1401 word_ends = FALSE;
1403 else
1404 word_ends = TRUE;
1405 /* The prefix flag is before compound flags. Once a valid prefix flag
1406 * has been found we try compound flags. */
1407 prefix_found = FALSE;
1409 #ifdef FEAT_MBYTE
1410 if (mode != FIND_KEEPWORD && has_mbyte)
1412 /* Compute byte length in original word, length may change
1413 * when folding case. This can be slow, take a shortcut when the
1414 * case-folded word is equal to the keep-case word. */
1415 p = mip->mi_word;
1416 if (STRNCMP(ptr, p, wlen) != 0)
1418 for (s = ptr; s < ptr + wlen; mb_ptr_adv(s))
1419 mb_ptr_adv(p);
1420 wlen = (int)(p - mip->mi_word);
1423 #endif
1425 /* Check flags and region. For FIND_PREFIX check the condition and
1426 * prefix ID.
1427 * Repeat this if there are more flags/region alternatives until there
1428 * is a match. */
1429 res = SP_BAD;
1430 for (len = byts[arridx - 1]; len > 0 && byts[arridx] == 0;
1431 --len, ++arridx)
1433 flags = idxs[arridx];
1435 /* For the fold-case tree check that the case of the checked word
1436 * matches with what the word in the tree requires.
1437 * For keep-case tree the case is always right. For prefixes we
1438 * don't bother to check. */
1439 if (mode == FIND_FOLDWORD)
1441 if (mip->mi_cend != mip->mi_word + wlen)
1443 /* mi_capflags was set for a different word length, need
1444 * to do it again. */
1445 mip->mi_cend = mip->mi_word + wlen;
1446 mip->mi_capflags = captype(mip->mi_word, mip->mi_cend);
1449 if (mip->mi_capflags == WF_KEEPCAP
1450 || !spell_valid_case(mip->mi_capflags, flags))
1451 continue;
1454 /* When mode is FIND_PREFIX the word must support the prefix:
1455 * check the prefix ID and the condition. Do that for the list at
1456 * mip->mi_prefarridx that find_prefix() filled. */
1457 else if (mode == FIND_PREFIX && !prefix_found)
1459 c = valid_word_prefix(mip->mi_prefcnt, mip->mi_prefarridx,
1460 flags,
1461 mip->mi_word + mip->mi_cprefixlen, slang,
1462 FALSE);
1463 if (c == 0)
1464 continue;
1466 /* Use the WF_RARE flag for a rare prefix. */
1467 if (c & WF_RAREPFX)
1468 flags |= WF_RARE;
1469 prefix_found = TRUE;
1472 if (slang->sl_nobreak)
1474 if ((mode == FIND_COMPOUND || mode == FIND_KEEPCOMPOUND)
1475 && (flags & WF_BANNED) == 0)
1477 /* NOBREAK: found a valid following word. That's all we
1478 * need to know, so return. */
1479 mip->mi_result = SP_OK;
1480 break;
1484 else if ((mode == FIND_COMPOUND || mode == FIND_KEEPCOMPOUND
1485 || !word_ends))
1487 /* If there is no compound flag or the word is shorter than
1488 * COMPOUNDMIN reject it quickly.
1489 * Makes you wonder why someone puts a compound flag on a word
1490 * that's too short... Myspell compatibility requires this
1491 * anyway. */
1492 if (((unsigned)flags >> 24) == 0
1493 || wlen - mip->mi_compoff < slang->sl_compminlen)
1494 continue;
1495 #ifdef FEAT_MBYTE
1496 /* For multi-byte chars check character length against
1497 * COMPOUNDMIN. */
1498 if (has_mbyte
1499 && slang->sl_compminlen > 0
1500 && mb_charlen_len(mip->mi_word + mip->mi_compoff,
1501 wlen - mip->mi_compoff) < slang->sl_compminlen)
1502 continue;
1503 #endif
1505 /* Limit the number of compound words to COMPOUNDWORDMAX if no
1506 * maximum for syllables is specified. */
1507 if (!word_ends && mip->mi_complen + mip->mi_compextra + 2
1508 > slang->sl_compmax
1509 && slang->sl_compsylmax == MAXWLEN)
1510 continue;
1512 /* Don't allow compounding on a side where an affix was added,
1513 * unless COMPOUNDPERMITFLAG was used. */
1514 if (mip->mi_complen > 0 && (flags & WF_NOCOMPBEF))
1515 continue;
1516 if (!word_ends && (flags & WF_NOCOMPAFT))
1517 continue;
1519 /* Quickly check if compounding is possible with this flag. */
1520 if (!byte_in_str(mip->mi_complen == 0
1521 ? slang->sl_compstartflags
1522 : slang->sl_compallflags,
1523 ((unsigned)flags >> 24)))
1524 continue;
1526 /* If there is a match with a CHECKCOMPOUNDPATTERN rule
1527 * discard the compound word. */
1528 if (match_checkcompoundpattern(ptr, wlen, &slang->sl_comppat))
1529 continue;
1531 if (mode == FIND_COMPOUND)
1533 int capflags;
1535 /* Need to check the caps type of the appended compound
1536 * word. */
1537 #ifdef FEAT_MBYTE
1538 if (has_mbyte && STRNCMP(ptr, mip->mi_word,
1539 mip->mi_compoff) != 0)
1541 /* case folding may have changed the length */
1542 p = mip->mi_word;
1543 for (s = ptr; s < ptr + mip->mi_compoff; mb_ptr_adv(s))
1544 mb_ptr_adv(p);
1546 else
1547 #endif
1548 p = mip->mi_word + mip->mi_compoff;
1549 capflags = captype(p, mip->mi_word + wlen);
1550 if (capflags == WF_KEEPCAP || (capflags == WF_ALLCAP
1551 && (flags & WF_FIXCAP) != 0))
1552 continue;
1554 if (capflags != WF_ALLCAP)
1556 /* When the character before the word is a word
1557 * character we do not accept a Onecap word. We do
1558 * accept a no-caps word, even when the dictionary
1559 * word specifies ONECAP. */
1560 mb_ptr_back(mip->mi_word, p);
1561 if (spell_iswordp_nmw(p)
1562 ? capflags == WF_ONECAP
1563 : (flags & WF_ONECAP) != 0
1564 && capflags != WF_ONECAP)
1565 continue;
1569 /* If the word ends the sequence of compound flags of the
1570 * words must match with one of the COMPOUNDRULE items and
1571 * the number of syllables must not be too large. */
1572 mip->mi_compflags[mip->mi_complen] = ((unsigned)flags >> 24);
1573 mip->mi_compflags[mip->mi_complen + 1] = NUL;
1574 if (word_ends)
1576 char_u fword[MAXWLEN];
1578 if (slang->sl_compsylmax < MAXWLEN)
1580 /* "fword" is only needed for checking syllables. */
1581 if (ptr == mip->mi_word)
1582 (void)spell_casefold(ptr, wlen, fword, MAXWLEN);
1583 else
1584 vim_strncpy(fword, ptr, endlen[endidxcnt]);
1586 if (!can_compound(slang, fword, mip->mi_compflags))
1587 continue;
1589 else if (slang->sl_comprules != NULL
1590 && !match_compoundrule(slang, mip->mi_compflags))
1591 /* The compound flags collected so far do not match any
1592 * COMPOUNDRULE, discard the compounded word. */
1593 continue;
1596 /* Check NEEDCOMPOUND: can't use word without compounding. */
1597 else if (flags & WF_NEEDCOMP)
1598 continue;
1600 nobreak_result = SP_OK;
1602 if (!word_ends)
1604 int save_result = mip->mi_result;
1605 char_u *save_end = mip->mi_end;
1606 langp_T *save_lp = mip->mi_lp;
1607 int lpi;
1609 /* Check that a valid word follows. If there is one and we
1610 * are compounding, it will set "mi_result", thus we are
1611 * always finished here. For NOBREAK we only check that a
1612 * valid word follows.
1613 * Recursive! */
1614 if (slang->sl_nobreak)
1615 mip->mi_result = SP_BAD;
1617 /* Find following word in case-folded tree. */
1618 mip->mi_compoff = endlen[endidxcnt];
1619 #ifdef FEAT_MBYTE
1620 if (has_mbyte && mode == FIND_KEEPWORD)
1622 /* Compute byte length in case-folded word from "wlen":
1623 * byte length in keep-case word. Length may change when
1624 * folding case. This can be slow, take a shortcut when
1625 * the case-folded word is equal to the keep-case word. */
1626 p = mip->mi_fword;
1627 if (STRNCMP(ptr, p, wlen) != 0)
1629 for (s = ptr; s < ptr + wlen; mb_ptr_adv(s))
1630 mb_ptr_adv(p);
1631 mip->mi_compoff = (int)(p - mip->mi_fword);
1634 #endif
1635 c = mip->mi_compoff;
1636 ++mip->mi_complen;
1637 if (flags & WF_COMPROOT)
1638 ++mip->mi_compextra;
1640 /* For NOBREAK we need to try all NOBREAK languages, at least
1641 * to find the ".add" file(s). */
1642 for (lpi = 0; lpi < mip->mi_buf->b_langp.ga_len; ++lpi)
1644 if (slang->sl_nobreak)
1646 mip->mi_lp = LANGP_ENTRY(mip->mi_buf->b_langp, lpi);
1647 if (mip->mi_lp->lp_slang->sl_fidxs == NULL
1648 || !mip->mi_lp->lp_slang->sl_nobreak)
1649 continue;
1652 find_word(mip, FIND_COMPOUND);
1654 /* When NOBREAK any word that matches is OK. Otherwise we
1655 * need to find the longest match, thus try with keep-case
1656 * and prefix too. */
1657 if (!slang->sl_nobreak || mip->mi_result == SP_BAD)
1659 /* Find following word in keep-case tree. */
1660 mip->mi_compoff = wlen;
1661 find_word(mip, FIND_KEEPCOMPOUND);
1663 #if 0 /* Disabled, a prefix must not appear halfway a compound word,
1664 unless the COMPOUNDPERMITFLAG is used and then it can't be a
1665 postponed prefix. */
1666 if (!slang->sl_nobreak || mip->mi_result == SP_BAD)
1668 /* Check for following word with prefix. */
1669 mip->mi_compoff = c;
1670 find_prefix(mip, FIND_COMPOUND);
1672 #endif
1675 if (!slang->sl_nobreak)
1676 break;
1678 --mip->mi_complen;
1679 if (flags & WF_COMPROOT)
1680 --mip->mi_compextra;
1681 mip->mi_lp = save_lp;
1683 if (slang->sl_nobreak)
1685 nobreak_result = mip->mi_result;
1686 mip->mi_result = save_result;
1687 mip->mi_end = save_end;
1689 else
1691 if (mip->mi_result == SP_OK)
1692 break;
1693 continue;
1697 if (flags & WF_BANNED)
1698 res = SP_BANNED;
1699 else if (flags & WF_REGION)
1701 /* Check region. */
1702 if ((mip->mi_lp->lp_region & (flags >> 16)) != 0)
1703 res = SP_OK;
1704 else
1705 res = SP_LOCAL;
1707 else if (flags & WF_RARE)
1708 res = SP_RARE;
1709 else
1710 res = SP_OK;
1712 /* Always use the longest match and the best result. For NOBREAK
1713 * we separately keep the longest match without a following good
1714 * word as a fall-back. */
1715 if (nobreak_result == SP_BAD)
1717 if (mip->mi_result2 > res)
1719 mip->mi_result2 = res;
1720 mip->mi_end2 = mip->mi_word + wlen;
1722 else if (mip->mi_result2 == res
1723 && mip->mi_end2 < mip->mi_word + wlen)
1724 mip->mi_end2 = mip->mi_word + wlen;
1726 else if (mip->mi_result > res)
1728 mip->mi_result = res;
1729 mip->mi_end = mip->mi_word + wlen;
1731 else if (mip->mi_result == res && mip->mi_end < mip->mi_word + wlen)
1732 mip->mi_end = mip->mi_word + wlen;
1734 if (mip->mi_result == SP_OK)
1735 break;
1738 if (mip->mi_result == SP_OK)
1739 break;
1744 * Return TRUE if there is a match between the word ptr[wlen] and
1745 * CHECKCOMPOUNDPATTERN rules, assuming that we will concatenate with another
1746 * word.
1747 * A match means that the first part of CHECKCOMPOUNDPATTERN matches at the
1748 * end of ptr[wlen] and the second part matches after it.
1750 static int
1751 match_checkcompoundpattern(ptr, wlen, gap)
1752 char_u *ptr;
1753 int wlen;
1754 garray_T *gap; /* &sl_comppat */
1756 int i;
1757 char_u *p;
1758 int len;
1760 for (i = 0; i + 1 < gap->ga_len; i += 2)
1762 p = ((char_u **)gap->ga_data)[i + 1];
1763 if (STRNCMP(ptr + wlen, p, STRLEN(p)) == 0)
1765 /* Second part matches at start of following compound word, now
1766 * check if first part matches at end of previous word. */
1767 p = ((char_u **)gap->ga_data)[i];
1768 len = (int)STRLEN(p);
1769 if (len <= wlen && STRNCMP(ptr + wlen - len, p, len) == 0)
1770 return TRUE;
1773 return FALSE;
1777 * Return TRUE if "flags" is a valid sequence of compound flags and "word"
1778 * does not have too many syllables.
1780 static int
1781 can_compound(slang, word, flags)
1782 slang_T *slang;
1783 char_u *word;
1784 char_u *flags;
1786 regmatch_T regmatch;
1787 #ifdef FEAT_MBYTE
1788 char_u uflags[MAXWLEN * 2];
1789 int i;
1790 #endif
1791 char_u *p;
1793 if (slang->sl_compprog == NULL)
1794 return FALSE;
1795 #ifdef FEAT_MBYTE
1796 if (enc_utf8)
1798 /* Need to convert the single byte flags to utf8 characters. */
1799 p = uflags;
1800 for (i = 0; flags[i] != NUL; ++i)
1801 p += mb_char2bytes(flags[i], p);
1802 *p = NUL;
1803 p = uflags;
1805 else
1806 #endif
1807 p = flags;
1808 regmatch.regprog = slang->sl_compprog;
1809 regmatch.rm_ic = FALSE;
1810 if (!vim_regexec(&regmatch, p, 0))
1811 return FALSE;
1813 /* Count the number of syllables. This may be slow, do it last. If there
1814 * are too many syllables AND the number of compound words is above
1815 * COMPOUNDWORDMAX then compounding is not allowed. */
1816 if (slang->sl_compsylmax < MAXWLEN
1817 && count_syllables(slang, word) > slang->sl_compsylmax)
1818 return (int)STRLEN(flags) < slang->sl_compmax;
1819 return TRUE;
1823 * Return TRUE when the sequence of flags in "compflags" plus "flag" can
1824 * possibly form a valid compounded word. This also checks the COMPOUNDRULE
1825 * lines if they don't contain wildcards.
1827 static int
1828 can_be_compound(sp, slang, compflags, flag)
1829 trystate_T *sp;
1830 slang_T *slang;
1831 char_u *compflags;
1832 int flag;
1834 /* If the flag doesn't appear in sl_compstartflags or sl_compallflags
1835 * then it can't possibly compound. */
1836 if (!byte_in_str(sp->ts_complen == sp->ts_compsplit
1837 ? slang->sl_compstartflags : slang->sl_compallflags, flag))
1838 return FALSE;
1840 /* If there are no wildcards, we can check if the flags collected so far
1841 * possibly can form a match with COMPOUNDRULE patterns. This only
1842 * makes sense when we have two or more words. */
1843 if (slang->sl_comprules != NULL && sp->ts_complen > sp->ts_compsplit)
1845 int v;
1847 compflags[sp->ts_complen] = flag;
1848 compflags[sp->ts_complen + 1] = NUL;
1849 v = match_compoundrule(slang, compflags + sp->ts_compsplit);
1850 compflags[sp->ts_complen] = NUL;
1851 return v;
1854 return TRUE;
1859 * Return TRUE if the compound flags in compflags[] match the start of any
1860 * compound rule. This is used to stop trying a compound if the flags
1861 * collected so far can't possibly match any compound rule.
1862 * Caller must check that slang->sl_comprules is not NULL.
1864 static int
1865 match_compoundrule(slang, compflags)
1866 slang_T *slang;
1867 char_u *compflags;
1869 char_u *p;
1870 int i;
1871 int c;
1873 /* loop over all the COMPOUNDRULE entries */
1874 for (p = slang->sl_comprules; *p != NUL; ++p)
1876 /* loop over the flags in the compound word we have made, match
1877 * them against the current rule entry */
1878 for (i = 0; ; ++i)
1880 c = compflags[i];
1881 if (c == NUL)
1882 /* found a rule that matches for the flags we have so far */
1883 return TRUE;
1884 if (*p == '/' || *p == NUL)
1885 break; /* end of rule, it's too short */
1886 if (*p == '[')
1888 int match = FALSE;
1890 /* compare against all the flags in [] */
1891 ++p;
1892 while (*p != ']' && *p != NUL)
1893 if (*p++ == c)
1894 match = TRUE;
1895 if (!match)
1896 break; /* none matches */
1898 else if (*p != c)
1899 break; /* flag of word doesn't match flag in pattern */
1900 ++p;
1903 /* Skip to the next "/", where the next pattern starts. */
1904 p = vim_strchr(p, '/');
1905 if (p == NULL)
1906 break;
1909 /* Checked all the rules and none of them match the flags, so there
1910 * can't possibly be a compound starting with these flags. */
1911 return FALSE;
1915 * Return non-zero if the prefix indicated by "arridx" matches with the prefix
1916 * ID in "flags" for the word "word".
1917 * The WF_RAREPFX flag is included in the return value for a rare prefix.
1919 static int
1920 valid_word_prefix(totprefcnt, arridx, flags, word, slang, cond_req)
1921 int totprefcnt; /* nr of prefix IDs */
1922 int arridx; /* idx in sl_pidxs[] */
1923 int flags;
1924 char_u *word;
1925 slang_T *slang;
1926 int cond_req; /* only use prefixes with a condition */
1928 int prefcnt;
1929 int pidx;
1930 regprog_T *rp;
1931 regmatch_T regmatch;
1932 int prefid;
1934 prefid = (unsigned)flags >> 24;
1935 for (prefcnt = totprefcnt - 1; prefcnt >= 0; --prefcnt)
1937 pidx = slang->sl_pidxs[arridx + prefcnt];
1939 /* Check the prefix ID. */
1940 if (prefid != (pidx & 0xff))
1941 continue;
1943 /* Check if the prefix doesn't combine and the word already has a
1944 * suffix. */
1945 if ((flags & WF_HAS_AFF) && (pidx & WF_PFX_NC))
1946 continue;
1948 /* Check the condition, if there is one. The condition index is
1949 * stored in the two bytes above the prefix ID byte. */
1950 rp = slang->sl_prefprog[((unsigned)pidx >> 8) & 0xffff];
1951 if (rp != NULL)
1953 regmatch.regprog = rp;
1954 regmatch.rm_ic = FALSE;
1955 if (!vim_regexec(&regmatch, word, 0))
1956 continue;
1958 else if (cond_req)
1959 continue;
1961 /* It's a match! Return the WF_ flags. */
1962 return pidx;
1964 return 0;
1968 * Check if the word at "mip->mi_word" has a matching prefix.
1969 * If it does, then check the following word.
1971 * If "mode" is "FIND_COMPOUND" then do the same after another word, find a
1972 * prefix in a compound word.
1974 * For a match mip->mi_result is updated.
1976 static void
1977 find_prefix(mip, mode)
1978 matchinf_T *mip;
1979 int mode;
1981 idx_T arridx = 0;
1982 int len;
1983 int wlen = 0;
1984 int flen;
1985 int c;
1986 char_u *ptr;
1987 idx_T lo, hi, m;
1988 slang_T *slang = mip->mi_lp->lp_slang;
1989 char_u *byts;
1990 idx_T *idxs;
1992 byts = slang->sl_pbyts;
1993 if (byts == NULL)
1994 return; /* array is empty */
1996 /* We use the case-folded word here, since prefixes are always
1997 * case-folded. */
1998 ptr = mip->mi_fword;
1999 flen = mip->mi_fwordlen; /* available case-folded bytes */
2000 if (mode == FIND_COMPOUND)
2002 /* Skip over the previously found word(s). */
2003 ptr += mip->mi_compoff;
2004 flen -= mip->mi_compoff;
2006 idxs = slang->sl_pidxs;
2009 * Repeat advancing in the tree until:
2010 * - there is a byte that doesn't match,
2011 * - we reach the end of the tree,
2012 * - or we reach the end of the line.
2014 for (;;)
2016 if (flen == 0 && *mip->mi_fend != NUL)
2017 flen = fold_more(mip);
2019 len = byts[arridx++];
2021 /* If the first possible byte is a zero the prefix could end here.
2022 * Check if the following word matches and supports the prefix. */
2023 if (byts[arridx] == 0)
2025 /* There can be several prefixes with different conditions. We
2026 * try them all, since we don't know which one will give the
2027 * longest match. The word is the same each time, pass the list
2028 * of possible prefixes to find_word(). */
2029 mip->mi_prefarridx = arridx;
2030 mip->mi_prefcnt = len;
2031 while (len > 0 && byts[arridx] == 0)
2033 ++arridx;
2034 --len;
2036 mip->mi_prefcnt -= len;
2038 /* Find the word that comes after the prefix. */
2039 mip->mi_prefixlen = wlen;
2040 if (mode == FIND_COMPOUND)
2041 /* Skip over the previously found word(s). */
2042 mip->mi_prefixlen += mip->mi_compoff;
2044 #ifdef FEAT_MBYTE
2045 if (has_mbyte)
2047 /* Case-folded length may differ from original length. */
2048 mip->mi_cprefixlen = nofold_len(mip->mi_fword,
2049 mip->mi_prefixlen, mip->mi_word);
2051 else
2052 mip->mi_cprefixlen = mip->mi_prefixlen;
2053 #endif
2054 find_word(mip, FIND_PREFIX);
2057 if (len == 0)
2058 break; /* no children, word must end here */
2061 /* Stop looking at end of the line. */
2062 if (ptr[wlen] == NUL)
2063 break;
2065 /* Perform a binary search in the list of accepted bytes. */
2066 c = ptr[wlen];
2067 lo = arridx;
2068 hi = arridx + len - 1;
2069 while (lo < hi)
2071 m = (lo + hi) / 2;
2072 if (byts[m] > c)
2073 hi = m - 1;
2074 else if (byts[m] < c)
2075 lo = m + 1;
2076 else
2078 lo = hi = m;
2079 break;
2083 /* Stop if there is no matching byte. */
2084 if (hi < lo || byts[lo] != c)
2085 break;
2087 /* Continue at the child (if there is one). */
2088 arridx = idxs[lo];
2089 ++wlen;
2090 --flen;
2095 * Need to fold at least one more character. Do until next non-word character
2096 * for efficiency. Include the non-word character too.
2097 * Return the length of the folded chars in bytes.
2099 static int
2100 fold_more(mip)
2101 matchinf_T *mip;
2103 int flen;
2104 char_u *p;
2106 p = mip->mi_fend;
2109 mb_ptr_adv(mip->mi_fend);
2110 } while (*mip->mi_fend != NUL && spell_iswordp(mip->mi_fend, mip->mi_buf));
2112 /* Include the non-word character so that we can check for the word end. */
2113 if (*mip->mi_fend != NUL)
2114 mb_ptr_adv(mip->mi_fend);
2116 (void)spell_casefold(p, (int)(mip->mi_fend - p),
2117 mip->mi_fword + mip->mi_fwordlen,
2118 MAXWLEN - mip->mi_fwordlen);
2119 flen = (int)STRLEN(mip->mi_fword + mip->mi_fwordlen);
2120 mip->mi_fwordlen += flen;
2121 return flen;
2125 * Check case flags for a word. Return TRUE if the word has the requested
2126 * case.
2128 static int
2129 spell_valid_case(wordflags, treeflags)
2130 int wordflags; /* flags for the checked word. */
2131 int treeflags; /* flags for the word in the spell tree */
2133 return ((wordflags == WF_ALLCAP && (treeflags & WF_FIXCAP) == 0)
2134 || ((treeflags & (WF_ALLCAP | WF_KEEPCAP)) == 0
2135 && ((treeflags & WF_ONECAP) == 0
2136 || (wordflags & WF_ONECAP) != 0)));
2140 * Return TRUE if spell checking is not enabled.
2142 static int
2143 no_spell_checking(wp)
2144 win_T *wp;
2146 if (!wp->w_p_spell || *wp->w_buffer->b_p_spl == NUL
2147 || wp->w_buffer->b_langp.ga_len == 0)
2149 EMSG(_("E756: Spell checking is not enabled"));
2150 return TRUE;
2152 return FALSE;
2156 * Move to next spell error.
2157 * "curline" is FALSE for "[s", "]s", "[S" and "]S".
2158 * "curline" is TRUE to find word under/after cursor in the same line.
2159 * For Insert mode completion "dir" is BACKWARD and "curline" is TRUE: move
2160 * to after badly spelled word before the cursor.
2161 * Return 0 if not found, length of the badly spelled word otherwise.
2164 spell_move_to(wp, dir, allwords, curline, attrp)
2165 win_T *wp;
2166 int dir; /* FORWARD or BACKWARD */
2167 int allwords; /* TRUE for "[s"/"]s", FALSE for "[S"/"]S" */
2168 int curline;
2169 hlf_T *attrp; /* return: attributes of bad word or NULL
2170 (only when "dir" is FORWARD) */
2172 linenr_T lnum;
2173 pos_T found_pos;
2174 int found_len = 0;
2175 char_u *line;
2176 char_u *p;
2177 char_u *endp;
2178 hlf_T attr;
2179 int len;
2180 # ifdef FEAT_SYN_HL
2181 int has_syntax = syntax_present(wp->w_buffer);
2182 # endif
2183 int col;
2184 int can_spell;
2185 char_u *buf = NULL;
2186 int buflen = 0;
2187 int skip = 0;
2188 int capcol = -1;
2189 int found_one = FALSE;
2190 int wrapped = FALSE;
2192 if (no_spell_checking(wp))
2193 return 0;
2196 * Start looking for bad word at the start of the line, because we can't
2197 * start halfway a word, we don't know where it starts or ends.
2199 * When searching backwards, we continue in the line to find the last
2200 * bad word (in the cursor line: before the cursor).
2202 * We concatenate the start of the next line, so that wrapped words work
2203 * (e.g. "et<line-break>cetera"). Doesn't work when searching backwards
2204 * though...
2206 lnum = wp->w_cursor.lnum;
2207 clearpos(&found_pos);
2209 while (!got_int)
2211 line = ml_get_buf(wp->w_buffer, lnum, FALSE);
2213 len = (int)STRLEN(line);
2214 if (buflen < len + MAXWLEN + 2)
2216 vim_free(buf);
2217 buflen = len + MAXWLEN + 2;
2218 buf = alloc(buflen);
2219 if (buf == NULL)
2220 break;
2223 /* In first line check first word for Capital. */
2224 if (lnum == 1)
2225 capcol = 0;
2227 /* For checking first word with a capital skip white space. */
2228 if (capcol == 0)
2229 capcol = (int)(skipwhite(line) - line);
2230 else if (curline && wp == curwin)
2232 /* For spellbadword(): check if first word needs a capital. */
2233 col = (int)(skipwhite(line) - line);
2234 if (check_need_cap(lnum, col))
2235 capcol = col;
2237 /* Need to get the line again, may have looked at the previous
2238 * one. */
2239 line = ml_get_buf(wp->w_buffer, lnum, FALSE);
2242 /* Copy the line into "buf" and append the start of the next line if
2243 * possible. */
2244 STRCPY(buf, line);
2245 if (lnum < wp->w_buffer->b_ml.ml_line_count)
2246 spell_cat_line(buf + STRLEN(buf),
2247 ml_get_buf(wp->w_buffer, lnum + 1, FALSE), MAXWLEN);
2249 p = buf + skip;
2250 endp = buf + len;
2251 while (p < endp)
2253 /* When searching backward don't search after the cursor. Unless
2254 * we wrapped around the end of the buffer. */
2255 if (dir == BACKWARD
2256 && lnum == wp->w_cursor.lnum
2257 && !wrapped
2258 && (colnr_T)(p - buf) >= wp->w_cursor.col)
2259 break;
2261 /* start of word */
2262 attr = HLF_COUNT;
2263 len = spell_check(wp, p, &attr, &capcol, FALSE);
2265 if (attr != HLF_COUNT)
2267 /* We found a bad word. Check the attribute. */
2268 if (allwords || attr == HLF_SPB)
2270 /* When searching forward only accept a bad word after
2271 * the cursor. */
2272 if (dir == BACKWARD
2273 || lnum != wp->w_cursor.lnum
2274 || (lnum == wp->w_cursor.lnum
2275 && (wrapped
2276 || (colnr_T)(curline ? p - buf + len
2277 : p - buf)
2278 > wp->w_cursor.col)))
2280 # ifdef FEAT_SYN_HL
2281 if (has_syntax)
2283 col = (int)(p - buf);
2284 (void)syn_get_id(wp, lnum, (colnr_T)col,
2285 FALSE, &can_spell, FALSE);
2286 if (!can_spell)
2287 attr = HLF_COUNT;
2289 else
2290 #endif
2291 can_spell = TRUE;
2293 if (can_spell)
2295 found_one = TRUE;
2296 found_pos.lnum = lnum;
2297 found_pos.col = (int)(p - buf);
2298 #ifdef FEAT_VIRTUALEDIT
2299 found_pos.coladd = 0;
2300 #endif
2301 if (dir == FORWARD)
2303 /* No need to search further. */
2304 wp->w_cursor = found_pos;
2305 vim_free(buf);
2306 if (attrp != NULL)
2307 *attrp = attr;
2308 return len;
2310 else if (curline)
2311 /* Insert mode completion: put cursor after
2312 * the bad word. */
2313 found_pos.col += len;
2314 found_len = len;
2317 else
2318 found_one = TRUE;
2322 /* advance to character after the word */
2323 p += len;
2324 capcol -= len;
2327 if (dir == BACKWARD && found_pos.lnum != 0)
2329 /* Use the last match in the line (before the cursor). */
2330 wp->w_cursor = found_pos;
2331 vim_free(buf);
2332 return found_len;
2335 if (curline)
2336 break; /* only check cursor line */
2338 /* Advance to next line. */
2339 if (dir == BACKWARD)
2341 /* If we are back at the starting line and searched it again there
2342 * is no match, give up. */
2343 if (lnum == wp->w_cursor.lnum && wrapped)
2344 break;
2346 if (lnum > 1)
2347 --lnum;
2348 else if (!p_ws)
2349 break; /* at first line and 'nowrapscan' */
2350 else
2352 /* Wrap around to the end of the buffer. May search the
2353 * starting line again and accept the last match. */
2354 lnum = wp->w_buffer->b_ml.ml_line_count;
2355 wrapped = TRUE;
2356 if (!shortmess(SHM_SEARCH))
2357 give_warning((char_u *)_(top_bot_msg), TRUE);
2359 capcol = -1;
2361 else
2363 if (lnum < wp->w_buffer->b_ml.ml_line_count)
2364 ++lnum;
2365 else if (!p_ws)
2366 break; /* at first line and 'nowrapscan' */
2367 else
2369 /* Wrap around to the start of the buffer. May search the
2370 * starting line again and accept the first match. */
2371 lnum = 1;
2372 wrapped = TRUE;
2373 if (!shortmess(SHM_SEARCH))
2374 give_warning((char_u *)_(bot_top_msg), TRUE);
2377 /* If we are back at the starting line and there is no match then
2378 * give up. */
2379 if (lnum == wp->w_cursor.lnum && (!found_one || wrapped))
2380 break;
2382 /* Skip the characters at the start of the next line that were
2383 * included in a match crossing line boundaries. */
2384 if (attr == HLF_COUNT)
2385 skip = (int)(p - endp);
2386 else
2387 skip = 0;
2389 /* Capcol skips over the inserted space. */
2390 --capcol;
2392 /* But after empty line check first word in next line */
2393 if (*skipwhite(line) == NUL)
2394 capcol = 0;
2397 line_breakcheck();
2400 vim_free(buf);
2401 return 0;
2405 * For spell checking: concatenate the start of the following line "line" into
2406 * "buf", blanking-out special characters. Copy less then "maxlen" bytes.
2407 * Keep the blanks at the start of the next line, this is used in win_line()
2408 * to skip those bytes if the word was OK.
2410 void
2411 spell_cat_line(buf, line, maxlen)
2412 char_u *buf;
2413 char_u *line;
2414 int maxlen;
2416 char_u *p;
2417 int n;
2419 p = skipwhite(line);
2420 while (vim_strchr((char_u *)"*#/\"\t", *p) != NULL)
2421 p = skipwhite(p + 1);
2423 if (*p != NUL)
2425 /* Only worth concatenating if there is something else than spaces to
2426 * concatenate. */
2427 n = (int)(p - line) + 1;
2428 if (n < maxlen - 1)
2430 vim_memset(buf, ' ', n);
2431 vim_strncpy(buf + n, p, maxlen - 1 - n);
2437 * Structure used for the cookie argument of do_in_runtimepath().
2439 typedef struct spelload_S
2441 char_u sl_lang[MAXWLEN + 1]; /* language name */
2442 slang_T *sl_slang; /* resulting slang_T struct */
2443 int sl_nobreak; /* NOBREAK language found */
2444 } spelload_T;
2447 * Load word list(s) for "lang" from Vim spell file(s).
2448 * "lang" must be the language without the region: e.g., "en".
2450 static void
2451 spell_load_lang(lang)
2452 char_u *lang;
2454 char_u fname_enc[85];
2455 int r;
2456 spelload_T sl;
2457 #ifdef FEAT_AUTOCMD
2458 int round;
2459 #endif
2461 /* Copy the language name to pass it to spell_load_cb() as a cookie.
2462 * It's truncated when an error is detected. */
2463 STRCPY(sl.sl_lang, lang);
2464 sl.sl_slang = NULL;
2465 sl.sl_nobreak = FALSE;
2467 #ifdef FEAT_AUTOCMD
2468 /* We may retry when no spell file is found for the language, an
2469 * autocommand may load it then. */
2470 for (round = 1; round <= 2; ++round)
2471 #endif
2474 * Find the first spell file for "lang" in 'runtimepath' and load it.
2476 vim_snprintf((char *)fname_enc, sizeof(fname_enc) - 5,
2477 "spell/%s.%s.spl", lang, spell_enc());
2478 r = do_in_runtimepath(fname_enc, FALSE, spell_load_cb, &sl);
2480 if (r == FAIL && *sl.sl_lang != NUL)
2482 /* Try loading the ASCII version. */
2483 vim_snprintf((char *)fname_enc, sizeof(fname_enc) - 5,
2484 "spell/%s.ascii.spl", lang);
2485 r = do_in_runtimepath(fname_enc, FALSE, spell_load_cb, &sl);
2487 #ifdef FEAT_AUTOCMD
2488 if (r == FAIL && *sl.sl_lang != NUL && round == 1
2489 && apply_autocmds(EVENT_SPELLFILEMISSING, lang,
2490 curbuf->b_fname, FALSE, curbuf))
2491 continue;
2492 break;
2493 #endif
2495 #ifdef FEAT_AUTOCMD
2496 break;
2497 #endif
2500 if (r == FAIL)
2502 smsg((char_u *)_("Warning: Cannot find word list \"%s.%s.spl\" or \"%s.ascii.spl\""),
2503 lang, spell_enc(), lang);
2505 else if (sl.sl_slang != NULL)
2507 /* At least one file was loaded, now load ALL the additions. */
2508 STRCPY(fname_enc + STRLEN(fname_enc) - 3, "add.spl");
2509 do_in_runtimepath(fname_enc, TRUE, spell_load_cb, &sl);
2514 * Return the encoding used for spell checking: Use 'encoding', except that we
2515 * use "latin1" for "latin9". And limit to 60 characters (just in case).
2517 static char_u *
2518 spell_enc()
2521 #ifdef FEAT_MBYTE
2522 if (STRLEN(p_enc) < 60 && STRCMP(p_enc, "iso-8859-15") != 0)
2523 return p_enc;
2524 #endif
2525 return (char_u *)"latin1";
2529 * Get the name of the .spl file for the internal wordlist into
2530 * "fname[MAXPATHL]".
2532 static void
2533 int_wordlist_spl(fname)
2534 char_u *fname;
2536 vim_snprintf((char *)fname, MAXPATHL, "%s.%s.spl",
2537 int_wordlist, spell_enc());
2541 * Allocate a new slang_T for language "lang". "lang" can be NULL.
2542 * Caller must fill "sl_next".
2544 static slang_T *
2545 slang_alloc(lang)
2546 char_u *lang;
2548 slang_T *lp;
2550 lp = (slang_T *)alloc_clear(sizeof(slang_T));
2551 if (lp != NULL)
2553 if (lang != NULL)
2554 lp->sl_name = vim_strsave(lang);
2555 ga_init2(&lp->sl_rep, sizeof(fromto_T), 10);
2556 ga_init2(&lp->sl_repsal, sizeof(fromto_T), 10);
2557 lp->sl_compmax = MAXWLEN;
2558 lp->sl_compsylmax = MAXWLEN;
2559 hash_init(&lp->sl_wordcount);
2562 return lp;
2566 * Free the contents of an slang_T and the structure itself.
2568 static void
2569 slang_free(lp)
2570 slang_T *lp;
2572 vim_free(lp->sl_name);
2573 vim_free(lp->sl_fname);
2574 slang_clear(lp);
2575 vim_free(lp);
2579 * Clear an slang_T so that the file can be reloaded.
2581 static void
2582 slang_clear(lp)
2583 slang_T *lp;
2585 garray_T *gap;
2586 fromto_T *ftp;
2587 salitem_T *smp;
2588 int i;
2589 int round;
2591 vim_free(lp->sl_fbyts);
2592 lp->sl_fbyts = NULL;
2593 vim_free(lp->sl_kbyts);
2594 lp->sl_kbyts = NULL;
2595 vim_free(lp->sl_pbyts);
2596 lp->sl_pbyts = NULL;
2598 vim_free(lp->sl_fidxs);
2599 lp->sl_fidxs = NULL;
2600 vim_free(lp->sl_kidxs);
2601 lp->sl_kidxs = NULL;
2602 vim_free(lp->sl_pidxs);
2603 lp->sl_pidxs = NULL;
2605 for (round = 1; round <= 2; ++round)
2607 gap = round == 1 ? &lp->sl_rep : &lp->sl_repsal;
2608 while (gap->ga_len > 0)
2610 ftp = &((fromto_T *)gap->ga_data)[--gap->ga_len];
2611 vim_free(ftp->ft_from);
2612 vim_free(ftp->ft_to);
2614 ga_clear(gap);
2617 gap = &lp->sl_sal;
2618 if (lp->sl_sofo)
2620 /* "ga_len" is set to 1 without adding an item for latin1 */
2621 if (gap->ga_data != NULL)
2622 /* SOFOFROM and SOFOTO items: free lists of wide characters. */
2623 for (i = 0; i < gap->ga_len; ++i)
2624 vim_free(((int **)gap->ga_data)[i]);
2626 else
2627 /* SAL items: free salitem_T items */
2628 while (gap->ga_len > 0)
2630 smp = &((salitem_T *)gap->ga_data)[--gap->ga_len];
2631 vim_free(smp->sm_lead);
2632 /* Don't free sm_oneof and sm_rules, they point into sm_lead. */
2633 vim_free(smp->sm_to);
2634 #ifdef FEAT_MBYTE
2635 vim_free(smp->sm_lead_w);
2636 vim_free(smp->sm_oneof_w);
2637 vim_free(smp->sm_to_w);
2638 #endif
2640 ga_clear(gap);
2642 for (i = 0; i < lp->sl_prefixcnt; ++i)
2643 vim_free(lp->sl_prefprog[i]);
2644 lp->sl_prefixcnt = 0;
2645 vim_free(lp->sl_prefprog);
2646 lp->sl_prefprog = NULL;
2648 vim_free(lp->sl_info);
2649 lp->sl_info = NULL;
2651 vim_free(lp->sl_midword);
2652 lp->sl_midword = NULL;
2654 vim_free(lp->sl_compprog);
2655 vim_free(lp->sl_comprules);
2656 vim_free(lp->sl_compstartflags);
2657 vim_free(lp->sl_compallflags);
2658 lp->sl_compprog = NULL;
2659 lp->sl_comprules = NULL;
2660 lp->sl_compstartflags = NULL;
2661 lp->sl_compallflags = NULL;
2663 vim_free(lp->sl_syllable);
2664 lp->sl_syllable = NULL;
2665 ga_clear(&lp->sl_syl_items);
2667 ga_clear_strings(&lp->sl_comppat);
2669 hash_clear_all(&lp->sl_wordcount, WC_KEY_OFF);
2670 hash_init(&lp->sl_wordcount);
2672 #ifdef FEAT_MBYTE
2673 hash_clear_all(&lp->sl_map_hash, 0);
2674 #endif
2676 /* Clear info from .sug file. */
2677 slang_clear_sug(lp);
2679 lp->sl_compmax = MAXWLEN;
2680 lp->sl_compminlen = 0;
2681 lp->sl_compsylmax = MAXWLEN;
2682 lp->sl_regions[0] = NUL;
2686 * Clear the info from the .sug file in "lp".
2688 static void
2689 slang_clear_sug(lp)
2690 slang_T *lp;
2692 vim_free(lp->sl_sbyts);
2693 lp->sl_sbyts = NULL;
2694 vim_free(lp->sl_sidxs);
2695 lp->sl_sidxs = NULL;
2696 close_spellbuf(lp->sl_sugbuf);
2697 lp->sl_sugbuf = NULL;
2698 lp->sl_sugloaded = FALSE;
2699 lp->sl_sugtime = 0;
2703 * Load one spell file and store the info into a slang_T.
2704 * Invoked through do_in_runtimepath().
2706 static void
2707 spell_load_cb(fname, cookie)
2708 char_u *fname;
2709 void *cookie;
2711 spelload_T *slp = (spelload_T *)cookie;
2712 slang_T *slang;
2714 slang = spell_load_file(fname, slp->sl_lang, NULL, FALSE);
2715 if (slang != NULL)
2717 /* When a previously loaded file has NOBREAK also use it for the
2718 * ".add" files. */
2719 if (slp->sl_nobreak && slang->sl_add)
2720 slang->sl_nobreak = TRUE;
2721 else if (slang->sl_nobreak)
2722 slp->sl_nobreak = TRUE;
2724 slp->sl_slang = slang;
2729 * Load one spell file and store the info into a slang_T.
2731 * This is invoked in three ways:
2732 * - From spell_load_cb() to load a spell file for the first time. "lang" is
2733 * the language name, "old_lp" is NULL. Will allocate an slang_T.
2734 * - To reload a spell file that was changed. "lang" is NULL and "old_lp"
2735 * points to the existing slang_T.
2736 * - Just after writing a .spl file; it's read back to produce the .sug file.
2737 * "old_lp" is NULL and "lang" is NULL. Will allocate an slang_T.
2739 * Returns the slang_T the spell file was loaded into. NULL for error.
2741 static slang_T *
2742 spell_load_file(fname, lang, old_lp, silent)
2743 char_u *fname;
2744 char_u *lang;
2745 slang_T *old_lp;
2746 int silent; /* no error if file doesn't exist */
2748 FILE *fd;
2749 char_u buf[VIMSPELLMAGICL];
2750 char_u *p;
2751 int i;
2752 int n;
2753 int len;
2754 char_u *save_sourcing_name = sourcing_name;
2755 linenr_T save_sourcing_lnum = sourcing_lnum;
2756 slang_T *lp = NULL;
2757 int c = 0;
2758 int res;
2760 fd = mch_fopen((char *)fname, "r");
2761 if (fd == NULL)
2763 if (!silent)
2764 EMSG2(_(e_notopen), fname);
2765 else if (p_verbose > 2)
2767 verbose_enter();
2768 smsg((char_u *)e_notopen, fname);
2769 verbose_leave();
2771 goto endFAIL;
2773 if (p_verbose > 2)
2775 verbose_enter();
2776 smsg((char_u *)_("Reading spell file \"%s\""), fname);
2777 verbose_leave();
2780 if (old_lp == NULL)
2782 lp = slang_alloc(lang);
2783 if (lp == NULL)
2784 goto endFAIL;
2786 /* Remember the file name, used to reload the file when it's updated. */
2787 lp->sl_fname = vim_strsave(fname);
2788 if (lp->sl_fname == NULL)
2789 goto endFAIL;
2791 /* Check for .add.spl. */
2792 lp->sl_add = strstr((char *)gettail(fname), ".add.") != NULL;
2794 else
2795 lp = old_lp;
2797 /* Set sourcing_name, so that error messages mention the file name. */
2798 sourcing_name = fname;
2799 sourcing_lnum = 0;
2802 * <HEADER>: <fileID>
2804 for (i = 0; i < VIMSPELLMAGICL; ++i)
2805 buf[i] = getc(fd); /* <fileID> */
2806 if (STRNCMP(buf, VIMSPELLMAGIC, VIMSPELLMAGICL) != 0)
2808 EMSG(_("E757: This does not look like a spell file"));
2809 goto endFAIL;
2811 c = getc(fd); /* <versionnr> */
2812 if (c < VIMSPELLVERSION)
2814 EMSG(_("E771: Old spell file, needs to be updated"));
2815 goto endFAIL;
2817 else if (c > VIMSPELLVERSION)
2819 EMSG(_("E772: Spell file is for newer version of Vim"));
2820 goto endFAIL;
2825 * <SECTIONS>: <section> ... <sectionend>
2826 * <section>: <sectionID> <sectionflags> <sectionlen> (section contents)
2828 for (;;)
2830 n = getc(fd); /* <sectionID> or <sectionend> */
2831 if (n == SN_END)
2832 break;
2833 c = getc(fd); /* <sectionflags> */
2834 len = get4c(fd); /* <sectionlen> */
2835 if (len < 0)
2836 goto truncerr;
2838 res = 0;
2839 switch (n)
2841 case SN_INFO:
2842 lp->sl_info = read_string(fd, len); /* <infotext> */
2843 if (lp->sl_info == NULL)
2844 goto endFAIL;
2845 break;
2847 case SN_REGION:
2848 res = read_region_section(fd, lp, len);
2849 break;
2851 case SN_CHARFLAGS:
2852 res = read_charflags_section(fd);
2853 break;
2855 case SN_MIDWORD:
2856 lp->sl_midword = read_string(fd, len); /* <midword> */
2857 if (lp->sl_midword == NULL)
2858 goto endFAIL;
2859 break;
2861 case SN_PREFCOND:
2862 res = read_prefcond_section(fd, lp);
2863 break;
2865 case SN_REP:
2866 res = read_rep_section(fd, &lp->sl_rep, lp->sl_rep_first);
2867 break;
2869 case SN_REPSAL:
2870 res = read_rep_section(fd, &lp->sl_repsal, lp->sl_repsal_first);
2871 break;
2873 case SN_SAL:
2874 res = read_sal_section(fd, lp);
2875 break;
2877 case SN_SOFO:
2878 res = read_sofo_section(fd, lp);
2879 break;
2881 case SN_MAP:
2882 p = read_string(fd, len); /* <mapstr> */
2883 if (p == NULL)
2884 goto endFAIL;
2885 set_map_str(lp, p);
2886 vim_free(p);
2887 break;
2889 case SN_WORDS:
2890 res = read_words_section(fd, lp, len);
2891 break;
2893 case SN_SUGFILE:
2894 lp->sl_sugtime = get8c(fd); /* <timestamp> */
2895 break;
2897 case SN_NOSPLITSUGS:
2898 lp->sl_nosplitsugs = TRUE; /* <timestamp> */
2899 break;
2901 case SN_COMPOUND:
2902 res = read_compound(fd, lp, len);
2903 break;
2905 case SN_NOBREAK:
2906 lp->sl_nobreak = TRUE;
2907 break;
2909 case SN_SYLLABLE:
2910 lp->sl_syllable = read_string(fd, len); /* <syllable> */
2911 if (lp->sl_syllable == NULL)
2912 goto endFAIL;
2913 if (init_syl_tab(lp) == FAIL)
2914 goto endFAIL;
2915 break;
2917 default:
2918 /* Unsupported section. When it's required give an error
2919 * message. When it's not required skip the contents. */
2920 if (c & SNF_REQUIRED)
2922 EMSG(_("E770: Unsupported section in spell file"));
2923 goto endFAIL;
2925 while (--len >= 0)
2926 if (getc(fd) < 0)
2927 goto truncerr;
2928 break;
2930 someerror:
2931 if (res == SP_FORMERROR)
2933 EMSG(_(e_format));
2934 goto endFAIL;
2936 if (res == SP_TRUNCERROR)
2938 truncerr:
2939 EMSG(_(e_spell_trunc));
2940 goto endFAIL;
2942 if (res == SP_OTHERERROR)
2943 goto endFAIL;
2946 /* <LWORDTREE> */
2947 res = spell_read_tree(fd, &lp->sl_fbyts, &lp->sl_fidxs, FALSE, 0);
2948 if (res != 0)
2949 goto someerror;
2951 /* <KWORDTREE> */
2952 res = spell_read_tree(fd, &lp->sl_kbyts, &lp->sl_kidxs, FALSE, 0);
2953 if (res != 0)
2954 goto someerror;
2956 /* <PREFIXTREE> */
2957 res = spell_read_tree(fd, &lp->sl_pbyts, &lp->sl_pidxs, TRUE,
2958 lp->sl_prefixcnt);
2959 if (res != 0)
2960 goto someerror;
2962 /* For a new file link it in the list of spell files. */
2963 if (old_lp == NULL && lang != NULL)
2965 lp->sl_next = first_lang;
2966 first_lang = lp;
2969 goto endOK;
2971 endFAIL:
2972 if (lang != NULL)
2973 /* truncating the name signals the error to spell_load_lang() */
2974 *lang = NUL;
2975 if (lp != NULL && old_lp == NULL)
2976 slang_free(lp);
2977 lp = NULL;
2979 endOK:
2980 if (fd != NULL)
2981 fclose(fd);
2982 sourcing_name = save_sourcing_name;
2983 sourcing_lnum = save_sourcing_lnum;
2985 return lp;
2989 * Read 2 bytes from "fd" and turn them into an int, MSB first.
2991 static int
2992 get2c(fd)
2993 FILE *fd;
2995 long n;
2997 n = getc(fd);
2998 n = (n << 8) + getc(fd);
2999 return n;
3003 * Read 3 bytes from "fd" and turn them into an int, MSB first.
3005 static int
3006 get3c(fd)
3007 FILE *fd;
3009 long n;
3011 n = getc(fd);
3012 n = (n << 8) + getc(fd);
3013 n = (n << 8) + getc(fd);
3014 return n;
3018 * Read 4 bytes from "fd" and turn them into an int, MSB first.
3020 static int
3021 get4c(fd)
3022 FILE *fd;
3024 long n;
3026 n = getc(fd);
3027 n = (n << 8) + getc(fd);
3028 n = (n << 8) + getc(fd);
3029 n = (n << 8) + getc(fd);
3030 return n;
3034 * Read 8 bytes from "fd" and turn them into a time_t, MSB first.
3036 static time_t
3037 get8c(fd)
3038 FILE *fd;
3040 time_t n = 0;
3041 int i;
3043 for (i = 0; i < 8; ++i)
3044 n = (n << 8) + getc(fd);
3045 return n;
3049 * Read a length field from "fd" in "cnt_bytes" bytes.
3050 * Allocate memory, read the string into it and add a NUL at the end.
3051 * Returns NULL when the count is zero.
3052 * Sets "*cntp" to SP_*ERROR when there is an error, length of the result
3053 * otherwise.
3055 static char_u *
3056 read_cnt_string(fd, cnt_bytes, cntp)
3057 FILE *fd;
3058 int cnt_bytes;
3059 int *cntp;
3061 int cnt = 0;
3062 int i;
3063 char_u *str;
3065 /* read the length bytes, MSB first */
3066 for (i = 0; i < cnt_bytes; ++i)
3067 cnt = (cnt << 8) + getc(fd);
3068 if (cnt < 0)
3070 *cntp = SP_TRUNCERROR;
3071 return NULL;
3073 *cntp = cnt;
3074 if (cnt == 0)
3075 return NULL; /* nothing to read, return NULL */
3077 str = read_string(fd, cnt);
3078 if (str == NULL)
3079 *cntp = SP_OTHERERROR;
3080 return str;
3084 * Read a string of length "cnt" from "fd" into allocated memory.
3085 * Returns NULL when out of memory or unable to read that many bytes.
3087 static char_u *
3088 read_string(fd, cnt)
3089 FILE *fd;
3090 int cnt;
3092 char_u *str;
3093 int i;
3094 int c;
3096 /* allocate memory */
3097 str = alloc((unsigned)cnt + 1);
3098 if (str != NULL)
3100 /* Read the string. Quit when running into the EOF. */
3101 for (i = 0; i < cnt; ++i)
3103 c = getc(fd);
3104 if (c == EOF)
3106 vim_free(str);
3107 return NULL;
3109 str[i] = c;
3111 str[i] = NUL;
3113 return str;
3117 * Read SN_REGION: <regionname> ...
3118 * Return SP_*ERROR flags.
3120 static int
3121 read_region_section(fd, lp, len)
3122 FILE *fd;
3123 slang_T *lp;
3124 int len;
3126 int i;
3128 if (len > 16)
3129 return SP_FORMERROR;
3130 for (i = 0; i < len; ++i)
3131 lp->sl_regions[i] = getc(fd); /* <regionname> */
3132 lp->sl_regions[len] = NUL;
3133 return 0;
3137 * Read SN_CHARFLAGS section: <charflagslen> <charflags>
3138 * <folcharslen> <folchars>
3139 * Return SP_*ERROR flags.
3141 static int
3142 read_charflags_section(fd)
3143 FILE *fd;
3145 char_u *flags;
3146 char_u *fol;
3147 int flagslen, follen;
3149 /* <charflagslen> <charflags> */
3150 flags = read_cnt_string(fd, 1, &flagslen);
3151 if (flagslen < 0)
3152 return flagslen;
3154 /* <folcharslen> <folchars> */
3155 fol = read_cnt_string(fd, 2, &follen);
3156 if (follen < 0)
3158 vim_free(flags);
3159 return follen;
3162 /* Set the word-char flags and fill SPELL_ISUPPER() table. */
3163 if (flags != NULL && fol != NULL)
3164 set_spell_charflags(flags, flagslen, fol);
3166 vim_free(flags);
3167 vim_free(fol);
3169 /* When <charflagslen> is zero then <fcharlen> must also be zero. */
3170 if ((flags == NULL) != (fol == NULL))
3171 return SP_FORMERROR;
3172 return 0;
3176 * Read SN_PREFCOND section.
3177 * Return SP_*ERROR flags.
3179 static int
3180 read_prefcond_section(fd, lp)
3181 FILE *fd;
3182 slang_T *lp;
3184 int cnt;
3185 int i;
3186 int n;
3187 char_u *p;
3188 char_u buf[MAXWLEN + 1];
3190 /* <prefcondcnt> <prefcond> ... */
3191 cnt = get2c(fd); /* <prefcondcnt> */
3192 if (cnt <= 0)
3193 return SP_FORMERROR;
3195 lp->sl_prefprog = (regprog_T **)alloc_clear(
3196 (unsigned)sizeof(regprog_T *) * cnt);
3197 if (lp->sl_prefprog == NULL)
3198 return SP_OTHERERROR;
3199 lp->sl_prefixcnt = cnt;
3201 for (i = 0; i < cnt; ++i)
3203 /* <prefcond> : <condlen> <condstr> */
3204 n = getc(fd); /* <condlen> */
3205 if (n < 0 || n >= MAXWLEN)
3206 return SP_FORMERROR;
3208 /* When <condlen> is zero we have an empty condition. Otherwise
3209 * compile the regexp program used to check for the condition. */
3210 if (n > 0)
3212 buf[0] = '^'; /* always match at one position only */
3213 p = buf + 1;
3214 while (n-- > 0)
3215 *p++ = getc(fd); /* <condstr> */
3216 *p = NUL;
3217 lp->sl_prefprog[i] = vim_regcomp(buf, RE_MAGIC + RE_STRING);
3220 return 0;
3224 * Read REP or REPSAL items section from "fd": <repcount> <rep> ...
3225 * Return SP_*ERROR flags.
3227 static int
3228 read_rep_section(fd, gap, first)
3229 FILE *fd;
3230 garray_T *gap;
3231 short *first;
3233 int cnt;
3234 fromto_T *ftp;
3235 int i;
3237 cnt = get2c(fd); /* <repcount> */
3238 if (cnt < 0)
3239 return SP_TRUNCERROR;
3241 if (ga_grow(gap, cnt) == FAIL)
3242 return SP_OTHERERROR;
3244 /* <rep> : <repfromlen> <repfrom> <reptolen> <repto> */
3245 for (; gap->ga_len < cnt; ++gap->ga_len)
3247 ftp = &((fromto_T *)gap->ga_data)[gap->ga_len];
3248 ftp->ft_from = read_cnt_string(fd, 1, &i);
3249 if (i < 0)
3250 return i;
3251 if (i == 0)
3252 return SP_FORMERROR;
3253 ftp->ft_to = read_cnt_string(fd, 1, &i);
3254 if (i <= 0)
3256 vim_free(ftp->ft_from);
3257 if (i < 0)
3258 return i;
3259 return SP_FORMERROR;
3263 /* Fill the first-index table. */
3264 for (i = 0; i < 256; ++i)
3265 first[i] = -1;
3266 for (i = 0; i < gap->ga_len; ++i)
3268 ftp = &((fromto_T *)gap->ga_data)[i];
3269 if (first[*ftp->ft_from] == -1)
3270 first[*ftp->ft_from] = i;
3272 return 0;
3276 * Read SN_SAL section: <salflags> <salcount> <sal> ...
3277 * Return SP_*ERROR flags.
3279 static int
3280 read_sal_section(fd, slang)
3281 FILE *fd;
3282 slang_T *slang;
3284 int i;
3285 int cnt;
3286 garray_T *gap;
3287 salitem_T *smp;
3288 int ccnt;
3289 char_u *p;
3290 int c = NUL;
3292 slang->sl_sofo = FALSE;
3294 i = getc(fd); /* <salflags> */
3295 if (i & SAL_F0LLOWUP)
3296 slang->sl_followup = TRUE;
3297 if (i & SAL_COLLAPSE)
3298 slang->sl_collapse = TRUE;
3299 if (i & SAL_REM_ACCENTS)
3300 slang->sl_rem_accents = TRUE;
3302 cnt = get2c(fd); /* <salcount> */
3303 if (cnt < 0)
3304 return SP_TRUNCERROR;
3306 gap = &slang->sl_sal;
3307 ga_init2(gap, sizeof(salitem_T), 10);
3308 if (ga_grow(gap, cnt + 1) == FAIL)
3309 return SP_OTHERERROR;
3311 /* <sal> : <salfromlen> <salfrom> <saltolen> <salto> */
3312 for (; gap->ga_len < cnt; ++gap->ga_len)
3314 smp = &((salitem_T *)gap->ga_data)[gap->ga_len];
3315 ccnt = getc(fd); /* <salfromlen> */
3316 if (ccnt < 0)
3317 return SP_TRUNCERROR;
3318 if ((p = alloc(ccnt + 2)) == NULL)
3319 return SP_OTHERERROR;
3320 smp->sm_lead = p;
3322 /* Read up to the first special char into sm_lead. */
3323 for (i = 0; i < ccnt; ++i)
3325 c = getc(fd); /* <salfrom> */
3326 if (vim_strchr((char_u *)"0123456789(-<^$", c) != NULL)
3327 break;
3328 *p++ = c;
3330 smp->sm_leadlen = (int)(p - smp->sm_lead);
3331 *p++ = NUL;
3333 /* Put (abc) chars in sm_oneof, if any. */
3334 if (c == '(')
3336 smp->sm_oneof = p;
3337 for (++i; i < ccnt; ++i)
3339 c = getc(fd); /* <salfrom> */
3340 if (c == ')')
3341 break;
3342 *p++ = c;
3344 *p++ = NUL;
3345 if (++i < ccnt)
3346 c = getc(fd);
3348 else
3349 smp->sm_oneof = NULL;
3351 /* Any following chars go in sm_rules. */
3352 smp->sm_rules = p;
3353 if (i < ccnt)
3354 /* store the char we got while checking for end of sm_lead */
3355 *p++ = c;
3356 for (++i; i < ccnt; ++i)
3357 *p++ = getc(fd); /* <salfrom> */
3358 *p++ = NUL;
3360 /* <saltolen> <salto> */
3361 smp->sm_to = read_cnt_string(fd, 1, &ccnt);
3362 if (ccnt < 0)
3364 vim_free(smp->sm_lead);
3365 return ccnt;
3368 #ifdef FEAT_MBYTE
3369 if (has_mbyte)
3371 /* convert the multi-byte strings to wide char strings */
3372 smp->sm_lead_w = mb_str2wide(smp->sm_lead);
3373 smp->sm_leadlen = mb_charlen(smp->sm_lead);
3374 if (smp->sm_oneof == NULL)
3375 smp->sm_oneof_w = NULL;
3376 else
3377 smp->sm_oneof_w = mb_str2wide(smp->sm_oneof);
3378 if (smp->sm_to == NULL)
3379 smp->sm_to_w = NULL;
3380 else
3381 smp->sm_to_w = mb_str2wide(smp->sm_to);
3382 if (smp->sm_lead_w == NULL
3383 || (smp->sm_oneof_w == NULL && smp->sm_oneof != NULL)
3384 || (smp->sm_to_w == NULL && smp->sm_to != NULL))
3386 vim_free(smp->sm_lead);
3387 vim_free(smp->sm_to);
3388 vim_free(smp->sm_lead_w);
3389 vim_free(smp->sm_oneof_w);
3390 vim_free(smp->sm_to_w);
3391 return SP_OTHERERROR;
3394 #endif
3397 if (gap->ga_len > 0)
3399 /* Add one extra entry to mark the end with an empty sm_lead. Avoids
3400 * that we need to check the index every time. */
3401 smp = &((salitem_T *)gap->ga_data)[gap->ga_len];
3402 if ((p = alloc(1)) == NULL)
3403 return SP_OTHERERROR;
3404 p[0] = NUL;
3405 smp->sm_lead = p;
3406 smp->sm_leadlen = 0;
3407 smp->sm_oneof = NULL;
3408 smp->sm_rules = p;
3409 smp->sm_to = NULL;
3410 #ifdef FEAT_MBYTE
3411 if (has_mbyte)
3413 smp->sm_lead_w = mb_str2wide(smp->sm_lead);
3414 smp->sm_leadlen = 0;
3415 smp->sm_oneof_w = NULL;
3416 smp->sm_to_w = NULL;
3418 #endif
3419 ++gap->ga_len;
3422 /* Fill the first-index table. */
3423 set_sal_first(slang);
3425 return 0;
3429 * Read SN_WORDS: <word> ...
3430 * Return SP_*ERROR flags.
3432 static int
3433 read_words_section(fd, lp, len)
3434 FILE *fd;
3435 slang_T *lp;
3436 int len;
3438 int done = 0;
3439 int i;
3440 int c;
3441 char_u word[MAXWLEN];
3443 while (done < len)
3445 /* Read one word at a time. */
3446 for (i = 0; ; ++i)
3448 c = getc(fd);
3449 if (c == EOF)
3450 return SP_TRUNCERROR;
3451 word[i] = c;
3452 if (word[i] == NUL)
3453 break;
3454 if (i == MAXWLEN - 1)
3455 return SP_FORMERROR;
3458 /* Init the count to 10. */
3459 count_common_word(lp, word, -1, 10);
3460 done += i + 1;
3462 return 0;
3466 * Add a word to the hashtable of common words.
3467 * If it's already there then the counter is increased.
3469 static void
3470 count_common_word(lp, word, len, count)
3471 slang_T *lp;
3472 char_u *word;
3473 int len; /* word length, -1 for upto NUL */
3474 int count; /* 1 to count once, 10 to init */
3476 hash_T hash;
3477 hashitem_T *hi;
3478 wordcount_T *wc;
3479 char_u buf[MAXWLEN];
3480 char_u *p;
3482 if (len == -1)
3483 p = word;
3484 else
3486 vim_strncpy(buf, word, len);
3487 p = buf;
3490 hash = hash_hash(p);
3491 hi = hash_lookup(&lp->sl_wordcount, p, hash);
3492 if (HASHITEM_EMPTY(hi))
3494 wc = (wordcount_T *)alloc((unsigned)(sizeof(wordcount_T) + STRLEN(p)));
3495 if (wc == NULL)
3496 return;
3497 STRCPY(wc->wc_word, p);
3498 wc->wc_count = count;
3499 hash_add_item(&lp->sl_wordcount, hi, wc->wc_word, hash);
3501 else
3503 wc = HI2WC(hi);
3504 if ((wc->wc_count += count) < (unsigned)count) /* check for overflow */
3505 wc->wc_count = MAXWORDCOUNT;
3510 * Adjust the score of common words.
3512 static int
3513 score_wordcount_adj(slang, score, word, split)
3514 slang_T *slang;
3515 int score;
3516 char_u *word;
3517 int split; /* word was split, less bonus */
3519 hashitem_T *hi;
3520 wordcount_T *wc;
3521 int bonus;
3522 int newscore;
3524 hi = hash_find(&slang->sl_wordcount, word);
3525 if (!HASHITEM_EMPTY(hi))
3527 wc = HI2WC(hi);
3528 if (wc->wc_count < SCORE_THRES2)
3529 bonus = SCORE_COMMON1;
3530 else if (wc->wc_count < SCORE_THRES3)
3531 bonus = SCORE_COMMON2;
3532 else
3533 bonus = SCORE_COMMON3;
3534 if (split)
3535 newscore = score - bonus / 2;
3536 else
3537 newscore = score - bonus;
3538 if (newscore < 0)
3539 return 0;
3540 return newscore;
3542 return score;
3546 * SN_SOFO: <sofofromlen> <sofofrom> <sofotolen> <sofoto>
3547 * Return SP_*ERROR flags.
3549 static int
3550 read_sofo_section(fd, slang)
3551 FILE *fd;
3552 slang_T *slang;
3554 int cnt;
3555 char_u *from, *to;
3556 int res;
3558 slang->sl_sofo = TRUE;
3560 /* <sofofromlen> <sofofrom> */
3561 from = read_cnt_string(fd, 2, &cnt);
3562 if (cnt < 0)
3563 return cnt;
3565 /* <sofotolen> <sofoto> */
3566 to = read_cnt_string(fd, 2, &cnt);
3567 if (cnt < 0)
3569 vim_free(from);
3570 return cnt;
3573 /* Store the info in slang->sl_sal and/or slang->sl_sal_first. */
3574 if (from != NULL && to != NULL)
3575 res = set_sofo(slang, from, to);
3576 else if (from != NULL || to != NULL)
3577 res = SP_FORMERROR; /* only one of two strings is an error */
3578 else
3579 res = 0;
3581 vim_free(from);
3582 vim_free(to);
3583 return res;
3587 * Read the compound section from the .spl file:
3588 * <compmax> <compminlen> <compsylmax> <compoptions> <compflags>
3589 * Returns SP_*ERROR flags.
3591 static int
3592 read_compound(fd, slang, len)
3593 FILE *fd;
3594 slang_T *slang;
3595 int len;
3597 int todo = len;
3598 int c;
3599 int atstart;
3600 char_u *pat;
3601 char_u *pp;
3602 char_u *cp;
3603 char_u *ap;
3604 char_u *crp;
3605 int cnt;
3606 garray_T *gap;
3608 if (todo < 2)
3609 return SP_FORMERROR; /* need at least two bytes */
3611 --todo;
3612 c = getc(fd); /* <compmax> */
3613 if (c < 2)
3614 c = MAXWLEN;
3615 slang->sl_compmax = c;
3617 --todo;
3618 c = getc(fd); /* <compminlen> */
3619 if (c < 1)
3620 c = 0;
3621 slang->sl_compminlen = c;
3623 --todo;
3624 c = getc(fd); /* <compsylmax> */
3625 if (c < 1)
3626 c = MAXWLEN;
3627 slang->sl_compsylmax = c;
3629 c = getc(fd); /* <compoptions> */
3630 if (c != 0)
3631 ungetc(c, fd); /* be backwards compatible with Vim 7.0b */
3632 else
3634 --todo;
3635 c = getc(fd); /* only use the lower byte for now */
3636 --todo;
3637 slang->sl_compoptions = c;
3639 gap = &slang->sl_comppat;
3640 c = get2c(fd); /* <comppatcount> */
3641 todo -= 2;
3642 ga_init2(gap, sizeof(char_u *), c);
3643 if (ga_grow(gap, c) == OK)
3644 while (--c >= 0)
3646 ((char_u **)(gap->ga_data))[gap->ga_len++] =
3647 read_cnt_string(fd, 1, &cnt);
3648 /* <comppatlen> <comppattext> */
3649 if (cnt < 0)
3650 return cnt;
3651 todo -= cnt + 1;
3654 if (todo < 0)
3655 return SP_FORMERROR;
3657 /* Turn the COMPOUNDRULE items into a regexp pattern:
3658 * "a[bc]/a*b+" -> "^\(a[bc]\|a*b\+\)$".
3659 * Inserting backslashes may double the length, "^\(\)$<Nul>" is 7 bytes.
3660 * Conversion to utf-8 may double the size. */
3661 c = todo * 2 + 7;
3662 #ifdef FEAT_MBYTE
3663 if (enc_utf8)
3664 c += todo * 2;
3665 #endif
3666 pat = alloc((unsigned)c);
3667 if (pat == NULL)
3668 return SP_OTHERERROR;
3670 /* We also need a list of all flags that can appear at the start and one
3671 * for all flags. */
3672 cp = alloc(todo + 1);
3673 if (cp == NULL)
3675 vim_free(pat);
3676 return SP_OTHERERROR;
3678 slang->sl_compstartflags = cp;
3679 *cp = NUL;
3681 ap = alloc(todo + 1);
3682 if (ap == NULL)
3684 vim_free(pat);
3685 return SP_OTHERERROR;
3687 slang->sl_compallflags = ap;
3688 *ap = NUL;
3690 /* And a list of all patterns in their original form, for checking whether
3691 * compounding may work in match_compoundrule(). This is freed when we
3692 * encounter a wildcard, the check doesn't work then. */
3693 crp = alloc(todo + 1);
3694 slang->sl_comprules = crp;
3696 pp = pat;
3697 *pp++ = '^';
3698 *pp++ = '\\';
3699 *pp++ = '(';
3701 atstart = 1;
3702 while (todo-- > 0)
3704 c = getc(fd); /* <compflags> */
3705 if (c == EOF)
3707 vim_free(pat);
3708 return SP_TRUNCERROR;
3711 /* Add all flags to "sl_compallflags". */
3712 if (vim_strchr((char_u *)"+*[]/", c) == NULL
3713 && !byte_in_str(slang->sl_compallflags, c))
3715 *ap++ = c;
3716 *ap = NUL;
3719 if (atstart != 0)
3721 /* At start of item: copy flags to "sl_compstartflags". For a
3722 * [abc] item set "atstart" to 2 and copy up to the ']'. */
3723 if (c == '[')
3724 atstart = 2;
3725 else if (c == ']')
3726 atstart = 0;
3727 else
3729 if (!byte_in_str(slang->sl_compstartflags, c))
3731 *cp++ = c;
3732 *cp = NUL;
3734 if (atstart == 1)
3735 atstart = 0;
3739 /* Copy flag to "sl_comprules", unless we run into a wildcard. */
3740 if (crp != NULL)
3742 if (c == '+' || c == '*')
3744 vim_free(slang->sl_comprules);
3745 slang->sl_comprules = NULL;
3746 crp = NULL;
3748 else
3749 *crp++ = c;
3752 if (c == '/') /* slash separates two items */
3754 *pp++ = '\\';
3755 *pp++ = '|';
3756 atstart = 1;
3758 else /* normal char, "[abc]" and '*' are copied as-is */
3760 if (c == '+' || c == '~')
3761 *pp++ = '\\'; /* "a+" becomes "a\+" */
3762 #ifdef FEAT_MBYTE
3763 if (enc_utf8)
3764 pp += mb_char2bytes(c, pp);
3765 else
3766 #endif
3767 *pp++ = c;
3771 *pp++ = '\\';
3772 *pp++ = ')';
3773 *pp++ = '$';
3774 *pp = NUL;
3776 if (crp != NULL)
3777 *crp = NUL;
3779 slang->sl_compprog = vim_regcomp(pat, RE_MAGIC + RE_STRING + RE_STRICT);
3780 vim_free(pat);
3781 if (slang->sl_compprog == NULL)
3782 return SP_FORMERROR;
3784 return 0;
3788 * Return TRUE if byte "n" appears in "str".
3789 * Like strchr() but independent of locale.
3791 static int
3792 byte_in_str(str, n)
3793 char_u *str;
3794 int n;
3796 char_u *p;
3798 for (p = str; *p != NUL; ++p)
3799 if (*p == n)
3800 return TRUE;
3801 return FALSE;
3804 #define SY_MAXLEN 30
3805 typedef struct syl_item_S
3807 char_u sy_chars[SY_MAXLEN]; /* the sequence of chars */
3808 int sy_len;
3809 } syl_item_T;
3812 * Truncate "slang->sl_syllable" at the first slash and put the following items
3813 * in "slang->sl_syl_items".
3815 static int
3816 init_syl_tab(slang)
3817 slang_T *slang;
3819 char_u *p;
3820 char_u *s;
3821 int l;
3822 syl_item_T *syl;
3824 ga_init2(&slang->sl_syl_items, sizeof(syl_item_T), 4);
3825 p = vim_strchr(slang->sl_syllable, '/');
3826 while (p != NULL)
3828 *p++ = NUL;
3829 if (*p == NUL) /* trailing slash */
3830 break;
3831 s = p;
3832 p = vim_strchr(p, '/');
3833 if (p == NULL)
3834 l = (int)STRLEN(s);
3835 else
3836 l = (int)(p - s);
3837 if (l >= SY_MAXLEN)
3838 return SP_FORMERROR;
3839 if (ga_grow(&slang->sl_syl_items, 1) == FAIL)
3840 return SP_OTHERERROR;
3841 syl = ((syl_item_T *)slang->sl_syl_items.ga_data)
3842 + slang->sl_syl_items.ga_len++;
3843 vim_strncpy(syl->sy_chars, s, l);
3844 syl->sy_len = l;
3846 return OK;
3850 * Count the number of syllables in "word".
3851 * When "word" contains spaces the syllables after the last space are counted.
3852 * Returns zero if syllables are not defines.
3854 static int
3855 count_syllables(slang, word)
3856 slang_T *slang;
3857 char_u *word;
3859 int cnt = 0;
3860 int skip = FALSE;
3861 char_u *p;
3862 int len;
3863 int i;
3864 syl_item_T *syl;
3865 int c;
3867 if (slang->sl_syllable == NULL)
3868 return 0;
3870 for (p = word; *p != NUL; p += len)
3872 /* When running into a space reset counter. */
3873 if (*p == ' ')
3875 len = 1;
3876 cnt = 0;
3877 continue;
3880 /* Find longest match of syllable items. */
3881 len = 0;
3882 for (i = 0; i < slang->sl_syl_items.ga_len; ++i)
3884 syl = ((syl_item_T *)slang->sl_syl_items.ga_data) + i;
3885 if (syl->sy_len > len
3886 && STRNCMP(p, syl->sy_chars, syl->sy_len) == 0)
3887 len = syl->sy_len;
3889 if (len != 0) /* found a match, count syllable */
3891 ++cnt;
3892 skip = FALSE;
3894 else
3896 /* No recognized syllable item, at least a syllable char then? */
3897 #ifdef FEAT_MBYTE
3898 c = mb_ptr2char(p);
3899 len = (*mb_ptr2len)(p);
3900 #else
3901 c = *p;
3902 len = 1;
3903 #endif
3904 if (vim_strchr(slang->sl_syllable, c) == NULL)
3905 skip = FALSE; /* No, search for next syllable */
3906 else if (!skip)
3908 ++cnt; /* Yes, count it */
3909 skip = TRUE; /* don't count following syllable chars */
3913 return cnt;
3917 * Set the SOFOFROM and SOFOTO items in language "lp".
3918 * Returns SP_*ERROR flags when there is something wrong.
3920 static int
3921 set_sofo(lp, from, to)
3922 slang_T *lp;
3923 char_u *from;
3924 char_u *to;
3926 int i;
3928 #ifdef FEAT_MBYTE
3929 garray_T *gap;
3930 char_u *s;
3931 char_u *p;
3932 int c;
3933 int *inp;
3935 if (has_mbyte)
3937 /* Use "sl_sal" as an array with 256 pointers to a list of wide
3938 * characters. The index is the low byte of the character.
3939 * The list contains from-to pairs with a terminating NUL.
3940 * sl_sal_first[] is used for latin1 "from" characters. */
3941 gap = &lp->sl_sal;
3942 ga_init2(gap, sizeof(int *), 1);
3943 if (ga_grow(gap, 256) == FAIL)
3944 return SP_OTHERERROR;
3945 vim_memset(gap->ga_data, 0, sizeof(int *) * 256);
3946 gap->ga_len = 256;
3948 /* First count the number of items for each list. Temporarily use
3949 * sl_sal_first[] for this. */
3950 for (p = from, s = to; *p != NUL && *s != NUL; )
3952 c = mb_cptr2char_adv(&p);
3953 mb_cptr_adv(s);
3954 if (c >= 256)
3955 ++lp->sl_sal_first[c & 0xff];
3957 if (*p != NUL || *s != NUL) /* lengths differ */
3958 return SP_FORMERROR;
3960 /* Allocate the lists. */
3961 for (i = 0; i < 256; ++i)
3962 if (lp->sl_sal_first[i] > 0)
3964 p = alloc(sizeof(int) * (lp->sl_sal_first[i] * 2 + 1));
3965 if (p == NULL)
3966 return SP_OTHERERROR;
3967 ((int **)gap->ga_data)[i] = (int *)p;
3968 *(int *)p = 0;
3971 /* Put the characters up to 255 in sl_sal_first[] the rest in a sl_sal
3972 * list. */
3973 vim_memset(lp->sl_sal_first, 0, sizeof(salfirst_T) * 256);
3974 for (p = from, s = to; *p != NUL && *s != NUL; )
3976 c = mb_cptr2char_adv(&p);
3977 i = mb_cptr2char_adv(&s);
3978 if (c >= 256)
3980 /* Append the from-to chars at the end of the list with
3981 * the low byte. */
3982 inp = ((int **)gap->ga_data)[c & 0xff];
3983 while (*inp != 0)
3984 ++inp;
3985 *inp++ = c; /* from char */
3986 *inp++ = i; /* to char */
3987 *inp++ = NUL; /* NUL at the end */
3989 else
3990 /* mapping byte to char is done in sl_sal_first[] */
3991 lp->sl_sal_first[c] = i;
3994 else
3995 #endif
3997 /* mapping bytes to bytes is done in sl_sal_first[] */
3998 if (STRLEN(from) != STRLEN(to))
3999 return SP_FORMERROR;
4001 for (i = 0; to[i] != NUL; ++i)
4002 lp->sl_sal_first[from[i]] = to[i];
4003 lp->sl_sal.ga_len = 1; /* indicates we have soundfolding */
4006 return 0;
4010 * Fill the first-index table for "lp".
4012 static void
4013 set_sal_first(lp)
4014 slang_T *lp;
4016 salfirst_T *sfirst;
4017 int i;
4018 salitem_T *smp;
4019 int c;
4020 garray_T *gap = &lp->sl_sal;
4022 sfirst = lp->sl_sal_first;
4023 for (i = 0; i < 256; ++i)
4024 sfirst[i] = -1;
4025 smp = (salitem_T *)gap->ga_data;
4026 for (i = 0; i < gap->ga_len; ++i)
4028 #ifdef FEAT_MBYTE
4029 if (has_mbyte)
4030 /* Use the lowest byte of the first character. For latin1 it's
4031 * the character, for other encodings it should differ for most
4032 * characters. */
4033 c = *smp[i].sm_lead_w & 0xff;
4034 else
4035 #endif
4036 c = *smp[i].sm_lead;
4037 if (sfirst[c] == -1)
4039 sfirst[c] = i;
4040 #ifdef FEAT_MBYTE
4041 if (has_mbyte)
4043 int n;
4045 /* Make sure all entries with this byte are following each
4046 * other. Move the ones that are in the wrong position. Do
4047 * keep the same ordering! */
4048 while (i + 1 < gap->ga_len
4049 && (*smp[i + 1].sm_lead_w & 0xff) == c)
4050 /* Skip over entry with same index byte. */
4051 ++i;
4053 for (n = 1; i + n < gap->ga_len; ++n)
4054 if ((*smp[i + n].sm_lead_w & 0xff) == c)
4056 salitem_T tsal;
4058 /* Move entry with same index byte after the entries
4059 * we already found. */
4060 ++i;
4061 --n;
4062 tsal = smp[i + n];
4063 mch_memmove(smp + i + 1, smp + i,
4064 sizeof(salitem_T) * n);
4065 smp[i] = tsal;
4068 #endif
4073 #ifdef FEAT_MBYTE
4075 * Turn a multi-byte string into a wide character string.
4076 * Return it in allocated memory (NULL for out-of-memory)
4078 static int *
4079 mb_str2wide(s)
4080 char_u *s;
4082 int *res;
4083 char_u *p;
4084 int i = 0;
4086 res = (int *)alloc(sizeof(int) * (mb_charlen(s) + 1));
4087 if (res != NULL)
4089 for (p = s; *p != NUL; )
4090 res[i++] = mb_ptr2char_adv(&p);
4091 res[i] = NUL;
4093 return res;
4095 #endif
4098 * Read a tree from the .spl or .sug file.
4099 * Allocates the memory and stores pointers in "bytsp" and "idxsp".
4100 * This is skipped when the tree has zero length.
4101 * Returns zero when OK, SP_ value for an error.
4103 static int
4104 spell_read_tree(fd, bytsp, idxsp, prefixtree, prefixcnt)
4105 FILE *fd;
4106 char_u **bytsp;
4107 idx_T **idxsp;
4108 int prefixtree; /* TRUE for the prefix tree */
4109 int prefixcnt; /* when "prefixtree" is TRUE: prefix count */
4111 int len;
4112 int idx;
4113 char_u *bp;
4114 idx_T *ip;
4116 /* The tree size was computed when writing the file, so that we can
4117 * allocate it as one long block. <nodecount> */
4118 len = get4c(fd);
4119 if (len < 0)
4120 return SP_TRUNCERROR;
4121 if (len > 0)
4123 /* Allocate the byte array. */
4124 bp = lalloc((long_u)len, TRUE);
4125 if (bp == NULL)
4126 return SP_OTHERERROR;
4127 *bytsp = bp;
4129 /* Allocate the index array. */
4130 ip = (idx_T *)lalloc_clear((long_u)(len * sizeof(int)), TRUE);
4131 if (ip == NULL)
4132 return SP_OTHERERROR;
4133 *idxsp = ip;
4135 /* Recursively read the tree and store it in the array. */
4136 idx = read_tree_node(fd, bp, ip, len, 0, prefixtree, prefixcnt);
4137 if (idx < 0)
4138 return idx;
4140 return 0;
4144 * Read one row of siblings from the spell file and store it in the byte array
4145 * "byts" and index array "idxs". Recursively read the children.
4147 * NOTE: The code here must match put_node()!
4149 * Returns the index (>= 0) following the siblings.
4150 * Returns SP_TRUNCERROR if the file is shorter than expected.
4151 * Returns SP_FORMERROR if there is a format error.
4153 static idx_T
4154 read_tree_node(fd, byts, idxs, maxidx, startidx, prefixtree, maxprefcondnr)
4155 FILE *fd;
4156 char_u *byts;
4157 idx_T *idxs;
4158 int maxidx; /* size of arrays */
4159 idx_T startidx; /* current index in "byts" and "idxs" */
4160 int prefixtree; /* TRUE for reading PREFIXTREE */
4161 int maxprefcondnr; /* maximum for <prefcondnr> */
4163 int len;
4164 int i;
4165 int n;
4166 idx_T idx = startidx;
4167 int c;
4168 int c2;
4169 #define SHARED_MASK 0x8000000
4171 len = getc(fd); /* <siblingcount> */
4172 if (len <= 0)
4173 return SP_TRUNCERROR;
4175 if (startidx + len >= maxidx)
4176 return SP_FORMERROR;
4177 byts[idx++] = len;
4179 /* Read the byte values, flag/region bytes and shared indexes. */
4180 for (i = 1; i <= len; ++i)
4182 c = getc(fd); /* <byte> */
4183 if (c < 0)
4184 return SP_TRUNCERROR;
4185 if (c <= BY_SPECIAL)
4187 if (c == BY_NOFLAGS && !prefixtree)
4189 /* No flags, all regions. */
4190 idxs[idx] = 0;
4191 c = 0;
4193 else if (c != BY_INDEX)
4195 if (prefixtree)
4197 /* Read the optional pflags byte, the prefix ID and the
4198 * condition nr. In idxs[] store the prefix ID in the low
4199 * byte, the condition index shifted up 8 bits, the flags
4200 * shifted up 24 bits. */
4201 if (c == BY_FLAGS)
4202 c = getc(fd) << 24; /* <pflags> */
4203 else
4204 c = 0;
4206 c |= getc(fd); /* <affixID> */
4208 n = get2c(fd); /* <prefcondnr> */
4209 if (n >= maxprefcondnr)
4210 return SP_FORMERROR;
4211 c |= (n << 8);
4213 else /* c must be BY_FLAGS or BY_FLAGS2 */
4215 /* Read flags and optional region and prefix ID. In
4216 * idxs[] the flags go in the low two bytes, region above
4217 * that and prefix ID above the region. */
4218 c2 = c;
4219 c = getc(fd); /* <flags> */
4220 if (c2 == BY_FLAGS2)
4221 c = (getc(fd) << 8) + c; /* <flags2> */
4222 if (c & WF_REGION)
4223 c = (getc(fd) << 16) + c; /* <region> */
4224 if (c & WF_AFX)
4225 c = (getc(fd) << 24) + c; /* <affixID> */
4228 idxs[idx] = c;
4229 c = 0;
4231 else /* c == BY_INDEX */
4233 /* <nodeidx> */
4234 n = get3c(fd);
4235 if (n < 0 || n >= maxidx)
4236 return SP_FORMERROR;
4237 idxs[idx] = n + SHARED_MASK;
4238 c = getc(fd); /* <xbyte> */
4241 byts[idx++] = c;
4244 /* Recursively read the children for non-shared siblings.
4245 * Skip the end-of-word ones (zero byte value) and the shared ones (and
4246 * remove SHARED_MASK) */
4247 for (i = 1; i <= len; ++i)
4248 if (byts[startidx + i] != 0)
4250 if (idxs[startidx + i] & SHARED_MASK)
4251 idxs[startidx + i] &= ~SHARED_MASK;
4252 else
4254 idxs[startidx + i] = idx;
4255 idx = read_tree_node(fd, byts, idxs, maxidx, idx,
4256 prefixtree, maxprefcondnr);
4257 if (idx < 0)
4258 break;
4262 return idx;
4266 * Parse 'spelllang' and set buf->b_langp accordingly.
4267 * Returns NULL if it's OK, an error message otherwise.
4269 char_u *
4270 did_set_spelllang(buf)
4271 buf_T *buf;
4273 garray_T ga;
4274 char_u *splp;
4275 char_u *region;
4276 char_u region_cp[3];
4277 int filename;
4278 int region_mask;
4279 slang_T *slang;
4280 int c;
4281 char_u lang[MAXWLEN + 1];
4282 char_u spf_name[MAXPATHL];
4283 int len;
4284 char_u *p;
4285 int round;
4286 char_u *spf;
4287 char_u *use_region = NULL;
4288 int dont_use_region = FALSE;
4289 int nobreak = FALSE;
4290 int i, j;
4291 langp_T *lp, *lp2;
4292 static int recursive = FALSE;
4293 char_u *ret_msg = NULL;
4294 char_u *spl_copy;
4296 /* We don't want to do this recursively. May happen when a language is
4297 * not available and the SpellFileMissing autocommand opens a new buffer
4298 * in which 'spell' is set. */
4299 if (recursive)
4300 return NULL;
4301 recursive = TRUE;
4303 ga_init2(&ga, sizeof(langp_T), 2);
4304 clear_midword(buf);
4306 /* Make a copy of 'spellang', the SpellFileMissing autocommands may change
4307 * it under our fingers. */
4308 spl_copy = vim_strsave(buf->b_p_spl);
4309 if (spl_copy == NULL)
4310 goto theend;
4312 /* loop over comma separated language names. */
4313 for (splp = spl_copy; *splp != NUL; )
4315 /* Get one language name. */
4316 copy_option_part(&splp, lang, MAXWLEN, ",");
4318 region = NULL;
4319 len = (int)STRLEN(lang);
4321 /* If the name ends in ".spl" use it as the name of the spell file.
4322 * If there is a region name let "region" point to it and remove it
4323 * from the name. */
4324 if (len > 4 && fnamecmp(lang + len - 4, ".spl") == 0)
4326 filename = TRUE;
4328 /* Locate a region and remove it from the file name. */
4329 p = vim_strchr(gettail(lang), '_');
4330 if (p != NULL && ASCII_ISALPHA(p[1]) && ASCII_ISALPHA(p[2])
4331 && !ASCII_ISALPHA(p[3]))
4333 vim_strncpy(region_cp, p + 1, 2);
4334 mch_memmove(p, p + 3, len - (p - lang) - 2);
4335 len -= 3;
4336 region = region_cp;
4338 else
4339 dont_use_region = TRUE;
4341 /* Check if we loaded this language before. */
4342 for (slang = first_lang; slang != NULL; slang = slang->sl_next)
4343 if (fullpathcmp(lang, slang->sl_fname, FALSE) == FPC_SAME)
4344 break;
4346 else
4348 filename = FALSE;
4349 if (len > 3 && lang[len - 3] == '_')
4351 region = lang + len - 2;
4352 len -= 3;
4353 lang[len] = NUL;
4355 else
4356 dont_use_region = TRUE;
4358 /* Check if we loaded this language before. */
4359 for (slang = first_lang; slang != NULL; slang = slang->sl_next)
4360 if (STRICMP(lang, slang->sl_name) == 0)
4361 break;
4364 if (region != NULL)
4366 /* If the region differs from what was used before then don't
4367 * use it for 'spellfile'. */
4368 if (use_region != NULL && STRCMP(region, use_region) != 0)
4369 dont_use_region = TRUE;
4370 use_region = region;
4373 /* If not found try loading the language now. */
4374 if (slang == NULL)
4376 if (filename)
4377 (void)spell_load_file(lang, lang, NULL, FALSE);
4378 else
4380 spell_load_lang(lang);
4381 #ifdef FEAT_AUTOCMD
4382 /* SpellFileMissing autocommands may do anything, including
4383 * destroying the buffer we are using... */
4384 if (!buf_valid(buf))
4386 ret_msg = (char_u *)"E797: SpellFileMissing autocommand deleted buffer";
4387 goto theend;
4389 #endif
4394 * Loop over the languages, there can be several files for "lang".
4396 for (slang = first_lang; slang != NULL; slang = slang->sl_next)
4397 if (filename ? fullpathcmp(lang, slang->sl_fname, FALSE) == FPC_SAME
4398 : STRICMP(lang, slang->sl_name) == 0)
4400 region_mask = REGION_ALL;
4401 if (!filename && region != NULL)
4403 /* find region in sl_regions */
4404 c = find_region(slang->sl_regions, region);
4405 if (c == REGION_ALL)
4407 if (slang->sl_add)
4409 if (*slang->sl_regions != NUL)
4410 /* This addition file is for other regions. */
4411 region_mask = 0;
4413 else
4414 /* This is probably an error. Give a warning and
4415 * accept the words anyway. */
4416 smsg((char_u *)
4417 _("Warning: region %s not supported"),
4418 region);
4420 else
4421 region_mask = 1 << c;
4424 if (region_mask != 0)
4426 if (ga_grow(&ga, 1) == FAIL)
4428 ga_clear(&ga);
4429 ret_msg = e_outofmem;
4430 goto theend;
4432 LANGP_ENTRY(ga, ga.ga_len)->lp_slang = slang;
4433 LANGP_ENTRY(ga, ga.ga_len)->lp_region = region_mask;
4434 ++ga.ga_len;
4435 use_midword(slang, buf);
4436 if (slang->sl_nobreak)
4437 nobreak = TRUE;
4442 /* round 0: load int_wordlist, if possible.
4443 * round 1: load first name in 'spellfile'.
4444 * round 2: load second name in 'spellfile.
4445 * etc. */
4446 spf = buf->b_p_spf;
4447 for (round = 0; round == 0 || *spf != NUL; ++round)
4449 if (round == 0)
4451 /* Internal wordlist, if there is one. */
4452 if (int_wordlist == NULL)
4453 continue;
4454 int_wordlist_spl(spf_name);
4456 else
4458 /* One entry in 'spellfile'. */
4459 copy_option_part(&spf, spf_name, MAXPATHL - 5, ",");
4460 STRCAT(spf_name, ".spl");
4462 /* If it was already found above then skip it. */
4463 for (c = 0; c < ga.ga_len; ++c)
4465 p = LANGP_ENTRY(ga, c)->lp_slang->sl_fname;
4466 if (p != NULL && fullpathcmp(spf_name, p, FALSE) == FPC_SAME)
4467 break;
4469 if (c < ga.ga_len)
4470 continue;
4473 /* Check if it was loaded already. */
4474 for (slang = first_lang; slang != NULL; slang = slang->sl_next)
4475 if (fullpathcmp(spf_name, slang->sl_fname, FALSE) == FPC_SAME)
4476 break;
4477 if (slang == NULL)
4479 /* Not loaded, try loading it now. The language name includes the
4480 * region name, the region is ignored otherwise. for int_wordlist
4481 * use an arbitrary name. */
4482 if (round == 0)
4483 STRCPY(lang, "internal wordlist");
4484 else
4486 vim_strncpy(lang, gettail(spf_name), MAXWLEN);
4487 p = vim_strchr(lang, '.');
4488 if (p != NULL)
4489 *p = NUL; /* truncate at ".encoding.add" */
4491 slang = spell_load_file(spf_name, lang, NULL, TRUE);
4493 /* If one of the languages has NOBREAK we assume the addition
4494 * files also have this. */
4495 if (slang != NULL && nobreak)
4496 slang->sl_nobreak = TRUE;
4498 if (slang != NULL && ga_grow(&ga, 1) == OK)
4500 region_mask = REGION_ALL;
4501 if (use_region != NULL && !dont_use_region)
4503 /* find region in sl_regions */
4504 c = find_region(slang->sl_regions, use_region);
4505 if (c != REGION_ALL)
4506 region_mask = 1 << c;
4507 else if (*slang->sl_regions != NUL)
4508 /* This spell file is for other regions. */
4509 region_mask = 0;
4512 if (region_mask != 0)
4514 LANGP_ENTRY(ga, ga.ga_len)->lp_slang = slang;
4515 LANGP_ENTRY(ga, ga.ga_len)->lp_sallang = NULL;
4516 LANGP_ENTRY(ga, ga.ga_len)->lp_replang = NULL;
4517 LANGP_ENTRY(ga, ga.ga_len)->lp_region = region_mask;
4518 ++ga.ga_len;
4519 use_midword(slang, buf);
4524 /* Everything is fine, store the new b_langp value. */
4525 ga_clear(&buf->b_langp);
4526 buf->b_langp = ga;
4528 /* For each language figure out what language to use for sound folding and
4529 * REP items. If the language doesn't support it itself use another one
4530 * with the same name. E.g. for "en-math" use "en". */
4531 for (i = 0; i < ga.ga_len; ++i)
4533 lp = LANGP_ENTRY(ga, i);
4535 /* sound folding */
4536 if (lp->lp_slang->sl_sal.ga_len > 0)
4537 /* language does sound folding itself */
4538 lp->lp_sallang = lp->lp_slang;
4539 else
4540 /* find first similar language that does sound folding */
4541 for (j = 0; j < ga.ga_len; ++j)
4543 lp2 = LANGP_ENTRY(ga, j);
4544 if (lp2->lp_slang->sl_sal.ga_len > 0
4545 && STRNCMP(lp->lp_slang->sl_name,
4546 lp2->lp_slang->sl_name, 2) == 0)
4548 lp->lp_sallang = lp2->lp_slang;
4549 break;
4553 /* REP items */
4554 if (lp->lp_slang->sl_rep.ga_len > 0)
4555 /* language has REP items itself */
4556 lp->lp_replang = lp->lp_slang;
4557 else
4558 /* find first similar language that has REP items */
4559 for (j = 0; j < ga.ga_len; ++j)
4561 lp2 = LANGP_ENTRY(ga, j);
4562 if (lp2->lp_slang->sl_rep.ga_len > 0
4563 && STRNCMP(lp->lp_slang->sl_name,
4564 lp2->lp_slang->sl_name, 2) == 0)
4566 lp->lp_replang = lp2->lp_slang;
4567 break;
4572 theend:
4573 vim_free(spl_copy);
4574 recursive = FALSE;
4575 return ret_msg;
4579 * Clear the midword characters for buffer "buf".
4581 static void
4582 clear_midword(buf)
4583 buf_T *buf;
4585 vim_memset(buf->b_spell_ismw, 0, 256);
4586 #ifdef FEAT_MBYTE
4587 vim_free(buf->b_spell_ismw_mb);
4588 buf->b_spell_ismw_mb = NULL;
4589 #endif
4593 * Use the "sl_midword" field of language "lp" for buffer "buf".
4594 * They add up to any currently used midword characters.
4596 static void
4597 use_midword(lp, buf)
4598 slang_T *lp;
4599 buf_T *buf;
4601 char_u *p;
4603 if (lp->sl_midword == NULL) /* there aren't any */
4604 return;
4606 for (p = lp->sl_midword; *p != NUL; )
4607 #ifdef FEAT_MBYTE
4608 if (has_mbyte)
4610 int c, l, n;
4611 char_u *bp;
4613 c = mb_ptr2char(p);
4614 l = (*mb_ptr2len)(p);
4615 if (c < 256 && l <= 2)
4616 buf->b_spell_ismw[c] = TRUE;
4617 else if (buf->b_spell_ismw_mb == NULL)
4618 /* First multi-byte char in "b_spell_ismw_mb". */
4619 buf->b_spell_ismw_mb = vim_strnsave(p, l);
4620 else
4622 /* Append multi-byte chars to "b_spell_ismw_mb". */
4623 n = (int)STRLEN(buf->b_spell_ismw_mb);
4624 bp = vim_strnsave(buf->b_spell_ismw_mb, n + l);
4625 if (bp != NULL)
4627 vim_free(buf->b_spell_ismw_mb);
4628 buf->b_spell_ismw_mb = bp;
4629 vim_strncpy(bp + n, p, l);
4632 p += l;
4634 else
4635 #endif
4636 buf->b_spell_ismw[*p++] = TRUE;
4640 * Find the region "region[2]" in "rp" (points to "sl_regions").
4641 * Each region is simply stored as the two characters of it's name.
4642 * Returns the index if found (first is 0), REGION_ALL if not found.
4644 static int
4645 find_region(rp, region)
4646 char_u *rp;
4647 char_u *region;
4649 int i;
4651 for (i = 0; ; i += 2)
4653 if (rp[i] == NUL)
4654 return REGION_ALL;
4655 if (rp[i] == region[0] && rp[i + 1] == region[1])
4656 break;
4658 return i / 2;
4662 * Return case type of word:
4663 * w word 0
4664 * Word WF_ONECAP
4665 * W WORD WF_ALLCAP
4666 * WoRd wOrd WF_KEEPCAP
4668 static int
4669 captype(word, end)
4670 char_u *word;
4671 char_u *end; /* When NULL use up to NUL byte. */
4673 char_u *p;
4674 int c;
4675 int firstcap;
4676 int allcap;
4677 int past_second = FALSE; /* past second word char */
4679 /* find first letter */
4680 for (p = word; !spell_iswordp_nmw(p); mb_ptr_adv(p))
4681 if (end == NULL ? *p == NUL : p >= end)
4682 return 0; /* only non-word characters, illegal word */
4683 #ifdef FEAT_MBYTE
4684 if (has_mbyte)
4685 c = mb_ptr2char_adv(&p);
4686 else
4687 #endif
4688 c = *p++;
4689 firstcap = allcap = SPELL_ISUPPER(c);
4692 * Need to check all letters to find a word with mixed upper/lower.
4693 * But a word with an upper char only at start is a ONECAP.
4695 for ( ; end == NULL ? *p != NUL : p < end; mb_ptr_adv(p))
4696 if (spell_iswordp_nmw(p))
4698 c = PTR2CHAR(p);
4699 if (!SPELL_ISUPPER(c))
4701 /* UUl -> KEEPCAP */
4702 if (past_second && allcap)
4703 return WF_KEEPCAP;
4704 allcap = FALSE;
4706 else if (!allcap)
4707 /* UlU -> KEEPCAP */
4708 return WF_KEEPCAP;
4709 past_second = TRUE;
4712 if (allcap)
4713 return WF_ALLCAP;
4714 if (firstcap)
4715 return WF_ONECAP;
4716 return 0;
4720 * Like captype() but for a KEEPCAP word add ONECAP if the word starts with a
4721 * capital. So that make_case_word() can turn WOrd into Word.
4722 * Add ALLCAP for "WOrD".
4724 static int
4725 badword_captype(word, end)
4726 char_u *word;
4727 char_u *end;
4729 int flags = captype(word, end);
4730 int c;
4731 int l, u;
4732 int first;
4733 char_u *p;
4735 if (flags & WF_KEEPCAP)
4737 /* Count the number of UPPER and lower case letters. */
4738 l = u = 0;
4739 first = FALSE;
4740 for (p = word; p < end; mb_ptr_adv(p))
4742 c = PTR2CHAR(p);
4743 if (SPELL_ISUPPER(c))
4745 ++u;
4746 if (p == word)
4747 first = TRUE;
4749 else
4750 ++l;
4753 /* If there are more UPPER than lower case letters suggest an
4754 * ALLCAP word. Otherwise, if the first letter is UPPER then
4755 * suggest ONECAP. Exception: "ALl" most likely should be "All",
4756 * require three upper case letters. */
4757 if (u > l && u > 2)
4758 flags |= WF_ALLCAP;
4759 else if (first)
4760 flags |= WF_ONECAP;
4762 if (u >= 2 && l >= 2) /* maCARONI maCAroni */
4763 flags |= WF_MIXCAP;
4765 return flags;
4768 # if defined(FEAT_MBYTE) || defined(EXITFREE) || defined(PROTO)
4770 * Free all languages.
4772 void
4773 spell_free_all()
4775 slang_T *slang;
4776 buf_T *buf;
4777 char_u fname[MAXPATHL];
4779 /* Go through all buffers and handle 'spelllang'. */
4780 for (buf = firstbuf; buf != NULL; buf = buf->b_next)
4781 ga_clear(&buf->b_langp);
4783 while (first_lang != NULL)
4785 slang = first_lang;
4786 first_lang = slang->sl_next;
4787 slang_free(slang);
4790 if (int_wordlist != NULL)
4792 /* Delete the internal wordlist and its .spl file */
4793 mch_remove(int_wordlist);
4794 int_wordlist_spl(fname);
4795 mch_remove(fname);
4796 vim_free(int_wordlist);
4797 int_wordlist = NULL;
4800 init_spell_chartab();
4802 vim_free(repl_to);
4803 repl_to = NULL;
4804 vim_free(repl_from);
4805 repl_from = NULL;
4807 # endif
4809 # if defined(FEAT_MBYTE) || defined(PROTO)
4811 * Clear all spelling tables and reload them.
4812 * Used after 'encoding' is set and when ":mkspell" was used.
4814 void
4815 spell_reload()
4817 buf_T *buf;
4818 win_T *wp;
4820 /* Initialize the table for spell_iswordp(). */
4821 init_spell_chartab();
4823 /* Unload all allocated memory. */
4824 spell_free_all();
4826 /* Go through all buffers and handle 'spelllang'. */
4827 for (buf = firstbuf; buf != NULL; buf = buf->b_next)
4829 /* Only load the wordlists when 'spelllang' is set and there is a
4830 * window for this buffer in which 'spell' is set. */
4831 if (*buf->b_p_spl != NUL)
4833 FOR_ALL_WINDOWS(wp)
4834 if (wp->w_buffer == buf && wp->w_p_spell)
4836 (void)did_set_spelllang(buf);
4837 # ifdef FEAT_WINDOWS
4838 break;
4839 # endif
4844 # endif
4847 * Reload the spell file "fname" if it's loaded.
4849 static void
4850 spell_reload_one(fname, added_word)
4851 char_u *fname;
4852 int added_word; /* invoked through "zg" */
4854 slang_T *slang;
4855 int didit = FALSE;
4857 for (slang = first_lang; slang != NULL; slang = slang->sl_next)
4859 if (fullpathcmp(fname, slang->sl_fname, FALSE) == FPC_SAME)
4861 slang_clear(slang);
4862 if (spell_load_file(fname, NULL, slang, FALSE) == NULL)
4863 /* reloading failed, clear the language */
4864 slang_clear(slang);
4865 redraw_all_later(SOME_VALID);
4866 didit = TRUE;
4870 /* When "zg" was used and the file wasn't loaded yet, should redo
4871 * 'spelllang' to load it now. */
4872 if (added_word && !didit)
4873 did_set_spelllang(curbuf);
4878 * Functions for ":mkspell".
4881 #define MAXLINELEN 500 /* Maximum length in bytes of a line in a .aff
4882 and .dic file. */
4884 * Main structure to store the contents of a ".aff" file.
4886 typedef struct afffile_S
4888 char_u *af_enc; /* "SET", normalized, alloc'ed string or NULL */
4889 int af_flagtype; /* AFT_CHAR, AFT_LONG, AFT_NUM or AFT_CAPLONG */
4890 unsigned af_rare; /* RARE ID for rare word */
4891 unsigned af_keepcase; /* KEEPCASE ID for keep-case word */
4892 unsigned af_bad; /* BAD ID for banned word */
4893 unsigned af_needaffix; /* NEEDAFFIX ID */
4894 unsigned af_circumfix; /* CIRCUMFIX ID */
4895 unsigned af_needcomp; /* NEEDCOMPOUND ID */
4896 unsigned af_comproot; /* COMPOUNDROOT ID */
4897 unsigned af_compforbid; /* COMPOUNDFORBIDFLAG ID */
4898 unsigned af_comppermit; /* COMPOUNDPERMITFLAG ID */
4899 unsigned af_nosuggest; /* NOSUGGEST ID */
4900 int af_pfxpostpone; /* postpone prefixes without chop string and
4901 without flags */
4902 hashtab_T af_pref; /* hashtable for prefixes, affheader_T */
4903 hashtab_T af_suff; /* hashtable for suffixes, affheader_T */
4904 hashtab_T af_comp; /* hashtable for compound flags, compitem_T */
4905 } afffile_T;
4907 #define AFT_CHAR 0 /* flags are one character */
4908 #define AFT_LONG 1 /* flags are two characters */
4909 #define AFT_CAPLONG 2 /* flags are one or two characters */
4910 #define AFT_NUM 3 /* flags are numbers, comma separated */
4912 typedef struct affentry_S affentry_T;
4913 /* Affix entry from ".aff" file. Used for prefixes and suffixes. */
4914 struct affentry_S
4916 affentry_T *ae_next; /* next affix with same name/number */
4917 char_u *ae_chop; /* text to chop off basic word (can be NULL) */
4918 char_u *ae_add; /* text to add to basic word (can be NULL) */
4919 char_u *ae_flags; /* flags on the affix (can be NULL) */
4920 char_u *ae_cond; /* condition (NULL for ".") */
4921 regprog_T *ae_prog; /* regexp program for ae_cond or NULL */
4922 char ae_compforbid; /* COMPOUNDFORBIDFLAG found */
4923 char ae_comppermit; /* COMPOUNDPERMITFLAG found */
4926 #ifdef FEAT_MBYTE
4927 # define AH_KEY_LEN 17 /* 2 x 8 bytes + NUL */
4928 #else
4929 # define AH_KEY_LEN 7 /* 6 digits + NUL */
4930 #endif
4932 /* Affix header from ".aff" file. Used for af_pref and af_suff. */
4933 typedef struct affheader_S
4935 char_u ah_key[AH_KEY_LEN]; /* key for hashtab == name of affix */
4936 unsigned ah_flag; /* affix name as number, uses "af_flagtype" */
4937 int ah_newID; /* prefix ID after renumbering; 0 if not used */
4938 int ah_combine; /* suffix may combine with prefix */
4939 int ah_follows; /* another affix block should be following */
4940 affentry_T *ah_first; /* first affix entry */
4941 } affheader_T;
4943 #define HI2AH(hi) ((affheader_T *)(hi)->hi_key)
4945 /* Flag used in compound items. */
4946 typedef struct compitem_S
4948 char_u ci_key[AH_KEY_LEN]; /* key for hashtab == name of compound */
4949 unsigned ci_flag; /* affix name as number, uses "af_flagtype" */
4950 int ci_newID; /* affix ID after renumbering. */
4951 } compitem_T;
4953 #define HI2CI(hi) ((compitem_T *)(hi)->hi_key)
4956 * Structure that is used to store the items in the word tree. This avoids
4957 * the need to keep track of each allocated thing, everything is freed all at
4958 * once after ":mkspell" is done.
4959 * Note: "sb_next" must be just before "sb_data" to make sure the alignment of
4960 * "sb_data" is correct for systems where pointers must be aligned on
4961 * pointer-size boundaries and sizeof(pointer) > sizeof(int) (e.g., Sparc).
4963 #define SBLOCKSIZE 16000 /* size of sb_data */
4964 typedef struct sblock_S sblock_T;
4965 struct sblock_S
4967 int sb_used; /* nr of bytes already in use */
4968 sblock_T *sb_next; /* next block in list */
4969 char_u sb_data[1]; /* data, actually longer */
4973 * A node in the tree.
4975 typedef struct wordnode_S wordnode_T;
4976 struct wordnode_S
4978 union /* shared to save space */
4980 char_u hashkey[6]; /* the hash key, only used while compressing */
4981 int index; /* index in written nodes (valid after first
4982 round) */
4983 } wn_u1;
4984 union /* shared to save space */
4986 wordnode_T *next; /* next node with same hash key */
4987 wordnode_T *wnode; /* parent node that will write this node */
4988 } wn_u2;
4989 wordnode_T *wn_child; /* child (next byte in word) */
4990 wordnode_T *wn_sibling; /* next sibling (alternate byte in word,
4991 always sorted) */
4992 int wn_refs; /* Nr. of references to this node. Only
4993 relevant for first node in a list of
4994 siblings, in following siblings it is
4995 always one. */
4996 char_u wn_byte; /* Byte for this node. NUL for word end */
4998 /* Info for when "wn_byte" is NUL.
4999 * In PREFIXTREE "wn_region" is used for the prefcondnr.
5000 * In the soundfolded word tree "wn_flags" has the MSW of the wordnr and
5001 * "wn_region" the LSW of the wordnr. */
5002 char_u wn_affixID; /* supported/required prefix ID or 0 */
5003 short_u wn_flags; /* WF_ flags */
5004 short wn_region; /* region mask */
5006 #ifdef SPELL_PRINTTREE
5007 int wn_nr; /* sequence nr for printing */
5008 #endif
5011 #define WN_MASK 0xffff /* mask relevant bits of "wn_flags" */
5013 #define HI2WN(hi) (wordnode_T *)((hi)->hi_key)
5016 * Info used while reading the spell files.
5018 typedef struct spellinfo_S
5020 wordnode_T *si_foldroot; /* tree with case-folded words */
5021 long si_foldwcount; /* nr of words in si_foldroot */
5023 wordnode_T *si_keeproot; /* tree with keep-case words */
5024 long si_keepwcount; /* nr of words in si_keeproot */
5026 wordnode_T *si_prefroot; /* tree with postponed prefixes */
5028 long si_sugtree; /* creating the soundfolding trie */
5030 sblock_T *si_blocks; /* memory blocks used */
5031 long si_blocks_cnt; /* memory blocks allocated */
5032 long si_compress_cnt; /* words to add before lowering
5033 compression limit */
5034 wordnode_T *si_first_free; /* List of nodes that have been freed during
5035 compression, linked by "wn_child" field. */
5036 long si_free_count; /* number of nodes in si_first_free */
5037 #ifdef SPELL_PRINTTREE
5038 int si_wordnode_nr; /* sequence nr for nodes */
5039 #endif
5040 buf_T *si_spellbuf; /* buffer used to store soundfold word table */
5042 int si_ascii; /* handling only ASCII words */
5043 int si_add; /* addition file */
5044 int si_clear_chartab; /* when TRUE clear char tables */
5045 int si_region; /* region mask */
5046 vimconv_T si_conv; /* for conversion to 'encoding' */
5047 int si_memtot; /* runtime memory used */
5048 int si_verbose; /* verbose messages */
5049 int si_msg_count; /* number of words added since last message */
5050 char_u *si_info; /* info text chars or NULL */
5051 int si_region_count; /* number of regions supported (1 when there
5052 are no regions) */
5053 char_u si_region_name[16]; /* region names; used only if
5054 * si_region_count > 1) */
5056 garray_T si_rep; /* list of fromto_T entries from REP lines */
5057 garray_T si_repsal; /* list of fromto_T entries from REPSAL lines */
5058 garray_T si_sal; /* list of fromto_T entries from SAL lines */
5059 char_u *si_sofofr; /* SOFOFROM text */
5060 char_u *si_sofoto; /* SOFOTO text */
5061 int si_nosugfile; /* NOSUGFILE item found */
5062 int si_nosplitsugs; /* NOSPLITSUGS item found */
5063 int si_followup; /* soundsalike: ? */
5064 int si_collapse; /* soundsalike: ? */
5065 hashtab_T si_commonwords; /* hashtable for common words */
5066 time_t si_sugtime; /* timestamp for .sug file */
5067 int si_rem_accents; /* soundsalike: remove accents */
5068 garray_T si_map; /* MAP info concatenated */
5069 char_u *si_midword; /* MIDWORD chars or NULL */
5070 int si_compmax; /* max nr of words for compounding */
5071 int si_compminlen; /* minimal length for compounding */
5072 int si_compsylmax; /* max nr of syllables for compounding */
5073 int si_compoptions; /* COMP_ flags */
5074 garray_T si_comppat; /* CHECKCOMPOUNDPATTERN items, each stored as
5075 a string */
5076 char_u *si_compflags; /* flags used for compounding */
5077 char_u si_nobreak; /* NOBREAK */
5078 char_u *si_syllable; /* syllable string */
5079 garray_T si_prefcond; /* table with conditions for postponed
5080 * prefixes, each stored as a string */
5081 int si_newprefID; /* current value for ah_newID */
5082 int si_newcompID; /* current value for compound ID */
5083 } spellinfo_T;
5085 static afffile_T *spell_read_aff __ARGS((spellinfo_T *spin, char_u *fname));
5086 static int is_aff_rule __ARGS((char_u **items, int itemcnt, char *rulename, int mincount));
5087 static void aff_process_flags __ARGS((afffile_T *affile, affentry_T *entry));
5088 static int spell_info_item __ARGS((char_u *s));
5089 static unsigned affitem2flag __ARGS((int flagtype, char_u *item, char_u *fname, int lnum));
5090 static unsigned get_affitem __ARGS((int flagtype, char_u **pp));
5091 static void process_compflags __ARGS((spellinfo_T *spin, afffile_T *aff, char_u *compflags));
5092 static void check_renumber __ARGS((spellinfo_T *spin));
5093 static int flag_in_afflist __ARGS((int flagtype, char_u *afflist, unsigned flag));
5094 static void aff_check_number __ARGS((int spinval, int affval, char *name));
5095 static void aff_check_string __ARGS((char_u *spinval, char_u *affval, char *name));
5096 static int str_equal __ARGS((char_u *s1, char_u *s2));
5097 static void add_fromto __ARGS((spellinfo_T *spin, garray_T *gap, char_u *from, char_u *to));
5098 static int sal_to_bool __ARGS((char_u *s));
5099 static int has_non_ascii __ARGS((char_u *s));
5100 static void spell_free_aff __ARGS((afffile_T *aff));
5101 static int spell_read_dic __ARGS((spellinfo_T *spin, char_u *fname, afffile_T *affile));
5102 static int get_affix_flags __ARGS((afffile_T *affile, char_u *afflist));
5103 static int get_pfxlist __ARGS((afffile_T *affile, char_u *afflist, char_u *store_afflist));
5104 static void get_compflags __ARGS((afffile_T *affile, char_u *afflist, char_u *store_afflist));
5105 static int store_aff_word __ARGS((spellinfo_T *spin, char_u *word, char_u *afflist, afffile_T *affile, hashtab_T *ht, hashtab_T *xht, int condit, int flags, char_u *pfxlist, int pfxlen));
5106 static int spell_read_wordfile __ARGS((spellinfo_T *spin, char_u *fname));
5107 static void *getroom __ARGS((spellinfo_T *spin, size_t len, int align));
5108 static char_u *getroom_save __ARGS((spellinfo_T *spin, char_u *s));
5109 static void free_blocks __ARGS((sblock_T *bl));
5110 static wordnode_T *wordtree_alloc __ARGS((spellinfo_T *spin));
5111 static int store_word __ARGS((spellinfo_T *spin, char_u *word, int flags, int region, char_u *pfxlist, int need_affix));
5112 static int tree_add_word __ARGS((spellinfo_T *spin, char_u *word, wordnode_T *tree, int flags, int region, int affixID));
5113 static wordnode_T *get_wordnode __ARGS((spellinfo_T *spin));
5114 static int deref_wordnode __ARGS((spellinfo_T *spin, wordnode_T *node));
5115 static void free_wordnode __ARGS((spellinfo_T *spin, wordnode_T *n));
5116 static void wordtree_compress __ARGS((spellinfo_T *spin, wordnode_T *root));
5117 static int node_compress __ARGS((spellinfo_T *spin, wordnode_T *node, hashtab_T *ht, int *tot));
5118 static int node_equal __ARGS((wordnode_T *n1, wordnode_T *n2));
5119 static void put_sugtime __ARGS((spellinfo_T *spin, FILE *fd));
5120 static int write_vim_spell __ARGS((spellinfo_T *spin, char_u *fname));
5121 static void clear_node __ARGS((wordnode_T *node));
5122 static int put_node __ARGS((FILE *fd, wordnode_T *node, int idx, int regionmask, int prefixtree));
5123 static void spell_make_sugfile __ARGS((spellinfo_T *spin, char_u *wfname));
5124 static int sug_filltree __ARGS((spellinfo_T *spin, slang_T *slang));
5125 static int sug_maketable __ARGS((spellinfo_T *spin));
5126 static int sug_filltable __ARGS((spellinfo_T *spin, wordnode_T *node, int startwordnr, garray_T *gap));
5127 static int offset2bytes __ARGS((int nr, char_u *buf));
5128 static int bytes2offset __ARGS((char_u **pp));
5129 static void sug_write __ARGS((spellinfo_T *spin, char_u *fname));
5130 static void mkspell __ARGS((int fcount, char_u **fnames, int ascii, int overwrite, int added_word));
5131 static void spell_message __ARGS((spellinfo_T *spin, char_u *str));
5132 static void init_spellfile __ARGS((void));
5134 /* In the postponed prefixes tree wn_flags is used to store the WFP_ flags,
5135 * but it must be negative to indicate the prefix tree to tree_add_word().
5136 * Use a negative number with the lower 8 bits zero. */
5137 #define PFX_FLAGS -256
5139 /* flags for "condit" argument of store_aff_word() */
5140 #define CONDIT_COMB 1 /* affix must combine */
5141 #define CONDIT_CFIX 2 /* affix must have CIRCUMFIX flag */
5142 #define CONDIT_SUF 4 /* add a suffix for matching flags */
5143 #define CONDIT_AFF 8 /* word already has an affix */
5146 * Tunable parameters for when the tree is compressed. See 'mkspellmem'.
5148 static long compress_start = 30000; /* memory / SBLOCKSIZE */
5149 static long compress_inc = 100; /* memory / SBLOCKSIZE */
5150 static long compress_added = 500000; /* word count */
5152 #ifdef SPELL_PRINTTREE
5154 * For debugging the tree code: print the current tree in a (more or less)
5155 * readable format, so that we can see what happens when adding a word and/or
5156 * compressing the tree.
5157 * Based on code from Olaf Seibert.
5159 #define PRINTLINESIZE 1000
5160 #define PRINTWIDTH 6
5162 #define PRINTSOME(l, depth, fmt, a1, a2) vim_snprintf(l + depth * PRINTWIDTH, \
5163 PRINTLINESIZE - PRINTWIDTH * depth, fmt, a1, a2)
5165 static char line1[PRINTLINESIZE];
5166 static char line2[PRINTLINESIZE];
5167 static char line3[PRINTLINESIZE];
5169 static void
5170 spell_clear_flags(wordnode_T *node)
5172 wordnode_T *np;
5174 for (np = node; np != NULL; np = np->wn_sibling)
5176 np->wn_u1.index = FALSE;
5177 spell_clear_flags(np->wn_child);
5181 static void
5182 spell_print_node(wordnode_T *node, int depth)
5184 if (node->wn_u1.index)
5186 /* Done this node before, print the reference. */
5187 PRINTSOME(line1, depth, "(%d)", node->wn_nr, 0);
5188 PRINTSOME(line2, depth, " ", 0, 0);
5189 PRINTSOME(line3, depth, " ", 0, 0);
5190 msg(line1);
5191 msg(line2);
5192 msg(line3);
5194 else
5196 node->wn_u1.index = TRUE;
5198 if (node->wn_byte != NUL)
5200 if (node->wn_child != NULL)
5201 PRINTSOME(line1, depth, " %c -> ", node->wn_byte, 0);
5202 else
5203 /* Cannot happen? */
5204 PRINTSOME(line1, depth, " %c ???", node->wn_byte, 0);
5206 else
5207 PRINTSOME(line1, depth, " $ ", 0, 0);
5209 PRINTSOME(line2, depth, "%d/%d ", node->wn_nr, node->wn_refs);
5211 if (node->wn_sibling != NULL)
5212 PRINTSOME(line3, depth, " | ", 0, 0);
5213 else
5214 PRINTSOME(line3, depth, " ", 0, 0);
5216 if (node->wn_byte == NUL)
5218 msg(line1);
5219 msg(line2);
5220 msg(line3);
5223 /* do the children */
5224 if (node->wn_byte != NUL && node->wn_child != NULL)
5225 spell_print_node(node->wn_child, depth + 1);
5227 /* do the siblings */
5228 if (node->wn_sibling != NULL)
5230 /* get rid of all parent details except | */
5231 STRCPY(line1, line3);
5232 STRCPY(line2, line3);
5233 spell_print_node(node->wn_sibling, depth);
5238 static void
5239 spell_print_tree(wordnode_T *root)
5241 if (root != NULL)
5243 /* Clear the "wn_u1.index" fields, used to remember what has been
5244 * done. */
5245 spell_clear_flags(root);
5247 /* Recursively print the tree. */
5248 spell_print_node(root, 0);
5251 #endif /* SPELL_PRINTTREE */
5254 * Read the affix file "fname".
5255 * Returns an afffile_T, NULL for complete failure.
5257 static afffile_T *
5258 spell_read_aff(spin, fname)
5259 spellinfo_T *spin;
5260 char_u *fname;
5262 FILE *fd;
5263 afffile_T *aff;
5264 char_u rline[MAXLINELEN];
5265 char_u *line;
5266 char_u *pc = NULL;
5267 #define MAXITEMCNT 30
5268 char_u *(items[MAXITEMCNT]);
5269 int itemcnt;
5270 char_u *p;
5271 int lnum = 0;
5272 affheader_T *cur_aff = NULL;
5273 int did_postpone_prefix = FALSE;
5274 int aff_todo = 0;
5275 hashtab_T *tp;
5276 char_u *low = NULL;
5277 char_u *fol = NULL;
5278 char_u *upp = NULL;
5279 int do_rep;
5280 int do_repsal;
5281 int do_sal;
5282 int do_mapline;
5283 int found_map = FALSE;
5284 hashitem_T *hi;
5285 int l;
5286 int compminlen = 0; /* COMPOUNDMIN value */
5287 int compsylmax = 0; /* COMPOUNDSYLMAX value */
5288 int compoptions = 0; /* COMP_ flags */
5289 int compmax = 0; /* COMPOUNDWORDMAX value */
5290 char_u *compflags = NULL; /* COMPOUNDFLAG and COMPOUNDRULE
5291 concatenated */
5292 char_u *midword = NULL; /* MIDWORD value */
5293 char_u *syllable = NULL; /* SYLLABLE value */
5294 char_u *sofofrom = NULL; /* SOFOFROM value */
5295 char_u *sofoto = NULL; /* SOFOTO value */
5298 * Open the file.
5300 fd = mch_fopen((char *)fname, "r");
5301 if (fd == NULL)
5303 EMSG2(_(e_notopen), fname);
5304 return NULL;
5307 vim_snprintf((char *)IObuff, IOSIZE, _("Reading affix file %s ..."), fname);
5308 spell_message(spin, IObuff);
5310 /* Only do REP lines when not done in another .aff file already. */
5311 do_rep = spin->si_rep.ga_len == 0;
5313 /* Only do REPSAL lines when not done in another .aff file already. */
5314 do_repsal = spin->si_repsal.ga_len == 0;
5316 /* Only do SAL lines when not done in another .aff file already. */
5317 do_sal = spin->si_sal.ga_len == 0;
5319 /* Only do MAP lines when not done in another .aff file already. */
5320 do_mapline = spin->si_map.ga_len == 0;
5323 * Allocate and init the afffile_T structure.
5325 aff = (afffile_T *)getroom(spin, sizeof(afffile_T), TRUE);
5326 if (aff == NULL)
5328 fclose(fd);
5329 return NULL;
5331 hash_init(&aff->af_pref);
5332 hash_init(&aff->af_suff);
5333 hash_init(&aff->af_comp);
5336 * Read all the lines in the file one by one.
5338 while (!vim_fgets(rline, MAXLINELEN, fd) && !got_int)
5340 line_breakcheck();
5341 ++lnum;
5343 /* Skip comment lines. */
5344 if (*rline == '#')
5345 continue;
5347 /* Convert from "SET" to 'encoding' when needed. */
5348 vim_free(pc);
5349 #ifdef FEAT_MBYTE
5350 if (spin->si_conv.vc_type != CONV_NONE)
5352 pc = string_convert(&spin->si_conv, rline, NULL);
5353 if (pc == NULL)
5355 smsg((char_u *)_("Conversion failure for word in %s line %d: %s"),
5356 fname, lnum, rline);
5357 continue;
5359 line = pc;
5361 else
5362 #endif
5364 pc = NULL;
5365 line = rline;
5368 /* Split the line up in white separated items. Put a NUL after each
5369 * item. */
5370 itemcnt = 0;
5371 for (p = line; ; )
5373 while (*p != NUL && *p <= ' ') /* skip white space and CR/NL */
5374 ++p;
5375 if (*p == NUL)
5376 break;
5377 if (itemcnt == MAXITEMCNT) /* too many items */
5378 break;
5379 items[itemcnt++] = p;
5380 /* A few items have arbitrary text argument, don't split them. */
5381 if (itemcnt == 2 && spell_info_item(items[0]))
5382 while (*p >= ' ' || *p == TAB) /* skip until CR/NL */
5383 ++p;
5384 else
5385 while (*p > ' ') /* skip until white space or CR/NL */
5386 ++p;
5387 if (*p == NUL)
5388 break;
5389 *p++ = NUL;
5392 /* Handle non-empty lines. */
5393 if (itemcnt > 0)
5395 if (is_aff_rule(items, itemcnt, "SET", 2) && aff->af_enc == NULL)
5397 #ifdef FEAT_MBYTE
5398 /* Setup for conversion from "ENC" to 'encoding'. */
5399 aff->af_enc = enc_canonize(items[1]);
5400 if (aff->af_enc != NULL && !spin->si_ascii
5401 && convert_setup(&spin->si_conv, aff->af_enc,
5402 p_enc) == FAIL)
5403 smsg((char_u *)_("Conversion in %s not supported: from %s to %s"),
5404 fname, aff->af_enc, p_enc);
5405 spin->si_conv.vc_fail = TRUE;
5406 #else
5407 smsg((char_u *)_("Conversion in %s not supported"), fname);
5408 #endif
5410 else if (is_aff_rule(items, itemcnt, "FLAG", 2)
5411 && aff->af_flagtype == AFT_CHAR)
5413 if (STRCMP(items[1], "long") == 0)
5414 aff->af_flagtype = AFT_LONG;
5415 else if (STRCMP(items[1], "num") == 0)
5416 aff->af_flagtype = AFT_NUM;
5417 else if (STRCMP(items[1], "caplong") == 0)
5418 aff->af_flagtype = AFT_CAPLONG;
5419 else
5420 smsg((char_u *)_("Invalid value for FLAG in %s line %d: %s"),
5421 fname, lnum, items[1]);
5422 if (aff->af_rare != 0
5423 || aff->af_keepcase != 0
5424 || aff->af_bad != 0
5425 || aff->af_needaffix != 0
5426 || aff->af_circumfix != 0
5427 || aff->af_needcomp != 0
5428 || aff->af_comproot != 0
5429 || aff->af_nosuggest != 0
5430 || compflags != NULL
5431 || aff->af_suff.ht_used > 0
5432 || aff->af_pref.ht_used > 0)
5433 smsg((char_u *)_("FLAG after using flags in %s line %d: %s"),
5434 fname, lnum, items[1]);
5436 else if (spell_info_item(items[0]))
5438 p = (char_u *)getroom(spin,
5439 (spin->si_info == NULL ? 0 : STRLEN(spin->si_info))
5440 + STRLEN(items[0])
5441 + STRLEN(items[1]) + 3, FALSE);
5442 if (p != NULL)
5444 if (spin->si_info != NULL)
5446 STRCPY(p, spin->si_info);
5447 STRCAT(p, "\n");
5449 STRCAT(p, items[0]);
5450 STRCAT(p, " ");
5451 STRCAT(p, items[1]);
5452 spin->si_info = p;
5455 else if (is_aff_rule(items, itemcnt, "MIDWORD", 2)
5456 && midword == NULL)
5458 midword = getroom_save(spin, items[1]);
5460 else if (is_aff_rule(items, itemcnt, "TRY", 2))
5462 /* ignored, we look in the tree for what chars may appear */
5464 /* TODO: remove "RAR" later */
5465 else if ((is_aff_rule(items, itemcnt, "RAR", 2)
5466 || is_aff_rule(items, itemcnt, "RARE", 2))
5467 && aff->af_rare == 0)
5469 aff->af_rare = affitem2flag(aff->af_flagtype, items[1],
5470 fname, lnum);
5472 /* TODO: remove "KEP" later */
5473 else if ((is_aff_rule(items, itemcnt, "KEP", 2)
5474 || is_aff_rule(items, itemcnt, "KEEPCASE", 2))
5475 && aff->af_keepcase == 0)
5477 aff->af_keepcase = affitem2flag(aff->af_flagtype, items[1],
5478 fname, lnum);
5480 else if ((is_aff_rule(items, itemcnt, "BAD", 2)
5481 || is_aff_rule(items, itemcnt, "FORBIDDENWORD", 2))
5482 && aff->af_bad == 0)
5484 aff->af_bad = affitem2flag(aff->af_flagtype, items[1],
5485 fname, lnum);
5487 else if (is_aff_rule(items, itemcnt, "NEEDAFFIX", 2)
5488 && aff->af_needaffix == 0)
5490 aff->af_needaffix = affitem2flag(aff->af_flagtype, items[1],
5491 fname, lnum);
5493 else if (is_aff_rule(items, itemcnt, "CIRCUMFIX", 2)
5494 && aff->af_circumfix == 0)
5496 aff->af_circumfix = affitem2flag(aff->af_flagtype, items[1],
5497 fname, lnum);
5499 else if (is_aff_rule(items, itemcnt, "NOSUGGEST", 2)
5500 && aff->af_nosuggest == 0)
5502 aff->af_nosuggest = affitem2flag(aff->af_flagtype, items[1],
5503 fname, lnum);
5505 else if ((is_aff_rule(items, itemcnt, "NEEDCOMPOUND", 2)
5506 || is_aff_rule(items, itemcnt, "ONLYINCOMPOUND", 2))
5507 && aff->af_needcomp == 0)
5509 aff->af_needcomp = affitem2flag(aff->af_flagtype, items[1],
5510 fname, lnum);
5512 else if (is_aff_rule(items, itemcnt, "COMPOUNDROOT", 2)
5513 && aff->af_comproot == 0)
5515 aff->af_comproot = affitem2flag(aff->af_flagtype, items[1],
5516 fname, lnum);
5518 else if (is_aff_rule(items, itemcnt, "COMPOUNDFORBIDFLAG", 2)
5519 && aff->af_compforbid == 0)
5521 aff->af_compforbid = affitem2flag(aff->af_flagtype, items[1],
5522 fname, lnum);
5523 if (aff->af_pref.ht_used > 0)
5524 smsg((char_u *)_("Defining COMPOUNDFORBIDFLAG after PFX item may give wrong results in %s line %d"),
5525 fname, lnum);
5527 else if (is_aff_rule(items, itemcnt, "COMPOUNDPERMITFLAG", 2)
5528 && aff->af_comppermit == 0)
5530 aff->af_comppermit = affitem2flag(aff->af_flagtype, items[1],
5531 fname, lnum);
5532 if (aff->af_pref.ht_used > 0)
5533 smsg((char_u *)_("Defining COMPOUNDPERMITFLAG after PFX item may give wrong results in %s line %d"),
5534 fname, lnum);
5536 else if (is_aff_rule(items, itemcnt, "COMPOUNDFLAG", 2)
5537 && compflags == NULL)
5539 /* Turn flag "c" into COMPOUNDRULE compatible string "c+",
5540 * "Na" into "Na+", "1234" into "1234+". */
5541 p = getroom(spin, STRLEN(items[1]) + 2, FALSE);
5542 if (p != NULL)
5544 STRCPY(p, items[1]);
5545 STRCAT(p, "+");
5546 compflags = p;
5549 else if (is_aff_rule(items, itemcnt, "COMPOUNDRULES", 2))
5551 /* We don't use the count, but do check that it's a number and
5552 * not COMPOUNDRULE mistyped. */
5553 if (atoi((char *)items[1]) == 0)
5554 smsg((char_u *)_("Wrong COMPOUNDRULES value in %s line %d: %s"),
5555 fname, lnum, items[1]);
5557 else if (is_aff_rule(items, itemcnt, "COMPOUNDRULE", 2))
5559 /* Concatenate this string to previously defined ones, using a
5560 * slash to separate them. */
5561 l = (int)STRLEN(items[1]) + 1;
5562 if (compflags != NULL)
5563 l += (int)STRLEN(compflags) + 1;
5564 p = getroom(spin, l, FALSE);
5565 if (p != NULL)
5567 if (compflags != NULL)
5569 STRCPY(p, compflags);
5570 STRCAT(p, "/");
5572 STRCAT(p, items[1]);
5573 compflags = p;
5576 else if (is_aff_rule(items, itemcnt, "COMPOUNDWORDMAX", 2)
5577 && compmax == 0)
5579 compmax = atoi((char *)items[1]);
5580 if (compmax == 0)
5581 smsg((char_u *)_("Wrong COMPOUNDWORDMAX value in %s line %d: %s"),
5582 fname, lnum, items[1]);
5584 else if (is_aff_rule(items, itemcnt, "COMPOUNDMIN", 2)
5585 && compminlen == 0)
5587 compminlen = atoi((char *)items[1]);
5588 if (compminlen == 0)
5589 smsg((char_u *)_("Wrong COMPOUNDMIN value in %s line %d: %s"),
5590 fname, lnum, items[1]);
5592 else if (is_aff_rule(items, itemcnt, "COMPOUNDSYLMAX", 2)
5593 && compsylmax == 0)
5595 compsylmax = atoi((char *)items[1]);
5596 if (compsylmax == 0)
5597 smsg((char_u *)_("Wrong COMPOUNDSYLMAX value in %s line %d: %s"),
5598 fname, lnum, items[1]);
5600 else if (is_aff_rule(items, itemcnt, "CHECKCOMPOUNDDUP", 1))
5602 compoptions |= COMP_CHECKDUP;
5604 else if (is_aff_rule(items, itemcnt, "CHECKCOMPOUNDREP", 1))
5606 compoptions |= COMP_CHECKREP;
5608 else if (is_aff_rule(items, itemcnt, "CHECKCOMPOUNDCASE", 1))
5610 compoptions |= COMP_CHECKCASE;
5612 else if (is_aff_rule(items, itemcnt, "CHECKCOMPOUNDTRIPLE", 1))
5614 compoptions |= COMP_CHECKTRIPLE;
5616 else if (is_aff_rule(items, itemcnt, "CHECKCOMPOUNDPATTERN", 2))
5618 if (atoi((char *)items[1]) == 0)
5619 smsg((char_u *)_("Wrong CHECKCOMPOUNDPATTERN value in %s line %d: %s"),
5620 fname, lnum, items[1]);
5622 else if (is_aff_rule(items, itemcnt, "CHECKCOMPOUNDPATTERN", 3))
5624 garray_T *gap = &spin->si_comppat;
5625 int i;
5627 /* Only add the couple if it isn't already there. */
5628 for (i = 0; i < gap->ga_len - 1; i += 2)
5629 if (STRCMP(((char_u **)(gap->ga_data))[i], items[1]) == 0
5630 && STRCMP(((char_u **)(gap->ga_data))[i + 1],
5631 items[2]) == 0)
5632 break;
5633 if (i >= gap->ga_len && ga_grow(gap, 2) == OK)
5635 ((char_u **)(gap->ga_data))[gap->ga_len++]
5636 = getroom_save(spin, items[1]);
5637 ((char_u **)(gap->ga_data))[gap->ga_len++]
5638 = getroom_save(spin, items[2]);
5641 else if (is_aff_rule(items, itemcnt, "SYLLABLE", 2)
5642 && syllable == NULL)
5644 syllable = getroom_save(spin, items[1]);
5646 else if (is_aff_rule(items, itemcnt, "NOBREAK", 1))
5648 spin->si_nobreak = TRUE;
5650 else if (is_aff_rule(items, itemcnt, "NOSPLITSUGS", 1))
5652 spin->si_nosplitsugs = TRUE;
5654 else if (is_aff_rule(items, itemcnt, "NOSUGFILE", 1))
5656 spin->si_nosugfile = TRUE;
5658 else if (is_aff_rule(items, itemcnt, "PFXPOSTPONE", 1))
5660 aff->af_pfxpostpone = TRUE;
5662 else if ((STRCMP(items[0], "PFX") == 0
5663 || STRCMP(items[0], "SFX") == 0)
5664 && aff_todo == 0
5665 && itemcnt >= 4)
5667 int lasti = 4;
5668 char_u key[AH_KEY_LEN];
5670 if (*items[0] == 'P')
5671 tp = &aff->af_pref;
5672 else
5673 tp = &aff->af_suff;
5675 /* Myspell allows the same affix name to be used multiple
5676 * times. The affix files that do this have an undocumented
5677 * "S" flag on all but the last block, thus we check for that
5678 * and store it in ah_follows. */
5679 vim_strncpy(key, items[1], AH_KEY_LEN - 1);
5680 hi = hash_find(tp, key);
5681 if (!HASHITEM_EMPTY(hi))
5683 cur_aff = HI2AH(hi);
5684 if (cur_aff->ah_combine != (*items[2] == 'Y'))
5685 smsg((char_u *)_("Different combining flag in continued affix block in %s line %d: %s"),
5686 fname, lnum, items[1]);
5687 if (!cur_aff->ah_follows)
5688 smsg((char_u *)_("Duplicate affix in %s line %d: %s"),
5689 fname, lnum, items[1]);
5691 else
5693 /* New affix letter. */
5694 cur_aff = (affheader_T *)getroom(spin,
5695 sizeof(affheader_T), TRUE);
5696 if (cur_aff == NULL)
5697 break;
5698 cur_aff->ah_flag = affitem2flag(aff->af_flagtype, items[1],
5699 fname, lnum);
5700 if (cur_aff->ah_flag == 0 || STRLEN(items[1]) >= AH_KEY_LEN)
5701 break;
5702 if (cur_aff->ah_flag == aff->af_bad
5703 || cur_aff->ah_flag == aff->af_rare
5704 || cur_aff->ah_flag == aff->af_keepcase
5705 || cur_aff->ah_flag == aff->af_needaffix
5706 || cur_aff->ah_flag == aff->af_circumfix
5707 || cur_aff->ah_flag == aff->af_nosuggest
5708 || cur_aff->ah_flag == aff->af_needcomp
5709 || cur_aff->ah_flag == aff->af_comproot)
5710 smsg((char_u *)_("Affix also used for BAD/RARE/KEEPCASE/NEEDAFFIX/NEEDCOMPOUND/NOSUGGEST in %s line %d: %s"),
5711 fname, lnum, items[1]);
5712 STRCPY(cur_aff->ah_key, items[1]);
5713 hash_add(tp, cur_aff->ah_key);
5715 cur_aff->ah_combine = (*items[2] == 'Y');
5718 /* Check for the "S" flag, which apparently means that another
5719 * block with the same affix name is following. */
5720 if (itemcnt > lasti && STRCMP(items[lasti], "S") == 0)
5722 ++lasti;
5723 cur_aff->ah_follows = TRUE;
5725 else
5726 cur_aff->ah_follows = FALSE;
5728 /* Myspell allows extra text after the item, but that might
5729 * mean mistakes go unnoticed. Require a comment-starter. */
5730 if (itemcnt > lasti && *items[lasti] != '#')
5731 smsg((char_u *)_(e_afftrailing), fname, lnum, items[lasti]);
5733 if (STRCMP(items[2], "Y") != 0 && STRCMP(items[2], "N") != 0)
5734 smsg((char_u *)_("Expected Y or N in %s line %d: %s"),
5735 fname, lnum, items[2]);
5737 if (*items[0] == 'P' && aff->af_pfxpostpone)
5739 if (cur_aff->ah_newID == 0)
5741 /* Use a new number in the .spl file later, to be able
5742 * to handle multiple .aff files. */
5743 check_renumber(spin);
5744 cur_aff->ah_newID = ++spin->si_newprefID;
5746 /* We only really use ah_newID if the prefix is
5747 * postponed. We know that only after handling all
5748 * the items. */
5749 did_postpone_prefix = FALSE;
5751 else
5752 /* Did use the ID in a previous block. */
5753 did_postpone_prefix = TRUE;
5756 aff_todo = atoi((char *)items[3]);
5758 else if ((STRCMP(items[0], "PFX") == 0
5759 || STRCMP(items[0], "SFX") == 0)
5760 && aff_todo > 0
5761 && STRCMP(cur_aff->ah_key, items[1]) == 0
5762 && itemcnt >= 5)
5764 affentry_T *aff_entry;
5765 int upper = FALSE;
5766 int lasti = 5;
5768 /* Myspell allows extra text after the item, but that might
5769 * mean mistakes go unnoticed. Require a comment-starter.
5770 * Hunspell uses a "-" item. */
5771 if (itemcnt > lasti && *items[lasti] != '#'
5772 && (STRCMP(items[lasti], "-") != 0
5773 || itemcnt != lasti + 1))
5774 smsg((char_u *)_(e_afftrailing), fname, lnum, items[lasti]);
5776 /* New item for an affix letter. */
5777 --aff_todo;
5778 aff_entry = (affentry_T *)getroom(spin,
5779 sizeof(affentry_T), TRUE);
5780 if (aff_entry == NULL)
5781 break;
5783 if (STRCMP(items[2], "0") != 0)
5784 aff_entry->ae_chop = getroom_save(spin, items[2]);
5785 if (STRCMP(items[3], "0") != 0)
5787 aff_entry->ae_add = getroom_save(spin, items[3]);
5789 /* Recognize flags on the affix: abcd/XYZ */
5790 aff_entry->ae_flags = vim_strchr(aff_entry->ae_add, '/');
5791 if (aff_entry->ae_flags != NULL)
5793 *aff_entry->ae_flags++ = NUL;
5794 aff_process_flags(aff, aff_entry);
5798 /* Don't use an affix entry with non-ASCII characters when
5799 * "spin->si_ascii" is TRUE. */
5800 if (!spin->si_ascii || !(has_non_ascii(aff_entry->ae_chop)
5801 || has_non_ascii(aff_entry->ae_add)))
5803 aff_entry->ae_next = cur_aff->ah_first;
5804 cur_aff->ah_first = aff_entry;
5806 if (STRCMP(items[4], ".") != 0)
5808 char_u buf[MAXLINELEN];
5810 aff_entry->ae_cond = getroom_save(spin, items[4]);
5811 if (*items[0] == 'P')
5812 sprintf((char *)buf, "^%s", items[4]);
5813 else
5814 sprintf((char *)buf, "%s$", items[4]);
5815 aff_entry->ae_prog = vim_regcomp(buf,
5816 RE_MAGIC + RE_STRING + RE_STRICT);
5817 if (aff_entry->ae_prog == NULL)
5818 smsg((char_u *)_("Broken condition in %s line %d: %s"),
5819 fname, lnum, items[4]);
5822 /* For postponed prefixes we need an entry in si_prefcond
5823 * for the condition. Use an existing one if possible.
5824 * Can't be done for an affix with flags, ignoring
5825 * COMPOUNDFORBIDFLAG and COMPOUNDPERMITFLAG. */
5826 if (*items[0] == 'P' && aff->af_pfxpostpone
5827 && aff_entry->ae_flags == NULL)
5829 /* When the chop string is one lower-case letter and
5830 * the add string ends in the upper-case letter we set
5831 * the "upper" flag, clear "ae_chop" and remove the
5832 * letters from "ae_add". The condition must either
5833 * be empty or start with the same letter. */
5834 if (aff_entry->ae_chop != NULL
5835 && aff_entry->ae_add != NULL
5836 #ifdef FEAT_MBYTE
5837 && aff_entry->ae_chop[(*mb_ptr2len)(
5838 aff_entry->ae_chop)] == NUL
5839 #else
5840 && aff_entry->ae_chop[1] == NUL
5841 #endif
5844 int c, c_up;
5846 c = PTR2CHAR(aff_entry->ae_chop);
5847 c_up = SPELL_TOUPPER(c);
5848 if (c_up != c
5849 && (aff_entry->ae_cond == NULL
5850 || PTR2CHAR(aff_entry->ae_cond) == c))
5852 p = aff_entry->ae_add
5853 + STRLEN(aff_entry->ae_add);
5854 mb_ptr_back(aff_entry->ae_add, p);
5855 if (PTR2CHAR(p) == c_up)
5857 upper = TRUE;
5858 aff_entry->ae_chop = NULL;
5859 *p = NUL;
5861 /* The condition is matched with the
5862 * actual word, thus must check for the
5863 * upper-case letter. */
5864 if (aff_entry->ae_cond != NULL)
5866 char_u buf[MAXLINELEN];
5867 #ifdef FEAT_MBYTE
5868 if (has_mbyte)
5870 onecap_copy(items[4], buf, TRUE);
5871 aff_entry->ae_cond = getroom_save(
5872 spin, buf);
5874 else
5875 #endif
5876 *aff_entry->ae_cond = c_up;
5877 if (aff_entry->ae_cond != NULL)
5879 sprintf((char *)buf, "^%s",
5880 aff_entry->ae_cond);
5881 vim_free(aff_entry->ae_prog);
5882 aff_entry->ae_prog = vim_regcomp(
5883 buf, RE_MAGIC + RE_STRING);
5890 if (aff_entry->ae_chop == NULL
5891 && aff_entry->ae_flags == NULL)
5893 int idx;
5894 char_u **pp;
5895 int n;
5897 /* Find a previously used condition. */
5898 for (idx = spin->si_prefcond.ga_len - 1; idx >= 0;
5899 --idx)
5901 p = ((char_u **)spin->si_prefcond.ga_data)[idx];
5902 if (str_equal(p, aff_entry->ae_cond))
5903 break;
5905 if (idx < 0 && ga_grow(&spin->si_prefcond, 1) == OK)
5907 /* Not found, add a new condition. */
5908 idx = spin->si_prefcond.ga_len++;
5909 pp = ((char_u **)spin->si_prefcond.ga_data)
5910 + idx;
5911 if (aff_entry->ae_cond == NULL)
5912 *pp = NULL;
5913 else
5914 *pp = getroom_save(spin,
5915 aff_entry->ae_cond);
5918 /* Add the prefix to the prefix tree. */
5919 if (aff_entry->ae_add == NULL)
5920 p = (char_u *)"";
5921 else
5922 p = aff_entry->ae_add;
5924 /* PFX_FLAGS is a negative number, so that
5925 * tree_add_word() knows this is the prefix tree. */
5926 n = PFX_FLAGS;
5927 if (!cur_aff->ah_combine)
5928 n |= WFP_NC;
5929 if (upper)
5930 n |= WFP_UP;
5931 if (aff_entry->ae_comppermit)
5932 n |= WFP_COMPPERMIT;
5933 if (aff_entry->ae_compforbid)
5934 n |= WFP_COMPFORBID;
5935 tree_add_word(spin, p, spin->si_prefroot, n,
5936 idx, cur_aff->ah_newID);
5937 did_postpone_prefix = TRUE;
5940 /* Didn't actually use ah_newID, backup si_newprefID. */
5941 if (aff_todo == 0 && !did_postpone_prefix)
5943 --spin->si_newprefID;
5944 cur_aff->ah_newID = 0;
5949 else if (is_aff_rule(items, itemcnt, "FOL", 2) && fol == NULL)
5951 fol = vim_strsave(items[1]);
5953 else if (is_aff_rule(items, itemcnt, "LOW", 2) && low == NULL)
5955 low = vim_strsave(items[1]);
5957 else if (is_aff_rule(items, itemcnt, "UPP", 2) && upp == NULL)
5959 upp = vim_strsave(items[1]);
5961 else if (is_aff_rule(items, itemcnt, "REP", 2)
5962 || is_aff_rule(items, itemcnt, "REPSAL", 2))
5964 /* Ignore REP/REPSAL count */;
5965 if (!isdigit(*items[1]))
5966 smsg((char_u *)_("Expected REP(SAL) count in %s line %d"),
5967 fname, lnum);
5969 else if ((STRCMP(items[0], "REP") == 0
5970 || STRCMP(items[0], "REPSAL") == 0)
5971 && itemcnt >= 3)
5973 /* REP/REPSAL item */
5974 /* Myspell ignores extra arguments, we require it starts with
5975 * # to detect mistakes. */
5976 if (itemcnt > 3 && items[3][0] != '#')
5977 smsg((char_u *)_(e_afftrailing), fname, lnum, items[3]);
5978 if (items[0][3] == 'S' ? do_repsal : do_rep)
5980 /* Replace underscore with space (can't include a space
5981 * directly). */
5982 for (p = items[1]; *p != NUL; mb_ptr_adv(p))
5983 if (*p == '_')
5984 *p = ' ';
5985 for (p = items[2]; *p != NUL; mb_ptr_adv(p))
5986 if (*p == '_')
5987 *p = ' ';
5988 add_fromto(spin, items[0][3] == 'S'
5989 ? &spin->si_repsal
5990 : &spin->si_rep, items[1], items[2]);
5993 else if (is_aff_rule(items, itemcnt, "MAP", 2))
5995 /* MAP item or count */
5996 if (!found_map)
5998 /* First line contains the count. */
5999 found_map = TRUE;
6000 if (!isdigit(*items[1]))
6001 smsg((char_u *)_("Expected MAP count in %s line %d"),
6002 fname, lnum);
6004 else if (do_mapline)
6006 int c;
6008 /* Check that every character appears only once. */
6009 for (p = items[1]; *p != NUL; )
6011 #ifdef FEAT_MBYTE
6012 c = mb_ptr2char_adv(&p);
6013 #else
6014 c = *p++;
6015 #endif
6016 if ((spin->si_map.ga_len > 0
6017 && vim_strchr(spin->si_map.ga_data, c)
6018 != NULL)
6019 || vim_strchr(p, c) != NULL)
6020 smsg((char_u *)_("Duplicate character in MAP in %s line %d"),
6021 fname, lnum);
6024 /* We simply concatenate all the MAP strings, separated by
6025 * slashes. */
6026 ga_concat(&spin->si_map, items[1]);
6027 ga_append(&spin->si_map, '/');
6030 /* Accept "SAL from to" and "SAL from to #comment". */
6031 else if (is_aff_rule(items, itemcnt, "SAL", 3))
6033 if (do_sal)
6035 /* SAL item (sounds-a-like)
6036 * Either one of the known keys or a from-to pair. */
6037 if (STRCMP(items[1], "followup") == 0)
6038 spin->si_followup = sal_to_bool(items[2]);
6039 else if (STRCMP(items[1], "collapse_result") == 0)
6040 spin->si_collapse = sal_to_bool(items[2]);
6041 else if (STRCMP(items[1], "remove_accents") == 0)
6042 spin->si_rem_accents = sal_to_bool(items[2]);
6043 else
6044 /* when "to" is "_" it means empty */
6045 add_fromto(spin, &spin->si_sal, items[1],
6046 STRCMP(items[2], "_") == 0 ? (char_u *)""
6047 : items[2]);
6050 else if (is_aff_rule(items, itemcnt, "SOFOFROM", 2)
6051 && sofofrom == NULL)
6053 sofofrom = getroom_save(spin, items[1]);
6055 else if (is_aff_rule(items, itemcnt, "SOFOTO", 2)
6056 && sofoto == NULL)
6058 sofoto = getroom_save(spin, items[1]);
6060 else if (STRCMP(items[0], "COMMON") == 0)
6062 int i;
6064 for (i = 1; i < itemcnt; ++i)
6066 if (HASHITEM_EMPTY(hash_find(&spin->si_commonwords,
6067 items[i])))
6069 p = vim_strsave(items[i]);
6070 if (p == NULL)
6071 break;
6072 hash_add(&spin->si_commonwords, p);
6076 else
6077 smsg((char_u *)_("Unrecognized or duplicate item in %s line %d: %s"),
6078 fname, lnum, items[0]);
6082 if (fol != NULL || low != NULL || upp != NULL)
6084 if (spin->si_clear_chartab)
6086 /* Clear the char type tables, don't want to use any of the
6087 * currently used spell properties. */
6088 init_spell_chartab();
6089 spin->si_clear_chartab = FALSE;
6093 * Don't write a word table for an ASCII file, so that we don't check
6094 * for conflicts with a word table that matches 'encoding'.
6095 * Don't write one for utf-8 either, we use utf_*() and
6096 * mb_get_class(), the list of chars in the file will be incomplete.
6098 if (!spin->si_ascii
6099 #ifdef FEAT_MBYTE
6100 && !enc_utf8
6101 #endif
6104 if (fol == NULL || low == NULL || upp == NULL)
6105 smsg((char_u *)_("Missing FOL/LOW/UPP line in %s"), fname);
6106 else
6107 (void)set_spell_chartab(fol, low, upp);
6110 vim_free(fol);
6111 vim_free(low);
6112 vim_free(upp);
6115 /* Use compound specifications of the .aff file for the spell info. */
6116 if (compmax != 0)
6118 aff_check_number(spin->si_compmax, compmax, "COMPOUNDWORDMAX");
6119 spin->si_compmax = compmax;
6122 if (compminlen != 0)
6124 aff_check_number(spin->si_compminlen, compminlen, "COMPOUNDMIN");
6125 spin->si_compminlen = compminlen;
6128 if (compsylmax != 0)
6130 if (syllable == NULL)
6131 smsg((char_u *)_("COMPOUNDSYLMAX used without SYLLABLE"));
6132 aff_check_number(spin->si_compsylmax, compsylmax, "COMPOUNDSYLMAX");
6133 spin->si_compsylmax = compsylmax;
6136 if (compoptions != 0)
6138 aff_check_number(spin->si_compoptions, compoptions, "COMPOUND options");
6139 spin->si_compoptions |= compoptions;
6142 if (compflags != NULL)
6143 process_compflags(spin, aff, compflags);
6145 /* Check that we didn't use too many renumbered flags. */
6146 if (spin->si_newcompID < spin->si_newprefID)
6148 if (spin->si_newcompID == 127 || spin->si_newcompID == 255)
6149 MSG(_("Too many postponed prefixes"));
6150 else if (spin->si_newprefID == 0 || spin->si_newprefID == 127)
6151 MSG(_("Too many compound flags"));
6152 else
6153 MSG(_("Too many postponed prefixes and/or compound flags"));
6156 if (syllable != NULL)
6158 aff_check_string(spin->si_syllable, syllable, "SYLLABLE");
6159 spin->si_syllable = syllable;
6162 if (sofofrom != NULL || sofoto != NULL)
6164 if (sofofrom == NULL || sofoto == NULL)
6165 smsg((char_u *)_("Missing SOFO%s line in %s"),
6166 sofofrom == NULL ? "FROM" : "TO", fname);
6167 else if (spin->si_sal.ga_len > 0)
6168 smsg((char_u *)_("Both SAL and SOFO lines in %s"), fname);
6169 else
6171 aff_check_string(spin->si_sofofr, sofofrom, "SOFOFROM");
6172 aff_check_string(spin->si_sofoto, sofoto, "SOFOTO");
6173 spin->si_sofofr = sofofrom;
6174 spin->si_sofoto = sofoto;
6178 if (midword != NULL)
6180 aff_check_string(spin->si_midword, midword, "MIDWORD");
6181 spin->si_midword = midword;
6184 vim_free(pc);
6185 fclose(fd);
6186 return aff;
6190 * Return TRUE when items[0] equals "rulename", there are "mincount" items or
6191 * a comment is following after item "mincount".
6193 static int
6194 is_aff_rule(items, itemcnt, rulename, mincount)
6195 char_u **items;
6196 int itemcnt;
6197 char *rulename;
6198 int mincount;
6200 return (STRCMP(items[0], rulename) == 0
6201 && (itemcnt == mincount
6202 || (itemcnt > mincount && items[mincount][0] == '#')));
6206 * For affix "entry" move COMPOUNDFORBIDFLAG and COMPOUNDPERMITFLAG from
6207 * ae_flags to ae_comppermit and ae_compforbid.
6209 static void
6210 aff_process_flags(affile, entry)
6211 afffile_T *affile;
6212 affentry_T *entry;
6214 char_u *p;
6215 char_u *prevp;
6216 unsigned flag;
6218 if (entry->ae_flags != NULL
6219 && (affile->af_compforbid != 0 || affile->af_comppermit != 0))
6221 for (p = entry->ae_flags; *p != NUL; )
6223 prevp = p;
6224 flag = get_affitem(affile->af_flagtype, &p);
6225 if (flag == affile->af_comppermit || flag == affile->af_compforbid)
6227 STRMOVE(prevp, p);
6228 p = prevp;
6229 if (flag == affile->af_comppermit)
6230 entry->ae_comppermit = TRUE;
6231 else
6232 entry->ae_compforbid = TRUE;
6234 if (affile->af_flagtype == AFT_NUM && *p == ',')
6235 ++p;
6237 if (*entry->ae_flags == NUL)
6238 entry->ae_flags = NULL; /* nothing left */
6243 * Return TRUE if "s" is the name of an info item in the affix file.
6245 static int
6246 spell_info_item(s)
6247 char_u *s;
6249 return STRCMP(s, "NAME") == 0
6250 || STRCMP(s, "HOME") == 0
6251 || STRCMP(s, "VERSION") == 0
6252 || STRCMP(s, "AUTHOR") == 0
6253 || STRCMP(s, "EMAIL") == 0
6254 || STRCMP(s, "COPYRIGHT") == 0;
6258 * Turn an affix flag name into a number, according to the FLAG type.
6259 * returns zero for failure.
6261 static unsigned
6262 affitem2flag(flagtype, item, fname, lnum)
6263 int flagtype;
6264 char_u *item;
6265 char_u *fname;
6266 int lnum;
6268 unsigned res;
6269 char_u *p = item;
6271 res = get_affitem(flagtype, &p);
6272 if (res == 0)
6274 if (flagtype == AFT_NUM)
6275 smsg((char_u *)_("Flag is not a number in %s line %d: %s"),
6276 fname, lnum, item);
6277 else
6278 smsg((char_u *)_("Illegal flag in %s line %d: %s"),
6279 fname, lnum, item);
6281 if (*p != NUL)
6283 smsg((char_u *)_(e_affname), fname, lnum, item);
6284 return 0;
6287 return res;
6291 * Get one affix name from "*pp" and advance the pointer.
6292 * Returns zero for an error, still advances the pointer then.
6294 static unsigned
6295 get_affitem(flagtype, pp)
6296 int flagtype;
6297 char_u **pp;
6299 int res;
6301 if (flagtype == AFT_NUM)
6303 if (!VIM_ISDIGIT(**pp))
6305 ++*pp; /* always advance, avoid getting stuck */
6306 return 0;
6308 res = getdigits(pp);
6310 else
6312 #ifdef FEAT_MBYTE
6313 res = mb_ptr2char_adv(pp);
6314 #else
6315 res = *(*pp)++;
6316 #endif
6317 if (flagtype == AFT_LONG || (flagtype == AFT_CAPLONG
6318 && res >= 'A' && res <= 'Z'))
6320 if (**pp == NUL)
6321 return 0;
6322 #ifdef FEAT_MBYTE
6323 res = mb_ptr2char_adv(pp) + (res << 16);
6324 #else
6325 res = *(*pp)++ + (res << 16);
6326 #endif
6329 return res;
6333 * Process the "compflags" string used in an affix file and append it to
6334 * spin->si_compflags.
6335 * The processing involves changing the affix names to ID numbers, so that
6336 * they fit in one byte.
6338 static void
6339 process_compflags(spin, aff, compflags)
6340 spellinfo_T *spin;
6341 afffile_T *aff;
6342 char_u *compflags;
6344 char_u *p;
6345 char_u *prevp;
6346 unsigned flag;
6347 compitem_T *ci;
6348 int id;
6349 int len;
6350 char_u *tp;
6351 char_u key[AH_KEY_LEN];
6352 hashitem_T *hi;
6354 /* Make room for the old and the new compflags, concatenated with a / in
6355 * between. Processing it makes it shorter, but we don't know by how
6356 * much, thus allocate the maximum. */
6357 len = (int)STRLEN(compflags) + 1;
6358 if (spin->si_compflags != NULL)
6359 len += (int)STRLEN(spin->si_compflags) + 1;
6360 p = getroom(spin, len, FALSE);
6361 if (p == NULL)
6362 return;
6363 if (spin->si_compflags != NULL)
6365 STRCPY(p, spin->si_compflags);
6366 STRCAT(p, "/");
6368 spin->si_compflags = p;
6369 tp = p + STRLEN(p);
6371 for (p = compflags; *p != NUL; )
6373 if (vim_strchr((char_u *)"/*+[]", *p) != NULL)
6374 /* Copy non-flag characters directly. */
6375 *tp++ = *p++;
6376 else
6378 /* First get the flag number, also checks validity. */
6379 prevp = p;
6380 flag = get_affitem(aff->af_flagtype, &p);
6381 if (flag != 0)
6383 /* Find the flag in the hashtable. If it was used before, use
6384 * the existing ID. Otherwise add a new entry. */
6385 vim_strncpy(key, prevp, p - prevp);
6386 hi = hash_find(&aff->af_comp, key);
6387 if (!HASHITEM_EMPTY(hi))
6388 id = HI2CI(hi)->ci_newID;
6389 else
6391 ci = (compitem_T *)getroom(spin, sizeof(compitem_T), TRUE);
6392 if (ci == NULL)
6393 break;
6394 STRCPY(ci->ci_key, key);
6395 ci->ci_flag = flag;
6396 /* Avoid using a flag ID that has a special meaning in a
6397 * regexp (also inside []). */
6400 check_renumber(spin);
6401 id = spin->si_newcompID--;
6402 } while (vim_strchr((char_u *)"/+*[]\\-^", id) != NULL);
6403 ci->ci_newID = id;
6404 hash_add(&aff->af_comp, ci->ci_key);
6406 *tp++ = id;
6408 if (aff->af_flagtype == AFT_NUM && *p == ',')
6409 ++p;
6413 *tp = NUL;
6417 * Check that the new IDs for postponed affixes and compounding don't overrun
6418 * each other. We have almost 255 available, but start at 0-127 to avoid
6419 * using two bytes for utf-8. When the 0-127 range is used up go to 128-255.
6420 * When that is used up an error message is given.
6422 static void
6423 check_renumber(spin)
6424 spellinfo_T *spin;
6426 if (spin->si_newprefID == spin->si_newcompID && spin->si_newcompID < 128)
6428 spin->si_newprefID = 127;
6429 spin->si_newcompID = 255;
6434 * Return TRUE if flag "flag" appears in affix list "afflist".
6436 static int
6437 flag_in_afflist(flagtype, afflist, flag)
6438 int flagtype;
6439 char_u *afflist;
6440 unsigned flag;
6442 char_u *p;
6443 unsigned n;
6445 switch (flagtype)
6447 case AFT_CHAR:
6448 return vim_strchr(afflist, flag) != NULL;
6450 case AFT_CAPLONG:
6451 case AFT_LONG:
6452 for (p = afflist; *p != NUL; )
6454 #ifdef FEAT_MBYTE
6455 n = mb_ptr2char_adv(&p);
6456 #else
6457 n = *p++;
6458 #endif
6459 if ((flagtype == AFT_LONG || (n >= 'A' && n <= 'Z'))
6460 && *p != NUL)
6461 #ifdef FEAT_MBYTE
6462 n = mb_ptr2char_adv(&p) + (n << 16);
6463 #else
6464 n = *p++ + (n << 16);
6465 #endif
6466 if (n == flag)
6467 return TRUE;
6469 break;
6471 case AFT_NUM:
6472 for (p = afflist; *p != NUL; )
6474 n = getdigits(&p);
6475 if (n == flag)
6476 return TRUE;
6477 if (*p != NUL) /* skip over comma */
6478 ++p;
6480 break;
6482 return FALSE;
6486 * Give a warning when "spinval" and "affval" numbers are set and not the same.
6488 static void
6489 aff_check_number(spinval, affval, name)
6490 int spinval;
6491 int affval;
6492 char *name;
6494 if (spinval != 0 && spinval != affval)
6495 smsg((char_u *)_("%s value differs from what is used in another .aff file"), name);
6499 * Give a warning when "spinval" and "affval" strings are set and not the same.
6501 static void
6502 aff_check_string(spinval, affval, name)
6503 char_u *spinval;
6504 char_u *affval;
6505 char *name;
6507 if (spinval != NULL && STRCMP(spinval, affval) != 0)
6508 smsg((char_u *)_("%s value differs from what is used in another .aff file"), name);
6512 * Return TRUE if strings "s1" and "s2" are equal. Also consider both being
6513 * NULL as equal.
6515 static int
6516 str_equal(s1, s2)
6517 char_u *s1;
6518 char_u *s2;
6520 if (s1 == NULL || s2 == NULL)
6521 return s1 == s2;
6522 return STRCMP(s1, s2) == 0;
6526 * Add a from-to item to "gap". Used for REP and SAL items.
6527 * They are stored case-folded.
6529 static void
6530 add_fromto(spin, gap, from, to)
6531 spellinfo_T *spin;
6532 garray_T *gap;
6533 char_u *from;
6534 char_u *to;
6536 fromto_T *ftp;
6537 char_u word[MAXWLEN];
6539 if (ga_grow(gap, 1) == OK)
6541 ftp = ((fromto_T *)gap->ga_data) + gap->ga_len;
6542 (void)spell_casefold(from, (int)STRLEN(from), word, MAXWLEN);
6543 ftp->ft_from = getroom_save(spin, word);
6544 (void)spell_casefold(to, (int)STRLEN(to), word, MAXWLEN);
6545 ftp->ft_to = getroom_save(spin, word);
6546 ++gap->ga_len;
6551 * Convert a boolean argument in a SAL line to TRUE or FALSE;
6553 static int
6554 sal_to_bool(s)
6555 char_u *s;
6557 return STRCMP(s, "1") == 0 || STRCMP(s, "true") == 0;
6561 * Return TRUE if string "s" contains a non-ASCII character (128 or higher).
6562 * When "s" is NULL FALSE is returned.
6564 static int
6565 has_non_ascii(s)
6566 char_u *s;
6568 char_u *p;
6570 if (s != NULL)
6571 for (p = s; *p != NUL; ++p)
6572 if (*p >= 128)
6573 return TRUE;
6574 return FALSE;
6578 * Free the structure filled by spell_read_aff().
6580 static void
6581 spell_free_aff(aff)
6582 afffile_T *aff;
6584 hashtab_T *ht;
6585 hashitem_T *hi;
6586 int todo;
6587 affheader_T *ah;
6588 affentry_T *ae;
6590 vim_free(aff->af_enc);
6592 /* All this trouble to free the "ae_prog" items... */
6593 for (ht = &aff->af_pref; ; ht = &aff->af_suff)
6595 todo = (int)ht->ht_used;
6596 for (hi = ht->ht_array; todo > 0; ++hi)
6598 if (!HASHITEM_EMPTY(hi))
6600 --todo;
6601 ah = HI2AH(hi);
6602 for (ae = ah->ah_first; ae != NULL; ae = ae->ae_next)
6603 vim_free(ae->ae_prog);
6606 if (ht == &aff->af_suff)
6607 break;
6610 hash_clear(&aff->af_pref);
6611 hash_clear(&aff->af_suff);
6612 hash_clear(&aff->af_comp);
6616 * Read dictionary file "fname".
6617 * Returns OK or FAIL;
6619 static int
6620 spell_read_dic(spin, fname, affile)
6621 spellinfo_T *spin;
6622 char_u *fname;
6623 afffile_T *affile;
6625 hashtab_T ht;
6626 char_u line[MAXLINELEN];
6627 char_u *p;
6628 char_u *afflist;
6629 char_u store_afflist[MAXWLEN];
6630 int pfxlen;
6631 int need_affix;
6632 char_u *dw;
6633 char_u *pc;
6634 char_u *w;
6635 int l;
6636 hash_T hash;
6637 hashitem_T *hi;
6638 FILE *fd;
6639 int lnum = 1;
6640 int non_ascii = 0;
6641 int retval = OK;
6642 char_u message[MAXLINELEN + MAXWLEN];
6643 int flags;
6644 int duplicate = 0;
6647 * Open the file.
6649 fd = mch_fopen((char *)fname, "r");
6650 if (fd == NULL)
6652 EMSG2(_(e_notopen), fname);
6653 return FAIL;
6656 /* The hashtable is only used to detect duplicated words. */
6657 hash_init(&ht);
6659 vim_snprintf((char *)IObuff, IOSIZE,
6660 _("Reading dictionary file %s ..."), fname);
6661 spell_message(spin, IObuff);
6663 /* start with a message for the first line */
6664 spin->si_msg_count = 999999;
6666 /* Read and ignore the first line: word count. */
6667 (void)vim_fgets(line, MAXLINELEN, fd);
6668 if (!vim_isdigit(*skipwhite(line)))
6669 EMSG2(_("E760: No word count in %s"), fname);
6672 * Read all the lines in the file one by one.
6673 * The words are converted to 'encoding' here, before being added to
6674 * the hashtable.
6676 while (!vim_fgets(line, MAXLINELEN, fd) && !got_int)
6678 line_breakcheck();
6679 ++lnum;
6680 if (line[0] == '#' || line[0] == '/')
6681 continue; /* comment line */
6683 /* Remove CR, LF and white space from the end. White space halfway
6684 * the word is kept to allow e.g., "et al.". */
6685 l = (int)STRLEN(line);
6686 while (l > 0 && line[l - 1] <= ' ')
6687 --l;
6688 if (l == 0)
6689 continue; /* empty line */
6690 line[l] = NUL;
6692 #ifdef FEAT_MBYTE
6693 /* Convert from "SET" to 'encoding' when needed. */
6694 if (spin->si_conv.vc_type != CONV_NONE)
6696 pc = string_convert(&spin->si_conv, line, NULL);
6697 if (pc == NULL)
6699 smsg((char_u *)_("Conversion failure for word in %s line %d: %s"),
6700 fname, lnum, line);
6701 continue;
6703 w = pc;
6705 else
6706 #endif
6708 pc = NULL;
6709 w = line;
6712 /* Truncate the word at the "/", set "afflist" to what follows.
6713 * Replace "\/" by "/" and "\\" by "\". */
6714 afflist = NULL;
6715 for (p = w; *p != NUL; mb_ptr_adv(p))
6717 if (*p == '\\' && (p[1] == '\\' || p[1] == '/'))
6718 STRMOVE(p, p + 1);
6719 else if (*p == '/')
6721 *p = NUL;
6722 afflist = p + 1;
6723 break;
6727 /* Skip non-ASCII words when "spin->si_ascii" is TRUE. */
6728 if (spin->si_ascii && has_non_ascii(w))
6730 ++non_ascii;
6731 vim_free(pc);
6732 continue;
6735 /* This takes time, print a message every 10000 words. */
6736 if (spin->si_verbose && spin->si_msg_count > 10000)
6738 spin->si_msg_count = 0;
6739 vim_snprintf((char *)message, sizeof(message),
6740 _("line %6d, word %6d - %s"),
6741 lnum, spin->si_foldwcount + spin->si_keepwcount, w);
6742 msg_start();
6743 msg_puts_long_attr(message, 0);
6744 msg_clr_eos();
6745 msg_didout = FALSE;
6746 msg_col = 0;
6747 out_flush();
6750 /* Store the word in the hashtable to be able to find duplicates. */
6751 dw = (char_u *)getroom_save(spin, w);
6752 if (dw == NULL)
6754 retval = FAIL;
6755 vim_free(pc);
6756 break;
6759 hash = hash_hash(dw);
6760 hi = hash_lookup(&ht, dw, hash);
6761 if (!HASHITEM_EMPTY(hi))
6763 if (p_verbose > 0)
6764 smsg((char_u *)_("Duplicate word in %s line %d: %s"),
6765 fname, lnum, dw);
6766 else if (duplicate == 0)
6767 smsg((char_u *)_("First duplicate word in %s line %d: %s"),
6768 fname, lnum, dw);
6769 ++duplicate;
6771 else
6772 hash_add_item(&ht, hi, dw, hash);
6774 flags = 0;
6775 store_afflist[0] = NUL;
6776 pfxlen = 0;
6777 need_affix = FALSE;
6778 if (afflist != NULL)
6780 /* Extract flags from the affix list. */
6781 flags |= get_affix_flags(affile, afflist);
6783 if (affile->af_needaffix != 0 && flag_in_afflist(
6784 affile->af_flagtype, afflist, affile->af_needaffix))
6785 need_affix = TRUE;
6787 if (affile->af_pfxpostpone)
6788 /* Need to store the list of prefix IDs with the word. */
6789 pfxlen = get_pfxlist(affile, afflist, store_afflist);
6791 if (spin->si_compflags != NULL)
6792 /* Need to store the list of compound flags with the word.
6793 * Concatenate them to the list of prefix IDs. */
6794 get_compflags(affile, afflist, store_afflist + pfxlen);
6797 /* Add the word to the word tree(s). */
6798 if (store_word(spin, dw, flags, spin->si_region,
6799 store_afflist, need_affix) == FAIL)
6800 retval = FAIL;
6802 if (afflist != NULL)
6804 /* Find all matching suffixes and add the resulting words.
6805 * Additionally do matching prefixes that combine. */
6806 if (store_aff_word(spin, dw, afflist, affile,
6807 &affile->af_suff, &affile->af_pref,
6808 CONDIT_SUF, flags, store_afflist, pfxlen) == FAIL)
6809 retval = FAIL;
6811 /* Find all matching prefixes and add the resulting words. */
6812 if (store_aff_word(spin, dw, afflist, affile,
6813 &affile->af_pref, NULL,
6814 CONDIT_SUF, flags, store_afflist, pfxlen) == FAIL)
6815 retval = FAIL;
6818 vim_free(pc);
6821 if (duplicate > 0)
6822 smsg((char_u *)_("%d duplicate word(s) in %s"), duplicate, fname);
6823 if (spin->si_ascii && non_ascii > 0)
6824 smsg((char_u *)_("Ignored %d word(s) with non-ASCII characters in %s"),
6825 non_ascii, fname);
6826 hash_clear(&ht);
6828 fclose(fd);
6829 return retval;
6833 * Check for affix flags in "afflist" that are turned into word flags.
6834 * Return WF_ flags.
6836 static int
6837 get_affix_flags(affile, afflist)
6838 afffile_T *affile;
6839 char_u *afflist;
6841 int flags = 0;
6843 if (affile->af_keepcase != 0 && flag_in_afflist(
6844 affile->af_flagtype, afflist, affile->af_keepcase))
6845 flags |= WF_KEEPCAP | WF_FIXCAP;
6846 if (affile->af_rare != 0 && flag_in_afflist(
6847 affile->af_flagtype, afflist, affile->af_rare))
6848 flags |= WF_RARE;
6849 if (affile->af_bad != 0 && flag_in_afflist(
6850 affile->af_flagtype, afflist, affile->af_bad))
6851 flags |= WF_BANNED;
6852 if (affile->af_needcomp != 0 && flag_in_afflist(
6853 affile->af_flagtype, afflist, affile->af_needcomp))
6854 flags |= WF_NEEDCOMP;
6855 if (affile->af_comproot != 0 && flag_in_afflist(
6856 affile->af_flagtype, afflist, affile->af_comproot))
6857 flags |= WF_COMPROOT;
6858 if (affile->af_nosuggest != 0 && flag_in_afflist(
6859 affile->af_flagtype, afflist, affile->af_nosuggest))
6860 flags |= WF_NOSUGGEST;
6861 return flags;
6865 * Get the list of prefix IDs from the affix list "afflist".
6866 * Used for PFXPOSTPONE.
6867 * Put the resulting flags in "store_afflist[MAXWLEN]" with a terminating NUL
6868 * and return the number of affixes.
6870 static int
6871 get_pfxlist(affile, afflist, store_afflist)
6872 afffile_T *affile;
6873 char_u *afflist;
6874 char_u *store_afflist;
6876 char_u *p;
6877 char_u *prevp;
6878 int cnt = 0;
6879 int id;
6880 char_u key[AH_KEY_LEN];
6881 hashitem_T *hi;
6883 for (p = afflist; *p != NUL; )
6885 prevp = p;
6886 if (get_affitem(affile->af_flagtype, &p) != 0)
6888 /* A flag is a postponed prefix flag if it appears in "af_pref"
6889 * and it's ID is not zero. */
6890 vim_strncpy(key, prevp, p - prevp);
6891 hi = hash_find(&affile->af_pref, key);
6892 if (!HASHITEM_EMPTY(hi))
6894 id = HI2AH(hi)->ah_newID;
6895 if (id != 0)
6896 store_afflist[cnt++] = id;
6899 if (affile->af_flagtype == AFT_NUM && *p == ',')
6900 ++p;
6903 store_afflist[cnt] = NUL;
6904 return cnt;
6908 * Get the list of compound IDs from the affix list "afflist" that are used
6909 * for compound words.
6910 * Puts the flags in "store_afflist[]".
6912 static void
6913 get_compflags(affile, afflist, store_afflist)
6914 afffile_T *affile;
6915 char_u *afflist;
6916 char_u *store_afflist;
6918 char_u *p;
6919 char_u *prevp;
6920 int cnt = 0;
6921 char_u key[AH_KEY_LEN];
6922 hashitem_T *hi;
6924 for (p = afflist; *p != NUL; )
6926 prevp = p;
6927 if (get_affitem(affile->af_flagtype, &p) != 0)
6929 /* A flag is a compound flag if it appears in "af_comp". */
6930 vim_strncpy(key, prevp, p - prevp);
6931 hi = hash_find(&affile->af_comp, key);
6932 if (!HASHITEM_EMPTY(hi))
6933 store_afflist[cnt++] = HI2CI(hi)->ci_newID;
6935 if (affile->af_flagtype == AFT_NUM && *p == ',')
6936 ++p;
6939 store_afflist[cnt] = NUL;
6943 * Apply affixes to a word and store the resulting words.
6944 * "ht" is the hashtable with affentry_T that need to be applied, either
6945 * prefixes or suffixes.
6946 * "xht", when not NULL, is the prefix hashtable, to be used additionally on
6947 * the resulting words for combining affixes.
6949 * Returns FAIL when out of memory.
6951 static int
6952 store_aff_word(spin, word, afflist, affile, ht, xht, condit, flags,
6953 pfxlist, pfxlen)
6954 spellinfo_T *spin; /* spell info */
6955 char_u *word; /* basic word start */
6956 char_u *afflist; /* list of names of supported affixes */
6957 afffile_T *affile;
6958 hashtab_T *ht;
6959 hashtab_T *xht;
6960 int condit; /* CONDIT_SUF et al. */
6961 int flags; /* flags for the word */
6962 char_u *pfxlist; /* list of prefix IDs */
6963 int pfxlen; /* nr of flags in "pfxlist" for prefixes, rest
6964 * is compound flags */
6966 int todo;
6967 hashitem_T *hi;
6968 affheader_T *ah;
6969 affentry_T *ae;
6970 regmatch_T regmatch;
6971 char_u newword[MAXWLEN];
6972 int retval = OK;
6973 int i, j;
6974 char_u *p;
6975 int use_flags;
6976 char_u *use_pfxlist;
6977 int use_pfxlen;
6978 int need_affix;
6979 char_u store_afflist[MAXWLEN];
6980 char_u pfx_pfxlist[MAXWLEN];
6981 size_t wordlen = STRLEN(word);
6982 int use_condit;
6984 todo = (int)ht->ht_used;
6985 for (hi = ht->ht_array; todo > 0 && retval == OK; ++hi)
6987 if (!HASHITEM_EMPTY(hi))
6989 --todo;
6990 ah = HI2AH(hi);
6992 /* Check that the affix combines, if required, and that the word
6993 * supports this affix. */
6994 if (((condit & CONDIT_COMB) == 0 || ah->ah_combine)
6995 && flag_in_afflist(affile->af_flagtype, afflist,
6996 ah->ah_flag))
6998 /* Loop over all affix entries with this name. */
6999 for (ae = ah->ah_first; ae != NULL; ae = ae->ae_next)
7001 /* Check the condition. It's not logical to match case
7002 * here, but it is required for compatibility with
7003 * Myspell.
7004 * Another requirement from Myspell is that the chop
7005 * string is shorter than the word itself.
7006 * For prefixes, when "PFXPOSTPONE" was used, only do
7007 * prefixes with a chop string and/or flags.
7008 * When a previously added affix had CIRCUMFIX this one
7009 * must have it too, if it had not then this one must not
7010 * have one either. */
7011 regmatch.regprog = ae->ae_prog;
7012 regmatch.rm_ic = FALSE;
7013 if ((xht != NULL || !affile->af_pfxpostpone
7014 || ae->ae_chop != NULL
7015 || ae->ae_flags != NULL)
7016 && (ae->ae_chop == NULL
7017 || STRLEN(ae->ae_chop) < wordlen)
7018 && (ae->ae_prog == NULL
7019 || vim_regexec(&regmatch, word, (colnr_T)0))
7020 && (((condit & CONDIT_CFIX) == 0)
7021 == ((condit & CONDIT_AFF) == 0
7022 || ae->ae_flags == NULL
7023 || !flag_in_afflist(affile->af_flagtype,
7024 ae->ae_flags, affile->af_circumfix))))
7026 /* Match. Remove the chop and add the affix. */
7027 if (xht == NULL)
7029 /* prefix: chop/add at the start of the word */
7030 if (ae->ae_add == NULL)
7031 *newword = NUL;
7032 else
7033 STRCPY(newword, ae->ae_add);
7034 p = word;
7035 if (ae->ae_chop != NULL)
7037 /* Skip chop string. */
7038 #ifdef FEAT_MBYTE
7039 if (has_mbyte)
7041 i = mb_charlen(ae->ae_chop);
7042 for ( ; i > 0; --i)
7043 mb_ptr_adv(p);
7045 else
7046 #endif
7047 p += STRLEN(ae->ae_chop);
7049 STRCAT(newword, p);
7051 else
7053 /* suffix: chop/add at the end of the word */
7054 STRCPY(newword, word);
7055 if (ae->ae_chop != NULL)
7057 /* Remove chop string. */
7058 p = newword + STRLEN(newword);
7059 i = (int)MB_CHARLEN(ae->ae_chop);
7060 for ( ; i > 0; --i)
7061 mb_ptr_back(newword, p);
7062 *p = NUL;
7064 if (ae->ae_add != NULL)
7065 STRCAT(newword, ae->ae_add);
7068 use_flags = flags;
7069 use_pfxlist = pfxlist;
7070 use_pfxlen = pfxlen;
7071 need_affix = FALSE;
7072 use_condit = condit | CONDIT_COMB | CONDIT_AFF;
7073 if (ae->ae_flags != NULL)
7075 /* Extract flags from the affix list. */
7076 use_flags |= get_affix_flags(affile, ae->ae_flags);
7078 if (affile->af_needaffix != 0 && flag_in_afflist(
7079 affile->af_flagtype, ae->ae_flags,
7080 affile->af_needaffix))
7081 need_affix = TRUE;
7083 /* When there is a CIRCUMFIX flag the other affix
7084 * must also have it and we don't add the word
7085 * with one affix. */
7086 if (affile->af_circumfix != 0 && flag_in_afflist(
7087 affile->af_flagtype, ae->ae_flags,
7088 affile->af_circumfix))
7090 use_condit |= CONDIT_CFIX;
7091 if ((condit & CONDIT_CFIX) == 0)
7092 need_affix = TRUE;
7095 if (affile->af_pfxpostpone
7096 || spin->si_compflags != NULL)
7098 if (affile->af_pfxpostpone)
7099 /* Get prefix IDS from the affix list. */
7100 use_pfxlen = get_pfxlist(affile,
7101 ae->ae_flags, store_afflist);
7102 else
7103 use_pfxlen = 0;
7104 use_pfxlist = store_afflist;
7106 /* Combine the prefix IDs. Avoid adding the
7107 * same ID twice. */
7108 for (i = 0; i < pfxlen; ++i)
7110 for (j = 0; j < use_pfxlen; ++j)
7111 if (pfxlist[i] == use_pfxlist[j])
7112 break;
7113 if (j == use_pfxlen)
7114 use_pfxlist[use_pfxlen++] = pfxlist[i];
7117 if (spin->si_compflags != NULL)
7118 /* Get compound IDS from the affix list. */
7119 get_compflags(affile, ae->ae_flags,
7120 use_pfxlist + use_pfxlen);
7122 /* Combine the list of compound flags.
7123 * Concatenate them to the prefix IDs list.
7124 * Avoid adding the same ID twice. */
7125 for (i = pfxlen; pfxlist[i] != NUL; ++i)
7127 for (j = use_pfxlen;
7128 use_pfxlist[j] != NUL; ++j)
7129 if (pfxlist[i] == use_pfxlist[j])
7130 break;
7131 if (use_pfxlist[j] == NUL)
7133 use_pfxlist[j++] = pfxlist[i];
7134 use_pfxlist[j] = NUL;
7140 /* Obey a "COMPOUNDFORBIDFLAG" of the affix: don't
7141 * use the compound flags. */
7142 if (use_pfxlist != NULL && ae->ae_compforbid)
7144 vim_strncpy(pfx_pfxlist, use_pfxlist, use_pfxlen);
7145 use_pfxlist = pfx_pfxlist;
7148 /* When there are postponed prefixes... */
7149 if (spin->si_prefroot != NULL
7150 && spin->si_prefroot->wn_sibling != NULL)
7152 /* ... add a flag to indicate an affix was used. */
7153 use_flags |= WF_HAS_AFF;
7155 /* ... don't use a prefix list if combining
7156 * affixes is not allowed. But do use the
7157 * compound flags after them. */
7158 if (!ah->ah_combine && use_pfxlist != NULL)
7159 use_pfxlist += use_pfxlen;
7162 /* When compounding is supported and there is no
7163 * "COMPOUNDPERMITFLAG" then forbid compounding on the
7164 * side where the affix is applied. */
7165 if (spin->si_compflags != NULL && !ae->ae_comppermit)
7167 if (xht != NULL)
7168 use_flags |= WF_NOCOMPAFT;
7169 else
7170 use_flags |= WF_NOCOMPBEF;
7173 /* Store the modified word. */
7174 if (store_word(spin, newword, use_flags,
7175 spin->si_region, use_pfxlist,
7176 need_affix) == FAIL)
7177 retval = FAIL;
7179 /* When added a prefix or a first suffix and the affix
7180 * has flags may add a(nother) suffix. RECURSIVE! */
7181 if ((condit & CONDIT_SUF) && ae->ae_flags != NULL)
7182 if (store_aff_word(spin, newword, ae->ae_flags,
7183 affile, &affile->af_suff, xht,
7184 use_condit & (xht == NULL
7185 ? ~0 : ~CONDIT_SUF),
7186 use_flags, use_pfxlist, pfxlen) == FAIL)
7187 retval = FAIL;
7189 /* When added a suffix and combining is allowed also
7190 * try adding a prefix additionally. Both for the
7191 * word flags and for the affix flags. RECURSIVE! */
7192 if (xht != NULL && ah->ah_combine)
7194 if (store_aff_word(spin, newword,
7195 afflist, affile,
7196 xht, NULL, use_condit,
7197 use_flags, use_pfxlist,
7198 pfxlen) == FAIL
7199 || (ae->ae_flags != NULL
7200 && store_aff_word(spin, newword,
7201 ae->ae_flags, affile,
7202 xht, NULL, use_condit,
7203 use_flags, use_pfxlist,
7204 pfxlen) == FAIL))
7205 retval = FAIL;
7213 return retval;
7217 * Read a file with a list of words.
7219 static int
7220 spell_read_wordfile(spin, fname)
7221 spellinfo_T *spin;
7222 char_u *fname;
7224 FILE *fd;
7225 long lnum = 0;
7226 char_u rline[MAXLINELEN];
7227 char_u *line;
7228 char_u *pc = NULL;
7229 char_u *p;
7230 int l;
7231 int retval = OK;
7232 int did_word = FALSE;
7233 int non_ascii = 0;
7234 int flags;
7235 int regionmask;
7238 * Open the file.
7240 fd = mch_fopen((char *)fname, "r");
7241 if (fd == NULL)
7243 EMSG2(_(e_notopen), fname);
7244 return FAIL;
7247 vim_snprintf((char *)IObuff, IOSIZE, _("Reading word file %s ..."), fname);
7248 spell_message(spin, IObuff);
7251 * Read all the lines in the file one by one.
7253 while (!vim_fgets(rline, MAXLINELEN, fd) && !got_int)
7255 line_breakcheck();
7256 ++lnum;
7258 /* Skip comment lines. */
7259 if (*rline == '#')
7260 continue;
7262 /* Remove CR, LF and white space from the end. */
7263 l = (int)STRLEN(rline);
7264 while (l > 0 && rline[l - 1] <= ' ')
7265 --l;
7266 if (l == 0)
7267 continue; /* empty or blank line */
7268 rline[l] = NUL;
7270 /* Convert from "/encoding={encoding}" to 'encoding' when needed. */
7271 vim_free(pc);
7272 #ifdef FEAT_MBYTE
7273 if (spin->si_conv.vc_type != CONV_NONE)
7275 pc = string_convert(&spin->si_conv, rline, NULL);
7276 if (pc == NULL)
7278 smsg((char_u *)_("Conversion failure for word in %s line %d: %s"),
7279 fname, lnum, rline);
7280 continue;
7282 line = pc;
7284 else
7285 #endif
7287 pc = NULL;
7288 line = rline;
7291 if (*line == '/')
7293 ++line;
7294 if (STRNCMP(line, "encoding=", 9) == 0)
7296 if (spin->si_conv.vc_type != CONV_NONE)
7297 smsg((char_u *)_("Duplicate /encoding= line ignored in %s line %d: %s"),
7298 fname, lnum, line - 1);
7299 else if (did_word)
7300 smsg((char_u *)_("/encoding= line after word ignored in %s line %d: %s"),
7301 fname, lnum, line - 1);
7302 else
7304 #ifdef FEAT_MBYTE
7305 char_u *enc;
7307 /* Setup for conversion to 'encoding'. */
7308 line += 9;
7309 enc = enc_canonize(line);
7310 if (enc != NULL && !spin->si_ascii
7311 && convert_setup(&spin->si_conv, enc,
7312 p_enc) == FAIL)
7313 smsg((char_u *)_("Conversion in %s not supported: from %s to %s"),
7314 fname, line, p_enc);
7315 vim_free(enc);
7316 spin->si_conv.vc_fail = TRUE;
7317 #else
7318 smsg((char_u *)_("Conversion in %s not supported"), fname);
7319 #endif
7321 continue;
7324 if (STRNCMP(line, "regions=", 8) == 0)
7326 if (spin->si_region_count > 1)
7327 smsg((char_u *)_("Duplicate /regions= line ignored in %s line %d: %s"),
7328 fname, lnum, line);
7329 else
7331 line += 8;
7332 if (STRLEN(line) > 16)
7333 smsg((char_u *)_("Too many regions in %s line %d: %s"),
7334 fname, lnum, line);
7335 else
7337 spin->si_region_count = (int)STRLEN(line) / 2;
7338 STRCPY(spin->si_region_name, line);
7340 /* Adjust the mask for a word valid in all regions. */
7341 spin->si_region = (1 << spin->si_region_count) - 1;
7344 continue;
7347 smsg((char_u *)_("/ line ignored in %s line %d: %s"),
7348 fname, lnum, line - 1);
7349 continue;
7352 flags = 0;
7353 regionmask = spin->si_region;
7355 /* Check for flags and region after a slash. */
7356 p = vim_strchr(line, '/');
7357 if (p != NULL)
7359 *p++ = NUL;
7360 while (*p != NUL)
7362 if (*p == '=') /* keep-case word */
7363 flags |= WF_KEEPCAP | WF_FIXCAP;
7364 else if (*p == '!') /* Bad, bad, wicked word. */
7365 flags |= WF_BANNED;
7366 else if (*p == '?') /* Rare word. */
7367 flags |= WF_RARE;
7368 else if (VIM_ISDIGIT(*p)) /* region number(s) */
7370 if ((flags & WF_REGION) == 0) /* first one */
7371 regionmask = 0;
7372 flags |= WF_REGION;
7374 l = *p - '0';
7375 if (l > spin->si_region_count)
7377 smsg((char_u *)_("Invalid region nr in %s line %d: %s"),
7378 fname, lnum, p);
7379 break;
7381 regionmask |= 1 << (l - 1);
7383 else
7385 smsg((char_u *)_("Unrecognized flags in %s line %d: %s"),
7386 fname, lnum, p);
7387 break;
7389 ++p;
7393 /* Skip non-ASCII words when "spin->si_ascii" is TRUE. */
7394 if (spin->si_ascii && has_non_ascii(line))
7396 ++non_ascii;
7397 continue;
7400 /* Normal word: store it. */
7401 if (store_word(spin, line, flags, regionmask, NULL, FALSE) == FAIL)
7403 retval = FAIL;
7404 break;
7406 did_word = TRUE;
7409 vim_free(pc);
7410 fclose(fd);
7412 if (spin->si_ascii && non_ascii > 0)
7414 vim_snprintf((char *)IObuff, IOSIZE,
7415 _("Ignored %d words with non-ASCII characters"), non_ascii);
7416 spell_message(spin, IObuff);
7419 return retval;
7423 * Get part of an sblock_T, "len" bytes long.
7424 * This avoids calling free() for every little struct we use (and keeping
7425 * track of them).
7426 * The memory is cleared to all zeros.
7427 * Returns NULL when out of memory.
7429 static void *
7430 getroom(spin, len, align)
7431 spellinfo_T *spin;
7432 size_t len; /* length needed */
7433 int align; /* align for pointer */
7435 char_u *p;
7436 sblock_T *bl = spin->si_blocks;
7438 if (align && bl != NULL)
7439 /* Round size up for alignment. On some systems structures need to be
7440 * aligned to the size of a pointer (e.g., SPARC). */
7441 bl->sb_used = (bl->sb_used + sizeof(char *) - 1)
7442 & ~(sizeof(char *) - 1);
7444 if (bl == NULL || bl->sb_used + len > SBLOCKSIZE)
7446 /* Allocate a block of memory. This is not freed until much later. */
7447 bl = (sblock_T *)alloc_clear((unsigned)(sizeof(sblock_T) + SBLOCKSIZE));
7448 if (bl == NULL)
7449 return NULL;
7450 bl->sb_next = spin->si_blocks;
7451 spin->si_blocks = bl;
7452 bl->sb_used = 0;
7453 ++spin->si_blocks_cnt;
7456 p = bl->sb_data + bl->sb_used;
7457 bl->sb_used += (int)len;
7459 return p;
7463 * Make a copy of a string into memory allocated with getroom().
7465 static char_u *
7466 getroom_save(spin, s)
7467 spellinfo_T *spin;
7468 char_u *s;
7470 char_u *sc;
7472 sc = (char_u *)getroom(spin, STRLEN(s) + 1, FALSE);
7473 if (sc != NULL)
7474 STRCPY(sc, s);
7475 return sc;
7480 * Free the list of allocated sblock_T.
7482 static void
7483 free_blocks(bl)
7484 sblock_T *bl;
7486 sblock_T *next;
7488 while (bl != NULL)
7490 next = bl->sb_next;
7491 vim_free(bl);
7492 bl = next;
7497 * Allocate the root of a word tree.
7499 static wordnode_T *
7500 wordtree_alloc(spin)
7501 spellinfo_T *spin;
7503 return (wordnode_T *)getroom(spin, sizeof(wordnode_T), TRUE);
7507 * Store a word in the tree(s).
7508 * Always store it in the case-folded tree. For a keep-case word this is
7509 * useful when the word can also be used with all caps (no WF_FIXCAP flag) and
7510 * used to find suggestions.
7511 * For a keep-case word also store it in the keep-case tree.
7512 * When "pfxlist" is not NULL store the word for each postponed prefix ID and
7513 * compound flag.
7515 static int
7516 store_word(spin, word, flags, region, pfxlist, need_affix)
7517 spellinfo_T *spin;
7518 char_u *word;
7519 int flags; /* extra flags, WF_BANNED */
7520 int region; /* supported region(s) */
7521 char_u *pfxlist; /* list of prefix IDs or NULL */
7522 int need_affix; /* only store word with affix ID */
7524 int len = (int)STRLEN(word);
7525 int ct = captype(word, word + len);
7526 char_u foldword[MAXWLEN];
7527 int res = OK;
7528 char_u *p;
7530 (void)spell_casefold(word, len, foldword, MAXWLEN);
7531 for (p = pfxlist; res == OK; ++p)
7533 if (!need_affix || (p != NULL && *p != NUL))
7534 res = tree_add_word(spin, foldword, spin->si_foldroot, ct | flags,
7535 region, p == NULL ? 0 : *p);
7536 if (p == NULL || *p == NUL)
7537 break;
7539 ++spin->si_foldwcount;
7541 if (res == OK && (ct == WF_KEEPCAP || (flags & WF_KEEPCAP)))
7543 for (p = pfxlist; res == OK; ++p)
7545 if (!need_affix || (p != NULL && *p != NUL))
7546 res = tree_add_word(spin, word, spin->si_keeproot, flags,
7547 region, p == NULL ? 0 : *p);
7548 if (p == NULL || *p == NUL)
7549 break;
7551 ++spin->si_keepwcount;
7553 return res;
7557 * Add word "word" to a word tree at "root".
7558 * When "flags" < 0 we are adding to the prefix tree where "flags" is used for
7559 * "rare" and "region" is the condition nr.
7560 * Returns FAIL when out of memory.
7562 static int
7563 tree_add_word(spin, word, root, flags, region, affixID)
7564 spellinfo_T *spin;
7565 char_u *word;
7566 wordnode_T *root;
7567 int flags;
7568 int region;
7569 int affixID;
7571 wordnode_T *node = root;
7572 wordnode_T *np;
7573 wordnode_T *copyp, **copyprev;
7574 wordnode_T **prev = NULL;
7575 int i;
7577 /* Add each byte of the word to the tree, including the NUL at the end. */
7578 for (i = 0; ; ++i)
7580 /* When there is more than one reference to this node we need to make
7581 * a copy, so that we can modify it. Copy the whole list of siblings
7582 * (we don't optimize for a partly shared list of siblings). */
7583 if (node != NULL && node->wn_refs > 1)
7585 --node->wn_refs;
7586 copyprev = prev;
7587 for (copyp = node; copyp != NULL; copyp = copyp->wn_sibling)
7589 /* Allocate a new node and copy the info. */
7590 np = get_wordnode(spin);
7591 if (np == NULL)
7592 return FAIL;
7593 np->wn_child = copyp->wn_child;
7594 if (np->wn_child != NULL)
7595 ++np->wn_child->wn_refs; /* child gets extra ref */
7596 np->wn_byte = copyp->wn_byte;
7597 if (np->wn_byte == NUL)
7599 np->wn_flags = copyp->wn_flags;
7600 np->wn_region = copyp->wn_region;
7601 np->wn_affixID = copyp->wn_affixID;
7604 /* Link the new node in the list, there will be one ref. */
7605 np->wn_refs = 1;
7606 if (copyprev != NULL)
7607 *copyprev = np;
7608 copyprev = &np->wn_sibling;
7610 /* Let "node" point to the head of the copied list. */
7611 if (copyp == node)
7612 node = np;
7616 /* Look for the sibling that has the same character. They are sorted
7617 * on byte value, thus stop searching when a sibling is found with a
7618 * higher byte value. For zero bytes (end of word) the sorting is
7619 * done on flags and then on affixID. */
7620 while (node != NULL
7621 && (node->wn_byte < word[i]
7622 || (node->wn_byte == NUL
7623 && (flags < 0
7624 ? node->wn_affixID < (unsigned)affixID
7625 : (node->wn_flags < (unsigned)(flags & WN_MASK)
7626 || (node->wn_flags == (flags & WN_MASK)
7627 && (spin->si_sugtree
7628 ? (node->wn_region & 0xffff) < region
7629 : node->wn_affixID
7630 < (unsigned)affixID)))))))
7632 prev = &node->wn_sibling;
7633 node = *prev;
7635 if (node == NULL
7636 || node->wn_byte != word[i]
7637 || (word[i] == NUL
7638 && (flags < 0
7639 || spin->si_sugtree
7640 || node->wn_flags != (flags & WN_MASK)
7641 || node->wn_affixID != affixID)))
7643 /* Allocate a new node. */
7644 np = get_wordnode(spin);
7645 if (np == NULL)
7646 return FAIL;
7647 np->wn_byte = word[i];
7649 /* If "node" is NULL this is a new child or the end of the sibling
7650 * list: ref count is one. Otherwise use ref count of sibling and
7651 * make ref count of sibling one (matters when inserting in front
7652 * of the list of siblings). */
7653 if (node == NULL)
7654 np->wn_refs = 1;
7655 else
7657 np->wn_refs = node->wn_refs;
7658 node->wn_refs = 1;
7660 *prev = np;
7661 np->wn_sibling = node;
7662 node = np;
7665 if (word[i] == NUL)
7667 node->wn_flags = flags;
7668 node->wn_region |= region;
7669 node->wn_affixID = affixID;
7670 break;
7672 prev = &node->wn_child;
7673 node = *prev;
7675 #ifdef SPELL_PRINTTREE
7676 smsg("Added \"%s\"", word);
7677 spell_print_tree(root->wn_sibling);
7678 #endif
7680 /* count nr of words added since last message */
7681 ++spin->si_msg_count;
7683 if (spin->si_compress_cnt > 1)
7685 if (--spin->si_compress_cnt == 1)
7686 /* Did enough words to lower the block count limit. */
7687 spin->si_blocks_cnt += compress_inc;
7691 * When we have allocated lots of memory we need to compress the word tree
7692 * to free up some room. But compression is slow, and we might actually
7693 * need that room, thus only compress in the following situations:
7694 * 1. When not compressed before (si_compress_cnt == 0): when using
7695 * "compress_start" blocks.
7696 * 2. When compressed before and used "compress_inc" blocks before
7697 * adding "compress_added" words (si_compress_cnt > 1).
7698 * 3. When compressed before, added "compress_added" words
7699 * (si_compress_cnt == 1) and the number of free nodes drops below the
7700 * maximum word length.
7702 #ifndef SPELL_PRINTTREE
7703 if (spin->si_compress_cnt == 1
7704 ? spin->si_free_count < MAXWLEN
7705 : spin->si_blocks_cnt >= compress_start)
7706 #endif
7708 /* Decrement the block counter. The effect is that we compress again
7709 * when the freed up room has been used and another "compress_inc"
7710 * blocks have been allocated. Unless "compress_added" words have
7711 * been added, then the limit is put back again. */
7712 spin->si_blocks_cnt -= compress_inc;
7713 spin->si_compress_cnt = compress_added;
7715 if (spin->si_verbose)
7717 msg_start();
7718 msg_puts((char_u *)_(msg_compressing));
7719 msg_clr_eos();
7720 msg_didout = FALSE;
7721 msg_col = 0;
7722 out_flush();
7725 /* Compress both trees. Either they both have many nodes, which makes
7726 * compression useful, or one of them is small, which means
7727 * compression goes fast. But when filling the souldfold word tree
7728 * there is no keep-case tree. */
7729 wordtree_compress(spin, spin->si_foldroot);
7730 if (affixID >= 0)
7731 wordtree_compress(spin, spin->si_keeproot);
7734 return OK;
7738 * Check the 'mkspellmem' option. Return FAIL if it's wrong.
7739 * Sets "sps_flags".
7742 spell_check_msm()
7744 char_u *p = p_msm;
7745 long start = 0;
7746 long incr = 0;
7747 long added = 0;
7749 if (!VIM_ISDIGIT(*p))
7750 return FAIL;
7751 /* block count = (value * 1024) / SBLOCKSIZE (but avoid overflow)*/
7752 start = (getdigits(&p) * 10) / (SBLOCKSIZE / 102);
7753 if (*p != ',')
7754 return FAIL;
7755 ++p;
7756 if (!VIM_ISDIGIT(*p))
7757 return FAIL;
7758 incr = (getdigits(&p) * 102) / (SBLOCKSIZE / 10);
7759 if (*p != ',')
7760 return FAIL;
7761 ++p;
7762 if (!VIM_ISDIGIT(*p))
7763 return FAIL;
7764 added = getdigits(&p) * 1024;
7765 if (*p != NUL)
7766 return FAIL;
7768 if (start == 0 || incr == 0 || added == 0 || incr > start)
7769 return FAIL;
7771 compress_start = start;
7772 compress_inc = incr;
7773 compress_added = added;
7774 return OK;
7779 * Get a wordnode_T, either from the list of previously freed nodes or
7780 * allocate a new one.
7782 static wordnode_T *
7783 get_wordnode(spin)
7784 spellinfo_T *spin;
7786 wordnode_T *n;
7788 if (spin->si_first_free == NULL)
7789 n = (wordnode_T *)getroom(spin, sizeof(wordnode_T), TRUE);
7790 else
7792 n = spin->si_first_free;
7793 spin->si_first_free = n->wn_child;
7794 vim_memset(n, 0, sizeof(wordnode_T));
7795 --spin->si_free_count;
7797 #ifdef SPELL_PRINTTREE
7798 n->wn_nr = ++spin->si_wordnode_nr;
7799 #endif
7800 return n;
7804 * Decrement the reference count on a node (which is the head of a list of
7805 * siblings). If the reference count becomes zero free the node and its
7806 * siblings.
7807 * Returns the number of nodes actually freed.
7809 static int
7810 deref_wordnode(spin, node)
7811 spellinfo_T *spin;
7812 wordnode_T *node;
7814 wordnode_T *np;
7815 int cnt = 0;
7817 if (--node->wn_refs == 0)
7819 for (np = node; np != NULL; np = np->wn_sibling)
7821 if (np->wn_child != NULL)
7822 cnt += deref_wordnode(spin, np->wn_child);
7823 free_wordnode(spin, np);
7824 ++cnt;
7826 ++cnt; /* length field */
7828 return cnt;
7832 * Free a wordnode_T for re-use later.
7833 * Only the "wn_child" field becomes invalid.
7835 static void
7836 free_wordnode(spin, n)
7837 spellinfo_T *spin;
7838 wordnode_T *n;
7840 n->wn_child = spin->si_first_free;
7841 spin->si_first_free = n;
7842 ++spin->si_free_count;
7846 * Compress a tree: find tails that are identical and can be shared.
7848 static void
7849 wordtree_compress(spin, root)
7850 spellinfo_T *spin;
7851 wordnode_T *root;
7853 hashtab_T ht;
7854 int n;
7855 int tot = 0;
7856 int perc;
7858 /* Skip the root itself, it's not actually used. The first sibling is the
7859 * start of the tree. */
7860 if (root->wn_sibling != NULL)
7862 hash_init(&ht);
7863 n = node_compress(spin, root->wn_sibling, &ht, &tot);
7865 #ifndef SPELL_PRINTTREE
7866 if (spin->si_verbose || p_verbose > 2)
7867 #endif
7869 if (tot > 1000000)
7870 perc = (tot - n) / (tot / 100);
7871 else if (tot == 0)
7872 perc = 0;
7873 else
7874 perc = (tot - n) * 100 / tot;
7875 vim_snprintf((char *)IObuff, IOSIZE,
7876 _("Compressed %d of %d nodes; %d (%d%%) remaining"),
7877 n, tot, tot - n, perc);
7878 spell_message(spin, IObuff);
7880 #ifdef SPELL_PRINTTREE
7881 spell_print_tree(root->wn_sibling);
7882 #endif
7883 hash_clear(&ht);
7888 * Compress a node, its siblings and its children, depth first.
7889 * Returns the number of compressed nodes.
7891 static int
7892 node_compress(spin, node, ht, tot)
7893 spellinfo_T *spin;
7894 wordnode_T *node;
7895 hashtab_T *ht;
7896 int *tot; /* total count of nodes before compressing,
7897 incremented while going through the tree */
7899 wordnode_T *np;
7900 wordnode_T *tp;
7901 wordnode_T *child;
7902 hash_T hash;
7903 hashitem_T *hi;
7904 int len = 0;
7905 unsigned nr, n;
7906 int compressed = 0;
7909 * Go through the list of siblings. Compress each child and then try
7910 * finding an identical child to replace it.
7911 * Note that with "child" we mean not just the node that is pointed to,
7912 * but the whole list of siblings of which the child node is the first.
7914 for (np = node; np != NULL && !got_int; np = np->wn_sibling)
7916 ++len;
7917 if ((child = np->wn_child) != NULL)
7919 /* Compress the child first. This fills hashkey. */
7920 compressed += node_compress(spin, child, ht, tot);
7922 /* Try to find an identical child. */
7923 hash = hash_hash(child->wn_u1.hashkey);
7924 hi = hash_lookup(ht, child->wn_u1.hashkey, hash);
7925 if (!HASHITEM_EMPTY(hi))
7927 /* There are children we encountered before with a hash value
7928 * identical to the current child. Now check if there is one
7929 * that is really identical. */
7930 for (tp = HI2WN(hi); tp != NULL; tp = tp->wn_u2.next)
7931 if (node_equal(child, tp))
7933 /* Found one! Now use that child in place of the
7934 * current one. This means the current child and all
7935 * its siblings is unlinked from the tree. */
7936 ++tp->wn_refs;
7937 compressed += deref_wordnode(spin, child);
7938 np->wn_child = tp;
7939 break;
7941 if (tp == NULL)
7943 /* No other child with this hash value equals the child of
7944 * the node, add it to the linked list after the first
7945 * item. */
7946 tp = HI2WN(hi);
7947 child->wn_u2.next = tp->wn_u2.next;
7948 tp->wn_u2.next = child;
7951 else
7952 /* No other child has this hash value, add it to the
7953 * hashtable. */
7954 hash_add_item(ht, hi, child->wn_u1.hashkey, hash);
7957 *tot += len + 1; /* add one for the node that stores the length */
7960 * Make a hash key for the node and its siblings, so that we can quickly
7961 * find a lookalike node. This must be done after compressing the sibling
7962 * list, otherwise the hash key would become invalid by the compression.
7964 node->wn_u1.hashkey[0] = len;
7965 nr = 0;
7966 for (np = node; np != NULL; np = np->wn_sibling)
7968 if (np->wn_byte == NUL)
7969 /* end node: use wn_flags, wn_region and wn_affixID */
7970 n = np->wn_flags + (np->wn_region << 8) + (np->wn_affixID << 16);
7971 else
7972 /* byte node: use the byte value and the child pointer */
7973 n = (unsigned)(np->wn_byte + ((long_u)np->wn_child << 8));
7974 nr = nr * 101 + n;
7977 /* Avoid NUL bytes, it terminates the hash key. */
7978 n = nr & 0xff;
7979 node->wn_u1.hashkey[1] = n == 0 ? 1 : n;
7980 n = (nr >> 8) & 0xff;
7981 node->wn_u1.hashkey[2] = n == 0 ? 1 : n;
7982 n = (nr >> 16) & 0xff;
7983 node->wn_u1.hashkey[3] = n == 0 ? 1 : n;
7984 n = (nr >> 24) & 0xff;
7985 node->wn_u1.hashkey[4] = n == 0 ? 1 : n;
7986 node->wn_u1.hashkey[5] = NUL;
7988 /* Check for CTRL-C pressed now and then. */
7989 fast_breakcheck();
7991 return compressed;
7995 * Return TRUE when two nodes have identical siblings and children.
7997 static int
7998 node_equal(n1, n2)
7999 wordnode_T *n1;
8000 wordnode_T *n2;
8002 wordnode_T *p1;
8003 wordnode_T *p2;
8005 for (p1 = n1, p2 = n2; p1 != NULL && p2 != NULL;
8006 p1 = p1->wn_sibling, p2 = p2->wn_sibling)
8007 if (p1->wn_byte != p2->wn_byte
8008 || (p1->wn_byte == NUL
8009 ? (p1->wn_flags != p2->wn_flags
8010 || p1->wn_region != p2->wn_region
8011 || p1->wn_affixID != p2->wn_affixID)
8012 : (p1->wn_child != p2->wn_child)))
8013 break;
8015 return p1 == NULL && p2 == NULL;
8019 * Write a number to file "fd", MSB first, in "len" bytes.
8021 void
8022 put_bytes(fd, nr, len)
8023 FILE *fd;
8024 long_u nr;
8025 int len;
8027 int i;
8029 for (i = len - 1; i >= 0; --i)
8030 putc((int)(nr >> (i * 8)), fd);
8033 #ifdef _MSC_VER
8034 # if (_MSC_VER <= 1200)
8035 /* This line is required for VC6 without the service pack. Also see the
8036 * matching #pragma below. */
8037 # pragma optimize("", off)
8038 # endif
8039 #endif
8042 * Write spin->si_sugtime to file "fd".
8044 static void
8045 put_sugtime(spin, fd)
8046 spellinfo_T *spin;
8047 FILE *fd;
8049 int c;
8050 int i;
8052 /* time_t can be up to 8 bytes in size, more than long_u, thus we
8053 * can't use put_bytes() here. */
8054 for (i = 7; i >= 0; --i)
8055 if (i + 1 > sizeof(time_t))
8056 /* ">>" doesn't work well when shifting more bits than avail */
8057 putc(0, fd);
8058 else
8060 c = (unsigned)spin->si_sugtime >> (i * 8);
8061 putc(c, fd);
8065 #ifdef _MSC_VER
8066 # if (_MSC_VER <= 1200)
8067 # pragma optimize("", on)
8068 # endif
8069 #endif
8071 static int
8072 #ifdef __BORLANDC__
8073 _RTLENTRYF
8074 #endif
8075 rep_compare __ARGS((const void *s1, const void *s2));
8078 * Function given to qsort() to sort the REP items on "from" string.
8080 static int
8081 #ifdef __BORLANDC__
8082 _RTLENTRYF
8083 #endif
8084 rep_compare(s1, s2)
8085 const void *s1;
8086 const void *s2;
8088 fromto_T *p1 = (fromto_T *)s1;
8089 fromto_T *p2 = (fromto_T *)s2;
8091 return STRCMP(p1->ft_from, p2->ft_from);
8095 * Write the Vim .spl file "fname".
8096 * Return FAIL or OK;
8098 static int
8099 write_vim_spell(spin, fname)
8100 spellinfo_T *spin;
8101 char_u *fname;
8103 FILE *fd;
8104 int regionmask;
8105 int round;
8106 wordnode_T *tree;
8107 int nodecount;
8108 int i;
8109 int l;
8110 garray_T *gap;
8111 fromto_T *ftp;
8112 char_u *p;
8113 int rr;
8114 int retval = OK;
8115 size_t fwv = 1; /* collect return value of fwrite() to avoid
8116 warnings from picky compiler */
8118 fd = mch_fopen((char *)fname, "w");
8119 if (fd == NULL)
8121 EMSG2(_(e_notopen), fname);
8122 return FAIL;
8125 /* <HEADER>: <fileID> <versionnr> */
8126 /* <fileID> */
8127 fwv &= fwrite(VIMSPELLMAGIC, VIMSPELLMAGICL, (size_t)1, fd);
8128 if (fwv != (size_t)1)
8129 /* Catch first write error, don't try writing more. */
8130 goto theend;
8132 putc(VIMSPELLVERSION, fd); /* <versionnr> */
8135 * <SECTIONS>: <section> ... <sectionend>
8138 /* SN_INFO: <infotext> */
8139 if (spin->si_info != NULL)
8141 putc(SN_INFO, fd); /* <sectionID> */
8142 putc(0, fd); /* <sectionflags> */
8144 i = (int)STRLEN(spin->si_info);
8145 put_bytes(fd, (long_u)i, 4); /* <sectionlen> */
8146 fwv &= fwrite(spin->si_info, (size_t)i, (size_t)1, fd); /* <infotext> */
8149 /* SN_REGION: <regionname> ...
8150 * Write the region names only if there is more than one. */
8151 if (spin->si_region_count > 1)
8153 putc(SN_REGION, fd); /* <sectionID> */
8154 putc(SNF_REQUIRED, fd); /* <sectionflags> */
8155 l = spin->si_region_count * 2;
8156 put_bytes(fd, (long_u)l, 4); /* <sectionlen> */
8157 fwv &= fwrite(spin->si_region_name, (size_t)l, (size_t)1, fd);
8158 /* <regionname> ... */
8159 regionmask = (1 << spin->si_region_count) - 1;
8161 else
8162 regionmask = 0;
8164 /* SN_CHARFLAGS: <charflagslen> <charflags> <folcharslen> <folchars>
8166 * The table with character flags and the table for case folding.
8167 * This makes sure the same characters are recognized as word characters
8168 * when generating an when using a spell file.
8169 * Skip this for ASCII, the table may conflict with the one used for
8170 * 'encoding'.
8171 * Also skip this for an .add.spl file, the main spell file must contain
8172 * the table (avoids that it conflicts). File is shorter too.
8174 if (!spin->si_ascii && !spin->si_add)
8176 char_u folchars[128 * 8];
8177 int flags;
8179 putc(SN_CHARFLAGS, fd); /* <sectionID> */
8180 putc(SNF_REQUIRED, fd); /* <sectionflags> */
8182 /* Form the <folchars> string first, we need to know its length. */
8183 l = 0;
8184 for (i = 128; i < 256; ++i)
8186 #ifdef FEAT_MBYTE
8187 if (has_mbyte)
8188 l += mb_char2bytes(spelltab.st_fold[i], folchars + l);
8189 else
8190 #endif
8191 folchars[l++] = spelltab.st_fold[i];
8193 put_bytes(fd, (long_u)(1 + 128 + 2 + l), 4); /* <sectionlen> */
8195 fputc(128, fd); /* <charflagslen> */
8196 for (i = 128; i < 256; ++i)
8198 flags = 0;
8199 if (spelltab.st_isw[i])
8200 flags |= CF_WORD;
8201 if (spelltab.st_isu[i])
8202 flags |= CF_UPPER;
8203 fputc(flags, fd); /* <charflags> */
8206 put_bytes(fd, (long_u)l, 2); /* <folcharslen> */
8207 fwv &= fwrite(folchars, (size_t)l, (size_t)1, fd); /* <folchars> */
8210 /* SN_MIDWORD: <midword> */
8211 if (spin->si_midword != NULL)
8213 putc(SN_MIDWORD, fd); /* <sectionID> */
8214 putc(SNF_REQUIRED, fd); /* <sectionflags> */
8216 i = (int)STRLEN(spin->si_midword);
8217 put_bytes(fd, (long_u)i, 4); /* <sectionlen> */
8218 fwv &= fwrite(spin->si_midword, (size_t)i, (size_t)1, fd);
8219 /* <midword> */
8222 /* SN_PREFCOND: <prefcondcnt> <prefcond> ... */
8223 if (spin->si_prefcond.ga_len > 0)
8225 putc(SN_PREFCOND, fd); /* <sectionID> */
8226 putc(SNF_REQUIRED, fd); /* <sectionflags> */
8228 l = write_spell_prefcond(NULL, &spin->si_prefcond);
8229 put_bytes(fd, (long_u)l, 4); /* <sectionlen> */
8231 write_spell_prefcond(fd, &spin->si_prefcond);
8234 /* SN_REP: <repcount> <rep> ...
8235 * SN_SAL: <salflags> <salcount> <sal> ...
8236 * SN_REPSAL: <repcount> <rep> ... */
8238 /* round 1: SN_REP section
8239 * round 2: SN_SAL section (unless SN_SOFO is used)
8240 * round 3: SN_REPSAL section */
8241 for (round = 1; round <= 3; ++round)
8243 if (round == 1)
8244 gap = &spin->si_rep;
8245 else if (round == 2)
8247 /* Don't write SN_SAL when using a SN_SOFO section */
8248 if (spin->si_sofofr != NULL && spin->si_sofoto != NULL)
8249 continue;
8250 gap = &spin->si_sal;
8252 else
8253 gap = &spin->si_repsal;
8255 /* Don't write the section if there are no items. */
8256 if (gap->ga_len == 0)
8257 continue;
8259 /* Sort the REP/REPSAL items. */
8260 if (round != 2)
8261 qsort(gap->ga_data, (size_t)gap->ga_len,
8262 sizeof(fromto_T), rep_compare);
8264 i = round == 1 ? SN_REP : (round == 2 ? SN_SAL : SN_REPSAL);
8265 putc(i, fd); /* <sectionID> */
8267 /* This is for making suggestions, section is not required. */
8268 putc(0, fd); /* <sectionflags> */
8270 /* Compute the length of what follows. */
8271 l = 2; /* count <repcount> or <salcount> */
8272 for (i = 0; i < gap->ga_len; ++i)
8274 ftp = &((fromto_T *)gap->ga_data)[i];
8275 l += 1 + (int)STRLEN(ftp->ft_from); /* count <*fromlen> and <*from> */
8276 l += 1 + (int)STRLEN(ftp->ft_to); /* count <*tolen> and <*to> */
8278 if (round == 2)
8279 ++l; /* count <salflags> */
8280 put_bytes(fd, (long_u)l, 4); /* <sectionlen> */
8282 if (round == 2)
8284 i = 0;
8285 if (spin->si_followup)
8286 i |= SAL_F0LLOWUP;
8287 if (spin->si_collapse)
8288 i |= SAL_COLLAPSE;
8289 if (spin->si_rem_accents)
8290 i |= SAL_REM_ACCENTS;
8291 putc(i, fd); /* <salflags> */
8294 put_bytes(fd, (long_u)gap->ga_len, 2); /* <repcount> or <salcount> */
8295 for (i = 0; i < gap->ga_len; ++i)
8297 /* <rep> : <repfromlen> <repfrom> <reptolen> <repto> */
8298 /* <sal> : <salfromlen> <salfrom> <saltolen> <salto> */
8299 ftp = &((fromto_T *)gap->ga_data)[i];
8300 for (rr = 1; rr <= 2; ++rr)
8302 p = rr == 1 ? ftp->ft_from : ftp->ft_to;
8303 l = (int)STRLEN(p);
8304 putc(l, fd);
8305 if (l > 0)
8306 fwv &= fwrite(p, l, (size_t)1, fd);
8312 /* SN_SOFO: <sofofromlen> <sofofrom> <sofotolen> <sofoto>
8313 * This is for making suggestions, section is not required. */
8314 if (spin->si_sofofr != NULL && spin->si_sofoto != NULL)
8316 putc(SN_SOFO, fd); /* <sectionID> */
8317 putc(0, fd); /* <sectionflags> */
8319 l = (int)STRLEN(spin->si_sofofr);
8320 put_bytes(fd, (long_u)(l + STRLEN(spin->si_sofoto) + 4), 4);
8321 /* <sectionlen> */
8323 put_bytes(fd, (long_u)l, 2); /* <sofofromlen> */
8324 fwv &= fwrite(spin->si_sofofr, l, (size_t)1, fd); /* <sofofrom> */
8326 l = (int)STRLEN(spin->si_sofoto);
8327 put_bytes(fd, (long_u)l, 2); /* <sofotolen> */
8328 fwv &= fwrite(spin->si_sofoto, l, (size_t)1, fd); /* <sofoto> */
8331 /* SN_WORDS: <word> ...
8332 * This is for making suggestions, section is not required. */
8333 if (spin->si_commonwords.ht_used > 0)
8335 putc(SN_WORDS, fd); /* <sectionID> */
8336 putc(0, fd); /* <sectionflags> */
8338 /* round 1: count the bytes
8339 * round 2: write the bytes */
8340 for (round = 1; round <= 2; ++round)
8342 int todo;
8343 int len = 0;
8344 hashitem_T *hi;
8346 todo = (int)spin->si_commonwords.ht_used;
8347 for (hi = spin->si_commonwords.ht_array; todo > 0; ++hi)
8348 if (!HASHITEM_EMPTY(hi))
8350 l = (int)STRLEN(hi->hi_key) + 1;
8351 len += l;
8352 if (round == 2) /* <word> */
8353 fwv &= fwrite(hi->hi_key, (size_t)l, (size_t)1, fd);
8354 --todo;
8356 if (round == 1)
8357 put_bytes(fd, (long_u)len, 4); /* <sectionlen> */
8361 /* SN_MAP: <mapstr>
8362 * This is for making suggestions, section is not required. */
8363 if (spin->si_map.ga_len > 0)
8365 putc(SN_MAP, fd); /* <sectionID> */
8366 putc(0, fd); /* <sectionflags> */
8367 l = spin->si_map.ga_len;
8368 put_bytes(fd, (long_u)l, 4); /* <sectionlen> */
8369 fwv &= fwrite(spin->si_map.ga_data, (size_t)l, (size_t)1, fd);
8370 /* <mapstr> */
8373 /* SN_SUGFILE: <timestamp>
8374 * This is used to notify that a .sug file may be available and at the
8375 * same time allows for checking that a .sug file that is found matches
8376 * with this .spl file. That's because the word numbers must be exactly
8377 * right. */
8378 if (!spin->si_nosugfile
8379 && (spin->si_sal.ga_len > 0
8380 || (spin->si_sofofr != NULL && spin->si_sofoto != NULL)))
8382 putc(SN_SUGFILE, fd); /* <sectionID> */
8383 putc(0, fd); /* <sectionflags> */
8384 put_bytes(fd, (long_u)8, 4); /* <sectionlen> */
8386 /* Set si_sugtime and write it to the file. */
8387 spin->si_sugtime = time(NULL);
8388 put_sugtime(spin, fd); /* <timestamp> */
8391 /* SN_NOSPLITSUGS: nothing
8392 * This is used to notify that no suggestions with word splits are to be
8393 * made. */
8394 if (spin->si_nosplitsugs)
8396 putc(SN_NOSPLITSUGS, fd); /* <sectionID> */
8397 putc(0, fd); /* <sectionflags> */
8398 put_bytes(fd, (long_u)0, 4); /* <sectionlen> */
8401 /* SN_COMPOUND: compound info.
8402 * We don't mark it required, when not supported all compound words will
8403 * be bad words. */
8404 if (spin->si_compflags != NULL)
8406 putc(SN_COMPOUND, fd); /* <sectionID> */
8407 putc(0, fd); /* <sectionflags> */
8409 l = (int)STRLEN(spin->si_compflags);
8410 for (i = 0; i < spin->si_comppat.ga_len; ++i)
8411 l += (int)STRLEN(((char_u **)(spin->si_comppat.ga_data))[i]) + 1;
8412 put_bytes(fd, (long_u)(l + 7), 4); /* <sectionlen> */
8414 putc(spin->si_compmax, fd); /* <compmax> */
8415 putc(spin->si_compminlen, fd); /* <compminlen> */
8416 putc(spin->si_compsylmax, fd); /* <compsylmax> */
8417 putc(0, fd); /* for Vim 7.0b compatibility */
8418 putc(spin->si_compoptions, fd); /* <compoptions> */
8419 put_bytes(fd, (long_u)spin->si_comppat.ga_len, 2);
8420 /* <comppatcount> */
8421 for (i = 0; i < spin->si_comppat.ga_len; ++i)
8423 p = ((char_u **)(spin->si_comppat.ga_data))[i];
8424 putc((int)STRLEN(p), fd); /* <comppatlen> */
8425 fwv &= fwrite(p, (size_t)STRLEN(p), (size_t)1, fd);
8426 /* <comppattext> */
8428 /* <compflags> */
8429 fwv &= fwrite(spin->si_compflags, (size_t)STRLEN(spin->si_compflags),
8430 (size_t)1, fd);
8433 /* SN_NOBREAK: NOBREAK flag */
8434 if (spin->si_nobreak)
8436 putc(SN_NOBREAK, fd); /* <sectionID> */
8437 putc(0, fd); /* <sectionflags> */
8439 /* It's empty, the presence of the section flags the feature. */
8440 put_bytes(fd, (long_u)0, 4); /* <sectionlen> */
8443 /* SN_SYLLABLE: syllable info.
8444 * We don't mark it required, when not supported syllables will not be
8445 * counted. */
8446 if (spin->si_syllable != NULL)
8448 putc(SN_SYLLABLE, fd); /* <sectionID> */
8449 putc(0, fd); /* <sectionflags> */
8451 l = (int)STRLEN(spin->si_syllable);
8452 put_bytes(fd, (long_u)l, 4); /* <sectionlen> */
8453 fwv &= fwrite(spin->si_syllable, (size_t)l, (size_t)1, fd);
8454 /* <syllable> */
8457 /* end of <SECTIONS> */
8458 putc(SN_END, fd); /* <sectionend> */
8462 * <LWORDTREE> <KWORDTREE> <PREFIXTREE>
8464 spin->si_memtot = 0;
8465 for (round = 1; round <= 3; ++round)
8467 if (round == 1)
8468 tree = spin->si_foldroot->wn_sibling;
8469 else if (round == 2)
8470 tree = spin->si_keeproot->wn_sibling;
8471 else
8472 tree = spin->si_prefroot->wn_sibling;
8474 /* Clear the index and wnode fields in the tree. */
8475 clear_node(tree);
8477 /* Count the number of nodes. Needed to be able to allocate the
8478 * memory when reading the nodes. Also fills in index for shared
8479 * nodes. */
8480 nodecount = put_node(NULL, tree, 0, regionmask, round == 3);
8482 /* number of nodes in 4 bytes */
8483 put_bytes(fd, (long_u)nodecount, 4); /* <nodecount> */
8484 spin->si_memtot += nodecount + nodecount * sizeof(int);
8486 /* Write the nodes. */
8487 (void)put_node(fd, tree, 0, regionmask, round == 3);
8490 /* Write another byte to check for errors (file system full). */
8491 if (putc(0, fd) == EOF)
8492 retval = FAIL;
8493 theend:
8494 if (fclose(fd) == EOF)
8495 retval = FAIL;
8497 if (fwv != (size_t)1)
8498 retval = FAIL;
8499 if (retval == FAIL)
8500 EMSG(_(e_write));
8502 return retval;
8506 * Clear the index and wnode fields of "node", it siblings and its
8507 * children. This is needed because they are a union with other items to save
8508 * space.
8510 static void
8511 clear_node(node)
8512 wordnode_T *node;
8514 wordnode_T *np;
8516 if (node != NULL)
8517 for (np = node; np != NULL; np = np->wn_sibling)
8519 np->wn_u1.index = 0;
8520 np->wn_u2.wnode = NULL;
8522 if (np->wn_byte != NUL)
8523 clear_node(np->wn_child);
8529 * Dump a word tree at node "node".
8531 * This first writes the list of possible bytes (siblings). Then for each
8532 * byte recursively write the children.
8534 * NOTE: The code here must match the code in read_tree_node(), since
8535 * assumptions are made about the indexes (so that we don't have to write them
8536 * in the file).
8538 * Returns the number of nodes used.
8540 static int
8541 put_node(fd, node, idx, regionmask, prefixtree)
8542 FILE *fd; /* NULL when only counting */
8543 wordnode_T *node;
8544 int idx;
8545 int regionmask;
8546 int prefixtree; /* TRUE for PREFIXTREE */
8548 int newindex = idx;
8549 int siblingcount = 0;
8550 wordnode_T *np;
8551 int flags;
8553 /* If "node" is zero the tree is empty. */
8554 if (node == NULL)
8555 return 0;
8557 /* Store the index where this node is written. */
8558 node->wn_u1.index = idx;
8560 /* Count the number of siblings. */
8561 for (np = node; np != NULL; np = np->wn_sibling)
8562 ++siblingcount;
8564 /* Write the sibling count. */
8565 if (fd != NULL)
8566 putc(siblingcount, fd); /* <siblingcount> */
8568 /* Write each sibling byte and optionally extra info. */
8569 for (np = node; np != NULL; np = np->wn_sibling)
8571 if (np->wn_byte == 0)
8573 if (fd != NULL)
8575 /* For a NUL byte (end of word) write the flags etc. */
8576 if (prefixtree)
8578 /* In PREFIXTREE write the required affixID and the
8579 * associated condition nr (stored in wn_region). The
8580 * byte value is misused to store the "rare" and "not
8581 * combining" flags */
8582 if (np->wn_flags == (short_u)PFX_FLAGS)
8583 putc(BY_NOFLAGS, fd); /* <byte> */
8584 else
8586 putc(BY_FLAGS, fd); /* <byte> */
8587 putc(np->wn_flags, fd); /* <pflags> */
8589 putc(np->wn_affixID, fd); /* <affixID> */
8590 put_bytes(fd, (long_u)np->wn_region, 2); /* <prefcondnr> */
8592 else
8594 /* For word trees we write the flag/region items. */
8595 flags = np->wn_flags;
8596 if (regionmask != 0 && np->wn_region != regionmask)
8597 flags |= WF_REGION;
8598 if (np->wn_affixID != 0)
8599 flags |= WF_AFX;
8600 if (flags == 0)
8602 /* word without flags or region */
8603 putc(BY_NOFLAGS, fd); /* <byte> */
8605 else
8607 if (np->wn_flags >= 0x100)
8609 putc(BY_FLAGS2, fd); /* <byte> */
8610 putc(flags, fd); /* <flags> */
8611 putc((unsigned)flags >> 8, fd); /* <flags2> */
8613 else
8615 putc(BY_FLAGS, fd); /* <byte> */
8616 putc(flags, fd); /* <flags> */
8618 if (flags & WF_REGION)
8619 putc(np->wn_region, fd); /* <region> */
8620 if (flags & WF_AFX)
8621 putc(np->wn_affixID, fd); /* <affixID> */
8626 else
8628 if (np->wn_child->wn_u1.index != 0
8629 && np->wn_child->wn_u2.wnode != node)
8631 /* The child is written elsewhere, write the reference. */
8632 if (fd != NULL)
8634 putc(BY_INDEX, fd); /* <byte> */
8635 /* <nodeidx> */
8636 put_bytes(fd, (long_u)np->wn_child->wn_u1.index, 3);
8639 else if (np->wn_child->wn_u2.wnode == NULL)
8640 /* We will write the child below and give it an index. */
8641 np->wn_child->wn_u2.wnode = node;
8643 if (fd != NULL)
8644 if (putc(np->wn_byte, fd) == EOF) /* <byte> or <xbyte> */
8646 EMSG(_(e_write));
8647 return 0;
8652 /* Space used in the array when reading: one for each sibling and one for
8653 * the count. */
8654 newindex += siblingcount + 1;
8656 /* Recursively dump the children of each sibling. */
8657 for (np = node; np != NULL; np = np->wn_sibling)
8658 if (np->wn_byte != 0 && np->wn_child->wn_u2.wnode == node)
8659 newindex = put_node(fd, np->wn_child, newindex, regionmask,
8660 prefixtree);
8662 return newindex;
8667 * ":mkspell [-ascii] outfile infile ..."
8668 * ":mkspell [-ascii] addfile"
8670 void
8671 ex_mkspell(eap)
8672 exarg_T *eap;
8674 int fcount;
8675 char_u **fnames;
8676 char_u *arg = eap->arg;
8677 int ascii = FALSE;
8679 if (STRNCMP(arg, "-ascii", 6) == 0)
8681 ascii = TRUE;
8682 arg = skipwhite(arg + 6);
8685 /* Expand all the remaining arguments (e.g., $VIMRUNTIME). */
8686 if (get_arglist_exp(arg, &fcount, &fnames) == OK)
8688 mkspell(fcount, fnames, ascii, eap->forceit, FALSE);
8689 FreeWild(fcount, fnames);
8694 * Create the .sug file.
8695 * Uses the soundfold info in "spin".
8696 * Writes the file with the name "wfname", with ".spl" changed to ".sug".
8698 static void
8699 spell_make_sugfile(spin, wfname)
8700 spellinfo_T *spin;
8701 char_u *wfname;
8703 char_u fname[MAXPATHL];
8704 int len;
8705 slang_T *slang;
8706 int free_slang = FALSE;
8709 * Read back the .spl file that was written. This fills the required
8710 * info for soundfolding. This also uses less memory than the
8711 * pointer-linked version of the trie. And it avoids having two versions
8712 * of the code for the soundfolding stuff.
8713 * It might have been done already by spell_reload_one().
8715 for (slang = first_lang; slang != NULL; slang = slang->sl_next)
8716 if (fullpathcmp(wfname, slang->sl_fname, FALSE) == FPC_SAME)
8717 break;
8718 if (slang == NULL)
8720 spell_message(spin, (char_u *)_("Reading back spell file..."));
8721 slang = spell_load_file(wfname, NULL, NULL, FALSE);
8722 if (slang == NULL)
8723 return;
8724 free_slang = TRUE;
8728 * Clear the info in "spin" that is used.
8730 spin->si_blocks = NULL;
8731 spin->si_blocks_cnt = 0;
8732 spin->si_compress_cnt = 0; /* will stay at 0 all the time*/
8733 spin->si_free_count = 0;
8734 spin->si_first_free = NULL;
8735 spin->si_foldwcount = 0;
8738 * Go through the trie of good words, soundfold each word and add it to
8739 * the soundfold trie.
8741 spell_message(spin, (char_u *)_("Performing soundfolding..."));
8742 if (sug_filltree(spin, slang) == FAIL)
8743 goto theend;
8746 * Create the table which links each soundfold word with a list of the
8747 * good words it may come from. Creates buffer "spin->si_spellbuf".
8748 * This also removes the wordnr from the NUL byte entries to make
8749 * compression possible.
8751 if (sug_maketable(spin) == FAIL)
8752 goto theend;
8754 smsg((char_u *)_("Number of words after soundfolding: %ld"),
8755 (long)spin->si_spellbuf->b_ml.ml_line_count);
8758 * Compress the soundfold trie.
8760 spell_message(spin, (char_u *)_(msg_compressing));
8761 wordtree_compress(spin, spin->si_foldroot);
8764 * Write the .sug file.
8765 * Make the file name by changing ".spl" to ".sug".
8767 STRCPY(fname, wfname);
8768 len = (int)STRLEN(fname);
8769 fname[len - 2] = 'u';
8770 fname[len - 1] = 'g';
8771 sug_write(spin, fname);
8773 theend:
8774 if (free_slang)
8775 slang_free(slang);
8776 free_blocks(spin->si_blocks);
8777 close_spellbuf(spin->si_spellbuf);
8781 * Build the soundfold trie for language "slang".
8783 static int
8784 sug_filltree(spin, slang)
8785 spellinfo_T *spin;
8786 slang_T *slang;
8788 char_u *byts;
8789 idx_T *idxs;
8790 int depth;
8791 idx_T arridx[MAXWLEN];
8792 int curi[MAXWLEN];
8793 char_u tword[MAXWLEN];
8794 char_u tsalword[MAXWLEN];
8795 int c;
8796 idx_T n;
8797 unsigned words_done = 0;
8798 int wordcount[MAXWLEN];
8800 /* We use si_foldroot for the souldfolded trie. */
8801 spin->si_foldroot = wordtree_alloc(spin);
8802 if (spin->si_foldroot == NULL)
8803 return FAIL;
8805 /* let tree_add_word() know we're adding to the soundfolded tree */
8806 spin->si_sugtree = TRUE;
8809 * Go through the whole case-folded tree, soundfold each word and put it
8810 * in the trie.
8812 byts = slang->sl_fbyts;
8813 idxs = slang->sl_fidxs;
8815 arridx[0] = 0;
8816 curi[0] = 1;
8817 wordcount[0] = 0;
8819 depth = 0;
8820 while (depth >= 0 && !got_int)
8822 if (curi[depth] > byts[arridx[depth]])
8824 /* Done all bytes at this node, go up one level. */
8825 idxs[arridx[depth]] = wordcount[depth];
8826 if (depth > 0)
8827 wordcount[depth - 1] += wordcount[depth];
8829 --depth;
8830 line_breakcheck();
8832 else
8835 /* Do one more byte at this node. */
8836 n = arridx[depth] + curi[depth];
8837 ++curi[depth];
8839 c = byts[n];
8840 if (c == 0)
8842 /* Sound-fold the word. */
8843 tword[depth] = NUL;
8844 spell_soundfold(slang, tword, TRUE, tsalword);
8846 /* We use the "flags" field for the MSB of the wordnr,
8847 * "region" for the LSB of the wordnr. */
8848 if (tree_add_word(spin, tsalword, spin->si_foldroot,
8849 words_done >> 16, words_done & 0xffff,
8850 0) == FAIL)
8851 return FAIL;
8853 ++words_done;
8854 ++wordcount[depth];
8856 /* Reset the block count each time to avoid compression
8857 * kicking in. */
8858 spin->si_blocks_cnt = 0;
8860 /* Skip over any other NUL bytes (same word with different
8861 * flags). */
8862 while (byts[n + 1] == 0)
8864 ++n;
8865 ++curi[depth];
8868 else
8870 /* Normal char, go one level deeper. */
8871 tword[depth++] = c;
8872 arridx[depth] = idxs[n];
8873 curi[depth] = 1;
8874 wordcount[depth] = 0;
8879 smsg((char_u *)_("Total number of words: %d"), words_done);
8881 return OK;
8885 * Make the table that links each word in the soundfold trie to the words it
8886 * can be produced from.
8887 * This is not unlike lines in a file, thus use a memfile to be able to access
8888 * the table efficiently.
8889 * Returns FAIL when out of memory.
8891 static int
8892 sug_maketable(spin)
8893 spellinfo_T *spin;
8895 garray_T ga;
8896 int res = OK;
8898 /* Allocate a buffer, open a memline for it and create the swap file
8899 * (uses a temp file, not a .swp file). */
8900 spin->si_spellbuf = open_spellbuf();
8901 if (spin->si_spellbuf == NULL)
8902 return FAIL;
8904 /* Use a buffer to store the line info, avoids allocating many small
8905 * pieces of memory. */
8906 ga_init2(&ga, 1, 100);
8908 /* recursively go through the tree */
8909 if (sug_filltable(spin, spin->si_foldroot->wn_sibling, 0, &ga) == -1)
8910 res = FAIL;
8912 ga_clear(&ga);
8913 return res;
8917 * Fill the table for one node and its children.
8918 * Returns the wordnr at the start of the node.
8919 * Returns -1 when out of memory.
8921 static int
8922 sug_filltable(spin, node, startwordnr, gap)
8923 spellinfo_T *spin;
8924 wordnode_T *node;
8925 int startwordnr;
8926 garray_T *gap; /* place to store line of numbers */
8928 wordnode_T *p, *np;
8929 int wordnr = startwordnr;
8930 int nr;
8931 int prev_nr;
8933 for (p = node; p != NULL; p = p->wn_sibling)
8935 if (p->wn_byte == NUL)
8937 gap->ga_len = 0;
8938 prev_nr = 0;
8939 for (np = p; np != NULL && np->wn_byte == NUL; np = np->wn_sibling)
8941 if (ga_grow(gap, 10) == FAIL)
8942 return -1;
8944 nr = (np->wn_flags << 16) + (np->wn_region & 0xffff);
8945 /* Compute the offset from the previous nr and store the
8946 * offset in a way that it takes a minimum number of bytes.
8947 * It's a bit like utf-8, but without the need to mark
8948 * following bytes. */
8949 nr -= prev_nr;
8950 prev_nr += nr;
8951 gap->ga_len += offset2bytes(nr,
8952 (char_u *)gap->ga_data + gap->ga_len);
8955 /* add the NUL byte */
8956 ((char_u *)gap->ga_data)[gap->ga_len++] = NUL;
8958 if (ml_append_buf(spin->si_spellbuf, (linenr_T)wordnr,
8959 gap->ga_data, gap->ga_len, TRUE) == FAIL)
8960 return -1;
8961 ++wordnr;
8963 /* Remove extra NUL entries, we no longer need them. We don't
8964 * bother freeing the nodes, the won't be reused anyway. */
8965 while (p->wn_sibling != NULL && p->wn_sibling->wn_byte == NUL)
8966 p->wn_sibling = p->wn_sibling->wn_sibling;
8968 /* Clear the flags on the remaining NUL node, so that compression
8969 * works a lot better. */
8970 p->wn_flags = 0;
8971 p->wn_region = 0;
8973 else
8975 wordnr = sug_filltable(spin, p->wn_child, wordnr, gap);
8976 if (wordnr == -1)
8977 return -1;
8980 return wordnr;
8984 * Convert an offset into a minimal number of bytes.
8985 * Similar to utf_char2byters, but use 8 bits in followup bytes and avoid NUL
8986 * bytes.
8988 static int
8989 offset2bytes(nr, buf)
8990 int nr;
8991 char_u *buf;
8993 int rem;
8994 int b1, b2, b3, b4;
8996 /* Split the number in parts of base 255. We need to avoid NUL bytes. */
8997 b1 = nr % 255 + 1;
8998 rem = nr / 255;
8999 b2 = rem % 255 + 1;
9000 rem = rem / 255;
9001 b3 = rem % 255 + 1;
9002 b4 = rem / 255 + 1;
9004 if (b4 > 1 || b3 > 0x1f) /* 4 bytes */
9006 buf[0] = 0xe0 + b4;
9007 buf[1] = b3;
9008 buf[2] = b2;
9009 buf[3] = b1;
9010 return 4;
9012 if (b3 > 1 || b2 > 0x3f ) /* 3 bytes */
9014 buf[0] = 0xc0 + b3;
9015 buf[1] = b2;
9016 buf[2] = b1;
9017 return 3;
9019 if (b2 > 1 || b1 > 0x7f ) /* 2 bytes */
9021 buf[0] = 0x80 + b2;
9022 buf[1] = b1;
9023 return 2;
9025 /* 1 byte */
9026 buf[0] = b1;
9027 return 1;
9031 * Opposite of offset2bytes().
9032 * "pp" points to the bytes and is advanced over it.
9033 * Returns the offset.
9035 static int
9036 bytes2offset(pp)
9037 char_u **pp;
9039 char_u *p = *pp;
9040 int nr;
9041 int c;
9043 c = *p++;
9044 if ((c & 0x80) == 0x00) /* 1 byte */
9046 nr = c - 1;
9048 else if ((c & 0xc0) == 0x80) /* 2 bytes */
9050 nr = (c & 0x3f) - 1;
9051 nr = nr * 255 + (*p++ - 1);
9053 else if ((c & 0xe0) == 0xc0) /* 3 bytes */
9055 nr = (c & 0x1f) - 1;
9056 nr = nr * 255 + (*p++ - 1);
9057 nr = nr * 255 + (*p++ - 1);
9059 else /* 4 bytes */
9061 nr = (c & 0x0f) - 1;
9062 nr = nr * 255 + (*p++ - 1);
9063 nr = nr * 255 + (*p++ - 1);
9064 nr = nr * 255 + (*p++ - 1);
9067 *pp = p;
9068 return nr;
9072 * Write the .sug file in "fname".
9074 static void
9075 sug_write(spin, fname)
9076 spellinfo_T *spin;
9077 char_u *fname;
9079 FILE *fd;
9080 wordnode_T *tree;
9081 int nodecount;
9082 int wcount;
9083 char_u *line;
9084 linenr_T lnum;
9085 int len;
9087 /* Create the file. Note that an existing file is silently overwritten! */
9088 fd = mch_fopen((char *)fname, "w");
9089 if (fd == NULL)
9091 EMSG2(_(e_notopen), fname);
9092 return;
9095 vim_snprintf((char *)IObuff, IOSIZE,
9096 _("Writing suggestion file %s ..."), fname);
9097 spell_message(spin, IObuff);
9100 * <SUGHEADER>: <fileID> <versionnr> <timestamp>
9102 if (fwrite(VIMSUGMAGIC, VIMSUGMAGICL, (size_t)1, fd) != 1) /* <fileID> */
9104 EMSG(_(e_write));
9105 goto theend;
9107 putc(VIMSUGVERSION, fd); /* <versionnr> */
9109 /* Write si_sugtime to the file. */
9110 put_sugtime(spin, fd); /* <timestamp> */
9113 * <SUGWORDTREE>
9115 spin->si_memtot = 0;
9116 tree = spin->si_foldroot->wn_sibling;
9118 /* Clear the index and wnode fields in the tree. */
9119 clear_node(tree);
9121 /* Count the number of nodes. Needed to be able to allocate the
9122 * memory when reading the nodes. Also fills in index for shared
9123 * nodes. */
9124 nodecount = put_node(NULL, tree, 0, 0, FALSE);
9126 /* number of nodes in 4 bytes */
9127 put_bytes(fd, (long_u)nodecount, 4); /* <nodecount> */
9128 spin->si_memtot += nodecount + nodecount * sizeof(int);
9130 /* Write the nodes. */
9131 (void)put_node(fd, tree, 0, 0, FALSE);
9134 * <SUGTABLE>: <sugwcount> <sugline> ...
9136 wcount = spin->si_spellbuf->b_ml.ml_line_count;
9137 put_bytes(fd, (long_u)wcount, 4); /* <sugwcount> */
9139 for (lnum = 1; lnum <= (linenr_T)wcount; ++lnum)
9141 /* <sugline>: <sugnr> ... NUL */
9142 line = ml_get_buf(spin->si_spellbuf, lnum, FALSE);
9143 len = (int)STRLEN(line) + 1;
9144 if (fwrite(line, (size_t)len, (size_t)1, fd) == 0)
9146 EMSG(_(e_write));
9147 goto theend;
9149 spin->si_memtot += len;
9152 /* Write another byte to check for errors. */
9153 if (putc(0, fd) == EOF)
9154 EMSG(_(e_write));
9156 vim_snprintf((char *)IObuff, IOSIZE,
9157 _("Estimated runtime memory use: %d bytes"), spin->si_memtot);
9158 spell_message(spin, IObuff);
9160 theend:
9161 /* close the file */
9162 fclose(fd);
9166 * Open a spell buffer. This is a nameless buffer that is not in the buffer
9167 * list and only contains text lines. Can use a swapfile to reduce memory
9168 * use.
9169 * Most other fields are invalid! Esp. watch out for string options being
9170 * NULL and there is no undo info.
9171 * Returns NULL when out of memory.
9173 static buf_T *
9174 open_spellbuf()
9176 buf_T *buf;
9178 buf = (buf_T *)alloc_clear(sizeof(buf_T));
9179 if (buf != NULL)
9181 buf->b_spell = TRUE;
9182 buf->b_p_swf = TRUE; /* may create a swap file */
9183 ml_open(buf);
9184 ml_open_file(buf); /* create swap file now */
9186 return buf;
9190 * Close the buffer used for spell info.
9192 static void
9193 close_spellbuf(buf)
9194 buf_T *buf;
9196 if (buf != NULL)
9198 ml_close(buf, TRUE);
9199 vim_free(buf);
9205 * Create a Vim spell file from one or more word lists.
9206 * "fnames[0]" is the output file name.
9207 * "fnames[fcount - 1]" is the last input file name.
9208 * Exception: when "fnames[0]" ends in ".add" it's used as the input file name
9209 * and ".spl" is appended to make the output file name.
9211 static void
9212 mkspell(fcount, fnames, ascii, overwrite, added_word)
9213 int fcount;
9214 char_u **fnames;
9215 int ascii; /* -ascii argument given */
9216 int overwrite; /* overwrite existing output file */
9217 int added_word; /* invoked through "zg" */
9219 char_u fname[MAXPATHL];
9220 char_u wfname[MAXPATHL];
9221 char_u **innames;
9222 int incount;
9223 afffile_T *(afile[8]);
9224 int i;
9225 int len;
9226 struct stat st;
9227 int error = FALSE;
9228 spellinfo_T spin;
9230 vim_memset(&spin, 0, sizeof(spin));
9231 spin.si_verbose = !added_word;
9232 spin.si_ascii = ascii;
9233 spin.si_followup = TRUE;
9234 spin.si_rem_accents = TRUE;
9235 ga_init2(&spin.si_rep, (int)sizeof(fromto_T), 20);
9236 ga_init2(&spin.si_repsal, (int)sizeof(fromto_T), 20);
9237 ga_init2(&spin.si_sal, (int)sizeof(fromto_T), 20);
9238 ga_init2(&spin.si_map, (int)sizeof(char_u), 100);
9239 ga_init2(&spin.si_comppat, (int)sizeof(char_u *), 20);
9240 ga_init2(&spin.si_prefcond, (int)sizeof(char_u *), 50);
9241 hash_init(&spin.si_commonwords);
9242 spin.si_newcompID = 127; /* start compound ID at first maximum */
9244 /* default: fnames[0] is output file, following are input files */
9245 innames = &fnames[1];
9246 incount = fcount - 1;
9248 if (fcount >= 1)
9250 len = (int)STRLEN(fnames[0]);
9251 if (fcount == 1 && len > 4 && STRCMP(fnames[0] + len - 4, ".add") == 0)
9253 /* For ":mkspell path/en.latin1.add" output file is
9254 * "path/en.latin1.add.spl". */
9255 innames = &fnames[0];
9256 incount = 1;
9257 vim_snprintf((char *)wfname, sizeof(wfname), "%s.spl", fnames[0]);
9259 else if (fcount == 1)
9261 /* For ":mkspell path/vim" output file is "path/vim.latin1.spl". */
9262 innames = &fnames[0];
9263 incount = 1;
9264 vim_snprintf((char *)wfname, sizeof(wfname), "%s.%s.spl", fnames[0],
9265 spin.si_ascii ? (char_u *)"ascii" : spell_enc());
9267 else if (len > 4 && STRCMP(fnames[0] + len - 4, ".spl") == 0)
9269 /* Name ends in ".spl", use as the file name. */
9270 vim_strncpy(wfname, fnames[0], sizeof(wfname) - 1);
9272 else
9273 /* Name should be language, make the file name from it. */
9274 vim_snprintf((char *)wfname, sizeof(wfname), "%s.%s.spl", fnames[0],
9275 spin.si_ascii ? (char_u *)"ascii" : spell_enc());
9277 /* Check for .ascii.spl. */
9278 if (strstr((char *)gettail(wfname), ".ascii.") != NULL)
9279 spin.si_ascii = TRUE;
9281 /* Check for .add.spl. */
9282 if (strstr((char *)gettail(wfname), ".add.") != NULL)
9283 spin.si_add = TRUE;
9286 if (incount <= 0)
9287 EMSG(_(e_invarg)); /* need at least output and input names */
9288 else if (vim_strchr(gettail(wfname), '_') != NULL)
9289 EMSG(_("E751: Output file name must not have region name"));
9290 else if (incount > 8)
9291 EMSG(_("E754: Only up to 8 regions supported"));
9292 else
9294 /* Check for overwriting before doing things that may take a lot of
9295 * time. */
9296 if (!overwrite && mch_stat((char *)wfname, &st) >= 0)
9298 EMSG(_(e_exists));
9299 return;
9301 if (mch_isdir(wfname))
9303 EMSG2(_(e_isadir2), wfname);
9304 return;
9308 * Init the aff and dic pointers.
9309 * Get the region names if there are more than 2 arguments.
9311 for (i = 0; i < incount; ++i)
9313 afile[i] = NULL;
9315 if (incount > 1)
9317 len = (int)STRLEN(innames[i]);
9318 if (STRLEN(gettail(innames[i])) < 5
9319 || innames[i][len - 3] != '_')
9321 EMSG2(_("E755: Invalid region in %s"), innames[i]);
9322 return;
9324 spin.si_region_name[i * 2] = TOLOWER_ASC(innames[i][len - 2]);
9325 spin.si_region_name[i * 2 + 1] =
9326 TOLOWER_ASC(innames[i][len - 1]);
9329 spin.si_region_count = incount;
9331 spin.si_foldroot = wordtree_alloc(&spin);
9332 spin.si_keeproot = wordtree_alloc(&spin);
9333 spin.si_prefroot = wordtree_alloc(&spin);
9334 if (spin.si_foldroot == NULL
9335 || spin.si_keeproot == NULL
9336 || spin.si_prefroot == NULL)
9338 free_blocks(spin.si_blocks);
9339 return;
9342 /* When not producing a .add.spl file clear the character table when
9343 * we encounter one in the .aff file. This means we dump the current
9344 * one in the .spl file if the .aff file doesn't define one. That's
9345 * better than guessing the contents, the table will match a
9346 * previously loaded spell file. */
9347 if (!spin.si_add)
9348 spin.si_clear_chartab = TRUE;
9351 * Read all the .aff and .dic files.
9352 * Text is converted to 'encoding'.
9353 * Words are stored in the case-folded and keep-case trees.
9355 for (i = 0; i < incount && !error; ++i)
9357 spin.si_conv.vc_type = CONV_NONE;
9358 spin.si_region = 1 << i;
9360 vim_snprintf((char *)fname, sizeof(fname), "%s.aff", innames[i]);
9361 if (mch_stat((char *)fname, &st) >= 0)
9363 /* Read the .aff file. Will init "spin->si_conv" based on the
9364 * "SET" line. */
9365 afile[i] = spell_read_aff(&spin, fname);
9366 if (afile[i] == NULL)
9367 error = TRUE;
9368 else
9370 /* Read the .dic file and store the words in the trees. */
9371 vim_snprintf((char *)fname, sizeof(fname), "%s.dic",
9372 innames[i]);
9373 if (spell_read_dic(&spin, fname, afile[i]) == FAIL)
9374 error = TRUE;
9377 else
9379 /* No .aff file, try reading the file as a word list. Store
9380 * the words in the trees. */
9381 if (spell_read_wordfile(&spin, innames[i]) == FAIL)
9382 error = TRUE;
9385 #ifdef FEAT_MBYTE
9386 /* Free any conversion stuff. */
9387 convert_setup(&spin.si_conv, NULL, NULL);
9388 #endif
9391 if (spin.si_compflags != NULL && spin.si_nobreak)
9392 MSG(_("Warning: both compounding and NOBREAK specified"));
9394 if (!error && !got_int)
9397 * Combine tails in the tree.
9399 spell_message(&spin, (char_u *)_(msg_compressing));
9400 wordtree_compress(&spin, spin.si_foldroot);
9401 wordtree_compress(&spin, spin.si_keeproot);
9402 wordtree_compress(&spin, spin.si_prefroot);
9405 if (!error && !got_int)
9408 * Write the info in the spell file.
9410 vim_snprintf((char *)IObuff, IOSIZE,
9411 _("Writing spell file %s ..."), wfname);
9412 spell_message(&spin, IObuff);
9414 error = write_vim_spell(&spin, wfname) == FAIL;
9416 spell_message(&spin, (char_u *)_("Done!"));
9417 vim_snprintf((char *)IObuff, IOSIZE,
9418 _("Estimated runtime memory use: %d bytes"), spin.si_memtot);
9419 spell_message(&spin, IObuff);
9422 * If the file is loaded need to reload it.
9424 if (!error)
9425 spell_reload_one(wfname, added_word);
9428 /* Free the allocated memory. */
9429 ga_clear(&spin.si_rep);
9430 ga_clear(&spin.si_repsal);
9431 ga_clear(&spin.si_sal);
9432 ga_clear(&spin.si_map);
9433 ga_clear(&spin.si_comppat);
9434 ga_clear(&spin.si_prefcond);
9435 hash_clear_all(&spin.si_commonwords, 0);
9437 /* Free the .aff file structures. */
9438 for (i = 0; i < incount; ++i)
9439 if (afile[i] != NULL)
9440 spell_free_aff(afile[i]);
9442 /* Free all the bits and pieces at once. */
9443 free_blocks(spin.si_blocks);
9446 * If there is soundfolding info and no NOSUGFILE item create the
9447 * .sug file with the soundfolded word trie.
9449 if (spin.si_sugtime != 0 && !error && !got_int)
9450 spell_make_sugfile(&spin, wfname);
9456 * Display a message for spell file processing when 'verbose' is set or using
9457 * ":mkspell". "str" can be IObuff.
9459 static void
9460 spell_message(spin, str)
9461 spellinfo_T *spin;
9462 char_u *str;
9464 if (spin->si_verbose || p_verbose > 2)
9466 if (!spin->si_verbose)
9467 verbose_enter();
9468 MSG(str);
9469 out_flush();
9470 if (!spin->si_verbose)
9471 verbose_leave();
9476 * ":[count]spellgood {word}"
9477 * ":[count]spellwrong {word}"
9478 * ":[count]spellundo {word}"
9480 void
9481 ex_spell(eap)
9482 exarg_T *eap;
9484 spell_add_word(eap->arg, (int)STRLEN(eap->arg), eap->cmdidx == CMD_spellwrong,
9485 eap->forceit ? 0 : (int)eap->line2,
9486 eap->cmdidx == CMD_spellundo);
9490 * Add "word[len]" to 'spellfile' as a good or bad word.
9492 void
9493 spell_add_word(word, len, bad, idx, undo)
9494 char_u *word;
9495 int len;
9496 int bad;
9497 int idx; /* "zG" and "zW": zero, otherwise index in
9498 'spellfile' */
9499 int undo; /* TRUE for "zug", "zuG", "zuw" and "zuW" */
9501 FILE *fd = NULL;
9502 buf_T *buf = NULL;
9503 int new_spf = FALSE;
9504 char_u *fname;
9505 char_u fnamebuf[MAXPATHL];
9506 char_u line[MAXWLEN * 2];
9507 long fpos, fpos_next = 0;
9508 int i;
9509 char_u *spf;
9511 if (idx == 0) /* use internal wordlist */
9513 if (int_wordlist == NULL)
9515 int_wordlist = vim_tempname('s');
9516 if (int_wordlist == NULL)
9517 return;
9519 fname = int_wordlist;
9521 else
9523 /* If 'spellfile' isn't set figure out a good default value. */
9524 if (*curbuf->b_p_spf == NUL)
9526 init_spellfile();
9527 new_spf = TRUE;
9530 if (*curbuf->b_p_spf == NUL)
9532 EMSG2(_(e_notset), "spellfile");
9533 return;
9536 for (spf = curbuf->b_p_spf, i = 1; *spf != NUL; ++i)
9538 copy_option_part(&spf, fnamebuf, MAXPATHL, ",");
9539 if (i == idx)
9540 break;
9541 if (*spf == NUL)
9543 EMSGN(_("E765: 'spellfile' does not have %ld entries"), idx);
9544 return;
9548 /* Check that the user isn't editing the .add file somewhere. */
9549 buf = buflist_findname_exp(fnamebuf);
9550 if (buf != NULL && buf->b_ml.ml_mfp == NULL)
9551 buf = NULL;
9552 if (buf != NULL && bufIsChanged(buf))
9554 EMSG(_(e_bufloaded));
9555 return;
9558 fname = fnamebuf;
9561 if (bad || undo)
9563 /* When the word appears as good word we need to remove that one,
9564 * since its flags sort before the one with WF_BANNED. */
9565 fd = mch_fopen((char *)fname, "r");
9566 if (fd != NULL)
9568 while (!vim_fgets(line, MAXWLEN * 2, fd))
9570 fpos = fpos_next;
9571 fpos_next = ftell(fd);
9572 if (STRNCMP(word, line, len) == 0
9573 && (line[len] == '/' || line[len] < ' '))
9575 /* Found duplicate word. Remove it by writing a '#' at
9576 * the start of the line. Mixing reading and writing
9577 * doesn't work for all systems, close the file first. */
9578 fclose(fd);
9579 fd = mch_fopen((char *)fname, "r+");
9580 if (fd == NULL)
9581 break;
9582 if (fseek(fd, fpos, SEEK_SET) == 0)
9584 fputc('#', fd);
9585 if (undo)
9587 home_replace(NULL, fname, NameBuff, MAXPATHL, TRUE);
9588 smsg((char_u *)_("Word removed from %s"), NameBuff);
9591 fseek(fd, fpos_next, SEEK_SET);
9594 fclose(fd);
9598 if (!undo)
9600 fd = mch_fopen((char *)fname, "a");
9601 if (fd == NULL && new_spf)
9603 char_u *p;
9605 /* We just initialized the 'spellfile' option and can't open the
9606 * file. We may need to create the "spell" directory first. We
9607 * already checked the runtime directory is writable in
9608 * init_spellfile(). */
9609 if (!dir_of_file_exists(fname) && (p = gettail_sep(fname)) != fname)
9611 int c = *p;
9613 /* The directory doesn't exist. Try creating it and opening
9614 * the file again. */
9615 *p = NUL;
9616 vim_mkdir(fname, 0755);
9617 *p = c;
9618 fd = mch_fopen((char *)fname, "a");
9622 if (fd == NULL)
9623 EMSG2(_(e_notopen), fname);
9624 else
9626 if (bad)
9627 fprintf(fd, "%.*s/!\n", len, word);
9628 else
9629 fprintf(fd, "%.*s\n", len, word);
9630 fclose(fd);
9632 home_replace(NULL, fname, NameBuff, MAXPATHL, TRUE);
9633 smsg((char_u *)_("Word added to %s"), NameBuff);
9637 if (fd != NULL)
9639 /* Update the .add.spl file. */
9640 mkspell(1, &fname, FALSE, TRUE, TRUE);
9642 /* If the .add file is edited somewhere, reload it. */
9643 if (buf != NULL)
9644 buf_reload(buf, buf->b_orig_mode);
9646 redraw_all_later(SOME_VALID);
9651 * Initialize 'spellfile' for the current buffer.
9653 static void
9654 init_spellfile()
9656 char_u buf[MAXPATHL];
9657 int l;
9658 char_u *fname;
9659 char_u *rtp;
9660 char_u *lend;
9661 int aspath = FALSE;
9662 char_u *lstart = curbuf->b_p_spl;
9664 if (*curbuf->b_p_spl != NUL && curbuf->b_langp.ga_len > 0)
9666 /* Find the end of the language name. Exclude the region. If there
9667 * is a path separator remember the start of the tail. */
9668 for (lend = curbuf->b_p_spl; *lend != NUL
9669 && vim_strchr((char_u *)",._", *lend) == NULL; ++lend)
9670 if (vim_ispathsep(*lend))
9672 aspath = TRUE;
9673 lstart = lend + 1;
9676 /* Loop over all entries in 'runtimepath'. Use the first one where we
9677 * are allowed to write. */
9678 rtp = p_rtp;
9679 while (*rtp != NUL)
9681 if (aspath)
9682 /* Use directory of an entry with path, e.g., for
9683 * "/dir/lg.utf-8.spl" use "/dir". */
9684 vim_strncpy(buf, curbuf->b_p_spl, lstart - curbuf->b_p_spl - 1);
9685 else
9686 /* Copy the path from 'runtimepath' to buf[]. */
9687 copy_option_part(&rtp, buf, MAXPATHL, ",");
9688 if (filewritable(buf) == 2)
9690 /* Use the first language name from 'spelllang' and the
9691 * encoding used in the first loaded .spl file. */
9692 if (aspath)
9693 vim_strncpy(buf, curbuf->b_p_spl, lend - curbuf->b_p_spl);
9694 else
9696 /* Create the "spell" directory if it doesn't exist yet. */
9697 l = (int)STRLEN(buf);
9698 vim_snprintf((char *)buf + l, MAXPATHL - l, "/spell");
9699 if (!filewritable(buf) != 2)
9700 vim_mkdir(buf, 0755);
9702 l = (int)STRLEN(buf);
9703 vim_snprintf((char *)buf + l, MAXPATHL - l,
9704 "/%.*s", (int)(lend - lstart), lstart);
9706 l = (int)STRLEN(buf);
9707 fname = LANGP_ENTRY(curbuf->b_langp, 0)->lp_slang->sl_fname;
9708 vim_snprintf((char *)buf + l, MAXPATHL - l, ".%s.add",
9709 fname != NULL
9710 && strstr((char *)gettail(fname), ".ascii.") != NULL
9711 ? (char_u *)"ascii" : spell_enc());
9712 set_option_value((char_u *)"spellfile", 0L, buf, OPT_LOCAL);
9713 break;
9715 aspath = FALSE;
9722 * Init the chartab used for spelling for ASCII.
9723 * EBCDIC is not supported!
9725 static void
9726 clear_spell_chartab(sp)
9727 spelltab_T *sp;
9729 int i;
9731 /* Init everything to FALSE. */
9732 vim_memset(sp->st_isw, FALSE, sizeof(sp->st_isw));
9733 vim_memset(sp->st_isu, FALSE, sizeof(sp->st_isu));
9734 for (i = 0; i < 256; ++i)
9736 sp->st_fold[i] = i;
9737 sp->st_upper[i] = i;
9740 /* We include digits. A word shouldn't start with a digit, but handling
9741 * that is done separately. */
9742 for (i = '0'; i <= '9'; ++i)
9743 sp->st_isw[i] = TRUE;
9744 for (i = 'A'; i <= 'Z'; ++i)
9746 sp->st_isw[i] = TRUE;
9747 sp->st_isu[i] = TRUE;
9748 sp->st_fold[i] = i + 0x20;
9750 for (i = 'a'; i <= 'z'; ++i)
9752 sp->st_isw[i] = TRUE;
9753 sp->st_upper[i] = i - 0x20;
9758 * Init the chartab used for spelling. Only depends on 'encoding'.
9759 * Called once while starting up and when 'encoding' changes.
9760 * The default is to use isalpha(), but the spell file should define the word
9761 * characters to make it possible that 'encoding' differs from the current
9762 * locale. For utf-8 we don't use isalpha() but our own functions.
9764 void
9765 init_spell_chartab()
9767 int i;
9769 did_set_spelltab = FALSE;
9770 clear_spell_chartab(&spelltab);
9771 #ifdef FEAT_MBYTE
9772 if (enc_dbcs)
9774 /* DBCS: assume double-wide characters are word characters. */
9775 for (i = 128; i <= 255; ++i)
9776 if (MB_BYTE2LEN(i) == 2)
9777 spelltab.st_isw[i] = TRUE;
9779 else if (enc_utf8)
9781 for (i = 128; i < 256; ++i)
9783 spelltab.st_isu[i] = utf_isupper(i);
9784 spelltab.st_isw[i] = spelltab.st_isu[i] || utf_islower(i);
9785 spelltab.st_fold[i] = utf_fold(i);
9786 spelltab.st_upper[i] = utf_toupper(i);
9789 else
9790 #endif
9792 /* Rough guess: use locale-dependent library functions. */
9793 for (i = 128; i < 256; ++i)
9795 if (MB_ISUPPER(i))
9797 spelltab.st_isw[i] = TRUE;
9798 spelltab.st_isu[i] = TRUE;
9799 spelltab.st_fold[i] = MB_TOLOWER(i);
9801 else if (MB_ISLOWER(i))
9803 spelltab.st_isw[i] = TRUE;
9804 spelltab.st_upper[i] = MB_TOUPPER(i);
9811 * Set the spell character tables from strings in the affix file.
9813 static int
9814 set_spell_chartab(fol, low, upp)
9815 char_u *fol;
9816 char_u *low;
9817 char_u *upp;
9819 /* We build the new tables here first, so that we can compare with the
9820 * previous one. */
9821 spelltab_T new_st;
9822 char_u *pf = fol, *pl = low, *pu = upp;
9823 int f, l, u;
9825 clear_spell_chartab(&new_st);
9827 while (*pf != NUL)
9829 if (*pl == NUL || *pu == NUL)
9831 EMSG(_(e_affform));
9832 return FAIL;
9834 #ifdef FEAT_MBYTE
9835 f = mb_ptr2char_adv(&pf);
9836 l = mb_ptr2char_adv(&pl);
9837 u = mb_ptr2char_adv(&pu);
9838 #else
9839 f = *pf++;
9840 l = *pl++;
9841 u = *pu++;
9842 #endif
9843 /* Every character that appears is a word character. */
9844 if (f < 256)
9845 new_st.st_isw[f] = TRUE;
9846 if (l < 256)
9847 new_st.st_isw[l] = TRUE;
9848 if (u < 256)
9849 new_st.st_isw[u] = TRUE;
9851 /* if "LOW" and "FOL" are not the same the "LOW" char needs
9852 * case-folding */
9853 if (l < 256 && l != f)
9855 if (f >= 256)
9857 EMSG(_(e_affrange));
9858 return FAIL;
9860 new_st.st_fold[l] = f;
9863 /* if "UPP" and "FOL" are not the same the "UPP" char needs
9864 * case-folding, it's upper case and the "UPP" is the upper case of
9865 * "FOL" . */
9866 if (u < 256 && u != f)
9868 if (f >= 256)
9870 EMSG(_(e_affrange));
9871 return FAIL;
9873 new_st.st_fold[u] = f;
9874 new_st.st_isu[u] = TRUE;
9875 new_st.st_upper[f] = u;
9879 if (*pl != NUL || *pu != NUL)
9881 EMSG(_(e_affform));
9882 return FAIL;
9885 return set_spell_finish(&new_st);
9889 * Set the spell character tables from strings in the .spl file.
9891 static void
9892 set_spell_charflags(flags, cnt, fol)
9893 char_u *flags;
9894 int cnt; /* length of "flags" */
9895 char_u *fol;
9897 /* We build the new tables here first, so that we can compare with the
9898 * previous one. */
9899 spelltab_T new_st;
9900 int i;
9901 char_u *p = fol;
9902 int c;
9904 clear_spell_chartab(&new_st);
9906 for (i = 0; i < 128; ++i)
9908 if (i < cnt)
9910 new_st.st_isw[i + 128] = (flags[i] & CF_WORD) != 0;
9911 new_st.st_isu[i + 128] = (flags[i] & CF_UPPER) != 0;
9914 if (*p != NUL)
9916 #ifdef FEAT_MBYTE
9917 c = mb_ptr2char_adv(&p);
9918 #else
9919 c = *p++;
9920 #endif
9921 new_st.st_fold[i + 128] = c;
9922 if (i + 128 != c && new_st.st_isu[i + 128] && c < 256)
9923 new_st.st_upper[c] = i + 128;
9927 (void)set_spell_finish(&new_st);
9930 static int
9931 set_spell_finish(new_st)
9932 spelltab_T *new_st;
9934 int i;
9936 if (did_set_spelltab)
9938 /* check that it's the same table */
9939 for (i = 0; i < 256; ++i)
9941 if (spelltab.st_isw[i] != new_st->st_isw[i]
9942 || spelltab.st_isu[i] != new_st->st_isu[i]
9943 || spelltab.st_fold[i] != new_st->st_fold[i]
9944 || spelltab.st_upper[i] != new_st->st_upper[i])
9946 EMSG(_("E763: Word characters differ between spell files"));
9947 return FAIL;
9951 else
9953 /* copy the new spelltab into the one being used */
9954 spelltab = *new_st;
9955 did_set_spelltab = TRUE;
9958 return OK;
9962 * Return TRUE if "p" points to a word character.
9963 * As a special case we see "midword" characters as word character when it is
9964 * followed by a word character. This finds they'there but not 'they there'.
9965 * Thus this only works properly when past the first character of the word.
9967 static int
9968 spell_iswordp(p, buf)
9969 char_u *p;
9970 buf_T *buf; /* buffer used */
9972 #ifdef FEAT_MBYTE
9973 char_u *s;
9974 int l;
9975 int c;
9977 if (has_mbyte)
9979 l = MB_BYTE2LEN(*p);
9980 s = p;
9981 if (l == 1)
9983 /* be quick for ASCII */
9984 if (buf->b_spell_ismw[*p])
9986 s = p + 1; /* skip a mid-word character */
9987 l = MB_BYTE2LEN(*s);
9990 else
9992 c = mb_ptr2char(p);
9993 if (c < 256 ? buf->b_spell_ismw[c]
9994 : (buf->b_spell_ismw_mb != NULL
9995 && vim_strchr(buf->b_spell_ismw_mb, c) != NULL))
9997 s = p + l;
9998 l = MB_BYTE2LEN(*s);
10002 c = mb_ptr2char(s);
10003 if (c > 255)
10004 return spell_mb_isword_class(mb_get_class(s));
10005 return spelltab.st_isw[c];
10007 #endif
10009 return spelltab.st_isw[buf->b_spell_ismw[*p] ? p[1] : p[0]];
10013 * Return TRUE if "p" points to a word character.
10014 * Unlike spell_iswordp() this doesn't check for "midword" characters.
10016 static int
10017 spell_iswordp_nmw(p)
10018 char_u *p;
10020 #ifdef FEAT_MBYTE
10021 int c;
10023 if (has_mbyte)
10025 c = mb_ptr2char(p);
10026 if (c > 255)
10027 return spell_mb_isword_class(mb_get_class(p));
10028 return spelltab.st_isw[c];
10030 #endif
10031 return spelltab.st_isw[*p];
10034 #ifdef FEAT_MBYTE
10036 * Return TRUE if word class indicates a word character.
10037 * Only for characters above 255.
10038 * Unicode subscript and superscript are not considered word characters.
10040 static int
10041 spell_mb_isword_class(cl)
10042 int cl;
10044 return cl >= 2 && cl != 0x2070 && cl != 0x2080;
10048 * Return TRUE if "p" points to a word character.
10049 * Wide version of spell_iswordp().
10051 static int
10052 spell_iswordp_w(p, buf)
10053 int *p;
10054 buf_T *buf;
10056 int *s;
10058 if (*p < 256 ? buf->b_spell_ismw[*p]
10059 : (buf->b_spell_ismw_mb != NULL
10060 && vim_strchr(buf->b_spell_ismw_mb, *p) != NULL))
10061 s = p + 1;
10062 else
10063 s = p;
10065 if (*s > 255)
10067 if (enc_utf8)
10068 return spell_mb_isword_class(utf_class(*s));
10069 if (enc_dbcs)
10070 return dbcs_class((unsigned)*s >> 8, *s & 0xff) >= 2;
10071 return 0;
10073 return spelltab.st_isw[*s];
10075 #endif
10078 * Write the table with prefix conditions to the .spl file.
10079 * When "fd" is NULL only count the length of what is written.
10081 static int
10082 write_spell_prefcond(fd, gap)
10083 FILE *fd;
10084 garray_T *gap;
10086 int i;
10087 char_u *p;
10088 int len;
10089 int totlen;
10090 size_t x = 1; /* collect return value of fwrite() */
10092 if (fd != NULL)
10093 put_bytes(fd, (long_u)gap->ga_len, 2); /* <prefcondcnt> */
10095 totlen = 2 + gap->ga_len; /* length of <prefcondcnt> and <condlen> bytes */
10097 for (i = 0; i < gap->ga_len; ++i)
10099 /* <prefcond> : <condlen> <condstr> */
10100 p = ((char_u **)gap->ga_data)[i];
10101 if (p != NULL)
10103 len = (int)STRLEN(p);
10104 if (fd != NULL)
10106 fputc(len, fd);
10107 x &= fwrite(p, (size_t)len, (size_t)1, fd);
10109 totlen += len;
10111 else if (fd != NULL)
10112 fputc(0, fd);
10115 return totlen;
10119 * Case-fold "str[len]" into "buf[buflen]". The result is NUL terminated.
10120 * Uses the character definitions from the .spl file.
10121 * When using a multi-byte 'encoding' the length may change!
10122 * Returns FAIL when something wrong.
10124 static int
10125 spell_casefold(str, len, buf, buflen)
10126 char_u *str;
10127 int len;
10128 char_u *buf;
10129 int buflen;
10131 int i;
10133 if (len >= buflen)
10135 buf[0] = NUL;
10136 return FAIL; /* result will not fit */
10139 #ifdef FEAT_MBYTE
10140 if (has_mbyte)
10142 int outi = 0;
10143 char_u *p;
10144 int c;
10146 /* Fold one character at a time. */
10147 for (p = str; p < str + len; )
10149 if (outi + MB_MAXBYTES > buflen)
10151 buf[outi] = NUL;
10152 return FAIL;
10154 c = mb_cptr2char_adv(&p);
10155 outi += mb_char2bytes(SPELL_TOFOLD(c), buf + outi);
10157 buf[outi] = NUL;
10159 else
10160 #endif
10162 /* Be quick for non-multibyte encodings. */
10163 for (i = 0; i < len; ++i)
10164 buf[i] = spelltab.st_fold[str[i]];
10165 buf[i] = NUL;
10168 return OK;
10171 /* values for sps_flags */
10172 #define SPS_BEST 1
10173 #define SPS_FAST 2
10174 #define SPS_DOUBLE 4
10176 static int sps_flags = SPS_BEST; /* flags from 'spellsuggest' */
10177 static int sps_limit = 9999; /* max nr of suggestions given */
10180 * Check the 'spellsuggest' option. Return FAIL if it's wrong.
10181 * Sets "sps_flags" and "sps_limit".
10184 spell_check_sps()
10186 char_u *p;
10187 char_u *s;
10188 char_u buf[MAXPATHL];
10189 int f;
10191 sps_flags = 0;
10192 sps_limit = 9999;
10194 for (p = p_sps; *p != NUL; )
10196 copy_option_part(&p, buf, MAXPATHL, ",");
10198 f = 0;
10199 if (VIM_ISDIGIT(*buf))
10201 s = buf;
10202 sps_limit = getdigits(&s);
10203 if (*s != NUL && !VIM_ISDIGIT(*s))
10204 f = -1;
10206 else if (STRCMP(buf, "best") == 0)
10207 f = SPS_BEST;
10208 else if (STRCMP(buf, "fast") == 0)
10209 f = SPS_FAST;
10210 else if (STRCMP(buf, "double") == 0)
10211 f = SPS_DOUBLE;
10212 else if (STRNCMP(buf, "expr:", 5) != 0
10213 && STRNCMP(buf, "file:", 5) != 0)
10214 f = -1;
10216 if (f == -1 || (sps_flags != 0 && f != 0))
10218 sps_flags = SPS_BEST;
10219 sps_limit = 9999;
10220 return FAIL;
10222 if (f != 0)
10223 sps_flags = f;
10226 if (sps_flags == 0)
10227 sps_flags = SPS_BEST;
10229 return OK;
10233 * "z?": Find badly spelled word under or after the cursor.
10234 * Give suggestions for the properly spelled word.
10235 * In Visual mode use the highlighted word as the bad word.
10236 * When "count" is non-zero use that suggestion.
10238 void
10239 spell_suggest(count)
10240 int count;
10242 char_u *line;
10243 pos_T prev_cursor = curwin->w_cursor;
10244 char_u wcopy[MAXWLEN + 2];
10245 char_u *p;
10246 int i;
10247 int c;
10248 suginfo_T sug;
10249 suggest_T *stp;
10250 int mouse_used;
10251 int need_cap;
10252 int limit;
10253 int selected = count;
10254 int badlen = 0;
10256 if (no_spell_checking(curwin))
10257 return;
10259 #ifdef FEAT_VISUAL
10260 if (VIsual_active)
10262 /* Use the Visually selected text as the bad word. But reject
10263 * a multi-line selection. */
10264 if (curwin->w_cursor.lnum != VIsual.lnum)
10266 vim_beep();
10267 return;
10269 badlen = (int)curwin->w_cursor.col - (int)VIsual.col;
10270 if (badlen < 0)
10271 badlen = -badlen;
10272 else
10273 curwin->w_cursor.col = VIsual.col;
10274 ++badlen;
10275 end_visual_mode();
10277 else
10278 #endif
10279 /* Find the start of the badly spelled word. */
10280 if (spell_move_to(curwin, FORWARD, TRUE, TRUE, NULL) == 0
10281 || curwin->w_cursor.col > prev_cursor.col)
10283 /* No bad word or it starts after the cursor: use the word under the
10284 * cursor. */
10285 curwin->w_cursor = prev_cursor;
10286 line = ml_get_curline();
10287 p = line + curwin->w_cursor.col;
10288 /* Backup to before start of word. */
10289 while (p > line && spell_iswordp_nmw(p))
10290 mb_ptr_back(line, p);
10291 /* Forward to start of word. */
10292 while (*p != NUL && !spell_iswordp_nmw(p))
10293 mb_ptr_adv(p);
10295 if (!spell_iswordp_nmw(p)) /* No word found. */
10297 beep_flush();
10298 return;
10300 curwin->w_cursor.col = (colnr_T)(p - line);
10303 /* Get the word and its length. */
10305 /* Figure out if the word should be capitalised. */
10306 need_cap = check_need_cap(curwin->w_cursor.lnum, curwin->w_cursor.col);
10308 line = ml_get_curline();
10310 /* Get the list of suggestions. Limit to 'lines' - 2 or the number in
10311 * 'spellsuggest', whatever is smaller. */
10312 if (sps_limit > (int)Rows - 2)
10313 limit = (int)Rows - 2;
10314 else
10315 limit = sps_limit;
10316 spell_find_suggest(line + curwin->w_cursor.col, badlen, &sug, limit,
10317 TRUE, need_cap, TRUE);
10319 if (sug.su_ga.ga_len == 0)
10320 MSG(_("Sorry, no suggestions"));
10321 else if (count > 0)
10323 if (count > sug.su_ga.ga_len)
10324 smsg((char_u *)_("Sorry, only %ld suggestions"),
10325 (long)sug.su_ga.ga_len);
10327 else
10329 vim_free(repl_from);
10330 repl_from = NULL;
10331 vim_free(repl_to);
10332 repl_to = NULL;
10334 #ifdef FEAT_RIGHTLEFT
10335 /* When 'rightleft' is set the list is drawn right-left. */
10336 cmdmsg_rl = curwin->w_p_rl;
10337 if (cmdmsg_rl)
10338 msg_col = Columns - 1;
10339 #endif
10341 /* List the suggestions. */
10342 msg_start();
10343 msg_row = Rows - 1; /* for when 'cmdheight' > 1 */
10344 lines_left = Rows; /* avoid more prompt */
10345 vim_snprintf((char *)IObuff, IOSIZE, _("Change \"%.*s\" to:"),
10346 sug.su_badlen, sug.su_badptr);
10347 #ifdef FEAT_RIGHTLEFT
10348 if (cmdmsg_rl && STRNCMP(IObuff, "Change", 6) == 0)
10350 /* And now the rabbit from the high hat: Avoid showing the
10351 * untranslated message rightleft. */
10352 vim_snprintf((char *)IObuff, IOSIZE, ":ot \"%.*s\" egnahC",
10353 sug.su_badlen, sug.su_badptr);
10355 #endif
10356 msg_puts(IObuff);
10357 msg_clr_eos();
10358 msg_putchar('\n');
10360 msg_scroll = TRUE;
10361 for (i = 0; i < sug.su_ga.ga_len; ++i)
10363 stp = &SUG(sug.su_ga, i);
10365 /* The suggested word may replace only part of the bad word, add
10366 * the not replaced part. */
10367 STRCPY(wcopy, stp->st_word);
10368 if (sug.su_badlen > stp->st_orglen)
10369 vim_strncpy(wcopy + stp->st_wordlen,
10370 sug.su_badptr + stp->st_orglen,
10371 sug.su_badlen - stp->st_orglen);
10372 vim_snprintf((char *)IObuff, IOSIZE, "%2d", i + 1);
10373 #ifdef FEAT_RIGHTLEFT
10374 if (cmdmsg_rl)
10375 rl_mirror(IObuff);
10376 #endif
10377 msg_puts(IObuff);
10379 vim_snprintf((char *)IObuff, IOSIZE, " \"%s\"", wcopy);
10380 msg_puts(IObuff);
10382 /* The word may replace more than "su_badlen". */
10383 if (sug.su_badlen < stp->st_orglen)
10385 vim_snprintf((char *)IObuff, IOSIZE, _(" < \"%.*s\""),
10386 stp->st_orglen, sug.su_badptr);
10387 msg_puts(IObuff);
10390 if (p_verbose > 0)
10392 /* Add the score. */
10393 if (sps_flags & (SPS_DOUBLE | SPS_BEST))
10394 vim_snprintf((char *)IObuff, IOSIZE, " (%s%d - %d)",
10395 stp->st_salscore ? "s " : "",
10396 stp->st_score, stp->st_altscore);
10397 else
10398 vim_snprintf((char *)IObuff, IOSIZE, " (%d)",
10399 stp->st_score);
10400 #ifdef FEAT_RIGHTLEFT
10401 if (cmdmsg_rl)
10402 /* Mirror the numbers, but keep the leading space. */
10403 rl_mirror(IObuff + 1);
10404 #endif
10405 msg_advance(30);
10406 msg_puts(IObuff);
10408 msg_putchar('\n');
10411 #ifdef FEAT_RIGHTLEFT
10412 cmdmsg_rl = FALSE;
10413 msg_col = 0;
10414 #endif
10415 /* Ask for choice. */
10416 selected = prompt_for_number(&mouse_used);
10417 if (mouse_used)
10418 selected -= lines_left;
10419 lines_left = Rows; /* avoid more prompt */
10422 if (selected > 0 && selected <= sug.su_ga.ga_len && u_save_cursor() == OK)
10424 /* Save the from and to text for :spellrepall. */
10425 stp = &SUG(sug.su_ga, selected - 1);
10426 if (sug.su_badlen > stp->st_orglen)
10428 /* Replacing less than "su_badlen", append the remainder to
10429 * repl_to. */
10430 repl_from = vim_strnsave(sug.su_badptr, sug.su_badlen);
10431 vim_snprintf((char *)IObuff, IOSIZE, "%s%.*s", stp->st_word,
10432 sug.su_badlen - stp->st_orglen,
10433 sug.su_badptr + stp->st_orglen);
10434 repl_to = vim_strsave(IObuff);
10436 else
10438 /* Replacing su_badlen or more, use the whole word. */
10439 repl_from = vim_strnsave(sug.su_badptr, stp->st_orglen);
10440 repl_to = vim_strsave(stp->st_word);
10443 /* Replace the word. */
10444 p = alloc((unsigned)STRLEN(line) - stp->st_orglen + stp->st_wordlen + 1);
10445 if (p != NULL)
10447 c = (int)(sug.su_badptr - line);
10448 mch_memmove(p, line, c);
10449 STRCPY(p + c, stp->st_word);
10450 STRCAT(p, sug.su_badptr + stp->st_orglen);
10451 ml_replace(curwin->w_cursor.lnum, p, FALSE);
10452 curwin->w_cursor.col = c;
10454 /* For redo we use a change-word command. */
10455 ResetRedobuff();
10456 AppendToRedobuff((char_u *)"ciw");
10457 AppendToRedobuffLit(p + c,
10458 stp->st_wordlen + sug.su_badlen - stp->st_orglen);
10459 AppendCharToRedobuff(ESC);
10461 /* After this "p" may be invalid. */
10462 changed_bytes(curwin->w_cursor.lnum, c);
10465 else
10466 curwin->w_cursor = prev_cursor;
10468 spell_find_cleanup(&sug);
10472 * Check if the word at line "lnum" column "col" is required to start with a
10473 * capital. This uses 'spellcapcheck' of the current buffer.
10475 static int
10476 check_need_cap(lnum, col)
10477 linenr_T lnum;
10478 colnr_T col;
10480 int need_cap = FALSE;
10481 char_u *line;
10482 char_u *line_copy = NULL;
10483 char_u *p;
10484 colnr_T endcol;
10485 regmatch_T regmatch;
10487 if (curbuf->b_cap_prog == NULL)
10488 return FALSE;
10490 line = ml_get_curline();
10491 endcol = 0;
10492 if ((int)(skipwhite(line) - line) >= (int)col)
10494 /* At start of line, check if previous line is empty or sentence
10495 * ends there. */
10496 if (lnum == 1)
10497 need_cap = TRUE;
10498 else
10500 line = ml_get(lnum - 1);
10501 if (*skipwhite(line) == NUL)
10502 need_cap = TRUE;
10503 else
10505 /* Append a space in place of the line break. */
10506 line_copy = concat_str(line, (char_u *)" ");
10507 line = line_copy;
10508 endcol = (colnr_T)STRLEN(line);
10512 else
10513 endcol = col;
10515 if (endcol > 0)
10517 /* Check if sentence ends before the bad word. */
10518 regmatch.regprog = curbuf->b_cap_prog;
10519 regmatch.rm_ic = FALSE;
10520 p = line + endcol;
10521 for (;;)
10523 mb_ptr_back(line, p);
10524 if (p == line || spell_iswordp_nmw(p))
10525 break;
10526 if (vim_regexec(&regmatch, p, 0)
10527 && regmatch.endp[0] == line + endcol)
10529 need_cap = TRUE;
10530 break;
10535 vim_free(line_copy);
10537 return need_cap;
10542 * ":spellrepall"
10544 /*ARGSUSED*/
10545 void
10546 ex_spellrepall(eap)
10547 exarg_T *eap;
10549 pos_T pos = curwin->w_cursor;
10550 char_u *frompat;
10551 int addlen;
10552 char_u *line;
10553 char_u *p;
10554 int save_ws = p_ws;
10555 linenr_T prev_lnum = 0;
10557 if (repl_from == NULL || repl_to == NULL)
10559 EMSG(_("E752: No previous spell replacement"));
10560 return;
10562 addlen = (int)(STRLEN(repl_to) - STRLEN(repl_from));
10564 frompat = alloc((unsigned)STRLEN(repl_from) + 7);
10565 if (frompat == NULL)
10566 return;
10567 sprintf((char *)frompat, "\\V\\<%s\\>", repl_from);
10568 p_ws = FALSE;
10570 sub_nsubs = 0;
10571 sub_nlines = 0;
10572 curwin->w_cursor.lnum = 0;
10573 while (!got_int)
10575 if (do_search(NULL, '/', frompat, 1L, SEARCH_KEEP, NULL) == 0
10576 || u_save_cursor() == FAIL)
10577 break;
10579 /* Only replace when the right word isn't there yet. This happens
10580 * when changing "etc" to "etc.". */
10581 line = ml_get_curline();
10582 if (addlen <= 0 || STRNCMP(line + curwin->w_cursor.col,
10583 repl_to, STRLEN(repl_to)) != 0)
10585 p = alloc((unsigned)STRLEN(line) + addlen + 1);
10586 if (p == NULL)
10587 break;
10588 mch_memmove(p, line, curwin->w_cursor.col);
10589 STRCPY(p + curwin->w_cursor.col, repl_to);
10590 STRCAT(p, line + curwin->w_cursor.col + STRLEN(repl_from));
10591 ml_replace(curwin->w_cursor.lnum, p, FALSE);
10592 changed_bytes(curwin->w_cursor.lnum, curwin->w_cursor.col);
10594 if (curwin->w_cursor.lnum != prev_lnum)
10596 ++sub_nlines;
10597 prev_lnum = curwin->w_cursor.lnum;
10599 ++sub_nsubs;
10601 curwin->w_cursor.col += (colnr_T)STRLEN(repl_to);
10604 p_ws = save_ws;
10605 curwin->w_cursor = pos;
10606 vim_free(frompat);
10608 if (sub_nsubs == 0)
10609 EMSG2(_("E753: Not found: %s"), repl_from);
10610 else
10611 do_sub_msg(FALSE);
10615 * Find spell suggestions for "word". Return them in the growarray "*gap" as
10616 * a list of allocated strings.
10618 void
10619 spell_suggest_list(gap, word, maxcount, need_cap, interactive)
10620 garray_T *gap;
10621 char_u *word;
10622 int maxcount; /* maximum nr of suggestions */
10623 int need_cap; /* 'spellcapcheck' matched */
10624 int interactive;
10626 suginfo_T sug;
10627 int i;
10628 suggest_T *stp;
10629 char_u *wcopy;
10631 spell_find_suggest(word, 0, &sug, maxcount, FALSE, need_cap, interactive);
10633 /* Make room in "gap". */
10634 ga_init2(gap, sizeof(char_u *), sug.su_ga.ga_len + 1);
10635 if (ga_grow(gap, sug.su_ga.ga_len) == OK)
10637 for (i = 0; i < sug.su_ga.ga_len; ++i)
10639 stp = &SUG(sug.su_ga, i);
10641 /* The suggested word may replace only part of "word", add the not
10642 * replaced part. */
10643 wcopy = alloc(stp->st_wordlen
10644 + (unsigned)STRLEN(sug.su_badptr + stp->st_orglen) + 1);
10645 if (wcopy == NULL)
10646 break;
10647 STRCPY(wcopy, stp->st_word);
10648 STRCPY(wcopy + stp->st_wordlen, sug.su_badptr + stp->st_orglen);
10649 ((char_u **)gap->ga_data)[gap->ga_len++] = wcopy;
10653 spell_find_cleanup(&sug);
10657 * Find spell suggestions for the word at the start of "badptr".
10658 * Return the suggestions in "su->su_ga".
10659 * The maximum number of suggestions is "maxcount".
10660 * Note: does use info for the current window.
10661 * This is based on the mechanisms of Aspell, but completely reimplemented.
10663 static void
10664 spell_find_suggest(badptr, badlen, su, maxcount, banbadword, need_cap, interactive)
10665 char_u *badptr;
10666 int badlen; /* length of bad word or 0 if unknown */
10667 suginfo_T *su;
10668 int maxcount;
10669 int banbadword; /* don't include badword in suggestions */
10670 int need_cap; /* word should start with capital */
10671 int interactive;
10673 hlf_T attr = HLF_COUNT;
10674 char_u buf[MAXPATHL];
10675 char_u *p;
10676 int do_combine = FALSE;
10677 char_u *sps_copy;
10678 #ifdef FEAT_EVAL
10679 static int expr_busy = FALSE;
10680 #endif
10681 int c;
10682 int i;
10683 langp_T *lp;
10686 * Set the info in "*su".
10688 vim_memset(su, 0, sizeof(suginfo_T));
10689 ga_init2(&su->su_ga, (int)sizeof(suggest_T), 10);
10690 ga_init2(&su->su_sga, (int)sizeof(suggest_T), 10);
10691 if (*badptr == NUL)
10692 return;
10693 hash_init(&su->su_banned);
10695 su->su_badptr = badptr;
10696 if (badlen != 0)
10697 su->su_badlen = badlen;
10698 else
10699 su->su_badlen = spell_check(curwin, su->su_badptr, &attr, NULL, FALSE);
10700 su->su_maxcount = maxcount;
10701 su->su_maxscore = SCORE_MAXINIT;
10703 if (su->su_badlen >= MAXWLEN)
10704 su->su_badlen = MAXWLEN - 1; /* just in case */
10705 vim_strncpy(su->su_badword, su->su_badptr, su->su_badlen);
10706 (void)spell_casefold(su->su_badptr, su->su_badlen,
10707 su->su_fbadword, MAXWLEN);
10708 /* get caps flags for bad word */
10709 su->su_badflags = badword_captype(su->su_badptr,
10710 su->su_badptr + su->su_badlen);
10711 if (need_cap)
10712 su->su_badflags |= WF_ONECAP;
10714 /* Find the default language for sound folding. We simply use the first
10715 * one in 'spelllang' that supports sound folding. That's good for when
10716 * using multiple files for one language, it's not that bad when mixing
10717 * languages (e.g., "pl,en"). */
10718 for (i = 0; i < curbuf->b_langp.ga_len; ++i)
10720 lp = LANGP_ENTRY(curbuf->b_langp, i);
10721 if (lp->lp_sallang != NULL)
10723 su->su_sallang = lp->lp_sallang;
10724 break;
10728 /* Soundfold the bad word with the default sound folding, so that we don't
10729 * have to do this many times. */
10730 if (su->su_sallang != NULL)
10731 spell_soundfold(su->su_sallang, su->su_fbadword, TRUE,
10732 su->su_sal_badword);
10734 /* If the word is not capitalised and spell_check() doesn't consider the
10735 * word to be bad then it might need to be capitalised. Add a suggestion
10736 * for that. */
10737 c = PTR2CHAR(su->su_badptr);
10738 if (!SPELL_ISUPPER(c) && attr == HLF_COUNT)
10740 make_case_word(su->su_badword, buf, WF_ONECAP);
10741 add_suggestion(su, &su->su_ga, buf, su->su_badlen, SCORE_ICASE,
10742 0, TRUE, su->su_sallang, FALSE);
10745 /* Ban the bad word itself. It may appear in another region. */
10746 if (banbadword)
10747 add_banned(su, su->su_badword);
10749 /* Make a copy of 'spellsuggest', because the expression may change it. */
10750 sps_copy = vim_strsave(p_sps);
10751 if (sps_copy == NULL)
10752 return;
10754 /* Loop over the items in 'spellsuggest'. */
10755 for (p = sps_copy; *p != NUL; )
10757 copy_option_part(&p, buf, MAXPATHL, ",");
10759 if (STRNCMP(buf, "expr:", 5) == 0)
10761 #ifdef FEAT_EVAL
10762 /* Evaluate an expression. Skip this when called recursively,
10763 * when using spellsuggest() in the expression. */
10764 if (!expr_busy)
10766 expr_busy = TRUE;
10767 spell_suggest_expr(su, buf + 5);
10768 expr_busy = FALSE;
10770 #endif
10772 else if (STRNCMP(buf, "file:", 5) == 0)
10773 /* Use list of suggestions in a file. */
10774 spell_suggest_file(su, buf + 5);
10775 else
10777 /* Use internal method. */
10778 spell_suggest_intern(su, interactive);
10779 if (sps_flags & SPS_DOUBLE)
10780 do_combine = TRUE;
10784 vim_free(sps_copy);
10786 if (do_combine)
10787 /* Combine the two list of suggestions. This must be done last,
10788 * because sorting changes the order again. */
10789 score_combine(su);
10792 #ifdef FEAT_EVAL
10794 * Find suggestions by evaluating expression "expr".
10796 static void
10797 spell_suggest_expr(su, expr)
10798 suginfo_T *su;
10799 char_u *expr;
10801 list_T *list;
10802 listitem_T *li;
10803 int score;
10804 char_u *p;
10806 /* The work is split up in a few parts to avoid having to export
10807 * suginfo_T.
10808 * First evaluate the expression and get the resulting list. */
10809 list = eval_spell_expr(su->su_badword, expr);
10810 if (list != NULL)
10812 /* Loop over the items in the list. */
10813 for (li = list->lv_first; li != NULL; li = li->li_next)
10814 if (li->li_tv.v_type == VAR_LIST)
10816 /* Get the word and the score from the items. */
10817 score = get_spellword(li->li_tv.vval.v_list, &p);
10818 if (score >= 0 && score <= su->su_maxscore)
10819 add_suggestion(su, &su->su_ga, p, su->su_badlen,
10820 score, 0, TRUE, su->su_sallang, FALSE);
10822 list_unref(list);
10825 /* Remove bogus suggestions, sort and truncate at "maxcount". */
10826 check_suggestions(su, &su->su_ga);
10827 (void)cleanup_suggestions(&su->su_ga, su->su_maxscore, su->su_maxcount);
10829 #endif
10832 * Find suggestions in file "fname". Used for "file:" in 'spellsuggest'.
10834 static void
10835 spell_suggest_file(su, fname)
10836 suginfo_T *su;
10837 char_u *fname;
10839 FILE *fd;
10840 char_u line[MAXWLEN * 2];
10841 char_u *p;
10842 int len;
10843 char_u cword[MAXWLEN];
10845 /* Open the file. */
10846 fd = mch_fopen((char *)fname, "r");
10847 if (fd == NULL)
10849 EMSG2(_(e_notopen), fname);
10850 return;
10853 /* Read it line by line. */
10854 while (!vim_fgets(line, MAXWLEN * 2, fd) && !got_int)
10856 line_breakcheck();
10858 p = vim_strchr(line, '/');
10859 if (p == NULL)
10860 continue; /* No Tab found, just skip the line. */
10861 *p++ = NUL;
10862 if (STRICMP(su->su_badword, line) == 0)
10864 /* Match! Isolate the good word, until CR or NL. */
10865 for (len = 0; p[len] >= ' '; ++len)
10867 p[len] = NUL;
10869 /* If the suggestion doesn't have specific case duplicate the case
10870 * of the bad word. */
10871 if (captype(p, NULL) == 0)
10873 make_case_word(p, cword, su->su_badflags);
10874 p = cword;
10877 add_suggestion(su, &su->su_ga, p, su->su_badlen,
10878 SCORE_FILE, 0, TRUE, su->su_sallang, FALSE);
10882 fclose(fd);
10884 /* Remove bogus suggestions, sort and truncate at "maxcount". */
10885 check_suggestions(su, &su->su_ga);
10886 (void)cleanup_suggestions(&su->su_ga, su->su_maxscore, su->su_maxcount);
10890 * Find suggestions for the internal method indicated by "sps_flags".
10892 static void
10893 spell_suggest_intern(su, interactive)
10894 suginfo_T *su;
10895 int interactive;
10898 * Load the .sug file(s) that are available and not done yet.
10900 suggest_load_files();
10903 * 1. Try special cases, such as repeating a word: "the the" -> "the".
10905 * Set a maximum score to limit the combination of operations that is
10906 * tried.
10908 suggest_try_special(su);
10911 * 2. Try inserting/deleting/swapping/changing a letter, use REP entries
10912 * from the .aff file and inserting a space (split the word).
10914 suggest_try_change(su);
10916 /* For the resulting top-scorers compute the sound-a-like score. */
10917 if (sps_flags & SPS_DOUBLE)
10918 score_comp_sal(su);
10921 * 3. Try finding sound-a-like words.
10923 if ((sps_flags & SPS_FAST) == 0)
10925 if (sps_flags & SPS_BEST)
10926 /* Adjust the word score for the suggestions found so far for how
10927 * they sounds like. */
10928 rescore_suggestions(su);
10931 * While going throught the soundfold tree "su_maxscore" is the score
10932 * for the soundfold word, limits the changes that are being tried,
10933 * and "su_sfmaxscore" the rescored score, which is set by
10934 * cleanup_suggestions().
10935 * First find words with a small edit distance, because this is much
10936 * faster and often already finds the top-N suggestions. If we didn't
10937 * find many suggestions try again with a higher edit distance.
10938 * "sl_sounddone" is used to avoid doing the same word twice.
10940 suggest_try_soundalike_prep();
10941 su->su_maxscore = SCORE_SFMAX1;
10942 su->su_sfmaxscore = SCORE_MAXINIT * 3;
10943 suggest_try_soundalike(su);
10944 if (su->su_ga.ga_len < SUG_CLEAN_COUNT(su))
10946 /* We didn't find enough matches, try again, allowing more
10947 * changes to the soundfold word. */
10948 su->su_maxscore = SCORE_SFMAX2;
10949 suggest_try_soundalike(su);
10950 if (su->su_ga.ga_len < SUG_CLEAN_COUNT(su))
10952 /* Still didn't find enough matches, try again, allowing even
10953 * more changes to the soundfold word. */
10954 su->su_maxscore = SCORE_SFMAX3;
10955 suggest_try_soundalike(su);
10958 su->su_maxscore = su->su_sfmaxscore;
10959 suggest_try_soundalike_finish();
10962 /* When CTRL-C was hit while searching do show the results. Only clear
10963 * got_int when using a command, not for spellsuggest(). */
10964 ui_breakcheck();
10965 if (interactive && got_int)
10967 (void)vgetc();
10968 got_int = FALSE;
10971 if ((sps_flags & SPS_DOUBLE) == 0 && su->su_ga.ga_len != 0)
10973 if (sps_flags & SPS_BEST)
10974 /* Adjust the word score for how it sounds like. */
10975 rescore_suggestions(su);
10977 /* Remove bogus suggestions, sort and truncate at "maxcount". */
10978 check_suggestions(su, &su->su_ga);
10979 (void)cleanup_suggestions(&su->su_ga, su->su_maxscore, su->su_maxcount);
10984 * Load the .sug files for languages that have one and weren't loaded yet.
10986 static void
10987 suggest_load_files()
10989 langp_T *lp;
10990 int lpi;
10991 slang_T *slang;
10992 char_u *dotp;
10993 FILE *fd;
10994 char_u buf[MAXWLEN];
10995 int i;
10996 time_t timestamp;
10997 int wcount;
10998 int wordnr;
10999 garray_T ga;
11000 int c;
11002 /* Do this for all languages that support sound folding. */
11003 for (lpi = 0; lpi < curbuf->b_langp.ga_len; ++lpi)
11005 lp = LANGP_ENTRY(curbuf->b_langp, lpi);
11006 slang = lp->lp_slang;
11007 if (slang->sl_sugtime != 0 && !slang->sl_sugloaded)
11009 /* Change ".spl" to ".sug" and open the file. When the file isn't
11010 * found silently skip it. Do set "sl_sugloaded" so that we
11011 * don't try again and again. */
11012 slang->sl_sugloaded = TRUE;
11014 dotp = vim_strrchr(slang->sl_fname, '.');
11015 if (dotp == NULL || fnamecmp(dotp, ".spl") != 0)
11016 continue;
11017 STRCPY(dotp, ".sug");
11018 fd = mch_fopen((char *)slang->sl_fname, "r");
11019 if (fd == NULL)
11020 goto nextone;
11023 * <SUGHEADER>: <fileID> <versionnr> <timestamp>
11025 for (i = 0; i < VIMSUGMAGICL; ++i)
11026 buf[i] = getc(fd); /* <fileID> */
11027 if (STRNCMP(buf, VIMSUGMAGIC, VIMSUGMAGICL) != 0)
11029 EMSG2(_("E778: This does not look like a .sug file: %s"),
11030 slang->sl_fname);
11031 goto nextone;
11033 c = getc(fd); /* <versionnr> */
11034 if (c < VIMSUGVERSION)
11036 EMSG2(_("E779: Old .sug file, needs to be updated: %s"),
11037 slang->sl_fname);
11038 goto nextone;
11040 else if (c > VIMSUGVERSION)
11042 EMSG2(_("E780: .sug file is for newer version of Vim: %s"),
11043 slang->sl_fname);
11044 goto nextone;
11047 /* Check the timestamp, it must be exactly the same as the one in
11048 * the .spl file. Otherwise the word numbers won't match. */
11049 timestamp = get8c(fd); /* <timestamp> */
11050 if (timestamp != slang->sl_sugtime)
11052 EMSG2(_("E781: .sug file doesn't match .spl file: %s"),
11053 slang->sl_fname);
11054 goto nextone;
11058 * <SUGWORDTREE>: <wordtree>
11059 * Read the trie with the soundfolded words.
11061 if (spell_read_tree(fd, &slang->sl_sbyts, &slang->sl_sidxs,
11062 FALSE, 0) != 0)
11064 someerror:
11065 EMSG2(_("E782: error while reading .sug file: %s"),
11066 slang->sl_fname);
11067 slang_clear_sug(slang);
11068 goto nextone;
11072 * <SUGTABLE>: <sugwcount> <sugline> ...
11074 * Read the table with word numbers. We use a file buffer for
11075 * this, because it's so much like a file with lines. Makes it
11076 * possible to swap the info and save on memory use.
11078 slang->sl_sugbuf = open_spellbuf();
11079 if (slang->sl_sugbuf == NULL)
11080 goto someerror;
11081 /* <sugwcount> */
11082 wcount = get4c(fd);
11083 if (wcount < 0)
11084 goto someerror;
11086 /* Read all the wordnr lists into the buffer, one NUL terminated
11087 * list per line. */
11088 ga_init2(&ga, 1, 100);
11089 for (wordnr = 0; wordnr < wcount; ++wordnr)
11091 ga.ga_len = 0;
11092 for (;;)
11094 c = getc(fd); /* <sugline> */
11095 if (c < 0 || ga_grow(&ga, 1) == FAIL)
11096 goto someerror;
11097 ((char_u *)ga.ga_data)[ga.ga_len++] = c;
11098 if (c == NUL)
11099 break;
11101 if (ml_append_buf(slang->sl_sugbuf, (linenr_T)wordnr,
11102 ga.ga_data, ga.ga_len, TRUE) == FAIL)
11103 goto someerror;
11105 ga_clear(&ga);
11108 * Need to put word counts in the word tries, so that we can find
11109 * a word by its number.
11111 tree_count_words(slang->sl_fbyts, slang->sl_fidxs);
11112 tree_count_words(slang->sl_sbyts, slang->sl_sidxs);
11114 nextone:
11115 if (fd != NULL)
11116 fclose(fd);
11117 STRCPY(dotp, ".spl");
11124 * Fill in the wordcount fields for a trie.
11125 * Returns the total number of words.
11127 static void
11128 tree_count_words(byts, idxs)
11129 char_u *byts;
11130 idx_T *idxs;
11132 int depth;
11133 idx_T arridx[MAXWLEN];
11134 int curi[MAXWLEN];
11135 int c;
11136 idx_T n;
11137 int wordcount[MAXWLEN];
11139 arridx[0] = 0;
11140 curi[0] = 1;
11141 wordcount[0] = 0;
11142 depth = 0;
11143 while (depth >= 0 && !got_int)
11145 if (curi[depth] > byts[arridx[depth]])
11147 /* Done all bytes at this node, go up one level. */
11148 idxs[arridx[depth]] = wordcount[depth];
11149 if (depth > 0)
11150 wordcount[depth - 1] += wordcount[depth];
11152 --depth;
11153 fast_breakcheck();
11155 else
11157 /* Do one more byte at this node. */
11158 n = arridx[depth] + curi[depth];
11159 ++curi[depth];
11161 c = byts[n];
11162 if (c == 0)
11164 /* End of word, count it. */
11165 ++wordcount[depth];
11167 /* Skip over any other NUL bytes (same word with different
11168 * flags). */
11169 while (byts[n + 1] == 0)
11171 ++n;
11172 ++curi[depth];
11175 else
11177 /* Normal char, go one level deeper to count the words. */
11178 ++depth;
11179 arridx[depth] = idxs[n];
11180 curi[depth] = 1;
11181 wordcount[depth] = 0;
11188 * Free the info put in "*su" by spell_find_suggest().
11190 static void
11191 spell_find_cleanup(su)
11192 suginfo_T *su;
11194 int i;
11196 /* Free the suggestions. */
11197 for (i = 0; i < su->su_ga.ga_len; ++i)
11198 vim_free(SUG(su->su_ga, i).st_word);
11199 ga_clear(&su->su_ga);
11200 for (i = 0; i < su->su_sga.ga_len; ++i)
11201 vim_free(SUG(su->su_sga, i).st_word);
11202 ga_clear(&su->su_sga);
11204 /* Free the banned words. */
11205 hash_clear_all(&su->su_banned, 0);
11209 * Make a copy of "word", with the first letter upper or lower cased, to
11210 * "wcopy[MAXWLEN]". "word" must not be empty.
11211 * The result is NUL terminated.
11213 static void
11214 onecap_copy(word, wcopy, upper)
11215 char_u *word;
11216 char_u *wcopy;
11217 int upper; /* TRUE: first letter made upper case */
11219 char_u *p;
11220 int c;
11221 int l;
11223 p = word;
11224 #ifdef FEAT_MBYTE
11225 if (has_mbyte)
11226 c = mb_cptr2char_adv(&p);
11227 else
11228 #endif
11229 c = *p++;
11230 if (upper)
11231 c = SPELL_TOUPPER(c);
11232 else
11233 c = SPELL_TOFOLD(c);
11234 #ifdef FEAT_MBYTE
11235 if (has_mbyte)
11236 l = mb_char2bytes(c, wcopy);
11237 else
11238 #endif
11240 l = 1;
11241 wcopy[0] = c;
11243 vim_strncpy(wcopy + l, p, MAXWLEN - l - 1);
11247 * Make a copy of "word" with all the letters upper cased into
11248 * "wcopy[MAXWLEN]". The result is NUL terminated.
11250 static void
11251 allcap_copy(word, wcopy)
11252 char_u *word;
11253 char_u *wcopy;
11255 char_u *s;
11256 char_u *d;
11257 int c;
11259 d = wcopy;
11260 for (s = word; *s != NUL; )
11262 #ifdef FEAT_MBYTE
11263 if (has_mbyte)
11264 c = mb_cptr2char_adv(&s);
11265 else
11266 #endif
11267 c = *s++;
11269 #ifdef FEAT_MBYTE
11270 /* We only change ß to SS when we are certain latin1 is used. It
11271 * would cause weird errors in other 8-bit encodings. */
11272 if (enc_latin1like && c == 0xdf)
11274 c = 'S';
11275 if (d - wcopy >= MAXWLEN - 1)
11276 break;
11277 *d++ = c;
11279 else
11280 #endif
11281 c = SPELL_TOUPPER(c);
11283 #ifdef FEAT_MBYTE
11284 if (has_mbyte)
11286 if (d - wcopy >= MAXWLEN - MB_MAXBYTES)
11287 break;
11288 d += mb_char2bytes(c, d);
11290 else
11291 #endif
11293 if (d - wcopy >= MAXWLEN - 1)
11294 break;
11295 *d++ = c;
11298 *d = NUL;
11302 * Try finding suggestions by recognizing specific situations.
11304 static void
11305 suggest_try_special(su)
11306 suginfo_T *su;
11308 char_u *p;
11309 size_t len;
11310 int c;
11311 char_u word[MAXWLEN];
11314 * Recognize a word that is repeated: "the the".
11316 p = skiptowhite(su->su_fbadword);
11317 len = p - su->su_fbadword;
11318 p = skipwhite(p);
11319 if (STRLEN(p) == len && STRNCMP(su->su_fbadword, p, len) == 0)
11321 /* Include badflags: if the badword is onecap or allcap
11322 * use that for the goodword too: "The the" -> "The". */
11323 c = su->su_fbadword[len];
11324 su->su_fbadword[len] = NUL;
11325 make_case_word(su->su_fbadword, word, su->su_badflags);
11326 su->su_fbadword[len] = c;
11328 /* Give a soundalike score of 0, compute the score as if deleting one
11329 * character. */
11330 add_suggestion(su, &su->su_ga, word, su->su_badlen,
11331 RESCORE(SCORE_REP, 0), 0, TRUE, su->su_sallang, FALSE);
11336 * Try finding suggestions by adding/removing/swapping letters.
11338 static void
11339 suggest_try_change(su)
11340 suginfo_T *su;
11342 char_u fword[MAXWLEN]; /* copy of the bad word, case-folded */
11343 int n;
11344 char_u *p;
11345 int lpi;
11346 langp_T *lp;
11348 /* We make a copy of the case-folded bad word, so that we can modify it
11349 * to find matches (esp. REP items). Append some more text, changing
11350 * chars after the bad word may help. */
11351 STRCPY(fword, su->su_fbadword);
11352 n = (int)STRLEN(fword);
11353 p = su->su_badptr + su->su_badlen;
11354 (void)spell_casefold(p, (int)STRLEN(p), fword + n, MAXWLEN - n);
11356 for (lpi = 0; lpi < curbuf->b_langp.ga_len; ++lpi)
11358 lp = LANGP_ENTRY(curbuf->b_langp, lpi);
11360 /* If reloading a spell file fails it's still in the list but
11361 * everything has been cleared. */
11362 if (lp->lp_slang->sl_fbyts == NULL)
11363 continue;
11365 /* Try it for this language. Will add possible suggestions. */
11366 suggest_trie_walk(su, lp, fword, FALSE);
11370 /* Check the maximum score, if we go over it we won't try this change. */
11371 #define TRY_DEEPER(su, stack, depth, add) \
11372 (stack[depth].ts_score + (add) < su->su_maxscore)
11375 * Try finding suggestions by adding/removing/swapping letters.
11377 * This uses a state machine. At each node in the tree we try various
11378 * operations. When trying if an operation works "depth" is increased and the
11379 * stack[] is used to store info. This allows combinations, thus insert one
11380 * character, replace one and delete another. The number of changes is
11381 * limited by su->su_maxscore.
11383 * After implementing this I noticed an article by Kemal Oflazer that
11384 * describes something similar: "Error-tolerant Finite State Recognition with
11385 * Applications to Morphological Analysis and Spelling Correction" (1996).
11386 * The implementation in the article is simplified and requires a stack of
11387 * unknown depth. The implementation here only needs a stack depth equal to
11388 * the length of the word.
11390 * This is also used for the sound-folded word, "soundfold" is TRUE then.
11391 * The mechanism is the same, but we find a match with a sound-folded word
11392 * that comes from one or more original words. Each of these words may be
11393 * added, this is done by add_sound_suggest().
11394 * Don't use:
11395 * the prefix tree or the keep-case tree
11396 * "su->su_badlen"
11397 * anything to do with upper and lower case
11398 * anything to do with word or non-word characters ("spell_iswordp()")
11399 * banned words
11400 * word flags (rare, region, compounding)
11401 * word splitting for now
11402 * "similar_chars()"
11403 * use "slang->sl_repsal" instead of "lp->lp_replang->sl_rep"
11405 static void
11406 suggest_trie_walk(su, lp, fword, soundfold)
11407 suginfo_T *su;
11408 langp_T *lp;
11409 char_u *fword;
11410 int soundfold;
11412 char_u tword[MAXWLEN]; /* good word collected so far */
11413 trystate_T stack[MAXWLEN];
11414 char_u preword[MAXWLEN * 3]; /* word found with proper case;
11415 * concatanation of prefix compound
11416 * words and split word. NUL terminated
11417 * when going deeper but not when coming
11418 * back. */
11419 char_u compflags[MAXWLEN]; /* compound flags, one for each word */
11420 trystate_T *sp;
11421 int newscore;
11422 int score;
11423 char_u *byts, *fbyts, *pbyts;
11424 idx_T *idxs, *fidxs, *pidxs;
11425 int depth;
11426 int c, c2, c3;
11427 int n = 0;
11428 int flags;
11429 garray_T *gap;
11430 idx_T arridx;
11431 int len;
11432 char_u *p;
11433 fromto_T *ftp;
11434 int fl = 0, tl;
11435 int repextra = 0; /* extra bytes in fword[] from REP item */
11436 slang_T *slang = lp->lp_slang;
11437 int fword_ends;
11438 int goodword_ends;
11439 #ifdef DEBUG_TRIEWALK
11440 /* Stores the name of the change made at each level. */
11441 char_u changename[MAXWLEN][80];
11442 #endif
11443 int breakcheckcount = 1000;
11444 int compound_ok;
11447 * Go through the whole case-fold tree, try changes at each node.
11448 * "tword[]" contains the word collected from nodes in the tree.
11449 * "fword[]" the word we are trying to match with (initially the bad
11450 * word).
11452 depth = 0;
11453 sp = &stack[0];
11454 vim_memset(sp, 0, sizeof(trystate_T));
11455 sp->ts_curi = 1;
11457 if (soundfold)
11459 /* Going through the soundfold tree. */
11460 byts = fbyts = slang->sl_sbyts;
11461 idxs = fidxs = slang->sl_sidxs;
11462 pbyts = NULL;
11463 pidxs = NULL;
11464 sp->ts_prefixdepth = PFD_NOPREFIX;
11465 sp->ts_state = STATE_START;
11467 else
11470 * When there are postponed prefixes we need to use these first. At
11471 * the end of the prefix we continue in the case-fold tree.
11473 fbyts = slang->sl_fbyts;
11474 fidxs = slang->sl_fidxs;
11475 pbyts = slang->sl_pbyts;
11476 pidxs = slang->sl_pidxs;
11477 if (pbyts != NULL)
11479 byts = pbyts;
11480 idxs = pidxs;
11481 sp->ts_prefixdepth = PFD_PREFIXTREE;
11482 sp->ts_state = STATE_NOPREFIX; /* try without prefix first */
11484 else
11486 byts = fbyts;
11487 idxs = fidxs;
11488 sp->ts_prefixdepth = PFD_NOPREFIX;
11489 sp->ts_state = STATE_START;
11494 * Loop to find all suggestions. At each round we either:
11495 * - For the current state try one operation, advance "ts_curi",
11496 * increase "depth".
11497 * - When a state is done go to the next, set "ts_state".
11498 * - When all states are tried decrease "depth".
11500 while (depth >= 0 && !got_int)
11502 sp = &stack[depth];
11503 switch (sp->ts_state)
11505 case STATE_START:
11506 case STATE_NOPREFIX:
11508 * Start of node: Deal with NUL bytes, which means
11509 * tword[] may end here.
11511 arridx = sp->ts_arridx; /* current node in the tree */
11512 len = byts[arridx]; /* bytes in this node */
11513 arridx += sp->ts_curi; /* index of current byte */
11515 if (sp->ts_prefixdepth == PFD_PREFIXTREE)
11517 /* Skip over the NUL bytes, we use them later. */
11518 for (n = 0; n < len && byts[arridx + n] == 0; ++n)
11520 sp->ts_curi += n;
11522 /* Always past NUL bytes now. */
11523 n = (int)sp->ts_state;
11524 sp->ts_state = STATE_ENDNUL;
11525 sp->ts_save_badflags = su->su_badflags;
11527 /* At end of a prefix or at start of prefixtree: check for
11528 * following word. */
11529 if (byts[arridx] == 0 || n == (int)STATE_NOPREFIX)
11531 /* Set su->su_badflags to the caps type at this position.
11532 * Use the caps type until here for the prefix itself. */
11533 #ifdef FEAT_MBYTE
11534 if (has_mbyte)
11535 n = nofold_len(fword, sp->ts_fidx, su->su_badptr);
11536 else
11537 #endif
11538 n = sp->ts_fidx;
11539 flags = badword_captype(su->su_badptr, su->su_badptr + n);
11540 su->su_badflags = badword_captype(su->su_badptr + n,
11541 su->su_badptr + su->su_badlen);
11542 #ifdef DEBUG_TRIEWALK
11543 sprintf(changename[depth], "prefix");
11544 #endif
11545 go_deeper(stack, depth, 0);
11546 ++depth;
11547 sp = &stack[depth];
11548 sp->ts_prefixdepth = depth - 1;
11549 byts = fbyts;
11550 idxs = fidxs;
11551 sp->ts_arridx = 0;
11553 /* Move the prefix to preword[] with the right case
11554 * and make find_keepcap_word() works. */
11555 tword[sp->ts_twordlen] = NUL;
11556 make_case_word(tword + sp->ts_splitoff,
11557 preword + sp->ts_prewordlen, flags);
11558 sp->ts_prewordlen = (char_u)STRLEN(preword);
11559 sp->ts_splitoff = sp->ts_twordlen;
11561 break;
11564 if (sp->ts_curi > len || byts[arridx] != 0)
11566 /* Past bytes in node and/or past NUL bytes. */
11567 sp->ts_state = STATE_ENDNUL;
11568 sp->ts_save_badflags = su->su_badflags;
11569 break;
11573 * End of word in tree.
11575 ++sp->ts_curi; /* eat one NUL byte */
11577 flags = (int)idxs[arridx];
11579 /* Skip words with the NOSUGGEST flag. */
11580 if (flags & WF_NOSUGGEST)
11581 break;
11583 fword_ends = (fword[sp->ts_fidx] == NUL
11584 || (soundfold
11585 ? vim_iswhite(fword[sp->ts_fidx])
11586 : !spell_iswordp(fword + sp->ts_fidx, curbuf)));
11587 tword[sp->ts_twordlen] = NUL;
11589 if (sp->ts_prefixdepth <= PFD_NOTSPECIAL
11590 && (sp->ts_flags & TSF_PREFIXOK) == 0)
11592 /* There was a prefix before the word. Check that the prefix
11593 * can be used with this word. */
11594 /* Count the length of the NULs in the prefix. If there are
11595 * none this must be the first try without a prefix. */
11596 n = stack[sp->ts_prefixdepth].ts_arridx;
11597 len = pbyts[n++];
11598 for (c = 0; c < len && pbyts[n + c] == 0; ++c)
11600 if (c > 0)
11602 c = valid_word_prefix(c, n, flags,
11603 tword + sp->ts_splitoff, slang, FALSE);
11604 if (c == 0)
11605 break;
11607 /* Use the WF_RARE flag for a rare prefix. */
11608 if (c & WF_RAREPFX)
11609 flags |= WF_RARE;
11611 /* Tricky: when checking for both prefix and compounding
11612 * we run into the prefix flag first.
11613 * Remember that it's OK, so that we accept the prefix
11614 * when arriving at a compound flag. */
11615 sp->ts_flags |= TSF_PREFIXOK;
11619 /* Check NEEDCOMPOUND: can't use word without compounding. Do try
11620 * appending another compound word below. */
11621 if (sp->ts_complen == sp->ts_compsplit && fword_ends
11622 && (flags & WF_NEEDCOMP))
11623 goodword_ends = FALSE;
11624 else
11625 goodword_ends = TRUE;
11627 p = NULL;
11628 compound_ok = TRUE;
11629 if (sp->ts_complen > sp->ts_compsplit)
11631 if (slang->sl_nobreak)
11633 /* There was a word before this word. When there was no
11634 * change in this word (it was correct) add the first word
11635 * as a suggestion. If this word was corrected too, we
11636 * need to check if a correct word follows. */
11637 if (sp->ts_fidx - sp->ts_splitfidx
11638 == sp->ts_twordlen - sp->ts_splitoff
11639 && STRNCMP(fword + sp->ts_splitfidx,
11640 tword + sp->ts_splitoff,
11641 sp->ts_fidx - sp->ts_splitfidx) == 0)
11643 preword[sp->ts_prewordlen] = NUL;
11644 newscore = score_wordcount_adj(slang, sp->ts_score,
11645 preword + sp->ts_prewordlen,
11646 sp->ts_prewordlen > 0);
11647 /* Add the suggestion if the score isn't too bad. */
11648 if (newscore <= su->su_maxscore)
11649 add_suggestion(su, &su->su_ga, preword,
11650 sp->ts_splitfidx - repextra,
11651 newscore, 0, FALSE,
11652 lp->lp_sallang, FALSE);
11653 break;
11656 else
11658 /* There was a compound word before this word. If this
11659 * word does not support compounding then give up
11660 * (splitting is tried for the word without compound
11661 * flag). */
11662 if (((unsigned)flags >> 24) == 0
11663 || sp->ts_twordlen - sp->ts_splitoff
11664 < slang->sl_compminlen)
11665 break;
11666 #ifdef FEAT_MBYTE
11667 /* For multi-byte chars check character length against
11668 * COMPOUNDMIN. */
11669 if (has_mbyte
11670 && slang->sl_compminlen > 0
11671 && mb_charlen(tword + sp->ts_splitoff)
11672 < slang->sl_compminlen)
11673 break;
11674 #endif
11676 compflags[sp->ts_complen] = ((unsigned)flags >> 24);
11677 compflags[sp->ts_complen + 1] = NUL;
11678 vim_strncpy(preword + sp->ts_prewordlen,
11679 tword + sp->ts_splitoff,
11680 sp->ts_twordlen - sp->ts_splitoff);
11682 /* Verify CHECKCOMPOUNDPATTERN rules. */
11683 if (match_checkcompoundpattern(preword, sp->ts_prewordlen,
11684 &slang->sl_comppat))
11685 compound_ok = FALSE;
11687 if (compound_ok)
11689 p = preword;
11690 while (*skiptowhite(p) != NUL)
11691 p = skipwhite(skiptowhite(p));
11692 if (fword_ends && !can_compound(slang, p,
11693 compflags + sp->ts_compsplit))
11694 /* Compound is not allowed. But it may still be
11695 * possible if we add another (short) word. */
11696 compound_ok = FALSE;
11699 /* Get pointer to last char of previous word. */
11700 p = preword + sp->ts_prewordlen;
11701 mb_ptr_back(preword, p);
11706 * Form the word with proper case in preword.
11707 * If there is a word from a previous split, append.
11708 * For the soundfold tree don't change the case, simply append.
11710 if (soundfold)
11711 STRCPY(preword + sp->ts_prewordlen, tword + sp->ts_splitoff);
11712 else if (flags & WF_KEEPCAP)
11713 /* Must find the word in the keep-case tree. */
11714 find_keepcap_word(slang, tword + sp->ts_splitoff,
11715 preword + sp->ts_prewordlen);
11716 else
11718 /* Include badflags: If the badword is onecap or allcap
11719 * use that for the goodword too. But if the badword is
11720 * allcap and it's only one char long use onecap. */
11721 c = su->su_badflags;
11722 if ((c & WF_ALLCAP)
11723 #ifdef FEAT_MBYTE
11724 && su->su_badlen == (*mb_ptr2len)(su->su_badptr)
11725 #else
11726 && su->su_badlen == 1
11727 #endif
11729 c = WF_ONECAP;
11730 c |= flags;
11732 /* When appending a compound word after a word character don't
11733 * use Onecap. */
11734 if (p != NULL && spell_iswordp_nmw(p))
11735 c &= ~WF_ONECAP;
11736 make_case_word(tword + sp->ts_splitoff,
11737 preword + sp->ts_prewordlen, c);
11740 if (!soundfold)
11742 /* Don't use a banned word. It may appear again as a good
11743 * word, thus remember it. */
11744 if (flags & WF_BANNED)
11746 add_banned(su, preword + sp->ts_prewordlen);
11747 break;
11749 if ((sp->ts_complen == sp->ts_compsplit
11750 && WAS_BANNED(su, preword + sp->ts_prewordlen))
11751 || WAS_BANNED(su, preword))
11753 if (slang->sl_compprog == NULL)
11754 break;
11755 /* the word so far was banned but we may try compounding */
11756 goodword_ends = FALSE;
11760 newscore = 0;
11761 if (!soundfold) /* soundfold words don't have flags */
11763 if ((flags & WF_REGION)
11764 && (((unsigned)flags >> 16) & lp->lp_region) == 0)
11765 newscore += SCORE_REGION;
11766 if (flags & WF_RARE)
11767 newscore += SCORE_RARE;
11769 if (!spell_valid_case(su->su_badflags,
11770 captype(preword + sp->ts_prewordlen, NULL)))
11771 newscore += SCORE_ICASE;
11774 /* TODO: how about splitting in the soundfold tree? */
11775 if (fword_ends
11776 && goodword_ends
11777 && sp->ts_fidx >= sp->ts_fidxtry
11778 && compound_ok)
11780 /* The badword also ends: add suggestions. */
11781 #ifdef DEBUG_TRIEWALK
11782 if (soundfold && STRCMP(preword, "smwrd") == 0)
11784 int j;
11786 /* print the stack of changes that brought us here */
11787 smsg("------ %s -------", fword);
11788 for (j = 0; j < depth; ++j)
11789 smsg("%s", changename[j]);
11791 #endif
11792 if (soundfold)
11794 /* For soundfolded words we need to find the original
11795 * words, the edit distance and then add them. */
11796 add_sound_suggest(su, preword, sp->ts_score, lp);
11798 else
11800 /* Give a penalty when changing non-word char to word
11801 * char, e.g., "thes," -> "these". */
11802 p = fword + sp->ts_fidx;
11803 mb_ptr_back(fword, p);
11804 if (!spell_iswordp(p, curbuf))
11806 p = preword + STRLEN(preword);
11807 mb_ptr_back(preword, p);
11808 if (spell_iswordp(p, curbuf))
11809 newscore += SCORE_NONWORD;
11812 /* Give a bonus to words seen before. */
11813 score = score_wordcount_adj(slang,
11814 sp->ts_score + newscore,
11815 preword + sp->ts_prewordlen,
11816 sp->ts_prewordlen > 0);
11818 /* Add the suggestion if the score isn't too bad. */
11819 if (score <= su->su_maxscore)
11821 add_suggestion(su, &su->su_ga, preword,
11822 sp->ts_fidx - repextra,
11823 score, 0, FALSE, lp->lp_sallang, FALSE);
11825 if (su->su_badflags & WF_MIXCAP)
11827 /* We really don't know if the word should be
11828 * upper or lower case, add both. */
11829 c = captype(preword, NULL);
11830 if (c == 0 || c == WF_ALLCAP)
11832 make_case_word(tword + sp->ts_splitoff,
11833 preword + sp->ts_prewordlen,
11834 c == 0 ? WF_ALLCAP : 0);
11836 add_suggestion(su, &su->su_ga, preword,
11837 sp->ts_fidx - repextra,
11838 score + SCORE_ICASE, 0, FALSE,
11839 lp->lp_sallang, FALSE);
11847 * Try word split and/or compounding.
11849 if ((sp->ts_fidx >= sp->ts_fidxtry || fword_ends)
11850 #ifdef FEAT_MBYTE
11851 /* Don't split halfway a character. */
11852 && (!has_mbyte || sp->ts_tcharlen == 0)
11853 #endif
11856 int try_compound;
11857 int try_split;
11859 /* If past the end of the bad word don't try a split.
11860 * Otherwise try changing the next word. E.g., find
11861 * suggestions for "the the" where the second "the" is
11862 * different. It's done like a split.
11863 * TODO: word split for soundfold words */
11864 try_split = (sp->ts_fidx - repextra < su->su_badlen)
11865 && !soundfold;
11867 /* Get here in several situations:
11868 * 1. The word in the tree ends:
11869 * If the word allows compounding try that. Otherwise try
11870 * a split by inserting a space. For both check that a
11871 * valid words starts at fword[sp->ts_fidx].
11872 * For NOBREAK do like compounding to be able to check if
11873 * the next word is valid.
11874 * 2. The badword does end, but it was due to a change (e.g.,
11875 * a swap). No need to split, but do check that the
11876 * following word is valid.
11877 * 3. The badword and the word in the tree end. It may still
11878 * be possible to compound another (short) word.
11880 try_compound = FALSE;
11881 if (!soundfold
11882 && slang->sl_compprog != NULL
11883 && ((unsigned)flags >> 24) != 0
11884 && sp->ts_twordlen - sp->ts_splitoff
11885 >= slang->sl_compminlen
11886 #ifdef FEAT_MBYTE
11887 && (!has_mbyte
11888 || slang->sl_compminlen == 0
11889 || mb_charlen(tword + sp->ts_splitoff)
11890 >= slang->sl_compminlen)
11891 #endif
11892 && (slang->sl_compsylmax < MAXWLEN
11893 || sp->ts_complen + 1 - sp->ts_compsplit
11894 < slang->sl_compmax)
11895 && (can_be_compound(sp, slang,
11896 compflags, ((unsigned)flags >> 24))))
11899 try_compound = TRUE;
11900 compflags[sp->ts_complen] = ((unsigned)flags >> 24);
11901 compflags[sp->ts_complen + 1] = NUL;
11904 /* For NOBREAK we never try splitting, it won't make any word
11905 * valid. */
11906 if (slang->sl_nobreak)
11907 try_compound = TRUE;
11909 /* If we could add a compound word, and it's also possible to
11910 * split at this point, do the split first and set
11911 * TSF_DIDSPLIT to avoid doing it again. */
11912 else if (!fword_ends
11913 && try_compound
11914 && (sp->ts_flags & TSF_DIDSPLIT) == 0)
11916 try_compound = FALSE;
11917 sp->ts_flags |= TSF_DIDSPLIT;
11918 --sp->ts_curi; /* do the same NUL again */
11919 compflags[sp->ts_complen] = NUL;
11921 else
11922 sp->ts_flags &= ~TSF_DIDSPLIT;
11924 if (try_split || try_compound)
11926 if (!try_compound && (!fword_ends || !goodword_ends))
11928 /* If we're going to split need to check that the
11929 * words so far are valid for compounding. If there
11930 * is only one word it must not have the NEEDCOMPOUND
11931 * flag. */
11932 if (sp->ts_complen == sp->ts_compsplit
11933 && (flags & WF_NEEDCOMP))
11934 break;
11935 p = preword;
11936 while (*skiptowhite(p) != NUL)
11937 p = skipwhite(skiptowhite(p));
11938 if (sp->ts_complen > sp->ts_compsplit
11939 && !can_compound(slang, p,
11940 compflags + sp->ts_compsplit))
11941 break;
11943 if (slang->sl_nosplitsugs)
11944 newscore += SCORE_SPLIT_NO;
11945 else
11946 newscore += SCORE_SPLIT;
11948 /* Give a bonus to words seen before. */
11949 newscore = score_wordcount_adj(slang, newscore,
11950 preword + sp->ts_prewordlen, TRUE);
11953 if (TRY_DEEPER(su, stack, depth, newscore))
11955 go_deeper(stack, depth, newscore);
11956 #ifdef DEBUG_TRIEWALK
11957 if (!try_compound && !fword_ends)
11958 sprintf(changename[depth], "%.*s-%s: split",
11959 sp->ts_twordlen, tword, fword + sp->ts_fidx);
11960 else
11961 sprintf(changename[depth], "%.*s-%s: compound",
11962 sp->ts_twordlen, tword, fword + sp->ts_fidx);
11963 #endif
11964 /* Save things to be restored at STATE_SPLITUNDO. */
11965 sp->ts_save_badflags = su->su_badflags;
11966 sp->ts_state = STATE_SPLITUNDO;
11968 ++depth;
11969 sp = &stack[depth];
11971 /* Append a space to preword when splitting. */
11972 if (!try_compound && !fword_ends)
11973 STRCAT(preword, " ");
11974 sp->ts_prewordlen = (char_u)STRLEN(preword);
11975 sp->ts_splitoff = sp->ts_twordlen;
11976 sp->ts_splitfidx = sp->ts_fidx;
11978 /* If the badword has a non-word character at this
11979 * position skip it. That means replacing the
11980 * non-word character with a space. Always skip a
11981 * character when the word ends. But only when the
11982 * good word can end. */
11983 if (((!try_compound && !spell_iswordp_nmw(fword
11984 + sp->ts_fidx))
11985 || fword_ends)
11986 && fword[sp->ts_fidx] != NUL
11987 && goodword_ends)
11989 int l;
11991 #ifdef FEAT_MBYTE
11992 if (has_mbyte)
11993 l = MB_BYTE2LEN(fword[sp->ts_fidx]);
11994 else
11995 #endif
11996 l = 1;
11997 if (fword_ends)
11999 /* Copy the skipped character to preword. */
12000 mch_memmove(preword + sp->ts_prewordlen,
12001 fword + sp->ts_fidx, l);
12002 sp->ts_prewordlen += l;
12003 preword[sp->ts_prewordlen] = NUL;
12005 else
12006 sp->ts_score -= SCORE_SPLIT - SCORE_SUBST;
12007 sp->ts_fidx += l;
12010 /* When compounding include compound flag in
12011 * compflags[] (already set above). When splitting we
12012 * may start compounding over again. */
12013 if (try_compound)
12014 ++sp->ts_complen;
12015 else
12016 sp->ts_compsplit = sp->ts_complen;
12017 sp->ts_prefixdepth = PFD_NOPREFIX;
12019 /* set su->su_badflags to the caps type at this
12020 * position */
12021 #ifdef FEAT_MBYTE
12022 if (has_mbyte)
12023 n = nofold_len(fword, sp->ts_fidx, su->su_badptr);
12024 else
12025 #endif
12026 n = sp->ts_fidx;
12027 su->su_badflags = badword_captype(su->su_badptr + n,
12028 su->su_badptr + su->su_badlen);
12030 /* Restart at top of the tree. */
12031 sp->ts_arridx = 0;
12033 /* If there are postponed prefixes, try these too. */
12034 if (pbyts != NULL)
12036 byts = pbyts;
12037 idxs = pidxs;
12038 sp->ts_prefixdepth = PFD_PREFIXTREE;
12039 sp->ts_state = STATE_NOPREFIX;
12044 break;
12046 case STATE_SPLITUNDO:
12047 /* Undo the changes done for word split or compound word. */
12048 su->su_badflags = sp->ts_save_badflags;
12050 /* Continue looking for NUL bytes. */
12051 sp->ts_state = STATE_START;
12053 /* In case we went into the prefix tree. */
12054 byts = fbyts;
12055 idxs = fidxs;
12056 break;
12058 case STATE_ENDNUL:
12059 /* Past the NUL bytes in the node. */
12060 su->su_badflags = sp->ts_save_badflags;
12061 if (fword[sp->ts_fidx] == NUL
12062 #ifdef FEAT_MBYTE
12063 && sp->ts_tcharlen == 0
12064 #endif
12067 /* The badword ends, can't use STATE_PLAIN. */
12068 sp->ts_state = STATE_DEL;
12069 break;
12071 sp->ts_state = STATE_PLAIN;
12072 /*FALLTHROUGH*/
12074 case STATE_PLAIN:
12076 * Go over all possible bytes at this node, add each to tword[]
12077 * and use child node. "ts_curi" is the index.
12079 arridx = sp->ts_arridx;
12080 if (sp->ts_curi > byts[arridx])
12082 /* Done all bytes at this node, do next state. When still at
12083 * already changed bytes skip the other tricks. */
12084 if (sp->ts_fidx >= sp->ts_fidxtry)
12085 sp->ts_state = STATE_DEL;
12086 else
12087 sp->ts_state = STATE_FINAL;
12089 else
12091 arridx += sp->ts_curi++;
12092 c = byts[arridx];
12094 /* Normal byte, go one level deeper. If it's not equal to the
12095 * byte in the bad word adjust the score. But don't even try
12096 * when the byte was already changed. And don't try when we
12097 * just deleted this byte, accepting it is always cheaper then
12098 * delete + substitute. */
12099 if (c == fword[sp->ts_fidx]
12100 #ifdef FEAT_MBYTE
12101 || (sp->ts_tcharlen > 0 && sp->ts_isdiff != DIFF_NONE)
12102 #endif
12104 newscore = 0;
12105 else
12106 newscore = SCORE_SUBST;
12107 if ((newscore == 0
12108 || (sp->ts_fidx >= sp->ts_fidxtry
12109 && ((sp->ts_flags & TSF_DIDDEL) == 0
12110 || c != fword[sp->ts_delidx])))
12111 && TRY_DEEPER(su, stack, depth, newscore))
12113 go_deeper(stack, depth, newscore);
12114 #ifdef DEBUG_TRIEWALK
12115 if (newscore > 0)
12116 sprintf(changename[depth], "%.*s-%s: subst %c to %c",
12117 sp->ts_twordlen, tword, fword + sp->ts_fidx,
12118 fword[sp->ts_fidx], c);
12119 else
12120 sprintf(changename[depth], "%.*s-%s: accept %c",
12121 sp->ts_twordlen, tword, fword + sp->ts_fidx,
12122 fword[sp->ts_fidx]);
12123 #endif
12124 ++depth;
12125 sp = &stack[depth];
12126 ++sp->ts_fidx;
12127 tword[sp->ts_twordlen++] = c;
12128 sp->ts_arridx = idxs[arridx];
12129 #ifdef FEAT_MBYTE
12130 if (newscore == SCORE_SUBST)
12131 sp->ts_isdiff = DIFF_YES;
12132 if (has_mbyte)
12134 /* Multi-byte characters are a bit complicated to
12135 * handle: They differ when any of the bytes differ
12136 * and then their length may also differ. */
12137 if (sp->ts_tcharlen == 0)
12139 /* First byte. */
12140 sp->ts_tcharidx = 0;
12141 sp->ts_tcharlen = MB_BYTE2LEN(c);
12142 sp->ts_fcharstart = sp->ts_fidx - 1;
12143 sp->ts_isdiff = (newscore != 0)
12144 ? DIFF_YES : DIFF_NONE;
12146 else if (sp->ts_isdiff == DIFF_INSERT)
12147 /* When inserting trail bytes don't advance in the
12148 * bad word. */
12149 --sp->ts_fidx;
12150 if (++sp->ts_tcharidx == sp->ts_tcharlen)
12152 /* Last byte of character. */
12153 if (sp->ts_isdiff == DIFF_YES)
12155 /* Correct ts_fidx for the byte length of the
12156 * character (we didn't check that before). */
12157 sp->ts_fidx = sp->ts_fcharstart
12158 + MB_BYTE2LEN(
12159 fword[sp->ts_fcharstart]);
12161 /* For changing a composing character adjust
12162 * the score from SCORE_SUBST to
12163 * SCORE_SUBCOMP. */
12164 if (enc_utf8
12165 && utf_iscomposing(
12166 mb_ptr2char(tword
12167 + sp->ts_twordlen
12168 - sp->ts_tcharlen))
12169 && utf_iscomposing(
12170 mb_ptr2char(fword
12171 + sp->ts_fcharstart)))
12172 sp->ts_score -=
12173 SCORE_SUBST - SCORE_SUBCOMP;
12175 /* For a similar character adjust score from
12176 * SCORE_SUBST to SCORE_SIMILAR. */
12177 else if (!soundfold
12178 && slang->sl_has_map
12179 && similar_chars(slang,
12180 mb_ptr2char(tword
12181 + sp->ts_twordlen
12182 - sp->ts_tcharlen),
12183 mb_ptr2char(fword
12184 + sp->ts_fcharstart)))
12185 sp->ts_score -=
12186 SCORE_SUBST - SCORE_SIMILAR;
12188 else if (sp->ts_isdiff == DIFF_INSERT
12189 && sp->ts_twordlen > sp->ts_tcharlen)
12191 p = tword + sp->ts_twordlen - sp->ts_tcharlen;
12192 c = mb_ptr2char(p);
12193 if (enc_utf8 && utf_iscomposing(c))
12195 /* Inserting a composing char doesn't
12196 * count that much. */
12197 sp->ts_score -= SCORE_INS - SCORE_INSCOMP;
12199 else
12201 /* If the previous character was the same,
12202 * thus doubling a character, give a bonus
12203 * to the score. Also for the soundfold
12204 * tree (might seem illogical but does
12205 * give better scores). */
12206 mb_ptr_back(tword, p);
12207 if (c == mb_ptr2char(p))
12208 sp->ts_score -= SCORE_INS
12209 - SCORE_INSDUP;
12213 /* Starting a new char, reset the length. */
12214 sp->ts_tcharlen = 0;
12217 else
12218 #endif
12220 /* If we found a similar char adjust the score.
12221 * We do this after calling go_deeper() because
12222 * it's slow. */
12223 if (newscore != 0
12224 && !soundfold
12225 && slang->sl_has_map
12226 && similar_chars(slang,
12227 c, fword[sp->ts_fidx - 1]))
12228 sp->ts_score -= SCORE_SUBST - SCORE_SIMILAR;
12232 break;
12234 case STATE_DEL:
12235 #ifdef FEAT_MBYTE
12236 /* When past the first byte of a multi-byte char don't try
12237 * delete/insert/swap a character. */
12238 if (has_mbyte && sp->ts_tcharlen > 0)
12240 sp->ts_state = STATE_FINAL;
12241 break;
12243 #endif
12245 * Try skipping one character in the bad word (delete it).
12247 sp->ts_state = STATE_INS_PREP;
12248 sp->ts_curi = 1;
12249 if (soundfold && sp->ts_fidx == 0 && fword[sp->ts_fidx] == '*')
12250 /* Deleting a vowel at the start of a word counts less, see
12251 * soundalike_score(). */
12252 newscore = 2 * SCORE_DEL / 3;
12253 else
12254 newscore = SCORE_DEL;
12255 if (fword[sp->ts_fidx] != NUL
12256 && TRY_DEEPER(su, stack, depth, newscore))
12258 go_deeper(stack, depth, newscore);
12259 #ifdef DEBUG_TRIEWALK
12260 sprintf(changename[depth], "%.*s-%s: delete %c",
12261 sp->ts_twordlen, tword, fword + sp->ts_fidx,
12262 fword[sp->ts_fidx]);
12263 #endif
12264 ++depth;
12266 /* Remember what character we deleted, so that we can avoid
12267 * inserting it again. */
12268 stack[depth].ts_flags |= TSF_DIDDEL;
12269 stack[depth].ts_delidx = sp->ts_fidx;
12271 /* Advance over the character in fword[]. Give a bonus to the
12272 * score if the same character is following "nn" -> "n". It's
12273 * a bit illogical for soundfold tree but it does give better
12274 * results. */
12275 #ifdef FEAT_MBYTE
12276 if (has_mbyte)
12278 c = mb_ptr2char(fword + sp->ts_fidx);
12279 stack[depth].ts_fidx += MB_BYTE2LEN(fword[sp->ts_fidx]);
12280 if (enc_utf8 && utf_iscomposing(c))
12281 stack[depth].ts_score -= SCORE_DEL - SCORE_DELCOMP;
12282 else if (c == mb_ptr2char(fword + stack[depth].ts_fidx))
12283 stack[depth].ts_score -= SCORE_DEL - SCORE_DELDUP;
12285 else
12286 #endif
12288 ++stack[depth].ts_fidx;
12289 if (fword[sp->ts_fidx] == fword[sp->ts_fidx + 1])
12290 stack[depth].ts_score -= SCORE_DEL - SCORE_DELDUP;
12292 break;
12294 /*FALLTHROUGH*/
12296 case STATE_INS_PREP:
12297 if (sp->ts_flags & TSF_DIDDEL)
12299 /* If we just deleted a byte then inserting won't make sense,
12300 * a substitute is always cheaper. */
12301 sp->ts_state = STATE_SWAP;
12302 break;
12305 /* skip over NUL bytes */
12306 n = sp->ts_arridx;
12307 for (;;)
12309 if (sp->ts_curi > byts[n])
12311 /* Only NUL bytes at this node, go to next state. */
12312 sp->ts_state = STATE_SWAP;
12313 break;
12315 if (byts[n + sp->ts_curi] != NUL)
12317 /* Found a byte to insert. */
12318 sp->ts_state = STATE_INS;
12319 break;
12321 ++sp->ts_curi;
12323 break;
12325 /*FALLTHROUGH*/
12327 case STATE_INS:
12328 /* Insert one byte. Repeat this for each possible byte at this
12329 * node. */
12330 n = sp->ts_arridx;
12331 if (sp->ts_curi > byts[n])
12333 /* Done all bytes at this node, go to next state. */
12334 sp->ts_state = STATE_SWAP;
12335 break;
12338 /* Do one more byte at this node, but:
12339 * - Skip NUL bytes.
12340 * - Skip the byte if it's equal to the byte in the word,
12341 * accepting that byte is always better.
12343 n += sp->ts_curi++;
12344 c = byts[n];
12345 if (soundfold && sp->ts_twordlen == 0 && c == '*')
12346 /* Inserting a vowel at the start of a word counts less,
12347 * see soundalike_score(). */
12348 newscore = 2 * SCORE_INS / 3;
12349 else
12350 newscore = SCORE_INS;
12351 if (c != fword[sp->ts_fidx]
12352 && TRY_DEEPER(su, stack, depth, newscore))
12354 go_deeper(stack, depth, newscore);
12355 #ifdef DEBUG_TRIEWALK
12356 sprintf(changename[depth], "%.*s-%s: insert %c",
12357 sp->ts_twordlen, tword, fword + sp->ts_fidx,
12359 #endif
12360 ++depth;
12361 sp = &stack[depth];
12362 tword[sp->ts_twordlen++] = c;
12363 sp->ts_arridx = idxs[n];
12364 #ifdef FEAT_MBYTE
12365 if (has_mbyte)
12367 fl = MB_BYTE2LEN(c);
12368 if (fl > 1)
12370 /* There are following bytes for the same character.
12371 * We must find all bytes before trying
12372 * delete/insert/swap/etc. */
12373 sp->ts_tcharlen = fl;
12374 sp->ts_tcharidx = 1;
12375 sp->ts_isdiff = DIFF_INSERT;
12378 else
12379 fl = 1;
12380 if (fl == 1)
12381 #endif
12383 /* If the previous character was the same, thus doubling a
12384 * character, give a bonus to the score. Also for
12385 * soundfold words (illogical but does give a better
12386 * score). */
12387 if (sp->ts_twordlen >= 2
12388 && tword[sp->ts_twordlen - 2] == c)
12389 sp->ts_score -= SCORE_INS - SCORE_INSDUP;
12392 break;
12394 case STATE_SWAP:
12396 * Swap two bytes in the bad word: "12" -> "21".
12397 * We change "fword" here, it's changed back afterwards at
12398 * STATE_UNSWAP.
12400 p = fword + sp->ts_fidx;
12401 c = *p;
12402 if (c == NUL)
12404 /* End of word, can't swap or replace. */
12405 sp->ts_state = STATE_FINAL;
12406 break;
12409 /* Don't swap if the first character is not a word character.
12410 * SWAP3 etc. also don't make sense then. */
12411 if (!soundfold && !spell_iswordp(p, curbuf))
12413 sp->ts_state = STATE_REP_INI;
12414 break;
12417 #ifdef FEAT_MBYTE
12418 if (has_mbyte)
12420 n = mb_cptr2len(p);
12421 c = mb_ptr2char(p);
12422 if (p[n] == NUL)
12423 c2 = NUL;
12424 else if (!soundfold && !spell_iswordp(p + n, curbuf))
12425 c2 = c; /* don't swap non-word char */
12426 else
12427 c2 = mb_ptr2char(p + n);
12429 else
12430 #endif
12432 if (p[1] == NUL)
12433 c2 = NUL;
12434 else if (!soundfold && !spell_iswordp(p + 1, curbuf))
12435 c2 = c; /* don't swap non-word char */
12436 else
12437 c2 = p[1];
12440 /* When the second character is NUL we can't swap. */
12441 if (c2 == NUL)
12443 sp->ts_state = STATE_REP_INI;
12444 break;
12447 /* When characters are identical, swap won't do anything.
12448 * Also get here if the second char is not a word character. */
12449 if (c == c2)
12451 sp->ts_state = STATE_SWAP3;
12452 break;
12454 if (c2 != NUL && TRY_DEEPER(su, stack, depth, SCORE_SWAP))
12456 go_deeper(stack, depth, SCORE_SWAP);
12457 #ifdef DEBUG_TRIEWALK
12458 sprintf(changename[depth], "%.*s-%s: swap %c and %c",
12459 sp->ts_twordlen, tword, fword + sp->ts_fidx,
12460 c, c2);
12461 #endif
12462 sp->ts_state = STATE_UNSWAP;
12463 ++depth;
12464 #ifdef FEAT_MBYTE
12465 if (has_mbyte)
12467 fl = mb_char2len(c2);
12468 mch_memmove(p, p + n, fl);
12469 mb_char2bytes(c, p + fl);
12470 stack[depth].ts_fidxtry = sp->ts_fidx + n + fl;
12472 else
12473 #endif
12475 p[0] = c2;
12476 p[1] = c;
12477 stack[depth].ts_fidxtry = sp->ts_fidx + 2;
12480 else
12481 /* If this swap doesn't work then SWAP3 won't either. */
12482 sp->ts_state = STATE_REP_INI;
12483 break;
12485 case STATE_UNSWAP:
12486 /* Undo the STATE_SWAP swap: "21" -> "12". */
12487 p = fword + sp->ts_fidx;
12488 #ifdef FEAT_MBYTE
12489 if (has_mbyte)
12491 n = MB_BYTE2LEN(*p);
12492 c = mb_ptr2char(p + n);
12493 mch_memmove(p + MB_BYTE2LEN(p[n]), p, n);
12494 mb_char2bytes(c, p);
12496 else
12497 #endif
12499 c = *p;
12500 *p = p[1];
12501 p[1] = c;
12503 /*FALLTHROUGH*/
12505 case STATE_SWAP3:
12506 /* Swap two bytes, skipping one: "123" -> "321". We change
12507 * "fword" here, it's changed back afterwards at STATE_UNSWAP3. */
12508 p = fword + sp->ts_fidx;
12509 #ifdef FEAT_MBYTE
12510 if (has_mbyte)
12512 n = mb_cptr2len(p);
12513 c = mb_ptr2char(p);
12514 fl = mb_cptr2len(p + n);
12515 c2 = mb_ptr2char(p + n);
12516 if (!soundfold && !spell_iswordp(p + n + fl, curbuf))
12517 c3 = c; /* don't swap non-word char */
12518 else
12519 c3 = mb_ptr2char(p + n + fl);
12521 else
12522 #endif
12524 c = *p;
12525 c2 = p[1];
12526 if (!soundfold && !spell_iswordp(p + 2, curbuf))
12527 c3 = c; /* don't swap non-word char */
12528 else
12529 c3 = p[2];
12532 /* When characters are identical: "121" then SWAP3 result is
12533 * identical, ROT3L result is same as SWAP: "211", ROT3L result is
12534 * same as SWAP on next char: "112". Thus skip all swapping.
12535 * Also skip when c3 is NUL.
12536 * Also get here when the third character is not a word character.
12537 * Second character may any char: "a.b" -> "b.a" */
12538 if (c == c3 || c3 == NUL)
12540 sp->ts_state = STATE_REP_INI;
12541 break;
12543 if (TRY_DEEPER(su, stack, depth, SCORE_SWAP3))
12545 go_deeper(stack, depth, SCORE_SWAP3);
12546 #ifdef DEBUG_TRIEWALK
12547 sprintf(changename[depth], "%.*s-%s: swap3 %c and %c",
12548 sp->ts_twordlen, tword, fword + sp->ts_fidx,
12549 c, c3);
12550 #endif
12551 sp->ts_state = STATE_UNSWAP3;
12552 ++depth;
12553 #ifdef FEAT_MBYTE
12554 if (has_mbyte)
12556 tl = mb_char2len(c3);
12557 mch_memmove(p, p + n + fl, tl);
12558 mb_char2bytes(c2, p + tl);
12559 mb_char2bytes(c, p + fl + tl);
12560 stack[depth].ts_fidxtry = sp->ts_fidx + n + fl + tl;
12562 else
12563 #endif
12565 p[0] = p[2];
12566 p[2] = c;
12567 stack[depth].ts_fidxtry = sp->ts_fidx + 3;
12570 else
12571 sp->ts_state = STATE_REP_INI;
12572 break;
12574 case STATE_UNSWAP3:
12575 /* Undo STATE_SWAP3: "321" -> "123" */
12576 p = fword + sp->ts_fidx;
12577 #ifdef FEAT_MBYTE
12578 if (has_mbyte)
12580 n = MB_BYTE2LEN(*p);
12581 c2 = mb_ptr2char(p + n);
12582 fl = MB_BYTE2LEN(p[n]);
12583 c = mb_ptr2char(p + n + fl);
12584 tl = MB_BYTE2LEN(p[n + fl]);
12585 mch_memmove(p + fl + tl, p, n);
12586 mb_char2bytes(c, p);
12587 mb_char2bytes(c2, p + tl);
12588 p = p + tl;
12590 else
12591 #endif
12593 c = *p;
12594 *p = p[2];
12595 p[2] = c;
12596 ++p;
12599 if (!soundfold && !spell_iswordp(p, curbuf))
12601 /* Middle char is not a word char, skip the rotate. First and
12602 * third char were already checked at swap and swap3. */
12603 sp->ts_state = STATE_REP_INI;
12604 break;
12607 /* Rotate three characters left: "123" -> "231". We change
12608 * "fword" here, it's changed back afterwards at STATE_UNROT3L. */
12609 if (TRY_DEEPER(su, stack, depth, SCORE_SWAP3))
12611 go_deeper(stack, depth, SCORE_SWAP3);
12612 #ifdef DEBUG_TRIEWALK
12613 p = fword + sp->ts_fidx;
12614 sprintf(changename[depth], "%.*s-%s: rotate left %c%c%c",
12615 sp->ts_twordlen, tword, fword + sp->ts_fidx,
12616 p[0], p[1], p[2]);
12617 #endif
12618 sp->ts_state = STATE_UNROT3L;
12619 ++depth;
12620 p = fword + sp->ts_fidx;
12621 #ifdef FEAT_MBYTE
12622 if (has_mbyte)
12624 n = mb_cptr2len(p);
12625 c = mb_ptr2char(p);
12626 fl = mb_cptr2len(p + n);
12627 fl += mb_cptr2len(p + n + fl);
12628 mch_memmove(p, p + n, fl);
12629 mb_char2bytes(c, p + fl);
12630 stack[depth].ts_fidxtry = sp->ts_fidx + n + fl;
12632 else
12633 #endif
12635 c = *p;
12636 *p = p[1];
12637 p[1] = p[2];
12638 p[2] = c;
12639 stack[depth].ts_fidxtry = sp->ts_fidx + 3;
12642 else
12643 sp->ts_state = STATE_REP_INI;
12644 break;
12646 case STATE_UNROT3L:
12647 /* Undo ROT3L: "231" -> "123" */
12648 p = fword + sp->ts_fidx;
12649 #ifdef FEAT_MBYTE
12650 if (has_mbyte)
12652 n = MB_BYTE2LEN(*p);
12653 n += MB_BYTE2LEN(p[n]);
12654 c = mb_ptr2char(p + n);
12655 tl = MB_BYTE2LEN(p[n]);
12656 mch_memmove(p + tl, p, n);
12657 mb_char2bytes(c, p);
12659 else
12660 #endif
12662 c = p[2];
12663 p[2] = p[1];
12664 p[1] = *p;
12665 *p = c;
12668 /* Rotate three bytes right: "123" -> "312". We change "fword"
12669 * here, it's changed back afterwards at STATE_UNROT3R. */
12670 if (TRY_DEEPER(su, stack, depth, SCORE_SWAP3))
12672 go_deeper(stack, depth, SCORE_SWAP3);
12673 #ifdef DEBUG_TRIEWALK
12674 p = fword + sp->ts_fidx;
12675 sprintf(changename[depth], "%.*s-%s: rotate right %c%c%c",
12676 sp->ts_twordlen, tword, fword + sp->ts_fidx,
12677 p[0], p[1], p[2]);
12678 #endif
12679 sp->ts_state = STATE_UNROT3R;
12680 ++depth;
12681 p = fword + sp->ts_fidx;
12682 #ifdef FEAT_MBYTE
12683 if (has_mbyte)
12685 n = mb_cptr2len(p);
12686 n += mb_cptr2len(p + n);
12687 c = mb_ptr2char(p + n);
12688 tl = mb_cptr2len(p + n);
12689 mch_memmove(p + tl, p, n);
12690 mb_char2bytes(c, p);
12691 stack[depth].ts_fidxtry = sp->ts_fidx + n + tl;
12693 else
12694 #endif
12696 c = p[2];
12697 p[2] = p[1];
12698 p[1] = *p;
12699 *p = c;
12700 stack[depth].ts_fidxtry = sp->ts_fidx + 3;
12703 else
12704 sp->ts_state = STATE_REP_INI;
12705 break;
12707 case STATE_UNROT3R:
12708 /* Undo ROT3R: "312" -> "123" */
12709 p = fword + sp->ts_fidx;
12710 #ifdef FEAT_MBYTE
12711 if (has_mbyte)
12713 c = mb_ptr2char(p);
12714 tl = MB_BYTE2LEN(*p);
12715 n = MB_BYTE2LEN(p[tl]);
12716 n += MB_BYTE2LEN(p[tl + n]);
12717 mch_memmove(p, p + tl, n);
12718 mb_char2bytes(c, p + n);
12720 else
12721 #endif
12723 c = *p;
12724 *p = p[1];
12725 p[1] = p[2];
12726 p[2] = c;
12728 /*FALLTHROUGH*/
12730 case STATE_REP_INI:
12731 /* Check if matching with REP items from the .aff file would work.
12732 * Quickly skip if:
12733 * - there are no REP items and we are not in the soundfold trie
12734 * - the score is going to be too high anyway
12735 * - already applied a REP item or swapped here */
12736 if ((lp->lp_replang == NULL && !soundfold)
12737 || sp->ts_score + SCORE_REP >= su->su_maxscore
12738 || sp->ts_fidx < sp->ts_fidxtry)
12740 sp->ts_state = STATE_FINAL;
12741 break;
12744 /* Use the first byte to quickly find the first entry that may
12745 * match. If the index is -1 there is none. */
12746 if (soundfold)
12747 sp->ts_curi = slang->sl_repsal_first[fword[sp->ts_fidx]];
12748 else
12749 sp->ts_curi = lp->lp_replang->sl_rep_first[fword[sp->ts_fidx]];
12751 if (sp->ts_curi < 0)
12753 sp->ts_state = STATE_FINAL;
12754 break;
12757 sp->ts_state = STATE_REP;
12758 /*FALLTHROUGH*/
12760 case STATE_REP:
12761 /* Try matching with REP items from the .aff file. For each match
12762 * replace the characters and check if the resulting word is
12763 * valid. */
12764 p = fword + sp->ts_fidx;
12766 if (soundfold)
12767 gap = &slang->sl_repsal;
12768 else
12769 gap = &lp->lp_replang->sl_rep;
12770 while (sp->ts_curi < gap->ga_len)
12772 ftp = (fromto_T *)gap->ga_data + sp->ts_curi++;
12773 if (*ftp->ft_from != *p)
12775 /* past possible matching entries */
12776 sp->ts_curi = gap->ga_len;
12777 break;
12779 if (STRNCMP(ftp->ft_from, p, STRLEN(ftp->ft_from)) == 0
12780 && TRY_DEEPER(su, stack, depth, SCORE_REP))
12782 go_deeper(stack, depth, SCORE_REP);
12783 #ifdef DEBUG_TRIEWALK
12784 sprintf(changename[depth], "%.*s-%s: replace %s with %s",
12785 sp->ts_twordlen, tword, fword + sp->ts_fidx,
12786 ftp->ft_from, ftp->ft_to);
12787 #endif
12788 /* Need to undo this afterwards. */
12789 sp->ts_state = STATE_REP_UNDO;
12791 /* Change the "from" to the "to" string. */
12792 ++depth;
12793 fl = (int)STRLEN(ftp->ft_from);
12794 tl = (int)STRLEN(ftp->ft_to);
12795 if (fl != tl)
12797 STRMOVE(p + tl, p + fl);
12798 repextra += tl - fl;
12800 mch_memmove(p, ftp->ft_to, tl);
12801 stack[depth].ts_fidxtry = sp->ts_fidx + tl;
12802 #ifdef FEAT_MBYTE
12803 stack[depth].ts_tcharlen = 0;
12804 #endif
12805 break;
12809 if (sp->ts_curi >= gap->ga_len && sp->ts_state == STATE_REP)
12810 /* No (more) matches. */
12811 sp->ts_state = STATE_FINAL;
12813 break;
12815 case STATE_REP_UNDO:
12816 /* Undo a REP replacement and continue with the next one. */
12817 if (soundfold)
12818 gap = &slang->sl_repsal;
12819 else
12820 gap = &lp->lp_replang->sl_rep;
12821 ftp = (fromto_T *)gap->ga_data + sp->ts_curi - 1;
12822 fl = (int)STRLEN(ftp->ft_from);
12823 tl = (int)STRLEN(ftp->ft_to);
12824 p = fword + sp->ts_fidx;
12825 if (fl != tl)
12827 STRMOVE(p + fl, p + tl);
12828 repextra -= tl - fl;
12830 mch_memmove(p, ftp->ft_from, fl);
12831 sp->ts_state = STATE_REP;
12832 break;
12834 default:
12835 /* Did all possible states at this level, go up one level. */
12836 --depth;
12838 if (depth >= 0 && stack[depth].ts_prefixdepth == PFD_PREFIXTREE)
12840 /* Continue in or go back to the prefix tree. */
12841 byts = pbyts;
12842 idxs = pidxs;
12845 /* Don't check for CTRL-C too often, it takes time. */
12846 if (--breakcheckcount == 0)
12848 ui_breakcheck();
12849 breakcheckcount = 1000;
12857 * Go one level deeper in the tree.
12859 static void
12860 go_deeper(stack, depth, score_add)
12861 trystate_T *stack;
12862 int depth;
12863 int score_add;
12865 stack[depth + 1] = stack[depth];
12866 stack[depth + 1].ts_state = STATE_START;
12867 stack[depth + 1].ts_score = stack[depth].ts_score + score_add;
12868 stack[depth + 1].ts_curi = 1; /* start just after length byte */
12869 stack[depth + 1].ts_flags = 0;
12872 #ifdef FEAT_MBYTE
12874 * Case-folding may change the number of bytes: Count nr of chars in
12875 * fword[flen] and return the byte length of that many chars in "word".
12877 static int
12878 nofold_len(fword, flen, word)
12879 char_u *fword;
12880 int flen;
12881 char_u *word;
12883 char_u *p;
12884 int i = 0;
12886 for (p = fword; p < fword + flen; mb_ptr_adv(p))
12887 ++i;
12888 for (p = word; i > 0; mb_ptr_adv(p))
12889 --i;
12890 return (int)(p - word);
12892 #endif
12895 * "fword" is a good word with case folded. Find the matching keep-case
12896 * words and put it in "kword".
12897 * Theoretically there could be several keep-case words that result in the
12898 * same case-folded word, but we only find one...
12900 static void
12901 find_keepcap_word(slang, fword, kword)
12902 slang_T *slang;
12903 char_u *fword;
12904 char_u *kword;
12906 char_u uword[MAXWLEN]; /* "fword" in upper-case */
12907 int depth;
12908 idx_T tryidx;
12910 /* The following arrays are used at each depth in the tree. */
12911 idx_T arridx[MAXWLEN];
12912 int round[MAXWLEN];
12913 int fwordidx[MAXWLEN];
12914 int uwordidx[MAXWLEN];
12915 int kwordlen[MAXWLEN];
12917 int flen, ulen;
12918 int l;
12919 int len;
12920 int c;
12921 idx_T lo, hi, m;
12922 char_u *p;
12923 char_u *byts = slang->sl_kbyts; /* array with bytes of the words */
12924 idx_T *idxs = slang->sl_kidxs; /* array with indexes */
12926 if (byts == NULL)
12928 /* array is empty: "cannot happen" */
12929 *kword = NUL;
12930 return;
12933 /* Make an all-cap version of "fword". */
12934 allcap_copy(fword, uword);
12937 * Each character needs to be tried both case-folded and upper-case.
12938 * All this gets very complicated if we keep in mind that changing case
12939 * may change the byte length of a multi-byte character...
12941 depth = 0;
12942 arridx[0] = 0;
12943 round[0] = 0;
12944 fwordidx[0] = 0;
12945 uwordidx[0] = 0;
12946 kwordlen[0] = 0;
12947 while (depth >= 0)
12949 if (fword[fwordidx[depth]] == NUL)
12951 /* We are at the end of "fword". If the tree allows a word to end
12952 * here we have found a match. */
12953 if (byts[arridx[depth] + 1] == 0)
12955 kword[kwordlen[depth]] = NUL;
12956 return;
12959 /* kword is getting too long, continue one level up */
12960 --depth;
12962 else if (++round[depth] > 2)
12964 /* tried both fold-case and upper-case character, continue one
12965 * level up */
12966 --depth;
12968 else
12971 * round[depth] == 1: Try using the folded-case character.
12972 * round[depth] == 2: Try using the upper-case character.
12974 #ifdef FEAT_MBYTE
12975 if (has_mbyte)
12977 flen = mb_cptr2len(fword + fwordidx[depth]);
12978 ulen = mb_cptr2len(uword + uwordidx[depth]);
12980 else
12981 #endif
12982 ulen = flen = 1;
12983 if (round[depth] == 1)
12985 p = fword + fwordidx[depth];
12986 l = flen;
12988 else
12990 p = uword + uwordidx[depth];
12991 l = ulen;
12994 for (tryidx = arridx[depth]; l > 0; --l)
12996 /* Perform a binary search in the list of accepted bytes. */
12997 len = byts[tryidx++];
12998 c = *p++;
12999 lo = tryidx;
13000 hi = tryidx + len - 1;
13001 while (lo < hi)
13003 m = (lo + hi) / 2;
13004 if (byts[m] > c)
13005 hi = m - 1;
13006 else if (byts[m] < c)
13007 lo = m + 1;
13008 else
13010 lo = hi = m;
13011 break;
13015 /* Stop if there is no matching byte. */
13016 if (hi < lo || byts[lo] != c)
13017 break;
13019 /* Continue at the child (if there is one). */
13020 tryidx = idxs[lo];
13023 if (l == 0)
13026 * Found the matching char. Copy it to "kword" and go a
13027 * level deeper.
13029 if (round[depth] == 1)
13031 STRNCPY(kword + kwordlen[depth], fword + fwordidx[depth],
13032 flen);
13033 kwordlen[depth + 1] = kwordlen[depth] + flen;
13035 else
13037 STRNCPY(kword + kwordlen[depth], uword + uwordidx[depth],
13038 ulen);
13039 kwordlen[depth + 1] = kwordlen[depth] + ulen;
13041 fwordidx[depth + 1] = fwordidx[depth] + flen;
13042 uwordidx[depth + 1] = uwordidx[depth] + ulen;
13044 ++depth;
13045 arridx[depth] = tryidx;
13046 round[depth] = 0;
13051 /* Didn't find it: "cannot happen". */
13052 *kword = NUL;
13056 * Compute the sound-a-like score for suggestions in su->su_ga and add them to
13057 * su->su_sga.
13059 static void
13060 score_comp_sal(su)
13061 suginfo_T *su;
13063 langp_T *lp;
13064 char_u badsound[MAXWLEN];
13065 int i;
13066 suggest_T *stp;
13067 suggest_T *sstp;
13068 int score;
13069 int lpi;
13071 if (ga_grow(&su->su_sga, su->su_ga.ga_len) == FAIL)
13072 return;
13074 /* Use the sound-folding of the first language that supports it. */
13075 for (lpi = 0; lpi < curbuf->b_langp.ga_len; ++lpi)
13077 lp = LANGP_ENTRY(curbuf->b_langp, lpi);
13078 if (lp->lp_slang->sl_sal.ga_len > 0)
13080 /* soundfold the bad word */
13081 spell_soundfold(lp->lp_slang, su->su_fbadword, TRUE, badsound);
13083 for (i = 0; i < su->su_ga.ga_len; ++i)
13085 stp = &SUG(su->su_ga, i);
13087 /* Case-fold the suggested word, sound-fold it and compute the
13088 * sound-a-like score. */
13089 score = stp_sal_score(stp, su, lp->lp_slang, badsound);
13090 if (score < SCORE_MAXMAX)
13092 /* Add the suggestion. */
13093 sstp = &SUG(su->su_sga, su->su_sga.ga_len);
13094 sstp->st_word = vim_strsave(stp->st_word);
13095 if (sstp->st_word != NULL)
13097 sstp->st_wordlen = stp->st_wordlen;
13098 sstp->st_score = score;
13099 sstp->st_altscore = 0;
13100 sstp->st_orglen = stp->st_orglen;
13101 ++su->su_sga.ga_len;
13105 break;
13111 * Combine the list of suggestions in su->su_ga and su->su_sga.
13112 * They are intwined.
13114 static void
13115 score_combine(su)
13116 suginfo_T *su;
13118 int i;
13119 int j;
13120 garray_T ga;
13121 garray_T *gap;
13122 langp_T *lp;
13123 suggest_T *stp;
13124 char_u *p;
13125 char_u badsound[MAXWLEN];
13126 int round;
13127 int lpi;
13128 slang_T *slang = NULL;
13130 /* Add the alternate score to su_ga. */
13131 for (lpi = 0; lpi < curbuf->b_langp.ga_len; ++lpi)
13133 lp = LANGP_ENTRY(curbuf->b_langp, lpi);
13134 if (lp->lp_slang->sl_sal.ga_len > 0)
13136 /* soundfold the bad word */
13137 slang = lp->lp_slang;
13138 spell_soundfold(slang, su->su_fbadword, TRUE, badsound);
13140 for (i = 0; i < su->su_ga.ga_len; ++i)
13142 stp = &SUG(su->su_ga, i);
13143 stp->st_altscore = stp_sal_score(stp, su, slang, badsound);
13144 if (stp->st_altscore == SCORE_MAXMAX)
13145 stp->st_score = (stp->st_score * 3 + SCORE_BIG) / 4;
13146 else
13147 stp->st_score = (stp->st_score * 3
13148 + stp->st_altscore) / 4;
13149 stp->st_salscore = FALSE;
13151 break;
13155 if (slang == NULL) /* Using "double" without sound folding. */
13157 (void)cleanup_suggestions(&su->su_ga, su->su_maxscore,
13158 su->su_maxcount);
13159 return;
13162 /* Add the alternate score to su_sga. */
13163 for (i = 0; i < su->su_sga.ga_len; ++i)
13165 stp = &SUG(su->su_sga, i);
13166 stp->st_altscore = spell_edit_score(slang,
13167 su->su_badword, stp->st_word);
13168 if (stp->st_score == SCORE_MAXMAX)
13169 stp->st_score = (SCORE_BIG * 7 + stp->st_altscore) / 8;
13170 else
13171 stp->st_score = (stp->st_score * 7 + stp->st_altscore) / 8;
13172 stp->st_salscore = TRUE;
13175 /* Remove bad suggestions, sort the suggestions and truncate at "maxcount"
13176 * for both lists. */
13177 check_suggestions(su, &su->su_ga);
13178 (void)cleanup_suggestions(&su->su_ga, su->su_maxscore, su->su_maxcount);
13179 check_suggestions(su, &su->su_sga);
13180 (void)cleanup_suggestions(&su->su_sga, su->su_maxscore, su->su_maxcount);
13182 ga_init2(&ga, (int)sizeof(suginfo_T), 1);
13183 if (ga_grow(&ga, su->su_ga.ga_len + su->su_sga.ga_len) == FAIL)
13184 return;
13186 stp = &SUG(ga, 0);
13187 for (i = 0; i < su->su_ga.ga_len || i < su->su_sga.ga_len; ++i)
13189 /* round 1: get a suggestion from su_ga
13190 * round 2: get a suggestion from su_sga */
13191 for (round = 1; round <= 2; ++round)
13193 gap = round == 1 ? &su->su_ga : &su->su_sga;
13194 if (i < gap->ga_len)
13196 /* Don't add a word if it's already there. */
13197 p = SUG(*gap, i).st_word;
13198 for (j = 0; j < ga.ga_len; ++j)
13199 if (STRCMP(stp[j].st_word, p) == 0)
13200 break;
13201 if (j == ga.ga_len)
13202 stp[ga.ga_len++] = SUG(*gap, i);
13203 else
13204 vim_free(p);
13209 ga_clear(&su->su_ga);
13210 ga_clear(&su->su_sga);
13212 /* Truncate the list to the number of suggestions that will be displayed. */
13213 if (ga.ga_len > su->su_maxcount)
13215 for (i = su->su_maxcount; i < ga.ga_len; ++i)
13216 vim_free(stp[i].st_word);
13217 ga.ga_len = su->su_maxcount;
13220 su->su_ga = ga;
13224 * For the goodword in "stp" compute the soundalike score compared to the
13225 * badword.
13227 static int
13228 stp_sal_score(stp, su, slang, badsound)
13229 suggest_T *stp;
13230 suginfo_T *su;
13231 slang_T *slang;
13232 char_u *badsound; /* sound-folded badword */
13234 char_u *p;
13235 char_u *pbad;
13236 char_u *pgood;
13237 char_u badsound2[MAXWLEN];
13238 char_u fword[MAXWLEN];
13239 char_u goodsound[MAXWLEN];
13240 char_u goodword[MAXWLEN];
13241 int lendiff;
13243 lendiff = (int)(su->su_badlen - stp->st_orglen);
13244 if (lendiff >= 0)
13245 pbad = badsound;
13246 else
13248 /* soundfold the bad word with more characters following */
13249 (void)spell_casefold(su->su_badptr, stp->st_orglen, fword, MAXWLEN);
13251 /* When joining two words the sound often changes a lot. E.g., "t he"
13252 * sounds like "t h" while "the" sounds like "@". Avoid that by
13253 * removing the space. Don't do it when the good word also contains a
13254 * space. */
13255 if (vim_iswhite(su->su_badptr[su->su_badlen])
13256 && *skiptowhite(stp->st_word) == NUL)
13257 for (p = fword; *(p = skiptowhite(p)) != NUL; )
13258 STRMOVE(p, p + 1);
13260 spell_soundfold(slang, fword, TRUE, badsound2);
13261 pbad = badsound2;
13264 if (lendiff > 0)
13266 /* Add part of the bad word to the good word, so that we soundfold
13267 * what replaces the bad word. */
13268 STRCPY(goodword, stp->st_word);
13269 vim_strncpy(goodword + stp->st_wordlen,
13270 su->su_badptr + su->su_badlen - lendiff, lendiff);
13271 pgood = goodword;
13273 else
13274 pgood = stp->st_word;
13276 /* Sound-fold the word and compute the score for the difference. */
13277 spell_soundfold(slang, pgood, FALSE, goodsound);
13279 return soundalike_score(goodsound, pbad);
13282 /* structure used to store soundfolded words that add_sound_suggest() has
13283 * handled already. */
13284 typedef struct
13286 short sft_score; /* lowest score used */
13287 char_u sft_word[1]; /* soundfolded word, actually longer */
13288 } sftword_T;
13290 static sftword_T dumsft;
13291 #define HIKEY2SFT(p) ((sftword_T *)(p - (dumsft.sft_word - (char_u *)&dumsft)))
13292 #define HI2SFT(hi) HIKEY2SFT((hi)->hi_key)
13295 * Prepare for calling suggest_try_soundalike().
13297 static void
13298 suggest_try_soundalike_prep()
13300 langp_T *lp;
13301 int lpi;
13302 slang_T *slang;
13304 /* Do this for all languages that support sound folding and for which a
13305 * .sug file has been loaded. */
13306 for (lpi = 0; lpi < curbuf->b_langp.ga_len; ++lpi)
13308 lp = LANGP_ENTRY(curbuf->b_langp, lpi);
13309 slang = lp->lp_slang;
13310 if (slang->sl_sal.ga_len > 0 && slang->sl_sbyts != NULL)
13311 /* prepare the hashtable used by add_sound_suggest() */
13312 hash_init(&slang->sl_sounddone);
13317 * Find suggestions by comparing the word in a sound-a-like form.
13318 * Note: This doesn't support postponed prefixes.
13320 static void
13321 suggest_try_soundalike(su)
13322 suginfo_T *su;
13324 char_u salword[MAXWLEN];
13325 langp_T *lp;
13326 int lpi;
13327 slang_T *slang;
13329 /* Do this for all languages that support sound folding and for which a
13330 * .sug file has been loaded. */
13331 for (lpi = 0; lpi < curbuf->b_langp.ga_len; ++lpi)
13333 lp = LANGP_ENTRY(curbuf->b_langp, lpi);
13334 slang = lp->lp_slang;
13335 if (slang->sl_sal.ga_len > 0 && slang->sl_sbyts != NULL)
13337 /* soundfold the bad word */
13338 spell_soundfold(slang, su->su_fbadword, TRUE, salword);
13340 /* try all kinds of inserts/deletes/swaps/etc. */
13341 /* TODO: also soundfold the next words, so that we can try joining
13342 * and splitting */
13343 suggest_trie_walk(su, lp, salword, TRUE);
13349 * Finish up after calling suggest_try_soundalike().
13351 static void
13352 suggest_try_soundalike_finish()
13354 langp_T *lp;
13355 int lpi;
13356 slang_T *slang;
13357 int todo;
13358 hashitem_T *hi;
13360 /* Do this for all languages that support sound folding and for which a
13361 * .sug file has been loaded. */
13362 for (lpi = 0; lpi < curbuf->b_langp.ga_len; ++lpi)
13364 lp = LANGP_ENTRY(curbuf->b_langp, lpi);
13365 slang = lp->lp_slang;
13366 if (slang->sl_sal.ga_len > 0 && slang->sl_sbyts != NULL)
13368 /* Free the info about handled words. */
13369 todo = (int)slang->sl_sounddone.ht_used;
13370 for (hi = slang->sl_sounddone.ht_array; todo > 0; ++hi)
13371 if (!HASHITEM_EMPTY(hi))
13373 vim_free(HI2SFT(hi));
13374 --todo;
13377 /* Clear the hashtable, it may also be used by another region. */
13378 hash_clear(&slang->sl_sounddone);
13379 hash_init(&slang->sl_sounddone);
13385 * A match with a soundfolded word is found. Add the good word(s) that
13386 * produce this soundfolded word.
13388 static void
13389 add_sound_suggest(su, goodword, score, lp)
13390 suginfo_T *su;
13391 char_u *goodword;
13392 int score; /* soundfold score */
13393 langp_T *lp;
13395 slang_T *slang = lp->lp_slang; /* language for sound folding */
13396 int sfwordnr;
13397 char_u *nrline;
13398 int orgnr;
13399 char_u theword[MAXWLEN];
13400 int i;
13401 int wlen;
13402 char_u *byts;
13403 idx_T *idxs;
13404 int n;
13405 int wordcount;
13406 int wc;
13407 int goodscore;
13408 hash_T hash;
13409 hashitem_T *hi;
13410 sftword_T *sft;
13411 int bc, gc;
13412 int limit;
13415 * It's very well possible that the same soundfold word is found several
13416 * times with different scores. Since the following is quite slow only do
13417 * the words that have a better score than before. Use a hashtable to
13418 * remember the words that have been done.
13420 hash = hash_hash(goodword);
13421 hi = hash_lookup(&slang->sl_sounddone, goodword, hash);
13422 if (HASHITEM_EMPTY(hi))
13424 sft = (sftword_T *)alloc((unsigned)(sizeof(sftword_T)
13425 + STRLEN(goodword)));
13426 if (sft != NULL)
13428 sft->sft_score = score;
13429 STRCPY(sft->sft_word, goodword);
13430 hash_add_item(&slang->sl_sounddone, hi, sft->sft_word, hash);
13433 else
13435 sft = HI2SFT(hi);
13436 if (score >= sft->sft_score)
13437 return;
13438 sft->sft_score = score;
13442 * Find the word nr in the soundfold tree.
13444 sfwordnr = soundfold_find(slang, goodword);
13445 if (sfwordnr < 0)
13447 EMSG2(_(e_intern2), "add_sound_suggest()");
13448 return;
13452 * go over the list of good words that produce this soundfold word
13454 nrline = ml_get_buf(slang->sl_sugbuf, (linenr_T)(sfwordnr + 1), FALSE);
13455 orgnr = 0;
13456 while (*nrline != NUL)
13458 /* The wordnr was stored in a minimal nr of bytes as an offset to the
13459 * previous wordnr. */
13460 orgnr += bytes2offset(&nrline);
13462 byts = slang->sl_fbyts;
13463 idxs = slang->sl_fidxs;
13465 /* Lookup the word "orgnr" one of the two tries. */
13466 n = 0;
13467 wlen = 0;
13468 wordcount = 0;
13469 for (;;)
13471 i = 1;
13472 if (wordcount == orgnr && byts[n + 1] == NUL)
13473 break; /* found end of word */
13475 if (byts[n + 1] == NUL)
13476 ++wordcount;
13478 /* skip over the NUL bytes */
13479 for ( ; byts[n + i] == NUL; ++i)
13480 if (i > byts[n]) /* safety check */
13482 STRCPY(theword + wlen, "BAD");
13483 goto badword;
13486 /* One of the siblings must have the word. */
13487 for ( ; i < byts[n]; ++i)
13489 wc = idxs[idxs[n + i]]; /* nr of words under this byte */
13490 if (wordcount + wc > orgnr)
13491 break;
13492 wordcount += wc;
13495 theword[wlen++] = byts[n + i];
13496 n = idxs[n + i];
13498 badword:
13499 theword[wlen] = NUL;
13501 /* Go over the possible flags and regions. */
13502 for (; i <= byts[n] && byts[n + i] == NUL; ++i)
13504 char_u cword[MAXWLEN];
13505 char_u *p;
13506 int flags = (int)idxs[n + i];
13508 /* Skip words with the NOSUGGEST flag */
13509 if (flags & WF_NOSUGGEST)
13510 continue;
13512 if (flags & WF_KEEPCAP)
13514 /* Must find the word in the keep-case tree. */
13515 find_keepcap_word(slang, theword, cword);
13516 p = cword;
13518 else
13520 flags |= su->su_badflags;
13521 if ((flags & WF_CAPMASK) != 0)
13523 /* Need to fix case according to "flags". */
13524 make_case_word(theword, cword, flags);
13525 p = cword;
13527 else
13528 p = theword;
13531 /* Add the suggestion. */
13532 if (sps_flags & SPS_DOUBLE)
13534 /* Add the suggestion if the score isn't too bad. */
13535 if (score <= su->su_maxscore)
13536 add_suggestion(su, &su->su_sga, p, su->su_badlen,
13537 score, 0, FALSE, slang, FALSE);
13539 else
13541 /* Add a penalty for words in another region. */
13542 if ((flags & WF_REGION)
13543 && (((unsigned)flags >> 16) & lp->lp_region) == 0)
13544 goodscore = SCORE_REGION;
13545 else
13546 goodscore = 0;
13548 /* Add a small penalty for changing the first letter from
13549 * lower to upper case. Helps for "tath" -> "Kath", which is
13550 * less common thatn "tath" -> "path". Don't do it when the
13551 * letter is the same, that has already been counted. */
13552 gc = PTR2CHAR(p);
13553 if (SPELL_ISUPPER(gc))
13555 bc = PTR2CHAR(su->su_badword);
13556 if (!SPELL_ISUPPER(bc)
13557 && SPELL_TOFOLD(bc) != SPELL_TOFOLD(gc))
13558 goodscore += SCORE_ICASE / 2;
13561 /* Compute the score for the good word. This only does letter
13562 * insert/delete/swap/replace. REP items are not considered,
13563 * which may make the score a bit higher.
13564 * Use a limit for the score to make it work faster. Use
13565 * MAXSCORE(), because RESCORE() will change the score.
13566 * If the limit is very high then the iterative method is
13567 * inefficient, using an array is quicker. */
13568 limit = MAXSCORE(su->su_sfmaxscore - goodscore, score);
13569 if (limit > SCORE_LIMITMAX)
13570 goodscore += spell_edit_score(slang, su->su_badword, p);
13571 else
13572 goodscore += spell_edit_score_limit(slang, su->su_badword,
13573 p, limit);
13575 /* When going over the limit don't bother to do the rest. */
13576 if (goodscore < SCORE_MAXMAX)
13578 /* Give a bonus to words seen before. */
13579 goodscore = score_wordcount_adj(slang, goodscore, p, FALSE);
13581 /* Add the suggestion if the score isn't too bad. */
13582 goodscore = RESCORE(goodscore, score);
13583 if (goodscore <= su->su_sfmaxscore)
13584 add_suggestion(su, &su->su_ga, p, su->su_badlen,
13585 goodscore, score, TRUE, slang, TRUE);
13589 /* smsg("word %s (%d): %s (%d)", sftword, sftnr, theword, orgnr); */
13594 * Find word "word" in fold-case tree for "slang" and return the word number.
13596 static int
13597 soundfold_find(slang, word)
13598 slang_T *slang;
13599 char_u *word;
13601 idx_T arridx = 0;
13602 int len;
13603 int wlen = 0;
13604 int c;
13605 char_u *ptr = word;
13606 char_u *byts;
13607 idx_T *idxs;
13608 int wordnr = 0;
13610 byts = slang->sl_sbyts;
13611 idxs = slang->sl_sidxs;
13613 for (;;)
13615 /* First byte is the number of possible bytes. */
13616 len = byts[arridx++];
13618 /* If the first possible byte is a zero the word could end here.
13619 * If the word ends we found the word. If not skip the NUL bytes. */
13620 c = ptr[wlen];
13621 if (byts[arridx] == NUL)
13623 if (c == NUL)
13624 break;
13626 /* Skip over the zeros, there can be several. */
13627 while (len > 0 && byts[arridx] == NUL)
13629 ++arridx;
13630 --len;
13632 if (len == 0)
13633 return -1; /* no children, word should have ended here */
13634 ++wordnr;
13637 /* If the word ends we didn't find it. */
13638 if (c == NUL)
13639 return -1;
13641 /* Perform a binary search in the list of accepted bytes. */
13642 if (c == TAB) /* <Tab> is handled like <Space> */
13643 c = ' ';
13644 while (byts[arridx] < c)
13646 /* The word count is in the first idxs[] entry of the child. */
13647 wordnr += idxs[idxs[arridx]];
13648 ++arridx;
13649 if (--len == 0) /* end of the bytes, didn't find it */
13650 return -1;
13652 if (byts[arridx] != c) /* didn't find the byte */
13653 return -1;
13655 /* Continue at the child (if there is one). */
13656 arridx = idxs[arridx];
13657 ++wlen;
13659 /* One space in the good word may stand for several spaces in the
13660 * checked word. */
13661 if (c == ' ')
13662 while (ptr[wlen] == ' ' || ptr[wlen] == TAB)
13663 ++wlen;
13666 return wordnr;
13670 * Copy "fword" to "cword", fixing case according to "flags".
13672 static void
13673 make_case_word(fword, cword, flags)
13674 char_u *fword;
13675 char_u *cword;
13676 int flags;
13678 if (flags & WF_ALLCAP)
13679 /* Make it all upper-case */
13680 allcap_copy(fword, cword);
13681 else if (flags & WF_ONECAP)
13682 /* Make the first letter upper-case */
13683 onecap_copy(fword, cword, TRUE);
13684 else
13685 /* Use goodword as-is. */
13686 STRCPY(cword, fword);
13690 * Use map string "map" for languages "lp".
13692 static void
13693 set_map_str(lp, map)
13694 slang_T *lp;
13695 char_u *map;
13697 char_u *p;
13698 int headc = 0;
13699 int c;
13700 int i;
13702 if (*map == NUL)
13704 lp->sl_has_map = FALSE;
13705 return;
13707 lp->sl_has_map = TRUE;
13709 /* Init the array and hash tables empty. */
13710 for (i = 0; i < 256; ++i)
13711 lp->sl_map_array[i] = 0;
13712 #ifdef FEAT_MBYTE
13713 hash_init(&lp->sl_map_hash);
13714 #endif
13717 * The similar characters are stored separated with slashes:
13718 * "aaa/bbb/ccc/". Fill sl_map_array[c] with the character before c and
13719 * before the same slash. For characters above 255 sl_map_hash is used.
13721 for (p = map; *p != NUL; )
13723 #ifdef FEAT_MBYTE
13724 c = mb_cptr2char_adv(&p);
13725 #else
13726 c = *p++;
13727 #endif
13728 if (c == '/')
13729 headc = 0;
13730 else
13732 if (headc == 0)
13733 headc = c;
13735 #ifdef FEAT_MBYTE
13736 /* Characters above 255 don't fit in sl_map_array[], put them in
13737 * the hash table. Each entry is the char, a NUL the headchar and
13738 * a NUL. */
13739 if (c >= 256)
13741 int cl = mb_char2len(c);
13742 int headcl = mb_char2len(headc);
13743 char_u *b;
13744 hash_T hash;
13745 hashitem_T *hi;
13747 b = alloc((unsigned)(cl + headcl + 2));
13748 if (b == NULL)
13749 return;
13750 mb_char2bytes(c, b);
13751 b[cl] = NUL;
13752 mb_char2bytes(headc, b + cl + 1);
13753 b[cl + 1 + headcl] = NUL;
13754 hash = hash_hash(b);
13755 hi = hash_lookup(&lp->sl_map_hash, b, hash);
13756 if (HASHITEM_EMPTY(hi))
13757 hash_add_item(&lp->sl_map_hash, hi, b, hash);
13758 else
13760 /* This should have been checked when generating the .spl
13761 * file. */
13762 EMSG(_("E783: duplicate char in MAP entry"));
13763 vim_free(b);
13766 else
13767 #endif
13768 lp->sl_map_array[c] = headc;
13774 * Return TRUE if "c1" and "c2" are similar characters according to the MAP
13775 * lines in the .aff file.
13777 static int
13778 similar_chars(slang, c1, c2)
13779 slang_T *slang;
13780 int c1;
13781 int c2;
13783 int m1, m2;
13784 #ifdef FEAT_MBYTE
13785 char_u buf[MB_MAXBYTES];
13786 hashitem_T *hi;
13788 if (c1 >= 256)
13790 buf[mb_char2bytes(c1, buf)] = 0;
13791 hi = hash_find(&slang->sl_map_hash, buf);
13792 if (HASHITEM_EMPTY(hi))
13793 m1 = 0;
13794 else
13795 m1 = mb_ptr2char(hi->hi_key + STRLEN(hi->hi_key) + 1);
13797 else
13798 #endif
13799 m1 = slang->sl_map_array[c1];
13800 if (m1 == 0)
13801 return FALSE;
13804 #ifdef FEAT_MBYTE
13805 if (c2 >= 256)
13807 buf[mb_char2bytes(c2, buf)] = 0;
13808 hi = hash_find(&slang->sl_map_hash, buf);
13809 if (HASHITEM_EMPTY(hi))
13810 m2 = 0;
13811 else
13812 m2 = mb_ptr2char(hi->hi_key + STRLEN(hi->hi_key) + 1);
13814 else
13815 #endif
13816 m2 = slang->sl_map_array[c2];
13818 return m1 == m2;
13822 * Add a suggestion to the list of suggestions.
13823 * For a suggestion that is already in the list the lowest score is remembered.
13825 static void
13826 add_suggestion(su, gap, goodword, badlenarg, score, altscore, had_bonus,
13827 slang, maxsf)
13828 suginfo_T *su;
13829 garray_T *gap; /* either su_ga or su_sga */
13830 char_u *goodword;
13831 int badlenarg; /* len of bad word replaced with "goodword" */
13832 int score;
13833 int altscore;
13834 int had_bonus; /* value for st_had_bonus */
13835 slang_T *slang; /* language for sound folding */
13836 int maxsf; /* su_maxscore applies to soundfold score,
13837 su_sfmaxscore to the total score. */
13839 int goodlen; /* len of goodword changed */
13840 int badlen; /* len of bad word changed */
13841 suggest_T *stp;
13842 suggest_T new_sug;
13843 int i;
13844 char_u *pgood, *pbad;
13846 /* Minimize "badlen" for consistency. Avoids that changing "the the" to
13847 * "thee the" is added next to changing the first "the" the "thee". */
13848 pgood = goodword + STRLEN(goodword);
13849 pbad = su->su_badptr + badlenarg;
13850 for (;;)
13852 goodlen = (int)(pgood - goodword);
13853 badlen = (int)(pbad - su->su_badptr);
13854 if (goodlen <= 0 || badlen <= 0)
13855 break;
13856 mb_ptr_back(goodword, pgood);
13857 mb_ptr_back(su->su_badptr, pbad);
13858 #ifdef FEAT_MBYTE
13859 if (has_mbyte)
13861 if (mb_ptr2char(pgood) != mb_ptr2char(pbad))
13862 break;
13864 else
13865 #endif
13866 if (*pgood != *pbad)
13867 break;
13870 if (badlen == 0 && goodlen == 0)
13871 /* goodword doesn't change anything; may happen for "the the" changing
13872 * the first "the" to itself. */
13873 return;
13875 if (gap->ga_len == 0)
13876 i = -1;
13877 else
13879 /* Check if the word is already there. Also check the length that is
13880 * being replaced "thes," -> "these" is a different suggestion from
13881 * "thes" -> "these". */
13882 stp = &SUG(*gap, 0);
13883 for (i = gap->ga_len; --i >= 0; ++stp)
13884 if (stp->st_wordlen == goodlen
13885 && stp->st_orglen == badlen
13886 && STRNCMP(stp->st_word, goodword, goodlen) == 0)
13889 * Found it. Remember the word with the lowest score.
13891 if (stp->st_slang == NULL)
13892 stp->st_slang = slang;
13894 new_sug.st_score = score;
13895 new_sug.st_altscore = altscore;
13896 new_sug.st_had_bonus = had_bonus;
13898 if (stp->st_had_bonus != had_bonus)
13900 /* Only one of the two had the soundalike score computed.
13901 * Need to do that for the other one now, otherwise the
13902 * scores can't be compared. This happens because
13903 * suggest_try_change() doesn't compute the soundalike
13904 * word to keep it fast, while some special methods set
13905 * the soundalike score to zero. */
13906 if (had_bonus)
13907 rescore_one(su, stp);
13908 else
13910 new_sug.st_word = stp->st_word;
13911 new_sug.st_wordlen = stp->st_wordlen;
13912 new_sug.st_slang = stp->st_slang;
13913 new_sug.st_orglen = badlen;
13914 rescore_one(su, &new_sug);
13918 if (stp->st_score > new_sug.st_score)
13920 stp->st_score = new_sug.st_score;
13921 stp->st_altscore = new_sug.st_altscore;
13922 stp->st_had_bonus = new_sug.st_had_bonus;
13924 break;
13928 if (i < 0 && ga_grow(gap, 1) == OK)
13930 /* Add a suggestion. */
13931 stp = &SUG(*gap, gap->ga_len);
13932 stp->st_word = vim_strnsave(goodword, goodlen);
13933 if (stp->st_word != NULL)
13935 stp->st_wordlen = goodlen;
13936 stp->st_score = score;
13937 stp->st_altscore = altscore;
13938 stp->st_had_bonus = had_bonus;
13939 stp->st_orglen = badlen;
13940 stp->st_slang = slang;
13941 ++gap->ga_len;
13943 /* If we have too many suggestions now, sort the list and keep
13944 * the best suggestions. */
13945 if (gap->ga_len > SUG_MAX_COUNT(su))
13947 if (maxsf)
13948 su->su_sfmaxscore = cleanup_suggestions(gap,
13949 su->su_sfmaxscore, SUG_CLEAN_COUNT(su));
13950 else
13952 i = su->su_maxscore;
13953 su->su_maxscore = cleanup_suggestions(gap,
13954 su->su_maxscore, SUG_CLEAN_COUNT(su));
13962 * Suggestions may in fact be flagged as errors. Esp. for banned words and
13963 * for split words, such as "the the". Remove these from the list here.
13965 static void
13966 check_suggestions(su, gap)
13967 suginfo_T *su;
13968 garray_T *gap; /* either su_ga or su_sga */
13970 suggest_T *stp;
13971 int i;
13972 char_u longword[MAXWLEN + 1];
13973 int len;
13974 hlf_T attr;
13976 stp = &SUG(*gap, 0);
13977 for (i = gap->ga_len - 1; i >= 0; --i)
13979 /* Need to append what follows to check for "the the". */
13980 STRCPY(longword, stp[i].st_word);
13981 len = stp[i].st_wordlen;
13982 vim_strncpy(longword + len, su->su_badptr + stp[i].st_orglen,
13983 MAXWLEN - len);
13984 attr = HLF_COUNT;
13985 (void)spell_check(curwin, longword, &attr, NULL, FALSE);
13986 if (attr != HLF_COUNT)
13988 /* Remove this entry. */
13989 vim_free(stp[i].st_word);
13990 --gap->ga_len;
13991 if (i < gap->ga_len)
13992 mch_memmove(stp + i, stp + i + 1,
13993 sizeof(suggest_T) * (gap->ga_len - i));
14000 * Add a word to be banned.
14002 static void
14003 add_banned(su, word)
14004 suginfo_T *su;
14005 char_u *word;
14007 char_u *s;
14008 hash_T hash;
14009 hashitem_T *hi;
14011 hash = hash_hash(word);
14012 hi = hash_lookup(&su->su_banned, word, hash);
14013 if (HASHITEM_EMPTY(hi))
14015 s = vim_strsave(word);
14016 if (s != NULL)
14017 hash_add_item(&su->su_banned, hi, s, hash);
14022 * Recompute the score for all suggestions if sound-folding is possible. This
14023 * is slow, thus only done for the final results.
14025 static void
14026 rescore_suggestions(su)
14027 suginfo_T *su;
14029 int i;
14031 if (su->su_sallang != NULL)
14032 for (i = 0; i < su->su_ga.ga_len; ++i)
14033 rescore_one(su, &SUG(su->su_ga, i));
14037 * Recompute the score for one suggestion if sound-folding is possible.
14039 static void
14040 rescore_one(su, stp)
14041 suginfo_T *su;
14042 suggest_T *stp;
14044 slang_T *slang = stp->st_slang;
14045 char_u sal_badword[MAXWLEN];
14046 char_u *p;
14048 /* Only rescore suggestions that have no sal score yet and do have a
14049 * language. */
14050 if (slang != NULL && slang->sl_sal.ga_len > 0 && !stp->st_had_bonus)
14052 if (slang == su->su_sallang)
14053 p = su->su_sal_badword;
14054 else
14056 spell_soundfold(slang, su->su_fbadword, TRUE, sal_badword);
14057 p = sal_badword;
14060 stp->st_altscore = stp_sal_score(stp, su, slang, p);
14061 if (stp->st_altscore == SCORE_MAXMAX)
14062 stp->st_altscore = SCORE_BIG;
14063 stp->st_score = RESCORE(stp->st_score, stp->st_altscore);
14064 stp->st_had_bonus = TRUE;
14068 static int
14069 #ifdef __BORLANDC__
14070 _RTLENTRYF
14071 #endif
14072 sug_compare __ARGS((const void *s1, const void *s2));
14075 * Function given to qsort() to sort the suggestions on st_score.
14076 * First on "st_score", then "st_altscore" then alphabetically.
14078 static int
14079 #ifdef __BORLANDC__
14080 _RTLENTRYF
14081 #endif
14082 sug_compare(s1, s2)
14083 const void *s1;
14084 const void *s2;
14086 suggest_T *p1 = (suggest_T *)s1;
14087 suggest_T *p2 = (suggest_T *)s2;
14088 int n = p1->st_score - p2->st_score;
14090 if (n == 0)
14092 n = p1->st_altscore - p2->st_altscore;
14093 if (n == 0)
14094 n = STRICMP(p1->st_word, p2->st_word);
14096 return n;
14100 * Cleanup the suggestions:
14101 * - Sort on score.
14102 * - Remove words that won't be displayed.
14103 * Returns the maximum score in the list or "maxscore" unmodified.
14105 static int
14106 cleanup_suggestions(gap, maxscore, keep)
14107 garray_T *gap;
14108 int maxscore;
14109 int keep; /* nr of suggestions to keep */
14111 suggest_T *stp = &SUG(*gap, 0);
14112 int i;
14114 /* Sort the list. */
14115 qsort(gap->ga_data, (size_t)gap->ga_len, sizeof(suggest_T), sug_compare);
14117 /* Truncate the list to the number of suggestions that will be displayed. */
14118 if (gap->ga_len > keep)
14120 for (i = keep; i < gap->ga_len; ++i)
14121 vim_free(stp[i].st_word);
14122 gap->ga_len = keep;
14123 return stp[keep - 1].st_score;
14125 return maxscore;
14128 #if defined(FEAT_EVAL) || defined(PROTO)
14130 * Soundfold a string, for soundfold().
14131 * Result is in allocated memory, NULL for an error.
14133 char_u *
14134 eval_soundfold(word)
14135 char_u *word;
14137 langp_T *lp;
14138 char_u sound[MAXWLEN];
14139 int lpi;
14141 if (curwin->w_p_spell && *curbuf->b_p_spl != NUL)
14142 /* Use the sound-folding of the first language that supports it. */
14143 for (lpi = 0; lpi < curbuf->b_langp.ga_len; ++lpi)
14145 lp = LANGP_ENTRY(curbuf->b_langp, lpi);
14146 if (lp->lp_slang->sl_sal.ga_len > 0)
14148 /* soundfold the word */
14149 spell_soundfold(lp->lp_slang, word, FALSE, sound);
14150 return vim_strsave(sound);
14154 /* No language with sound folding, return word as-is. */
14155 return vim_strsave(word);
14157 #endif
14160 * Turn "inword" into its sound-a-like equivalent in "res[MAXWLEN]".
14162 * There are many ways to turn a word into a sound-a-like representation. The
14163 * oldest is Soundex (1918!). A nice overview can be found in "Approximate
14164 * swedish name matching - survey and test of different algorithms" by Klas
14165 * Erikson.
14167 * We support two methods:
14168 * 1. SOFOFROM/SOFOTO do a simple character mapping.
14169 * 2. SAL items define a more advanced sound-folding (and much slower).
14171 static void
14172 spell_soundfold(slang, inword, folded, res)
14173 slang_T *slang;
14174 char_u *inword;
14175 int folded; /* "inword" is already case-folded */
14176 char_u *res;
14178 char_u fword[MAXWLEN];
14179 char_u *word;
14181 if (slang->sl_sofo)
14182 /* SOFOFROM and SOFOTO used */
14183 spell_soundfold_sofo(slang, inword, res);
14184 else
14186 /* SAL items used. Requires the word to be case-folded. */
14187 if (folded)
14188 word = inword;
14189 else
14191 (void)spell_casefold(inword, (int)STRLEN(inword), fword, MAXWLEN);
14192 word = fword;
14195 #ifdef FEAT_MBYTE
14196 if (has_mbyte)
14197 spell_soundfold_wsal(slang, word, res);
14198 else
14199 #endif
14200 spell_soundfold_sal(slang, word, res);
14205 * Perform sound folding of "inword" into "res" according to SOFOFROM and
14206 * SOFOTO lines.
14208 static void
14209 spell_soundfold_sofo(slang, inword, res)
14210 slang_T *slang;
14211 char_u *inword;
14212 char_u *res;
14214 char_u *s;
14215 int ri = 0;
14216 int c;
14218 #ifdef FEAT_MBYTE
14219 if (has_mbyte)
14221 int prevc = 0;
14222 int *ip;
14224 /* The sl_sal_first[] table contains the translation for chars up to
14225 * 255, sl_sal the rest. */
14226 for (s = inword; *s != NUL; )
14228 c = mb_cptr2char_adv(&s);
14229 if (enc_utf8 ? utf_class(c) == 0 : vim_iswhite(c))
14230 c = ' ';
14231 else if (c < 256)
14232 c = slang->sl_sal_first[c];
14233 else
14235 ip = ((int **)slang->sl_sal.ga_data)[c & 0xff];
14236 if (ip == NULL) /* empty list, can't match */
14237 c = NUL;
14238 else
14239 for (;;) /* find "c" in the list */
14241 if (*ip == 0) /* not found */
14243 c = NUL;
14244 break;
14246 if (*ip == c) /* match! */
14248 c = ip[1];
14249 break;
14251 ip += 2;
14255 if (c != NUL && c != prevc)
14257 ri += mb_char2bytes(c, res + ri);
14258 if (ri + MB_MAXBYTES > MAXWLEN)
14259 break;
14260 prevc = c;
14264 else
14265 #endif
14267 /* The sl_sal_first[] table contains the translation. */
14268 for (s = inword; (c = *s) != NUL; ++s)
14270 if (vim_iswhite(c))
14271 c = ' ';
14272 else
14273 c = slang->sl_sal_first[c];
14274 if (c != NUL && (ri == 0 || res[ri - 1] != c))
14275 res[ri++] = c;
14279 res[ri] = NUL;
14282 static void
14283 spell_soundfold_sal(slang, inword, res)
14284 slang_T *slang;
14285 char_u *inword;
14286 char_u *res;
14288 salitem_T *smp;
14289 char_u word[MAXWLEN];
14290 char_u *s = inword;
14291 char_u *t;
14292 char_u *pf;
14293 int i, j, z;
14294 int reslen;
14295 int n, k = 0;
14296 int z0;
14297 int k0;
14298 int n0;
14299 int c;
14300 int pri;
14301 int p0 = -333;
14302 int c0;
14304 /* Remove accents, if wanted. We actually remove all non-word characters.
14305 * But keep white space. We need a copy, the word may be changed here. */
14306 if (slang->sl_rem_accents)
14308 t = word;
14309 while (*s != NUL)
14311 if (vim_iswhite(*s))
14313 *t++ = ' ';
14314 s = skipwhite(s);
14316 else
14318 if (spell_iswordp_nmw(s))
14319 *t++ = *s;
14320 ++s;
14323 *t = NUL;
14325 else
14326 STRCPY(word, s);
14328 smp = (salitem_T *)slang->sl_sal.ga_data;
14331 * This comes from Aspell phonet.cpp. Converted from C++ to C.
14332 * Changed to keep spaces.
14334 i = reslen = z = 0;
14335 while ((c = word[i]) != NUL)
14337 /* Start with the first rule that has the character in the word. */
14338 n = slang->sl_sal_first[c];
14339 z0 = 0;
14341 if (n >= 0)
14343 /* check all rules for the same letter */
14344 for (; (s = smp[n].sm_lead)[0] == c; ++n)
14346 /* Quickly skip entries that don't match the word. Most
14347 * entries are less then three chars, optimize for that. */
14348 k = smp[n].sm_leadlen;
14349 if (k > 1)
14351 if (word[i + 1] != s[1])
14352 continue;
14353 if (k > 2)
14355 for (j = 2; j < k; ++j)
14356 if (word[i + j] != s[j])
14357 break;
14358 if (j < k)
14359 continue;
14363 if ((pf = smp[n].sm_oneof) != NULL)
14365 /* Check for match with one of the chars in "sm_oneof". */
14366 while (*pf != NUL && *pf != word[i + k])
14367 ++pf;
14368 if (*pf == NUL)
14369 continue;
14370 ++k;
14372 s = smp[n].sm_rules;
14373 pri = 5; /* default priority */
14375 p0 = *s;
14376 k0 = k;
14377 while (*s == '-' && k > 1)
14379 k--;
14380 s++;
14382 if (*s == '<')
14383 s++;
14384 if (VIM_ISDIGIT(*s))
14386 /* determine priority */
14387 pri = *s - '0';
14388 s++;
14390 if (*s == '^' && *(s + 1) == '^')
14391 s++;
14393 if (*s == NUL
14394 || (*s == '^'
14395 && (i == 0 || !(word[i - 1] == ' '
14396 || spell_iswordp(word + i - 1, curbuf)))
14397 && (*(s + 1) != '$'
14398 || (!spell_iswordp(word + i + k0, curbuf))))
14399 || (*s == '$' && i > 0
14400 && spell_iswordp(word + i - 1, curbuf)
14401 && (!spell_iswordp(word + i + k0, curbuf))))
14403 /* search for followup rules, if: */
14404 /* followup and k > 1 and NO '-' in searchstring */
14405 c0 = word[i + k - 1];
14406 n0 = slang->sl_sal_first[c0];
14408 if (slang->sl_followup && k > 1 && n0 >= 0
14409 && p0 != '-' && word[i + k] != NUL)
14411 /* test follow-up rule for "word[i + k]" */
14412 for ( ; (s = smp[n0].sm_lead)[0] == c0; ++n0)
14414 /* Quickly skip entries that don't match the word.
14415 * */
14416 k0 = smp[n0].sm_leadlen;
14417 if (k0 > 1)
14419 if (word[i + k] != s[1])
14420 continue;
14421 if (k0 > 2)
14423 pf = word + i + k + 1;
14424 for (j = 2; j < k0; ++j)
14425 if (*pf++ != s[j])
14426 break;
14427 if (j < k0)
14428 continue;
14431 k0 += k - 1;
14433 if ((pf = smp[n0].sm_oneof) != NULL)
14435 /* Check for match with one of the chars in
14436 * "sm_oneof". */
14437 while (*pf != NUL && *pf != word[i + k0])
14438 ++pf;
14439 if (*pf == NUL)
14440 continue;
14441 ++k0;
14444 p0 = 5;
14445 s = smp[n0].sm_rules;
14446 while (*s == '-')
14448 /* "k0" gets NOT reduced because
14449 * "if (k0 == k)" */
14450 s++;
14452 if (*s == '<')
14453 s++;
14454 if (VIM_ISDIGIT(*s))
14456 p0 = *s - '0';
14457 s++;
14460 if (*s == NUL
14461 /* *s == '^' cuts */
14462 || (*s == '$'
14463 && !spell_iswordp(word + i + k0,
14464 curbuf)))
14466 if (k0 == k)
14467 /* this is just a piece of the string */
14468 continue;
14470 if (p0 < pri)
14471 /* priority too low */
14472 continue;
14473 /* rule fits; stop search */
14474 break;
14478 if (p0 >= pri && smp[n0].sm_lead[0] == c0)
14479 continue;
14482 /* replace string */
14483 s = smp[n].sm_to;
14484 if (s == NULL)
14485 s = (char_u *)"";
14486 pf = smp[n].sm_rules;
14487 p0 = (vim_strchr(pf, '<') != NULL) ? 1 : 0;
14488 if (p0 == 1 && z == 0)
14490 /* rule with '<' is used */
14491 if (reslen > 0 && *s != NUL && (res[reslen - 1] == c
14492 || res[reslen - 1] == *s))
14493 reslen--;
14494 z0 = 1;
14495 z = 1;
14496 k0 = 0;
14497 while (*s != NUL && word[i + k0] != NUL)
14499 word[i + k0] = *s;
14500 k0++;
14501 s++;
14503 if (k > k0)
14504 STRMOVE(word + i + k0, word + i + k);
14506 /* new "actual letter" */
14507 c = word[i];
14509 else
14511 /* no '<' rule used */
14512 i += k - 1;
14513 z = 0;
14514 while (*s != NUL && s[1] != NUL && reslen < MAXWLEN)
14516 if (reslen == 0 || res[reslen - 1] != *s)
14517 res[reslen++] = *s;
14518 s++;
14520 /* new "actual letter" */
14521 c = *s;
14522 if (strstr((char *)pf, "^^") != NULL)
14524 if (c != NUL)
14525 res[reslen++] = c;
14526 STRMOVE(word, word + i + 1);
14527 i = 0;
14528 z0 = 1;
14531 break;
14535 else if (vim_iswhite(c))
14537 c = ' ';
14538 k = 1;
14541 if (z0 == 0)
14543 if (k && !p0 && reslen < MAXWLEN && c != NUL
14544 && (!slang->sl_collapse || reslen == 0
14545 || res[reslen - 1] != c))
14546 /* condense only double letters */
14547 res[reslen++] = c;
14549 i++;
14550 z = 0;
14551 k = 0;
14555 res[reslen] = NUL;
14558 #ifdef FEAT_MBYTE
14560 * Turn "inword" into its sound-a-like equivalent in "res[MAXWLEN]".
14561 * Multi-byte version of spell_soundfold().
14563 static void
14564 spell_soundfold_wsal(slang, inword, res)
14565 slang_T *slang;
14566 char_u *inword;
14567 char_u *res;
14569 salitem_T *smp = (salitem_T *)slang->sl_sal.ga_data;
14570 int word[MAXWLEN];
14571 int wres[MAXWLEN];
14572 int l;
14573 char_u *s;
14574 int *ws;
14575 char_u *t;
14576 int *pf;
14577 int i, j, z;
14578 int reslen;
14579 int n, k = 0;
14580 int z0;
14581 int k0;
14582 int n0;
14583 int c;
14584 int pri;
14585 int p0 = -333;
14586 int c0;
14587 int did_white = FALSE;
14590 * Convert the multi-byte string to a wide-character string.
14591 * Remove accents, if wanted. We actually remove all non-word characters.
14592 * But keep white space.
14594 n = 0;
14595 for (s = inword; *s != NUL; )
14597 t = s;
14598 c = mb_cptr2char_adv(&s);
14599 if (slang->sl_rem_accents)
14601 if (enc_utf8 ? utf_class(c) == 0 : vim_iswhite(c))
14603 if (did_white)
14604 continue;
14605 c = ' ';
14606 did_white = TRUE;
14608 else
14610 did_white = FALSE;
14611 if (!spell_iswordp_nmw(t))
14612 continue;
14615 word[n++] = c;
14617 word[n] = NUL;
14620 * This comes from Aspell phonet.cpp.
14621 * Converted from C++ to C. Added support for multi-byte chars.
14622 * Changed to keep spaces.
14624 i = reslen = z = 0;
14625 while ((c = word[i]) != NUL)
14627 /* Start with the first rule that has the character in the word. */
14628 n = slang->sl_sal_first[c & 0xff];
14629 z0 = 0;
14631 if (n >= 0)
14633 /* check all rules for the same index byte */
14634 for (; ((ws = smp[n].sm_lead_w)[0] & 0xff) == (c & 0xff); ++n)
14636 /* Quickly skip entries that don't match the word. Most
14637 * entries are less then three chars, optimize for that. */
14638 if (c != ws[0])
14639 continue;
14640 k = smp[n].sm_leadlen;
14641 if (k > 1)
14643 if (word[i + 1] != ws[1])
14644 continue;
14645 if (k > 2)
14647 for (j = 2; j < k; ++j)
14648 if (word[i + j] != ws[j])
14649 break;
14650 if (j < k)
14651 continue;
14655 if ((pf = smp[n].sm_oneof_w) != NULL)
14657 /* Check for match with one of the chars in "sm_oneof". */
14658 while (*pf != NUL && *pf != word[i + k])
14659 ++pf;
14660 if (*pf == NUL)
14661 continue;
14662 ++k;
14664 s = smp[n].sm_rules;
14665 pri = 5; /* default priority */
14667 p0 = *s;
14668 k0 = k;
14669 while (*s == '-' && k > 1)
14671 k--;
14672 s++;
14674 if (*s == '<')
14675 s++;
14676 if (VIM_ISDIGIT(*s))
14678 /* determine priority */
14679 pri = *s - '0';
14680 s++;
14682 if (*s == '^' && *(s + 1) == '^')
14683 s++;
14685 if (*s == NUL
14686 || (*s == '^'
14687 && (i == 0 || !(word[i - 1] == ' '
14688 || spell_iswordp_w(word + i - 1, curbuf)))
14689 && (*(s + 1) != '$'
14690 || (!spell_iswordp_w(word + i + k0, curbuf))))
14691 || (*s == '$' && i > 0
14692 && spell_iswordp_w(word + i - 1, curbuf)
14693 && (!spell_iswordp_w(word + i + k0, curbuf))))
14695 /* search for followup rules, if: */
14696 /* followup and k > 1 and NO '-' in searchstring */
14697 c0 = word[i + k - 1];
14698 n0 = slang->sl_sal_first[c0 & 0xff];
14700 if (slang->sl_followup && k > 1 && n0 >= 0
14701 && p0 != '-' && word[i + k] != NUL)
14703 /* Test follow-up rule for "word[i + k]"; loop over
14704 * all entries with the same index byte. */
14705 for ( ; ((ws = smp[n0].sm_lead_w)[0] & 0xff)
14706 == (c0 & 0xff); ++n0)
14708 /* Quickly skip entries that don't match the word.
14710 if (c0 != ws[0])
14711 continue;
14712 k0 = smp[n0].sm_leadlen;
14713 if (k0 > 1)
14715 if (word[i + k] != ws[1])
14716 continue;
14717 if (k0 > 2)
14719 pf = word + i + k + 1;
14720 for (j = 2; j < k0; ++j)
14721 if (*pf++ != ws[j])
14722 break;
14723 if (j < k0)
14724 continue;
14727 k0 += k - 1;
14729 if ((pf = smp[n0].sm_oneof_w) != NULL)
14731 /* Check for match with one of the chars in
14732 * "sm_oneof". */
14733 while (*pf != NUL && *pf != word[i + k0])
14734 ++pf;
14735 if (*pf == NUL)
14736 continue;
14737 ++k0;
14740 p0 = 5;
14741 s = smp[n0].sm_rules;
14742 while (*s == '-')
14744 /* "k0" gets NOT reduced because
14745 * "if (k0 == k)" */
14746 s++;
14748 if (*s == '<')
14749 s++;
14750 if (VIM_ISDIGIT(*s))
14752 p0 = *s - '0';
14753 s++;
14756 if (*s == NUL
14757 /* *s == '^' cuts */
14758 || (*s == '$'
14759 && !spell_iswordp_w(word + i + k0,
14760 curbuf)))
14762 if (k0 == k)
14763 /* this is just a piece of the string */
14764 continue;
14766 if (p0 < pri)
14767 /* priority too low */
14768 continue;
14769 /* rule fits; stop search */
14770 break;
14774 if (p0 >= pri && (smp[n0].sm_lead_w[0] & 0xff)
14775 == (c0 & 0xff))
14776 continue;
14779 /* replace string */
14780 ws = smp[n].sm_to_w;
14781 s = smp[n].sm_rules;
14782 p0 = (vim_strchr(s, '<') != NULL) ? 1 : 0;
14783 if (p0 == 1 && z == 0)
14785 /* rule with '<' is used */
14786 if (reslen > 0 && ws != NULL && *ws != NUL
14787 && (wres[reslen - 1] == c
14788 || wres[reslen - 1] == *ws))
14789 reslen--;
14790 z0 = 1;
14791 z = 1;
14792 k0 = 0;
14793 if (ws != NULL)
14794 while (*ws != NUL && word[i + k0] != NUL)
14796 word[i + k0] = *ws;
14797 k0++;
14798 ws++;
14800 if (k > k0)
14801 mch_memmove(word + i + k0, word + i + k,
14802 sizeof(int) * (STRLEN(word + i + k) + 1));
14804 /* new "actual letter" */
14805 c = word[i];
14807 else
14809 /* no '<' rule used */
14810 i += k - 1;
14811 z = 0;
14812 if (ws != NULL)
14813 while (*ws != NUL && ws[1] != NUL
14814 && reslen < MAXWLEN)
14816 if (reslen == 0 || wres[reslen - 1] != *ws)
14817 wres[reslen++] = *ws;
14818 ws++;
14820 /* new "actual letter" */
14821 if (ws == NULL)
14822 c = NUL;
14823 else
14824 c = *ws;
14825 if (strstr((char *)s, "^^") != NULL)
14827 if (c != NUL)
14828 wres[reslen++] = c;
14829 mch_memmove(word, word + i + 1,
14830 sizeof(int) * (STRLEN(word + i + 1) + 1));
14831 i = 0;
14832 z0 = 1;
14835 break;
14839 else if (vim_iswhite(c))
14841 c = ' ';
14842 k = 1;
14845 if (z0 == 0)
14847 if (k && !p0 && reslen < MAXWLEN && c != NUL
14848 && (!slang->sl_collapse || reslen == 0
14849 || wres[reslen - 1] != c))
14850 /* condense only double letters */
14851 wres[reslen++] = c;
14853 i++;
14854 z = 0;
14855 k = 0;
14859 /* Convert wide characters in "wres" to a multi-byte string in "res". */
14860 l = 0;
14861 for (n = 0; n < reslen; ++n)
14863 l += mb_char2bytes(wres[n], res + l);
14864 if (l + MB_MAXBYTES > MAXWLEN)
14865 break;
14867 res[l] = NUL;
14869 #endif
14872 * Compute a score for two sound-a-like words.
14873 * This permits up to two inserts/deletes/swaps/etc. to keep things fast.
14874 * Instead of a generic loop we write out the code. That keeps it fast by
14875 * avoiding checks that will not be possible.
14877 static int
14878 soundalike_score(goodstart, badstart)
14879 char_u *goodstart; /* sound-folded good word */
14880 char_u *badstart; /* sound-folded bad word */
14882 char_u *goodsound = goodstart;
14883 char_u *badsound = badstart;
14884 int goodlen;
14885 int badlen;
14886 int n;
14887 char_u *pl, *ps;
14888 char_u *pl2, *ps2;
14889 int score = 0;
14891 /* adding/inserting "*" at the start (word starts with vowel) shouldn't be
14892 * counted so much, vowels halfway the word aren't counted at all. */
14893 if ((*badsound == '*' || *goodsound == '*') && *badsound != *goodsound)
14895 if (badsound[1] == goodsound[1]
14896 || (badsound[1] != NUL
14897 && goodsound[1] != NUL
14898 && badsound[2] == goodsound[2]))
14900 /* handle like a substitute */
14902 else
14904 score = 2 * SCORE_DEL / 3;
14905 if (*badsound == '*')
14906 ++badsound;
14907 else
14908 ++goodsound;
14912 goodlen = (int)STRLEN(goodsound);
14913 badlen = (int)STRLEN(badsound);
14915 /* Return quickly if the lengths are too different to be fixed by two
14916 * changes. */
14917 n = goodlen - badlen;
14918 if (n < -2 || n > 2)
14919 return SCORE_MAXMAX;
14921 if (n > 0)
14923 pl = goodsound; /* goodsound is longest */
14924 ps = badsound;
14926 else
14928 pl = badsound; /* badsound is longest */
14929 ps = goodsound;
14932 /* Skip over the identical part. */
14933 while (*pl == *ps && *pl != NUL)
14935 ++pl;
14936 ++ps;
14939 switch (n)
14941 case -2:
14942 case 2:
14944 * Must delete two characters from "pl".
14946 ++pl; /* first delete */
14947 while (*pl == *ps)
14949 ++pl;
14950 ++ps;
14952 /* strings must be equal after second delete */
14953 if (STRCMP(pl + 1, ps) == 0)
14954 return score + SCORE_DEL * 2;
14956 /* Failed to compare. */
14957 break;
14959 case -1:
14960 case 1:
14962 * Minimal one delete from "pl" required.
14965 /* 1: delete */
14966 pl2 = pl + 1;
14967 ps2 = ps;
14968 while (*pl2 == *ps2)
14970 if (*pl2 == NUL) /* reached the end */
14971 return score + SCORE_DEL;
14972 ++pl2;
14973 ++ps2;
14976 /* 2: delete then swap, then rest must be equal */
14977 if (pl2[0] == ps2[1] && pl2[1] == ps2[0]
14978 && STRCMP(pl2 + 2, ps2 + 2) == 0)
14979 return score + SCORE_DEL + SCORE_SWAP;
14981 /* 3: delete then substitute, then the rest must be equal */
14982 if (STRCMP(pl2 + 1, ps2 + 1) == 0)
14983 return score + SCORE_DEL + SCORE_SUBST;
14985 /* 4: first swap then delete */
14986 if (pl[0] == ps[1] && pl[1] == ps[0])
14988 pl2 = pl + 2; /* swap, skip two chars */
14989 ps2 = ps + 2;
14990 while (*pl2 == *ps2)
14992 ++pl2;
14993 ++ps2;
14995 /* delete a char and then strings must be equal */
14996 if (STRCMP(pl2 + 1, ps2) == 0)
14997 return score + SCORE_SWAP + SCORE_DEL;
15000 /* 5: first substitute then delete */
15001 pl2 = pl + 1; /* substitute, skip one char */
15002 ps2 = ps + 1;
15003 while (*pl2 == *ps2)
15005 ++pl2;
15006 ++ps2;
15008 /* delete a char and then strings must be equal */
15009 if (STRCMP(pl2 + 1, ps2) == 0)
15010 return score + SCORE_SUBST + SCORE_DEL;
15012 /* Failed to compare. */
15013 break;
15015 case 0:
15017 * Lengths are equal, thus changes must result in same length: An
15018 * insert is only possible in combination with a delete.
15019 * 1: check if for identical strings
15021 if (*pl == NUL)
15022 return score;
15024 /* 2: swap */
15025 if (pl[0] == ps[1] && pl[1] == ps[0])
15027 pl2 = pl + 2; /* swap, skip two chars */
15028 ps2 = ps + 2;
15029 while (*pl2 == *ps2)
15031 if (*pl2 == NUL) /* reached the end */
15032 return score + SCORE_SWAP;
15033 ++pl2;
15034 ++ps2;
15036 /* 3: swap and swap again */
15037 if (pl2[0] == ps2[1] && pl2[1] == ps2[0]
15038 && STRCMP(pl2 + 2, ps2 + 2) == 0)
15039 return score + SCORE_SWAP + SCORE_SWAP;
15041 /* 4: swap and substitute */
15042 if (STRCMP(pl2 + 1, ps2 + 1) == 0)
15043 return score + SCORE_SWAP + SCORE_SUBST;
15046 /* 5: substitute */
15047 pl2 = pl + 1;
15048 ps2 = ps + 1;
15049 while (*pl2 == *ps2)
15051 if (*pl2 == NUL) /* reached the end */
15052 return score + SCORE_SUBST;
15053 ++pl2;
15054 ++ps2;
15057 /* 6: substitute and swap */
15058 if (pl2[0] == ps2[1] && pl2[1] == ps2[0]
15059 && STRCMP(pl2 + 2, ps2 + 2) == 0)
15060 return score + SCORE_SUBST + SCORE_SWAP;
15062 /* 7: substitute and substitute */
15063 if (STRCMP(pl2 + 1, ps2 + 1) == 0)
15064 return score + SCORE_SUBST + SCORE_SUBST;
15066 /* 8: insert then delete */
15067 pl2 = pl;
15068 ps2 = ps + 1;
15069 while (*pl2 == *ps2)
15071 ++pl2;
15072 ++ps2;
15074 if (STRCMP(pl2 + 1, ps2) == 0)
15075 return score + SCORE_INS + SCORE_DEL;
15077 /* 9: delete then insert */
15078 pl2 = pl + 1;
15079 ps2 = ps;
15080 while (*pl2 == *ps2)
15082 ++pl2;
15083 ++ps2;
15085 if (STRCMP(pl2, ps2 + 1) == 0)
15086 return score + SCORE_INS + SCORE_DEL;
15088 /* Failed to compare. */
15089 break;
15092 return SCORE_MAXMAX;
15096 * Compute the "edit distance" to turn "badword" into "goodword". The less
15097 * deletes/inserts/substitutes/swaps are required the lower the score.
15099 * The algorithm is described by Du and Chang, 1992.
15100 * The implementation of the algorithm comes from Aspell editdist.cpp,
15101 * edit_distance(). It has been converted from C++ to C and modified to
15102 * support multi-byte characters.
15104 static int
15105 spell_edit_score(slang, badword, goodword)
15106 slang_T *slang;
15107 char_u *badword;
15108 char_u *goodword;
15110 int *cnt;
15111 int badlen, goodlen; /* lengths including NUL */
15112 int j, i;
15113 int t;
15114 int bc, gc;
15115 int pbc, pgc;
15116 #ifdef FEAT_MBYTE
15117 char_u *p;
15118 int wbadword[MAXWLEN];
15119 int wgoodword[MAXWLEN];
15121 if (has_mbyte)
15123 /* Get the characters from the multi-byte strings and put them in an
15124 * int array for easy access. */
15125 for (p = badword, badlen = 0; *p != NUL; )
15126 wbadword[badlen++] = mb_cptr2char_adv(&p);
15127 wbadword[badlen++] = 0;
15128 for (p = goodword, goodlen = 0; *p != NUL; )
15129 wgoodword[goodlen++] = mb_cptr2char_adv(&p);
15130 wgoodword[goodlen++] = 0;
15132 else
15133 #endif
15135 badlen = (int)STRLEN(badword) + 1;
15136 goodlen = (int)STRLEN(goodword) + 1;
15139 /* We use "cnt" as an array: CNT(badword_idx, goodword_idx). */
15140 #define CNT(a, b) cnt[(a) + (b) * (badlen + 1)]
15141 cnt = (int *)lalloc((long_u)(sizeof(int) * (badlen + 1) * (goodlen + 1)),
15142 TRUE);
15143 if (cnt == NULL)
15144 return 0; /* out of memory */
15146 CNT(0, 0) = 0;
15147 for (j = 1; j <= goodlen; ++j)
15148 CNT(0, j) = CNT(0, j - 1) + SCORE_INS;
15150 for (i = 1; i <= badlen; ++i)
15152 CNT(i, 0) = CNT(i - 1, 0) + SCORE_DEL;
15153 for (j = 1; j <= goodlen; ++j)
15155 #ifdef FEAT_MBYTE
15156 if (has_mbyte)
15158 bc = wbadword[i - 1];
15159 gc = wgoodword[j - 1];
15161 else
15162 #endif
15164 bc = badword[i - 1];
15165 gc = goodword[j - 1];
15167 if (bc == gc)
15168 CNT(i, j) = CNT(i - 1, j - 1);
15169 else
15171 /* Use a better score when there is only a case difference. */
15172 if (SPELL_TOFOLD(bc) == SPELL_TOFOLD(gc))
15173 CNT(i, j) = SCORE_ICASE + CNT(i - 1, j - 1);
15174 else
15176 /* For a similar character use SCORE_SIMILAR. */
15177 if (slang != NULL
15178 && slang->sl_has_map
15179 && similar_chars(slang, gc, bc))
15180 CNT(i, j) = SCORE_SIMILAR + CNT(i - 1, j - 1);
15181 else
15182 CNT(i, j) = SCORE_SUBST + CNT(i - 1, j - 1);
15185 if (i > 1 && j > 1)
15187 #ifdef FEAT_MBYTE
15188 if (has_mbyte)
15190 pbc = wbadword[i - 2];
15191 pgc = wgoodword[j - 2];
15193 else
15194 #endif
15196 pbc = badword[i - 2];
15197 pgc = goodword[j - 2];
15199 if (bc == pgc && pbc == gc)
15201 t = SCORE_SWAP + CNT(i - 2, j - 2);
15202 if (t < CNT(i, j))
15203 CNT(i, j) = t;
15206 t = SCORE_DEL + CNT(i - 1, j);
15207 if (t < CNT(i, j))
15208 CNT(i, j) = t;
15209 t = SCORE_INS + CNT(i, j - 1);
15210 if (t < CNT(i, j))
15211 CNT(i, j) = t;
15216 i = CNT(badlen - 1, goodlen - 1);
15217 vim_free(cnt);
15218 return i;
15221 typedef struct
15223 int badi;
15224 int goodi;
15225 int score;
15226 } limitscore_T;
15229 * Like spell_edit_score(), but with a limit on the score to make it faster.
15230 * May return SCORE_MAXMAX when the score is higher than "limit".
15232 * This uses a stack for the edits still to be tried.
15233 * The idea comes from Aspell leditdist.cpp. Rewritten in C and added support
15234 * for multi-byte characters.
15236 static int
15237 spell_edit_score_limit(slang, badword, goodword, limit)
15238 slang_T *slang;
15239 char_u *badword;
15240 char_u *goodword;
15241 int limit;
15243 limitscore_T stack[10]; /* allow for over 3 * 2 edits */
15244 int stackidx;
15245 int bi, gi;
15246 int bi2, gi2;
15247 int bc, gc;
15248 int score;
15249 int score_off;
15250 int minscore;
15251 int round;
15253 #ifdef FEAT_MBYTE
15254 /* Multi-byte characters require a bit more work, use a different function
15255 * to avoid testing "has_mbyte" quite often. */
15256 if (has_mbyte)
15257 return spell_edit_score_limit_w(slang, badword, goodword, limit);
15258 #endif
15261 * The idea is to go from start to end over the words. So long as
15262 * characters are equal just continue, this always gives the lowest score.
15263 * When there is a difference try several alternatives. Each alternative
15264 * increases "score" for the edit distance. Some of the alternatives are
15265 * pushed unto a stack and tried later, some are tried right away. At the
15266 * end of the word the score for one alternative is known. The lowest
15267 * possible score is stored in "minscore".
15269 stackidx = 0;
15270 bi = 0;
15271 gi = 0;
15272 score = 0;
15273 minscore = limit + 1;
15275 for (;;)
15277 /* Skip over an equal part, score remains the same. */
15278 for (;;)
15280 bc = badword[bi];
15281 gc = goodword[gi];
15282 if (bc != gc) /* stop at a char that's different */
15283 break;
15284 if (bc == NUL) /* both words end */
15286 if (score < minscore)
15287 minscore = score;
15288 goto pop; /* do next alternative */
15290 ++bi;
15291 ++gi;
15294 if (gc == NUL) /* goodword ends, delete badword chars */
15298 if ((score += SCORE_DEL) >= minscore)
15299 goto pop; /* do next alternative */
15300 } while (badword[++bi] != NUL);
15301 minscore = score;
15303 else if (bc == NUL) /* badword ends, insert badword chars */
15307 if ((score += SCORE_INS) >= minscore)
15308 goto pop; /* do next alternative */
15309 } while (goodword[++gi] != NUL);
15310 minscore = score;
15312 else /* both words continue */
15314 /* If not close to the limit, perform a change. Only try changes
15315 * that may lead to a lower score than "minscore".
15316 * round 0: try deleting a char from badword
15317 * round 1: try inserting a char in badword */
15318 for (round = 0; round <= 1; ++round)
15320 score_off = score + (round == 0 ? SCORE_DEL : SCORE_INS);
15321 if (score_off < minscore)
15323 if (score_off + SCORE_EDIT_MIN >= minscore)
15325 /* Near the limit, rest of the words must match. We
15326 * can check that right now, no need to push an item
15327 * onto the stack. */
15328 bi2 = bi + 1 - round;
15329 gi2 = gi + round;
15330 while (goodword[gi2] == badword[bi2])
15332 if (goodword[gi2] == NUL)
15334 minscore = score_off;
15335 break;
15337 ++bi2;
15338 ++gi2;
15341 else
15343 /* try deleting/inserting a character later */
15344 stack[stackidx].badi = bi + 1 - round;
15345 stack[stackidx].goodi = gi + round;
15346 stack[stackidx].score = score_off;
15347 ++stackidx;
15352 if (score + SCORE_SWAP < minscore)
15354 /* If swapping two characters makes a match then the
15355 * substitution is more expensive, thus there is no need to
15356 * try both. */
15357 if (gc == badword[bi + 1] && bc == goodword[gi + 1])
15359 /* Swap two characters, that is: skip them. */
15360 gi += 2;
15361 bi += 2;
15362 score += SCORE_SWAP;
15363 continue;
15367 /* Substitute one character for another which is the same
15368 * thing as deleting a character from both goodword and badword.
15369 * Use a better score when there is only a case difference. */
15370 if (SPELL_TOFOLD(bc) == SPELL_TOFOLD(gc))
15371 score += SCORE_ICASE;
15372 else
15374 /* For a similar character use SCORE_SIMILAR. */
15375 if (slang != NULL
15376 && slang->sl_has_map
15377 && similar_chars(slang, gc, bc))
15378 score += SCORE_SIMILAR;
15379 else
15380 score += SCORE_SUBST;
15383 if (score < minscore)
15385 /* Do the substitution. */
15386 ++gi;
15387 ++bi;
15388 continue;
15391 pop:
15393 * Get here to try the next alternative, pop it from the stack.
15395 if (stackidx == 0) /* stack is empty, finished */
15396 break;
15398 /* pop an item from the stack */
15399 --stackidx;
15400 gi = stack[stackidx].goodi;
15401 bi = stack[stackidx].badi;
15402 score = stack[stackidx].score;
15405 /* When the score goes over "limit" it may actually be much higher.
15406 * Return a very large number to avoid going below the limit when giving a
15407 * bonus. */
15408 if (minscore > limit)
15409 return SCORE_MAXMAX;
15410 return minscore;
15413 #ifdef FEAT_MBYTE
15415 * Multi-byte version of spell_edit_score_limit().
15416 * Keep it in sync with the above!
15418 static int
15419 spell_edit_score_limit_w(slang, badword, goodword, limit)
15420 slang_T *slang;
15421 char_u *badword;
15422 char_u *goodword;
15423 int limit;
15425 limitscore_T stack[10]; /* allow for over 3 * 2 edits */
15426 int stackidx;
15427 int bi, gi;
15428 int bi2, gi2;
15429 int bc, gc;
15430 int score;
15431 int score_off;
15432 int minscore;
15433 int round;
15434 char_u *p;
15435 int wbadword[MAXWLEN];
15436 int wgoodword[MAXWLEN];
15438 /* Get the characters from the multi-byte strings and put them in an
15439 * int array for easy access. */
15440 bi = 0;
15441 for (p = badword; *p != NUL; )
15442 wbadword[bi++] = mb_cptr2char_adv(&p);
15443 wbadword[bi++] = 0;
15444 gi = 0;
15445 for (p = goodword; *p != NUL; )
15446 wgoodword[gi++] = mb_cptr2char_adv(&p);
15447 wgoodword[gi++] = 0;
15450 * The idea is to go from start to end over the words. So long as
15451 * characters are equal just continue, this always gives the lowest score.
15452 * When there is a difference try several alternatives. Each alternative
15453 * increases "score" for the edit distance. Some of the alternatives are
15454 * pushed unto a stack and tried later, some are tried right away. At the
15455 * end of the word the score for one alternative is known. The lowest
15456 * possible score is stored in "minscore".
15458 stackidx = 0;
15459 bi = 0;
15460 gi = 0;
15461 score = 0;
15462 minscore = limit + 1;
15464 for (;;)
15466 /* Skip over an equal part, score remains the same. */
15467 for (;;)
15469 bc = wbadword[bi];
15470 gc = wgoodword[gi];
15472 if (bc != gc) /* stop at a char that's different */
15473 break;
15474 if (bc == NUL) /* both words end */
15476 if (score < minscore)
15477 minscore = score;
15478 goto pop; /* do next alternative */
15480 ++bi;
15481 ++gi;
15484 if (gc == NUL) /* goodword ends, delete badword chars */
15488 if ((score += SCORE_DEL) >= minscore)
15489 goto pop; /* do next alternative */
15490 } while (wbadword[++bi] != NUL);
15491 minscore = score;
15493 else if (bc == NUL) /* badword ends, insert badword chars */
15497 if ((score += SCORE_INS) >= minscore)
15498 goto pop; /* do next alternative */
15499 } while (wgoodword[++gi] != NUL);
15500 minscore = score;
15502 else /* both words continue */
15504 /* If not close to the limit, perform a change. Only try changes
15505 * that may lead to a lower score than "minscore".
15506 * round 0: try deleting a char from badword
15507 * round 1: try inserting a char in badword */
15508 for (round = 0; round <= 1; ++round)
15510 score_off = score + (round == 0 ? SCORE_DEL : SCORE_INS);
15511 if (score_off < minscore)
15513 if (score_off + SCORE_EDIT_MIN >= minscore)
15515 /* Near the limit, rest of the words must match. We
15516 * can check that right now, no need to push an item
15517 * onto the stack. */
15518 bi2 = bi + 1 - round;
15519 gi2 = gi + round;
15520 while (wgoodword[gi2] == wbadword[bi2])
15522 if (wgoodword[gi2] == NUL)
15524 minscore = score_off;
15525 break;
15527 ++bi2;
15528 ++gi2;
15531 else
15533 /* try deleting a character from badword later */
15534 stack[stackidx].badi = bi + 1 - round;
15535 stack[stackidx].goodi = gi + round;
15536 stack[stackidx].score = score_off;
15537 ++stackidx;
15542 if (score + SCORE_SWAP < minscore)
15544 /* If swapping two characters makes a match then the
15545 * substitution is more expensive, thus there is no need to
15546 * try both. */
15547 if (gc == wbadword[bi + 1] && bc == wgoodword[gi + 1])
15549 /* Swap two characters, that is: skip them. */
15550 gi += 2;
15551 bi += 2;
15552 score += SCORE_SWAP;
15553 continue;
15557 /* Substitute one character for another which is the same
15558 * thing as deleting a character from both goodword and badword.
15559 * Use a better score when there is only a case difference. */
15560 if (SPELL_TOFOLD(bc) == SPELL_TOFOLD(gc))
15561 score += SCORE_ICASE;
15562 else
15564 /* For a similar character use SCORE_SIMILAR. */
15565 if (slang != NULL
15566 && slang->sl_has_map
15567 && similar_chars(slang, gc, bc))
15568 score += SCORE_SIMILAR;
15569 else
15570 score += SCORE_SUBST;
15573 if (score < minscore)
15575 /* Do the substitution. */
15576 ++gi;
15577 ++bi;
15578 continue;
15581 pop:
15583 * Get here to try the next alternative, pop it from the stack.
15585 if (stackidx == 0) /* stack is empty, finished */
15586 break;
15588 /* pop an item from the stack */
15589 --stackidx;
15590 gi = stack[stackidx].goodi;
15591 bi = stack[stackidx].badi;
15592 score = stack[stackidx].score;
15595 /* When the score goes over "limit" it may actually be much higher.
15596 * Return a very large number to avoid going below the limit when giving a
15597 * bonus. */
15598 if (minscore > limit)
15599 return SCORE_MAXMAX;
15600 return minscore;
15602 #endif
15605 * ":spellinfo"
15607 /*ARGSUSED*/
15608 void
15609 ex_spellinfo(eap)
15610 exarg_T *eap;
15612 int lpi;
15613 langp_T *lp;
15614 char_u *p;
15616 if (no_spell_checking(curwin))
15617 return;
15619 msg_start();
15620 for (lpi = 0; lpi < curbuf->b_langp.ga_len && !got_int; ++lpi)
15622 lp = LANGP_ENTRY(curbuf->b_langp, lpi);
15623 msg_puts((char_u *)"file: ");
15624 msg_puts(lp->lp_slang->sl_fname);
15625 msg_putchar('\n');
15626 p = lp->lp_slang->sl_info;
15627 if (p != NULL)
15629 msg_puts(p);
15630 msg_putchar('\n');
15633 msg_end();
15636 #define DUMPFLAG_KEEPCASE 1 /* round 2: keep-case tree */
15637 #define DUMPFLAG_COUNT 2 /* include word count */
15638 #define DUMPFLAG_ICASE 4 /* ignore case when finding matches */
15639 #define DUMPFLAG_ONECAP 8 /* pattern starts with capital */
15640 #define DUMPFLAG_ALLCAP 16 /* pattern is all capitals */
15643 * ":spelldump"
15645 void
15646 ex_spelldump(eap)
15647 exarg_T *eap;
15649 buf_T *buf = curbuf;
15651 if (no_spell_checking(curwin))
15652 return;
15654 /* Create a new empty buffer by splitting the window. */
15655 do_cmdline_cmd((char_u *)"new");
15656 if (!bufempty() || !buf_valid(buf))
15657 return;
15659 spell_dump_compl(buf, NULL, 0, NULL, eap->forceit ? DUMPFLAG_COUNT : 0);
15661 /* Delete the empty line that we started with. */
15662 if (curbuf->b_ml.ml_line_count > 1)
15663 ml_delete(curbuf->b_ml.ml_line_count, FALSE);
15665 redraw_later(NOT_VALID);
15669 * Go through all possible words and:
15670 * 1. When "pat" is NULL: dump a list of all words in the current buffer.
15671 * "ic" and "dir" are not used.
15672 * 2. When "pat" is not NULL: add matching words to insert mode completion.
15674 void
15675 spell_dump_compl(buf, pat, ic, dir, dumpflags_arg)
15676 buf_T *buf; /* buffer with spell checking */
15677 char_u *pat; /* leading part of the word */
15678 int ic; /* ignore case */
15679 int *dir; /* direction for adding matches */
15680 int dumpflags_arg; /* DUMPFLAG_* */
15682 langp_T *lp;
15683 slang_T *slang;
15684 idx_T arridx[MAXWLEN];
15685 int curi[MAXWLEN];
15686 char_u word[MAXWLEN];
15687 int c;
15688 char_u *byts;
15689 idx_T *idxs;
15690 linenr_T lnum = 0;
15691 int round;
15692 int depth;
15693 int n;
15694 int flags;
15695 char_u *region_names = NULL; /* region names being used */
15696 int do_region = TRUE; /* dump region names and numbers */
15697 char_u *p;
15698 int lpi;
15699 int dumpflags = dumpflags_arg;
15700 int patlen;
15702 /* When ignoring case or when the pattern starts with capital pass this on
15703 * to dump_word(). */
15704 if (pat != NULL)
15706 if (ic)
15707 dumpflags |= DUMPFLAG_ICASE;
15708 else
15710 n = captype(pat, NULL);
15711 if (n == WF_ONECAP)
15712 dumpflags |= DUMPFLAG_ONECAP;
15713 else if (n == WF_ALLCAP
15714 #ifdef FEAT_MBYTE
15715 && (int)STRLEN(pat) > mb_ptr2len(pat)
15716 #else
15717 && (int)STRLEN(pat) > 1
15718 #endif
15720 dumpflags |= DUMPFLAG_ALLCAP;
15724 /* Find out if we can support regions: All languages must support the same
15725 * regions or none at all. */
15726 for (lpi = 0; lpi < buf->b_langp.ga_len; ++lpi)
15728 lp = LANGP_ENTRY(buf->b_langp, lpi);
15729 p = lp->lp_slang->sl_regions;
15730 if (p[0] != 0)
15732 if (region_names == NULL) /* first language with regions */
15733 region_names = p;
15734 else if (STRCMP(region_names, p) != 0)
15736 do_region = FALSE; /* region names are different */
15737 break;
15742 if (do_region && region_names != NULL)
15744 if (pat == NULL)
15746 vim_snprintf((char *)IObuff, IOSIZE, "/regions=%s", region_names);
15747 ml_append(lnum++, IObuff, (colnr_T)0, FALSE);
15750 else
15751 do_region = FALSE;
15754 * Loop over all files loaded for the entries in 'spelllang'.
15756 for (lpi = 0; lpi < buf->b_langp.ga_len; ++lpi)
15758 lp = LANGP_ENTRY(buf->b_langp, lpi);
15759 slang = lp->lp_slang;
15760 if (slang->sl_fbyts == NULL) /* reloading failed */
15761 continue;
15763 if (pat == NULL)
15765 vim_snprintf((char *)IObuff, IOSIZE, "# file: %s", slang->sl_fname);
15766 ml_append(lnum++, IObuff, (colnr_T)0, FALSE);
15769 /* When matching with a pattern and there are no prefixes only use
15770 * parts of the tree that match "pat". */
15771 if (pat != NULL && slang->sl_pbyts == NULL)
15772 patlen = (int)STRLEN(pat);
15773 else
15774 patlen = -1;
15776 /* round 1: case-folded tree
15777 * round 2: keep-case tree */
15778 for (round = 1; round <= 2; ++round)
15780 if (round == 1)
15782 dumpflags &= ~DUMPFLAG_KEEPCASE;
15783 byts = slang->sl_fbyts;
15784 idxs = slang->sl_fidxs;
15786 else
15788 dumpflags |= DUMPFLAG_KEEPCASE;
15789 byts = slang->sl_kbyts;
15790 idxs = slang->sl_kidxs;
15792 if (byts == NULL)
15793 continue; /* array is empty */
15795 depth = 0;
15796 arridx[0] = 0;
15797 curi[0] = 1;
15798 while (depth >= 0 && !got_int
15799 && (pat == NULL || !compl_interrupted))
15801 if (curi[depth] > byts[arridx[depth]])
15803 /* Done all bytes at this node, go up one level. */
15804 --depth;
15805 line_breakcheck();
15806 ins_compl_check_keys(50);
15808 else
15810 /* Do one more byte at this node. */
15811 n = arridx[depth] + curi[depth];
15812 ++curi[depth];
15813 c = byts[n];
15814 if (c == 0)
15816 /* End of word, deal with the word.
15817 * Don't use keep-case words in the fold-case tree,
15818 * they will appear in the keep-case tree.
15819 * Only use the word when the region matches. */
15820 flags = (int)idxs[n];
15821 if ((round == 2 || (flags & WF_KEEPCAP) == 0)
15822 && (flags & WF_NEEDCOMP) == 0
15823 && (do_region
15824 || (flags & WF_REGION) == 0
15825 || (((unsigned)flags >> 16)
15826 & lp->lp_region) != 0))
15828 word[depth] = NUL;
15829 if (!do_region)
15830 flags &= ~WF_REGION;
15832 /* Dump the basic word if there is no prefix or
15833 * when it's the first one. */
15834 c = (unsigned)flags >> 24;
15835 if (c == 0 || curi[depth] == 2)
15837 dump_word(slang, word, pat, dir,
15838 dumpflags, flags, lnum);
15839 if (pat == NULL)
15840 ++lnum;
15843 /* Apply the prefix, if there is one. */
15844 if (c != 0)
15845 lnum = dump_prefixes(slang, word, pat, dir,
15846 dumpflags, flags, lnum);
15849 else
15851 /* Normal char, go one level deeper. */
15852 word[depth++] = c;
15853 arridx[depth] = idxs[n];
15854 curi[depth] = 1;
15856 /* Check if this characters matches with the pattern.
15857 * If not skip the whole tree below it.
15858 * Always ignore case here, dump_word() will check
15859 * proper case later. This isn't exactly right when
15860 * length changes for multi-byte characters with
15861 * ignore case... */
15862 if (depth <= patlen
15863 && MB_STRNICMP(word, pat, depth) != 0)
15864 --depth;
15873 * Dump one word: apply case modifications and append a line to the buffer.
15874 * When "lnum" is zero add insert mode completion.
15876 static void
15877 dump_word(slang, word, pat, dir, dumpflags, wordflags, lnum)
15878 slang_T *slang;
15879 char_u *word;
15880 char_u *pat;
15881 int *dir;
15882 int dumpflags;
15883 int wordflags;
15884 linenr_T lnum;
15886 int keepcap = FALSE;
15887 char_u *p;
15888 char_u *tw;
15889 char_u cword[MAXWLEN];
15890 char_u badword[MAXWLEN + 10];
15891 int i;
15892 int flags = wordflags;
15894 if (dumpflags & DUMPFLAG_ONECAP)
15895 flags |= WF_ONECAP;
15896 if (dumpflags & DUMPFLAG_ALLCAP)
15897 flags |= WF_ALLCAP;
15899 if ((dumpflags & DUMPFLAG_KEEPCASE) == 0 && (flags & WF_CAPMASK) != 0)
15901 /* Need to fix case according to "flags". */
15902 make_case_word(word, cword, flags);
15903 p = cword;
15905 else
15907 p = word;
15908 if ((dumpflags & DUMPFLAG_KEEPCASE)
15909 && ((captype(word, NULL) & WF_KEEPCAP) == 0
15910 || (flags & WF_FIXCAP) != 0))
15911 keepcap = TRUE;
15913 tw = p;
15915 if (pat == NULL)
15917 /* Add flags and regions after a slash. */
15918 if ((flags & (WF_BANNED | WF_RARE | WF_REGION)) || keepcap)
15920 STRCPY(badword, p);
15921 STRCAT(badword, "/");
15922 if (keepcap)
15923 STRCAT(badword, "=");
15924 if (flags & WF_BANNED)
15925 STRCAT(badword, "!");
15926 else if (flags & WF_RARE)
15927 STRCAT(badword, "?");
15928 if (flags & WF_REGION)
15929 for (i = 0; i < 7; ++i)
15930 if (flags & (0x10000 << i))
15931 sprintf((char *)badword + STRLEN(badword), "%d", i + 1);
15932 p = badword;
15935 if (dumpflags & DUMPFLAG_COUNT)
15937 hashitem_T *hi;
15939 /* Include the word count for ":spelldump!". */
15940 hi = hash_find(&slang->sl_wordcount, tw);
15941 if (!HASHITEM_EMPTY(hi))
15943 vim_snprintf((char *)IObuff, IOSIZE, "%s\t%d",
15944 tw, HI2WC(hi)->wc_count);
15945 p = IObuff;
15949 ml_append(lnum, p, (colnr_T)0, FALSE);
15951 else if (((dumpflags & DUMPFLAG_ICASE)
15952 ? MB_STRNICMP(p, pat, STRLEN(pat)) == 0
15953 : STRNCMP(p, pat, STRLEN(pat)) == 0)
15954 && ins_compl_add_infercase(p, (int)STRLEN(p),
15955 p_ic, NULL, *dir, 0) == OK)
15956 /* if dir was BACKWARD then honor it just once */
15957 *dir = FORWARD;
15961 * For ":spelldump": Find matching prefixes for "word". Prepend each to
15962 * "word" and append a line to the buffer.
15963 * When "lnum" is zero add insert mode completion.
15964 * Return the updated line number.
15966 static linenr_T
15967 dump_prefixes(slang, word, pat, dir, dumpflags, flags, startlnum)
15968 slang_T *slang;
15969 char_u *word; /* case-folded word */
15970 char_u *pat;
15971 int *dir;
15972 int dumpflags;
15973 int flags; /* flags with prefix ID */
15974 linenr_T startlnum;
15976 idx_T arridx[MAXWLEN];
15977 int curi[MAXWLEN];
15978 char_u prefix[MAXWLEN];
15979 char_u word_up[MAXWLEN];
15980 int has_word_up = FALSE;
15981 int c;
15982 char_u *byts;
15983 idx_T *idxs;
15984 linenr_T lnum = startlnum;
15985 int depth;
15986 int n;
15987 int len;
15988 int i;
15990 /* If the word starts with a lower-case letter make the word with an
15991 * upper-case letter in word_up[]. */
15992 c = PTR2CHAR(word);
15993 if (SPELL_TOUPPER(c) != c)
15995 onecap_copy(word, word_up, TRUE);
15996 has_word_up = TRUE;
15999 byts = slang->sl_pbyts;
16000 idxs = slang->sl_pidxs;
16001 if (byts != NULL) /* array not is empty */
16004 * Loop over all prefixes, building them byte-by-byte in prefix[].
16005 * When at the end of a prefix check that it supports "flags".
16007 depth = 0;
16008 arridx[0] = 0;
16009 curi[0] = 1;
16010 while (depth >= 0 && !got_int)
16012 n = arridx[depth];
16013 len = byts[n];
16014 if (curi[depth] > len)
16016 /* Done all bytes at this node, go up one level. */
16017 --depth;
16018 line_breakcheck();
16020 else
16022 /* Do one more byte at this node. */
16023 n += curi[depth];
16024 ++curi[depth];
16025 c = byts[n];
16026 if (c == 0)
16028 /* End of prefix, find out how many IDs there are. */
16029 for (i = 1; i < len; ++i)
16030 if (byts[n + i] != 0)
16031 break;
16032 curi[depth] += i - 1;
16034 c = valid_word_prefix(i, n, flags, word, slang, FALSE);
16035 if (c != 0)
16037 vim_strncpy(prefix + depth, word, MAXWLEN - depth - 1);
16038 dump_word(slang, prefix, pat, dir, dumpflags,
16039 (c & WF_RAREPFX) ? (flags | WF_RARE)
16040 : flags, lnum);
16041 if (lnum != 0)
16042 ++lnum;
16045 /* Check for prefix that matches the word when the
16046 * first letter is upper-case, but only if the prefix has
16047 * a condition. */
16048 if (has_word_up)
16050 c = valid_word_prefix(i, n, flags, word_up, slang,
16051 TRUE);
16052 if (c != 0)
16054 vim_strncpy(prefix + depth, word_up,
16055 MAXWLEN - depth - 1);
16056 dump_word(slang, prefix, pat, dir, dumpflags,
16057 (c & WF_RAREPFX) ? (flags | WF_RARE)
16058 : flags, lnum);
16059 if (lnum != 0)
16060 ++lnum;
16064 else
16066 /* Normal char, go one level deeper. */
16067 prefix[depth++] = c;
16068 arridx[depth] = idxs[n];
16069 curi[depth] = 1;
16075 return lnum;
16079 * Move "p" to the end of word "start".
16080 * Uses the spell-checking word characters.
16082 char_u *
16083 spell_to_word_end(start, buf)
16084 char_u *start;
16085 buf_T *buf;
16087 char_u *p = start;
16089 while (*p != NUL && spell_iswordp(p, buf))
16090 mb_ptr_adv(p);
16091 return p;
16094 #if defined(FEAT_INS_EXPAND) || defined(PROTO)
16096 * For Insert mode completion CTRL-X s:
16097 * Find start of the word in front of column "startcol".
16098 * We don't check if it is badly spelled, with completion we can only change
16099 * the word in front of the cursor.
16100 * Returns the column number of the word.
16103 spell_word_start(startcol)
16104 int startcol;
16106 char_u *line;
16107 char_u *p;
16108 int col = 0;
16110 if (no_spell_checking(curwin))
16111 return startcol;
16113 /* Find a word character before "startcol". */
16114 line = ml_get_curline();
16115 for (p = line + startcol; p > line; )
16117 mb_ptr_back(line, p);
16118 if (spell_iswordp_nmw(p))
16119 break;
16122 /* Go back to start of the word. */
16123 while (p > line)
16125 col = (int)(p - line);
16126 mb_ptr_back(line, p);
16127 if (!spell_iswordp(p, curbuf))
16128 break;
16129 col = 0;
16132 return col;
16136 * Need to check for 'spellcapcheck' now, the word is removed before
16137 * expand_spelling() is called. Therefore the ugly global variable.
16139 static int spell_expand_need_cap;
16141 void
16142 spell_expand_check_cap(col)
16143 colnr_T col;
16145 spell_expand_need_cap = check_need_cap(curwin->w_cursor.lnum, col);
16149 * Get list of spelling suggestions.
16150 * Used for Insert mode completion CTRL-X ?.
16151 * Returns the number of matches. The matches are in "matchp[]", array of
16152 * allocated strings.
16154 /*ARGSUSED*/
16156 expand_spelling(lnum, col, pat, matchp)
16157 linenr_T lnum;
16158 int col;
16159 char_u *pat;
16160 char_u ***matchp;
16162 garray_T ga;
16164 spell_suggest_list(&ga, pat, 100, spell_expand_need_cap, TRUE);
16165 *matchp = ga.ga_data;
16166 return ga.ga_len;
16168 #endif
16170 #endif /* FEAT_SPELL */