1 /* Extended regular expression matching and search library.
2 Copyright (C) 2002, 2003, 2004, 2005 Free Software Foundation, Inc.
3 This file is part of the GNU C Library.
4 Contributed by Isamu Hasegawa <isamu@yamato.ibm.com>.
6 The GNU C Library is free software; you can redistribute it and/or
7 modify it under the terms of the GNU Lesser General Public
8 License as published by the Free Software Foundation; either
9 version 2.1 of the License, or (at your option) any later version.
11 The GNU C Library is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 Lesser General Public License for more details.
16 You should have received a copy of the GNU Lesser General Public
17 License along with the GNU C Library; if not, write to the Free
18 Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
21 #ifndef _REGEX_INTERNAL_H
22 #define _REGEX_INTERNAL_H 1
30 #if defined HAVE_LANGINFO_H || defined HAVE_LANGINFO_CODESET || defined _LIBC
31 # include <langinfo.h>
33 #if defined HAVE_LOCALE_H || defined _LIBC
36 #if defined HAVE_WCHAR_H || defined _LIBC
38 #endif /* HAVE_WCHAR_H || _LIBC */
39 #if defined HAVE_WCTYPE_H || defined _LIBC
41 #endif /* HAVE_WCTYPE_H || _LIBC */
42 #if defined HAVE_STDBOOL_H || defined _LIBC
44 #endif /* HAVE_STDBOOL_H || _LIBC */
46 # include <bits/libc-lock.h>
48 # define __libc_lock_define(CLASS,NAME)
49 # define __libc_lock_init(NAME) do { } while (0)
50 # define __libc_lock_lock(NAME) do { } while (0)
51 # define __libc_lock_unlock(NAME) do { } while (0)
54 /* In case that the system doesn't have isblank(). */
55 #if !defined _LIBC && !defined HAVE_ISBLANK && !defined isblank
56 # define isblank(ch) ((ch) == ' ' || (ch) == '\t')
60 # ifndef _RE_DEFINE_LOCALE_FUNCTIONS
61 # define _RE_DEFINE_LOCALE_FUNCTIONS 1
62 # include <locale/localeinfo.h>
63 # include <locale/elem-hash.h>
64 # include <locale/coll-lookup.h>
68 /* This is for other GNU distributions with internationalized messages. */
69 #if (HAVE_LIBINTL_H && ENABLE_NLS) || defined _LIBC
73 # define gettext(msgid) \
74 INTUSE(__dcgettext) (_libc_intl_domainname, msgid, LC_MESSAGES)
77 # define gettext(msgid) (msgid)
81 /* This define is so xgettext can find the internationalizable
83 # define gettext_noop(String) String
86 #if (defined MB_CUR_MAX && HAVE_LOCALE_H && HAVE_WCTYPE_H && HAVE_WCHAR_H && HAVE_WCRTOMB && HAVE_MBRTOWC && HAVE_WCSCOLL) || _LIBC
87 # define RE_ENABLE_I18N
91 # define BE(expr, val) __builtin_expect (expr, val)
93 # define BE(expr, val) (expr)
97 /* Number of single byte character. */
100 #define COLL_ELEM_LEN_MAX 8
102 /* The character which represents newline. */
103 #define NEWLINE_CHAR '\n'
104 #define WIDE_NEWLINE_CHAR L'\n'
106 /* Rename to standard API for using out of glibc. */
108 # define __wctype wctype
109 # define __iswctype iswctype
110 # define __btowc btowc
111 # define __mempcpy mempcpy
112 # define __wcrtomb wcrtomb
113 # define __regfree regfree
114 # define attribute_hidden
115 #endif /* not _LIBC */
118 # define __attribute(arg) __attribute__ (arg)
120 # define __attribute(arg)
123 extern const char __re_error_msgid
[] attribute_hidden
;
124 extern const size_t __re_error_msgid_idx
[] attribute_hidden
;
126 /* An integer used to represent a set of bits. It must be unsigned,
127 and must be at least as wide as unsigned int. */
128 typedef unsigned long int bitset_word_t
;
129 /* All bits set in a bitset_word_t. */
130 #define BITSET_WORD_MAX ULONG_MAX
131 /* Number of bits in a bitset_word_t. */
132 #define BITSET_WORD_BITS (sizeof (bitset_word_t) * CHAR_BIT)
133 /* Number of bitset_word_t in a bit_set. */
134 #define BITSET_WORDS (SBC_MAX / BITSET_WORD_BITS)
135 typedef bitset_word_t bitset_t
[BITSET_WORDS
];
136 typedef bitset_word_t
*re_bitset_ptr_t
;
137 typedef const bitset_word_t
*re_const_bitset_ptr_t
;
139 #define bitset_set(set,i) \
140 (set[i / BITSET_WORD_BITS] |= (bitset_word_t) 1 << i % BITSET_WORD_BITS)
141 #define bitset_clear(set,i) \
142 (set[i / BITSET_WORD_BITS] &= ~((bitset_word_t) 1 << i % BITSET_WORD_BITS))
143 #define bitset_contain(set,i) \
144 (set[i / BITSET_WORD_BITS] & ((bitset_word_t) 1 << i % BITSET_WORD_BITS))
145 #define bitset_empty(set) memset (set, '\0', sizeof (bitset_t))
146 #define bitset_set_all(set) memset (set, '\xff', sizeof (bitset_t))
147 #define bitset_copy(dest,src) memcpy (dest, src, sizeof (bitset_t))
149 #define PREV_WORD_CONSTRAINT 0x0001
150 #define PREV_NOTWORD_CONSTRAINT 0x0002
151 #define NEXT_WORD_CONSTRAINT 0x0004
152 #define NEXT_NOTWORD_CONSTRAINT 0x0008
153 #define PREV_NEWLINE_CONSTRAINT 0x0010
154 #define NEXT_NEWLINE_CONSTRAINT 0x0020
155 #define PREV_BEGBUF_CONSTRAINT 0x0040
156 #define NEXT_ENDBUF_CONSTRAINT 0x0080
157 #define WORD_DELIM_CONSTRAINT 0x0100
158 #define NOT_WORD_DELIM_CONSTRAINT 0x0200
162 INSIDE_WORD
= PREV_WORD_CONSTRAINT
| NEXT_WORD_CONSTRAINT
,
163 WORD_FIRST
= PREV_NOTWORD_CONSTRAINT
| NEXT_WORD_CONSTRAINT
,
164 WORD_LAST
= PREV_WORD_CONSTRAINT
| NEXT_NOTWORD_CONSTRAINT
,
165 INSIDE_NOTWORD
= PREV_NOTWORD_CONSTRAINT
| NEXT_NOTWORD_CONSTRAINT
,
166 LINE_FIRST
= PREV_NEWLINE_CONSTRAINT
,
167 LINE_LAST
= NEXT_NEWLINE_CONSTRAINT
,
168 BUF_FIRST
= PREV_BEGBUF_CONSTRAINT
,
169 BUF_LAST
= NEXT_ENDBUF_CONSTRAINT
,
170 WORD_DELIM
= WORD_DELIM_CONSTRAINT
,
171 NOT_WORD_DELIM
= NOT_WORD_DELIM_CONSTRAINT
185 /* Node type, These are used by token, node, tree. */
191 #ifdef RE_ENABLE_I18N
194 #endif /* RE_ENABLE_I18N */
196 /* We define EPSILON_BIT as a macro so that OP_OPEN_SUBEXP is used
197 when the debugger shows values of this enum type. */
198 #define EPSILON_BIT 8
199 OP_OPEN_SUBEXP
= EPSILON_BIT
| 0,
200 OP_CLOSE_SUBEXP
= EPSILON_BIT
| 1,
201 OP_ALT
= EPSILON_BIT
| 2,
202 OP_DUP_ASTERISK
= EPSILON_BIT
| 3,
203 ANCHOR
= EPSILON_BIT
| 4,
205 /* Tree type, these are used only by tree. */
209 /* Token type, these are used only by token. */
221 OP_CLOSE_EQUIV_CLASS
,
232 #ifdef RE_ENABLE_I18N
235 /* Multibyte characters. */
238 /* Collating symbols. */
243 /* Equivalence classes. */
245 int32_t *equiv_classes
;
248 /* Range expressions. */
250 uint32_t *range_starts
;
251 uint32_t *range_ends
;
252 # else /* not _LIBC */
253 wchar_t *range_starts
;
255 # endif /* not _LIBC */
257 /* Character classes. */
258 wctype_t *char_classes
;
260 /* If this character set is the non-matching list. */
261 unsigned int non_match
: 1;
263 /* # of multibyte characters. */
266 /* # of collating symbols. */
269 /* # of equivalence classes. */
272 /* # of range expressions. */
275 /* # of character classes. */
278 #endif /* RE_ENABLE_I18N */
284 unsigned char c
; /* for CHARACTER */
285 re_bitset_ptr_t sbcset
; /* for SIMPLE_BRACKET */
286 #ifdef RE_ENABLE_I18N
287 re_charset_t
*mbcset
; /* for COMPLEX_BRACKET */
288 #endif /* RE_ENABLE_I18N */
289 int idx
; /* for BACK_REF */
290 re_context_type ctx_type
; /* for ANCHOR */
293 re_token_type_t type
: 8;
295 re_token_type_t type
;
297 unsigned int constraint
: 10; /* context constraint */
298 unsigned int duplicated
: 1;
299 unsigned int opt_subexp
: 1;
300 #ifdef RE_ENABLE_I18N
301 unsigned int accept_mb
: 1;
302 /* These 2 bits can be moved into the union if needed (e.g. if running out
303 of bits; move opr.c to opr.c.c and move the flags to opr.c.flags). */
304 unsigned int mb_partial
: 1;
306 unsigned int word_char
: 1;
309 #define IS_EPSILON_NODE(type) ((type) & EPSILON_BIT)
313 /* Indicate the raw buffer which is the original string passed as an
314 argument of regexec(), re_search(), etc.. */
315 const unsigned char *raw_mbs
;
316 /* Store the multibyte string. In case of "case insensitive mode" like
317 REG_ICASE, upper cases of the string are stored, otherwise MBS points
318 the same address that RAW_MBS points. */
320 #ifdef RE_ENABLE_I18N
321 /* Store the wide character string which is corresponding to MBS. */
326 /* Index in RAW_MBS. Each character mbs[i] corresponds to
327 raw_mbs[raw_mbs_idx + i]. */
329 /* The length of the valid characters in the buffers. */
331 /* The corresponding number of bytes in raw_mbs array. */
333 /* The length of the buffers MBS and WCS. */
335 /* The index in MBS, which is updated by re_string_fetch_byte. */
337 /* length of RAW_MBS array. */
339 /* This is RAW_LEN - RAW_MBS_IDX + VALID_LEN - VALID_RAW_LEN. */
341 /* End of the buffer may be shorter than its length in the cases such
342 as re_match_2, re_search_2. Then, we use STOP for end of the buffer
345 /* This is RAW_STOP - RAW_MBS_IDX adjusted through OFFSETS. */
348 /* The context of mbs[0]. We store the context independently, since
349 the context of mbs[0] may be different from raw_mbs[0], which is
350 the beginning of the input string. */
351 unsigned int tip_context
;
352 /* The translation passed as a part of an argument of re_compile_pattern. */
353 RE_TRANSLATE_TYPE trans
;
354 /* Copy of re_dfa_t's word_char. */
355 re_const_bitset_ptr_t word_char
;
356 /* 1 if REG_ICASE. */
358 unsigned char is_utf8
;
359 unsigned char map_notascii
;
360 unsigned char mbs_allocated
;
361 unsigned char offsets_needed
;
362 unsigned char newline_anchor
;
363 unsigned char word_ops_used
;
366 typedef struct re_string_t re_string_t
;
370 typedef struct re_dfa_t re_dfa_t
;
374 # define internal_function __attribute ((regparm (3), stdcall))
376 # define internal_function
380 static reg_errcode_t
re_string_realloc_buffers (re_string_t
*pstr
,
383 #ifdef RE_ENABLE_I18N
384 static void build_wcs_buffer (re_string_t
*pstr
) internal_function
;
385 static int build_wcs_upper_buffer (re_string_t
*pstr
) internal_function
;
386 #endif /* RE_ENABLE_I18N */
387 static void build_upper_buffer (re_string_t
*pstr
) internal_function
;
388 static void re_string_translate_buffer (re_string_t
*pstr
) internal_function
;
389 static unsigned int re_string_context_at (const re_string_t
*input
, int idx
,
391 internal_function
__attribute ((pure
));
392 #define re_string_peek_byte(pstr, offset) \
393 ((pstr)->mbs[(pstr)->cur_idx + offset])
394 #define re_string_fetch_byte(pstr) \
395 ((pstr)->mbs[(pstr)->cur_idx++])
396 #define re_string_first_byte(pstr, idx) \
397 ((idx) == (pstr)->valid_len || (pstr)->wcs[idx] != WEOF)
398 #define re_string_is_single_byte_char(pstr, idx) \
399 ((pstr)->wcs[idx] != WEOF && ((pstr)->valid_len == (idx) + 1 \
400 || (pstr)->wcs[(idx) + 1] != WEOF))
401 #define re_string_eoi(pstr) ((pstr)->stop <= (pstr)->cur_idx)
402 #define re_string_cur_idx(pstr) ((pstr)->cur_idx)
403 #define re_string_get_buffer(pstr) ((pstr)->mbs)
404 #define re_string_length(pstr) ((pstr)->len)
405 #define re_string_byte_at(pstr,idx) ((pstr)->mbs[idx])
406 #define re_string_skip_bytes(pstr,idx) ((pstr)->cur_idx += (idx))
407 #define re_string_set_index(pstr,idx) ((pstr)->cur_idx = (idx))
413 /* The OS usually guarantees only one guard page at the bottom of the stack,
414 and a page size can be as small as 4096 bytes. So we cannot safely
415 allocate anything larger than 4096 bytes. Also care for the possibility
416 of a few compiler-allocated temporary stack slots. */
417 # define __libc_use_alloca(n) ((n) < 4032)
419 /* alloca is implemented with malloc, so just use malloc. */
420 # define __libc_use_alloca(n) 0
424 #define re_malloc(t,n) ((t *) malloc ((n) * sizeof (t)))
425 #define re_realloc(p,t,n) ((t *) realloc (p, (n) * sizeof (t)))
426 #define re_free(p) free (p)
430 struct bin_tree_t
*parent
;
431 struct bin_tree_t
*left
;
432 struct bin_tree_t
*right
;
433 struct bin_tree_t
*first
;
434 struct bin_tree_t
*next
;
438 /* `node_idx' is the index in dfa->nodes, if `type' == 0.
439 Otherwise `type' indicate the type of this node. */
442 typedef struct bin_tree_t bin_tree_t
;
444 #define BIN_TREE_STORAGE_SIZE \
445 ((1024 - sizeof (void *)) / sizeof (bin_tree_t))
447 struct bin_tree_storage_t
449 struct bin_tree_storage_t
*next
;
450 bin_tree_t data
[BIN_TREE_STORAGE_SIZE
];
452 typedef struct bin_tree_storage_t bin_tree_storage_t
;
454 #define CONTEXT_WORD 1
455 #define CONTEXT_NEWLINE (CONTEXT_WORD << 1)
456 #define CONTEXT_BEGBUF (CONTEXT_NEWLINE << 1)
457 #define CONTEXT_ENDBUF (CONTEXT_BEGBUF << 1)
459 #define IS_WORD_CONTEXT(c) ((c) & CONTEXT_WORD)
460 #define IS_NEWLINE_CONTEXT(c) ((c) & CONTEXT_NEWLINE)
461 #define IS_BEGBUF_CONTEXT(c) ((c) & CONTEXT_BEGBUF)
462 #define IS_ENDBUF_CONTEXT(c) ((c) & CONTEXT_ENDBUF)
463 #define IS_ORDINARY_CONTEXT(c) ((c) == 0)
465 #define IS_WORD_CHAR(ch) (isalnum (ch) || (ch) == '_')
466 #define IS_NEWLINE(ch) ((ch) == NEWLINE_CHAR)
467 #define IS_WIDE_WORD_CHAR(ch) (iswalnum (ch) || (ch) == L'_')
468 #define IS_WIDE_NEWLINE(ch) ((ch) == WIDE_NEWLINE_CHAR)
470 #define NOT_SATISFY_PREV_CONSTRAINT(constraint,context) \
471 ((((constraint) & PREV_WORD_CONSTRAINT) && !IS_WORD_CONTEXT (context)) \
472 || ((constraint & PREV_NOTWORD_CONSTRAINT) && IS_WORD_CONTEXT (context)) \
473 || ((constraint & PREV_NEWLINE_CONSTRAINT) && !IS_NEWLINE_CONTEXT (context))\
474 || ((constraint & PREV_BEGBUF_CONSTRAINT) && !IS_BEGBUF_CONTEXT (context)))
476 #define NOT_SATISFY_NEXT_CONSTRAINT(constraint,context) \
477 ((((constraint) & NEXT_WORD_CONSTRAINT) && !IS_WORD_CONTEXT (context)) \
478 || (((constraint) & NEXT_NOTWORD_CONSTRAINT) && IS_WORD_CONTEXT (context)) \
479 || (((constraint) & NEXT_NEWLINE_CONSTRAINT) && !IS_NEWLINE_CONTEXT (context)) \
480 || (((constraint) & NEXT_ENDBUF_CONSTRAINT) && !IS_ENDBUF_CONTEXT (context)))
486 re_node_set non_eps_nodes
;
487 re_node_set inveclosure
;
488 re_node_set
*entrance_nodes
;
489 struct re_dfastate_t
**trtable
, **word_trtable
;
490 unsigned int context
: 4;
491 unsigned int halt
: 1;
492 /* If this state can accept `multi byte'.
493 Note that we refer to multibyte characters, and multi character
494 collating elements as `multi byte'. */
495 unsigned int accept_mb
: 1;
496 /* If this state has backreference node(s). */
497 unsigned int has_backref
: 1;
498 unsigned int has_constraint
: 1;
500 typedef struct re_dfastate_t re_dfastate_t
;
502 struct re_state_table_entry
506 re_dfastate_t
**array
;
509 /* Array type used in re_sub_match_last_t and re_sub_match_top_t. */
515 re_dfastate_t
**array
;
518 /* Store information about the node NODE whose type is OP_CLOSE_SUBEXP. */
523 int str_idx
; /* The position NODE match at. */
525 } re_sub_match_last_t
;
527 /* Store information about the node NODE whose type is OP_OPEN_SUBEXP.
528 And information about the node, whose type is OP_CLOSE_SUBEXP,
529 corresponding to NODE is stored in LASTS. */
536 int alasts
; /* Allocation size of LASTS. */
537 int nlasts
; /* The number of LASTS. */
538 re_sub_match_last_t
**lasts
;
539 } re_sub_match_top_t
;
541 struct re_backref_cache_entry
549 unsigned short int eps_reachable_subexps_map
;
554 /* The string object corresponding to the input string. */
556 #if defined _LIBC || (defined __STDC_VERSION__ && __STDC_VERSION__ >= 199901L)
557 const re_dfa_t
*const dfa
;
561 /* EFLAGS of the argument of regexec. */
563 /* Where the matching ends. */
566 /* The state log used by the matcher. */
567 re_dfastate_t
**state_log
;
569 /* Back reference cache. */
572 struct re_backref_cache_entry
*bkref_ents
;
576 re_sub_match_top_t
**sub_tops
;
577 } re_match_context_t
;
581 re_dfastate_t
**sifted_states
;
582 re_dfastate_t
**limited_states
;
588 struct re_fail_stack_ent_t
593 re_node_set eps_via_nodes
;
596 struct re_fail_stack_t
600 struct re_fail_stack_ent_t
*stack
;
611 re_node_set
*eclosures
;
612 re_node_set
*inveclosures
;
613 struct re_state_table_entry
*state_table
;
614 re_dfastate_t
*init_state
;
615 re_dfastate_t
*init_state_word
;
616 re_dfastate_t
*init_state_nl
;
617 re_dfastate_t
*init_state_begbuf
;
618 bin_tree_t
*str_tree
;
619 bin_tree_storage_t
*str_tree_storage
;
620 re_bitset_ptr_t sb_char
;
621 int str_tree_storage_idx
;
623 /* number of subexpressions `re_nsub' is in regex_t. */
624 unsigned int state_hash_mask
;
626 int nbackref
; /* The number of backreference in this dfa. */
628 /* Bitmap expressing which backreference is used. */
629 bitset_word_t used_bkref_map
;
630 bitset_word_t completed_bkref_map
;
632 unsigned int has_plural_match
: 1;
633 /* If this dfa has "multibyte node", which is a backreference or
634 a node which can accept multibyte character or multi character
635 collating element. */
636 unsigned int has_mb_node
: 1;
637 unsigned int is_utf8
: 1;
638 unsigned int map_notascii
: 1;
639 unsigned int word_ops_used
: 1;
647 __libc_lock_define (, lock
)
650 #define re_node_set_init_empty(set) memset (set, '\0', sizeof (re_node_set))
651 #define re_node_set_remove(set,id) \
652 (re_node_set_remove_at (set, re_node_set_contains (set, id) - 1))
653 #define re_node_set_empty(p) ((p)->nelem = 0)
654 #define re_node_set_free(set) re_free ((set)->elems)
668 bracket_elem_type type
;
678 /* Inline functions for bitset operation. */
680 bitset_not (bitset_t set
)
683 for (bitset_i
= 0; bitset_i
< BITSET_WORDS
; ++bitset_i
)
684 set
[bitset_i
] = ~set
[bitset_i
];
688 bitset_merge (bitset_t dest
, const bitset_t src
)
691 for (bitset_i
= 0; bitset_i
< BITSET_WORDS
; ++bitset_i
)
692 dest
[bitset_i
] |= src
[bitset_i
];
696 bitset_mask (bitset_t dest
, const bitset_t src
)
699 for (bitset_i
= 0; bitset_i
< BITSET_WORDS
; ++bitset_i
)
700 dest
[bitset_i
] &= src
[bitset_i
];
703 #ifdef RE_ENABLE_I18N
704 /* Inline functions for re_string. */
706 internal_function
__attribute ((pure
))
707 re_string_char_size_at (const re_string_t
*pstr
, int idx
)
710 if (pstr
->mb_cur_max
== 1)
712 for (byte_idx
= 1; idx
+ byte_idx
< pstr
->valid_len
; ++byte_idx
)
713 if (pstr
->wcs
[idx
+ byte_idx
] != WEOF
)
719 internal_function
__attribute ((pure
))
720 re_string_wchar_at (const re_string_t
*pstr
, int idx
)
722 if (pstr
->mb_cur_max
== 1)
723 return (wint_t) pstr
->mbs
[idx
];
724 return (wint_t) pstr
->wcs
[idx
];
728 internal_function
__attribute ((pure
))
729 re_string_elem_size_at (const re_string_t
*pstr
, int idx
)
732 const unsigned char *p
, *extra
;
733 const int32_t *table
, *indirect
;
735 # include <locale/weight.h>
736 uint_fast32_t nrules
= _NL_CURRENT_WORD (LC_COLLATE
, _NL_COLLATE_NRULES
);
740 table
= (const int32_t *) _NL_CURRENT (LC_COLLATE
, _NL_COLLATE_TABLEMB
);
741 extra
= (const unsigned char *)
742 _NL_CURRENT (LC_COLLATE
, _NL_COLLATE_EXTRAMB
);
743 indirect
= (const int32_t *) _NL_CURRENT (LC_COLLATE
,
744 _NL_COLLATE_INDIRECTMB
);
747 return p
- pstr
->mbs
- idx
;
753 #endif /* RE_ENABLE_I18N */
755 #endif /* _REGEX_INTERNAL_H */