posix/regex_internal.h

   1 /* Extended regular expression matching and search library.
   2    Copyright (C) 2002 Free Software Foundation, Inc.
   3    This file is part of the GNU C Library.
   4    Contributed by Isamu Hasegawa <isamu@yamato.ibm.com>.
   5
   6    The GNU C Library is free software; you can redistribute it and/or
   7    modify it under the terms of the GNU Lesser General Public
   8    License as published by the Free Software Foundation; either
   9    version 2.1 of the License, or (at your option) any later version.
  10
  11    The GNU C Library is distributed in the hope that it will be useful,
  12    but WITHOUT ANY WARRANTY; without even the implied warranty of
  13    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14    Lesser General Public License for more details.
  15
  16    You should have received a copy of the GNU Lesser General Public
  17    License along with the GNU C Library; if not, write to the Free
  18    Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
  19    02111-1307 USA.  */
  20
  21 #ifndef _REGEX_INTERNAL_H
  22 #define _REGEX_INTERNAL_H 1
  23
  24 /* Number of bits in a byte.  */
  25 #define BYTE_BITS 8
  26 /* Number of single byte character.  */
  27 #define SBC_MAX 256
  28
  29 #define COLL_ELEM_LEN_MAX 8
  30
  31 /* The character which represents newline.  */
  32 #define NEWLINE_CHAR '\n'
  33
  34 /* Rename to standard API for using out of glibc.  */
  35 #ifndef _LIBC
  36 # define __wctype wctype
  37 # define __iswctype iswctype
  38 # define __btowc btowc
  39 # define __mempcpy memcpy
  40 #endif
  41
  42 extern const char re_error_msgid[];
  43 extern const size_t re_error_msgid_idx[];
  44
  45 /* Number of bits in an unsinged int.  */
  46 #define UINT_BITS (sizeof (unsigned int) * BYTE_BITS)
  47 /* Number of unsigned int in an bit_set.  */
  48 #define BITSET_UINTS ((SBC_MAX + UINT_BITS - 1) / UINT_BITS)
  49 typedef unsigned int bitset[BITSET_UINTS];
  50 typedef unsigned int *re_bitset_ptr_t;
  51
  52 #define bitset_set(set,i) (set[i / UINT_BITS] |= 1 << i % UINT_BITS)
  53 #define bitset_clear(set,i) (set[i / UINT_BITS] &= ~(1 << i % UINT_BITS))
  54 #define bitset_contain(set,i) (set[i / UINT_BITS] & (1 << i % UINT_BITS))
  55 #define bitset_empty(set) memset (set, 0, sizeof (unsigned int) * BITSET_UINTS)
  56 #define bitset_set_all(set) \
  57   memset (set, 255, sizeof (unsigned int) * BITSET_UINTS)
  58 #define bitset_copy(dest,src) \
  59   memcpy (dest, src, sizeof (unsigned int) * BITSET_UINTS)
  60 static inline void bitset_not (bitset set);
  61 static inline void bitset_merge (bitset dest, const bitset src);
  62 static inline void bitset_not_merge (bitset dest, const bitset src);
  63
  64 #define PREV_WORD_CONSTRAINT 0x0001
  65 #define PREV_NOTWORD_CONSTRAINT 0x0002
  66 #define NEXT_WORD_CONSTRAINT 0x0004
  67 #define NEXT_NOTWORD_CONSTRAINT 0x0008
  68 #define PREV_NEWLINE_CONSTRAINT 0x0010
  69 #define NEXT_NEWLINE_CONSTRAINT 0x0020
  70 #define PREV_BEGBUF_CONSTRAINT 0x0040
  71 #define NEXT_ENDBUF_CONSTRAINT 0x0080
  72 #define DUMMY_CONSTRAINT 0x0100
  73
  74 typedef enum
  75 {
  76   INSIDE_WORD = PREV_WORD_CONSTRAINT | NEXT_WORD_CONSTRAINT,
  77   WORD_FIRST = PREV_NOTWORD_CONSTRAINT | NEXT_WORD_CONSTRAINT,
  78   WORD_LAST = PREV_WORD_CONSTRAINT | NEXT_NOTWORD_CONSTRAINT,
  79   LINE_FIRST = PREV_NEWLINE_CONSTRAINT,
  80   LINE_LAST = NEXT_NEWLINE_CONSTRAINT,
  81   BUF_FIRST = PREV_BEGBUF_CONSTRAINT,
  82   BUF_LAST = NEXT_ENDBUF_CONSTRAINT,
  83   WORD_DELIM = DUMMY_CONSTRAINT
  84 } re_context_type;
  85
  86 typedef struct
  87 {
  88   int alloc;
  89   int nelem;
  90   int *elems;
  91 } re_node_set;
  92
  93 typedef enum
  94 {
  95   NON_TYPE = 0,
  96
  97   /* Token type, these are used only by token.  */
  98   OP_OPEN_SUBEXP,
  99   OP_CLOSE_SUBEXP,
 100   OP_OPEN_BRACKET,
 101   OP_CLOSE_BRACKET,
 102   OP_CHARSET_RANGE,
 103   OP_OPEN_DUP_NUM,
 104   OP_CLOSE_DUP_NUM,
 105   OP_NON_MATCH_LIST,
 106   OP_OPEN_COLL_ELEM,
 107   OP_CLOSE_COLL_ELEM,
 108   OP_OPEN_EQUIV_CLASS,
 109   OP_CLOSE_EQUIV_CLASS,
 110   OP_OPEN_CHAR_CLASS,
 111   OP_CLOSE_CHAR_CLASS,
 112   OP_WORD,
 113   OP_NOTWORD,
 114   BACK_SLASH,
 115
 116   /* Tree type, these are used only by tree. */
 117   CONCAT,
 118   ALT,
 119   SUBEXP,
 120   SIMPLE_BRACKET,
 121   COMPLEX_BRACKET,
 122
 123   /* Node type, These are used by token, node, tree.  */
 124   OP_PERIOD,
 125   CHARACTER,
 126   END_OF_RE,
 127   OP_ALT,
 128   OP_DUP_ASTERISK,
 129   OP_DUP_PLUS,
 130   OP_DUP_QUESTION,
 131   OP_BACK_REF,
 132   ANCHOR,
 133   OP_CONTEXT_NODE,
 134
 135   /* Dummy marker.  */
 136   END_OF_RE_TOKEN_T
 137 } re_token_type_t;
 138
 139 typedef struct
 140 {
 141   /* If this character set is the non-matching list.  */
 142   unsigned int non_match : 1;
 143
 144   /* Multibyte characters.  */
 145   wchar_t *mbchars;
 146   int nmbchars;
 147
 148   /* Collating symbols.  */
 149 #ifdef _LIBC
 150   int32_t *coll_syms;
 151 #endif
 152   int ncoll_syms;
 153
 154   /* Equivalence classes. */
 155 #ifdef _LIBC
 156   int32_t *equiv_classes;
 157 #endif
 158   int nequiv_classes;
 159
 160   /* Range expressions. */
 161 #ifdef _LIBC
 162   uint32_t *range_starts;
 163   uint32_t *range_ends;
 164 #endif
 165   int nranges;
 166
 167   /* Character classes. */
 168   wctype_t *char_classes;
 169   int nchar_classes;
 170 } re_charset_t;
 171
 172 typedef struct
 173 {
 174   re_token_type_t type;
 175   union
 176   {
 177     unsigned char c;            /* for CHARACTER */
 178     re_bitset_ptr_t sbcset;     /* for SIMPLE_BRACKET */
 179     re_charset_t *mbcset;       /* for COMPLEX_BRACKET */
 180     int idx;                    /* for BACK_REF */
 181     re_context_type ctx_type;   /* for ANCHOR */
 182     struct
 183     {
 184       int entity;               /* for OP_CONTEXT_NODE, index of the entity */
 185       re_node_set *bkref_eclosure;
 186     } *ctx_info;
 187   } opr;
 188   unsigned int constraint : 10; /* context constraint */
 189   unsigned int duplicated : 1;
 190 #ifdef RE_ENABLE_I18N
 191   unsigned int mb_partial : 1;
 192 #endif
 193 } re_token_t;
 194
 195 #define IS_EPSILON_NODE(type) \
 196   ((type) == OP_ALT || (type) == OP_DUP_ASTERISK || (type) == OP_DUP_PLUS || \
 197    (type) == OP_DUP_QUESTION || (type) == ANCHOR)
 198
 199 #define ACCEPT_MB_NODE(type) \
 200   ((type) == COMPLEX_BRACKET || (type) == OP_PERIOD)
 201
 202 struct re_string_t
 203 {
 204   /* Store the multibyte string.  In case of "case insensitive mode" like
 205      REG_ICASE, upper cases of the string are stored.  */
 206   const unsigned char *mbs;
 207   /* Store the case sensitive multibyte string.  In case of
 208      "case insensitive mode", the original string are stored,
 209      otherwise MBS_CASE points the same address that MBS points.  */
 210   const unsigned char *mbs_case;
 211   int cur_idx;
 212   int len;
 213 #ifdef RE_ENABLE_I18N
 214   /* Store the wide character string which is corresponding to MBS.  */
 215   wchar_t *wcs;
 216 #endif
 217   /* 1 if mbs is allocated by regex library.  */
 218   unsigned int mbs_alloc : 1;
 219   /* 1 if mbs_case is allocated by regex library.  */
 220   unsigned int mbs_case_alloc : 1;
 221 };
 222 typedef struct re_string_t re_string_t;
 223
 224 static reg_errcode_t re_string_construct (re_string_t *pstr,
 225                                           const unsigned char *str, int len,
 226                                           RE_TRANSLATE_TYPE trans);
 227 static reg_errcode_t re_string_construct_toupper (re_string_t *pstr,
 228                                                   const unsigned char *str,
 229                                                   int len,
 230                                                   RE_TRANSLATE_TYPE trans);
 231 static void re_string_destruct (re_string_t *pstr);
 232 #ifdef RE_ENABLE_I18N
 233 static int re_string_elem_size_at (const re_string_t *pstr, int idx);
 234 static inline int re_string_char_size_at (const re_string_t *pstr, int idx);
 235 static inline wint_t re_string_wchar_at (const re_string_t *pstr, int idx);
 236 #endif /* RE_ENABLE_I18N */
 237 static unsigned int re_string_context_at (const re_string_t *input, int idx,
 238                                           int eflags, int newline_anchor);
 239 #define re_string_peek_byte(pstr, offset) \
 240   ((pstr)->mbs[(pstr)->cur_idx + offset])
 241 #define re_string_peek_byte_case(pstr, offset) \
 242   ((pstr)->mbs_case[(pstr)->cur_idx + offset])
 243 #define re_string_fetch_byte(pstr) \
 244   ((pstr)->mbs[(pstr)->cur_idx++])
 245 #define re_string_fetch_byte_case(pstr) \
 246   ((pstr)->mbs_case[(pstr)->cur_idx++])
 247 #define re_string_first_byte(pstr, idx) \
 248   ((idx) == (pstr)->len || (pstr)->wcs[idx] != WEOF)
 249 #define re_string_is_single_byte_char(pstr, idx) \
 250   ((pstr)->wcs[idx] != WEOF && ((pstr)->len == (idx) \
 251                                 || (pstr)->wcs[(idx) + 1] != WEOF))
 252 #define re_string_eoi(pstr) ((pstr)->len == (pstr)->cur_idx)
 253 #define re_string_cur_idx(pstr) ((pstr)->cur_idx)
 254 #define re_string_get_buffer(pstr) ((pstr)->mbs)
 255 #define re_string_length(pstr) ((pstr)->len)
 256 #define re_string_byte_at(pstr,idx) \
 257   ((pstr)->mbs[idx])
 258 #define re_string_skip_bytes(pstr,idx) ((pstr)->cur_idx += (idx))
 259 #define re_string_set_index(pstr,idx) ((pstr)->cur_idx = (idx))
 260
 261 #define re_malloc(t,n) ((t *) malloc ((n) * sizeof (t)))
 262 #define re_realloc(p,t,n) ((t *) realloc (p, (n) * sizeof (t)))
 263 #define re_free(p) free (p)
 264
 265 struct bin_tree_t
 266 {
 267   struct bin_tree_t *parent;
 268   struct bin_tree_t *left;
 269   struct bin_tree_t *right;
 270
 271   /* `node_idx' is the index in dfa->nodes, if `type' == 0.
 272      Otherwise `type' indicate the type of this node.  */
 273   re_token_type_t type;
 274   int node_idx;
 275
 276   int first;
 277   int next;
 278   re_node_set eclosure;
 279 };
 280 typedef struct bin_tree_t bin_tree_t;
 281
 282 struct re_backref_cache_entry
 283 {
 284   int node;
 285   int from;
 286   int to;
 287   int flag;
 288 };
 289
 290 typedef struct
 291 {
 292   int eflags;
 293   int match_first;
 294   int match_last;
 295   int state_log_top;
 296   /* Back reference cache.  */
 297   int nbkref_ents;
 298   int abkref_ents;
 299   struct re_backref_cache_entry *bkref_ents;
 300   int max_bkref_len;
 301 } re_match_context_t;
 302
 303
 304 #define CONTEXT_WORD 1
 305 #define CONTEXT_NEWLINE (CONTEXT_WORD << 1)
 306 #define CONTEXT_BEGBUF (CONTEXT_NEWLINE << 1)
 307 #define CONTEXT_ENDBUF (CONTEXT_BEGBUF << 1)
 308
 309 #define IS_WORD_CONTEXT(c) ((c) & CONTEXT_WORD)
 310 #define IS_NEWLINE_CONTEXT(c) ((c) & CONTEXT_NEWLINE)
 311 #define IS_BEGBUF_CONTEXT(c) ((c) & CONTEXT_BEGBUF)
 312 #define IS_ENDBUF_CONTEXT(c) ((c) & CONTEXT_ENDBUF)
 313 #define IS_ORDINARY_CONTEXT(c) ((c) == 0)
 314
 315 #define IS_WORD_CHAR(ch) (isalnum (ch) || (ch) == '_')
 316 #define IS_NEWLINE(ch) ((ch) == NEWLINE_CHAR)
 317
 318 #define NOT_SATISFY_PREV_CONSTRAINT(constraint,context) \
 319  ((((constraint) & PREV_WORD_CONSTRAINT) && !IS_WORD_CONTEXT (context)) \
 320   || ((constraint & PREV_NOTWORD_CONSTRAINT) && IS_WORD_CONTEXT (context)) \
 321   || ((constraint & PREV_NEWLINE_CONSTRAINT) && !IS_NEWLINE_CONTEXT (context))\
 322   || ((constraint & PREV_BEGBUF_CONSTRAINT) && !IS_BEGBUF_CONTEXT (context)))
 323
 324 #define NOT_SATISFY_NEXT_CONSTRAINT(constraint,context) \
 325  ((((constraint) & NEXT_WORD_CONSTRAINT) && !IS_WORD_CONTEXT (context)) \
 326   || (((constraint) & NEXT_NOTWORD_CONSTRAINT) && IS_WORD_CONTEXT (context)) \
 327   || (((constraint) & NEXT_NEWLINE_CONSTRAINT) && !IS_NEWLINE_CONTEXT (context)) \
 328   || (((constraint) & NEXT_ENDBUF_CONSTRAINT) && !IS_ENDBUF_CONTEXT (context)))
 329
 330 struct re_dfastate_t
 331 {
 332   unsigned int hash;
 333   re_node_set nodes;
 334   re_node_set *entrance_nodes;
 335   struct re_dfastate_t **trtable;
 336   struct re_dfastate_t **trtable_search;
 337   /* If this state is a special state.
 338      A state is a special state if the state is the halt state, or
 339      a anchor.  */
 340   unsigned int context : 2;
 341   unsigned int halt : 1;
 342   /* If this state can accept `multi byte'.
 343      Note that we refer to multibyte characters, and multi character
 344      collating elements as `multi byte'.  */
 345   unsigned int accept_mb : 1;
 346   /* If this state has backreference node(s).  */
 347   unsigned int has_backref : 1;
 348   unsigned int has_constraint : 1;
 349 };
 350 typedef struct re_dfastate_t re_dfastate_t;
 351
 352 typedef struct
 353 {
 354   /* start <= node < end  */
 355   int start;
 356   int end;
 357 } re_subexp_t;
 358
 359 struct re_state_table_entry
 360 {
 361   int num;
 362   int alloc;
 363   union
 364   {
 365     re_dfastate_t *state;
 366     re_dfastate_t **array;
 367   } entry;
 368 };
 369
 370 struct re_dfa_t
 371 {
 372   re_bitset_ptr_t word_char;
 373
 374   /* number of subexpressions `re_nsub' is in regex_t.  */
 375   int subexps_alloc;
 376   re_subexp_t *subexps;
 377
 378   re_token_t *nodes;
 379   int nodes_alloc;
 380   int nodes_len;
 381   bin_tree_t *str_tree;
 382   int *firsts;
 383   int *nexts;
 384   re_node_set *edests;
 385   re_node_set *eclosures;
 386   re_node_set *inveclosures;
 387   struct re_state_table_entry *state_table;
 388   unsigned int state_hash_mask;
 389   re_dfastate_t *init_state;
 390   re_dfastate_t *init_state_word;
 391   re_dfastate_t *init_state_nl;
 392   re_dfastate_t *init_state_begbuf;
 393   int states_alloc;
 394   int init_node;
 395   int nbackref; /* The number of backreference in this dfa.  */
 396   /* If this dfa has "multibyte node", which is a backreference or
 397      a node which can accept multibyte character or multi character
 398      collating element.  */
 399   unsigned int has_mb_node : 1;
 400 };
 401 typedef struct re_dfa_t re_dfa_t;
 402
 403 static reg_errcode_t re_node_set_alloc (re_node_set *set, int size);
 404 static reg_errcode_t re_node_set_init_1 (re_node_set *set, int elem);
 405 static reg_errcode_t re_node_set_init_2 (re_node_set *set, int elem1,
 406                                          int elem2);
 407 #define re_node_set_init_empty(set) memset (set, '\0', sizeof (re_node_set))
 408 static reg_errcode_t re_node_set_init_copy (re_node_set *dest,
 409                                             const re_node_set *src);
 410 static reg_errcode_t re_node_set_intersect (re_node_set *dest,
 411                                             const re_node_set *src1,
 412                                             const re_node_set *src2);
 413 static reg_errcode_t re_node_set_add_intersect (re_node_set *dest,
 414                                                 const re_node_set *src1,
 415                                                 const re_node_set *src2);
 416 static reg_errcode_t re_node_set_init_union (re_node_set *dest,
 417                                              const re_node_set *src1,
 418                                              const re_node_set *src2);
 419 static reg_errcode_t re_node_set_merge (re_node_set *dest,
 420                                         const re_node_set *src);
 421 static int re_node_set_insert (re_node_set *set, int elem);
 422 static int re_node_set_compare (const re_node_set *set1,
 423                                 const re_node_set *set2);
 424 static int re_node_set_contains (const re_node_set *set, int elem);
 425 static void re_node_set_remove_at (re_node_set *set, int idx);
 426 #define re_node_set_empty(p) ((p)->nelem = 0)
 427 #define re_node_set_free(set) re_free ((set)->elems)
 428 static int re_dfa_add_node (re_dfa_t *dfa, re_token_t token, int mode);
 429 static re_dfastate_t *re_acquire_state (re_dfa_t *dfa,
 430                                         const re_node_set *nodes);
 431 static re_dfastate_t *re_acquire_state_context (re_dfa_t *dfa,
 432                                                 const re_node_set *nodes,
 433                                                 unsigned int context);
 434 \f
 435
 436 typedef enum
 437 {
 438   SB_CHAR,
 439   MB_CHAR,
 440   EQUIV_CLASS,
 441   COLL_SYM,
 442   CHAR_CLASS
 443 } bracket_elem_type;
 444
 445 typedef struct
 446 {
 447   bracket_elem_type type;
 448   union
 449   {
 450     unsigned char ch;
 451     unsigned char *name;
 452     wchar_t wch;
 453   } opr;
 454 } bracket_elem_t;
 455
 456
 457 /* Inline functions for bitset operation.  */
 458 static inline void
 459 bitset_not (set)
 460      bitset set;
 461 {
 462   int bitset_i;
 463   for (bitset_i = 0; bitset_i < BITSET_UINTS; ++bitset_i)
 464     set[bitset_i] = ~set[bitset_i];
 465 }
 466
 467 static inline void
 468 bitset_merge (dest, src)
 469      bitset dest;
 470      const bitset src;
 471 {
 472   int bitset_i;
 473   for (bitset_i = 0; bitset_i < BITSET_UINTS; ++bitset_i)
 474     dest[bitset_i] |= src[bitset_i];
 475 }
 476
 477 static inline void
 478 bitset_not_merge (dest, src)
 479      bitset dest;
 480      const bitset src;
 481 {
 482   int i;
 483   for (i = 0; i < BITSET_UINTS; ++i)
 484     dest[i] |= ~src[i];
 485 }
 486
 487 #ifdef RE_ENABLE_I18N
 488 /* Inline functions for re_string.  */
 489 static inline int
 490 re_string_char_size_at (pstr, idx)
 491      const re_string_t *pstr;
 492      int idx;
 493 {
 494   int byte_idx;
 495   if (MB_CUR_MAX == 1)
 496     return 1;
 497   for (byte_idx = 1; idx + byte_idx < pstr->len; ++byte_idx)
 498     if (pstr->wcs[idx + byte_idx] != WEOF)
 499       break;
 500   return byte_idx;
 501 }
 502
 503 static inline wint_t
 504 re_string_wchar_at (pstr, idx)
 505      const re_string_t *pstr;
 506      int idx;
 507 {
 508   if (MB_CUR_MAX == 1)
 509     return (wint_t) pstr->mbs[idx];
 510   return (wint_t) pstr->wcs[idx];
 511 }
 512
 513 static int
 514 re_string_elem_size_at (pstr, idx)
 515      const re_string_t *pstr;
 516      int idx;
 517 {
 518 #ifdef _LIBC
 519   const unsigned char *p;
 520   const char *extra;
 521   const int32_t *table, *indirect;
 522   int32_t tmp;
 523 # include <locale/weight.h>
 524   uint_fast32_t nrules = _NL_CURRENT_WORD (LC_COLLATE, _NL_COLLATE_NRULES);
 525
 526   if (nrules != 0)
 527     {
 528       table = (const int32_t *) _NL_CURRENT (LC_COLLATE, _NL_COLLATE_TABLEMB);
 529       extra = (const char *) _NL_CURRENT (LC_COLLATE, _NL_COLLATE_EXTRAMB);
 530       indirect = (const int32_t *) _NL_CURRENT (LC_COLLATE,
 531                                                 _NL_COLLATE_INDIRECTMB);
 532       p = pstr->mbs + idx;
 533       tmp = findidx (&p);
 534       return p - (const unsigned char *) pstr->mbs - idx;
 535     }
 536   else
 537 #endif /* _LIBC */
 538     return 1;
 539 }
 540 #endif /* RE_ENABLE_I18N */
 541
 542 #endif /*  _REGEX_INTERNAL_H */