1 ------------------------------------------------------------------------------
3 -- GNAT LIBRARY COMPONENTS --
5 -- G N A T . R E G P A T --
9 -- Copyright (C) 1986 by University of Toronto. --
10 -- Copyright (C) 1996-2006, AdaCore --
12 -- GNAT is free software; you can redistribute it and/or modify it under --
13 -- terms of the GNU General Public License as published by the Free Soft- --
14 -- ware Foundation; either version 2, or (at your option) any later ver- --
15 -- sion. GNAT is distributed in the hope that it will be useful, but WITH- --
16 -- OUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY --
17 -- or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License --
18 -- for more details. You should have received a copy of the GNU General --
19 -- Public License distributed with GNAT; see file COPYING. If not, write --
20 -- to the Free Software Foundation, 51 Franklin Street, Fifth Floor, --
21 -- Boston, MA 02110-1301, USA. --
23 -- As a special exception, if other files instantiate generics from this --
24 -- unit, or you link this unit with other files to produce an executable, --
25 -- this unit does not by itself cause the resulting executable to be --
26 -- covered by the GNU General Public License. This exception does not --
27 -- however invalidate any other reasons why the executable file might be --
28 -- covered by the GNU Public License. --
30 -- GNAT was originally developed by the GNAT team at New York University. --
31 -- Extensive contributions were provided by Ada Core Technologies Inc. --
33 ------------------------------------------------------------------------------
35 -- This package implements roughly the same set of regular expressions as
36 -- are available in the Perl or Python programming languages.
38 -- This is an extension of the original V7 style regular expression library
39 -- written in C by Henry Spencer. Apart from the translation to Ada, the
40 -- interface has been considerably changed to use the Ada String type
41 -- instead of C-style nul-terminated strings.
43 ------------------------------------------------------------
44 -- Summary of Pattern Matching Packages in GNAT Hierarchy --
45 ------------------------------------------------------------
47 -- There are three related packages that perform pattern maching functions.
48 -- the following is an outline of these packages, to help you determine
49 -- which is best for your needs.
51 -- GNAT.Regexp (files g-regexp.ads/g-regexp.adb)
52 -- This is a simple package providing Unix-style regular expression
53 -- matching with the restriction that it matches entire strings. It
54 -- is particularly useful for file name matching, and in particular
55 -- it provides "globbing patterns" that are useful in implementing
56 -- unix or DOS style wild card matching for file names.
58 -- GNAT.Regpat (files g-regpat.ads/g-regpat.adb)
59 -- This is a more complete implementation of Unix-style regular
60 -- expressions, copied from the Perl regular expression engine,
61 -- written originally in C by Henry Spencer. It is functionally the
62 -- same as that library.
64 -- GNAT.Spitbol.Patterns (files g-spipat.ads/g-spipat.adb)
65 -- This is a completely general pattern matching package based on the
66 -- pattern language of SNOBOL4, as implemented in SPITBOL. The pattern
67 -- language is modeled on context free grammars, with context sensitive
68 -- extensions that provide full (type 0) computational capabilities.
70 package GNAT
.Regpat
is
73 -- The grammar is the following:
76 -- ::= ^ expr -- anchor at the beginning of string
77 -- ::= expr $ -- anchor at the end of string
80 -- ::= term | term -- alternation (term or term ...)
83 -- ::= item item ... -- concatenation (item then item)
85 -- item ::= elmt -- match elmt
86 -- ::= elmt * -- zero or more elmt's
87 -- ::= elmt + -- one or more elmt's
88 -- ::= elmt ? -- matches elmt or nothing
89 -- ::= elmt *? -- zero or more times, minimum number
90 -- ::= elmt +? -- one or more times, minimum number
91 -- ::= elmt ?? -- zero or one time, minimum number
92 -- ::= elmt { num } -- matches elmt exactly num times
93 -- ::= elmt { num , } -- matches elmt at least num times
94 -- ::= elmt { num , num2 } -- matches between num and num2 times
95 -- ::= elmt { num }? -- matches elmt exactly num times
96 -- ::= elmt { num , }? -- matches elmt at least num times
98 -- ::= elmt { num , num2 }? -- matches between num and num2 times
101 -- elmt ::= nchr -- matches given character
102 -- ::= [range range ...] -- matches any character listed
103 -- ::= [^ range range ...] -- matches any character not listed
104 -- ::= . -- matches any single character
105 -- -- except newlines
106 -- ::= ( expr ) -- parens used for grouping
107 -- ::= \ num -- reference to num-th parenthesis
109 -- range ::= char - char -- matches chars in given range
111 -- ::= [: posix :] -- any character in the POSIX range
112 -- ::= [:^ posix :] -- not in the POSIX range
114 -- posix ::= alnum -- alphanumeric characters
115 -- ::= alpha -- alphabetic characters
116 -- ::= ascii -- ascii characters (0 .. 127)
117 -- ::= cntrl -- control chars (0..31, 127..159)
118 -- ::= digit -- digits ('0' .. '9')
119 -- ::= graph -- graphic chars (32..126, 160..255)
120 -- ::= lower -- lower case characters
121 -- ::= print -- printable characters (32..127)
122 -- ::= punct -- printable, except alphanumeric
123 -- ::= space -- space characters
124 -- ::= upper -- upper case characters
125 -- ::= word -- alphanumeric characters
126 -- ::= xdigit -- hexadecimal chars (0..9, a..f)
128 -- char ::= any character, including special characters
129 -- ASCII.NUL is not supported.
131 -- nchr ::= any character except \()[].*+?^ or \char to match char
132 -- \n means a newline (ASCII.LF)
133 -- \t means a tab (ASCII.HT)
134 -- \r means a return (ASCII.CR)
135 -- \b matches the empty string at the beginning or end of a
136 -- word. A word is defined as a set of alphanumerical
137 -- characters (see \w below).
138 -- \B matches the empty string only when *not* at the
139 -- beginning or end of a word.
140 -- \d matches any digit character ([0-9])
141 -- \D matches any non digit character ([^0-9])
142 -- \s matches any white space character. This is equivalent
143 -- to [ \t\n\r\f\v] (tab, form-feed, vertical-tab,...
144 -- \S matches any non-white space character.
145 -- \w matches any alphanumeric character or underscore.
146 -- This include accented letters, as defined in the
147 -- package Ada.Characters.Handling.
148 -- \W matches any non-alphanumeric character.
149 -- \A match the empty string only at the beginning of the
150 -- string, whatever flags are used for Compile (the
151 -- behavior of ^ can change, see Regexp_Flags below).
152 -- \G match the empty string only at the end of the
153 -- string, whatever flags are used for Compile (the
154 -- behavior of $ can change, see Regexp_Flags below).
155 -- ... ::= is used to indication repetition (one or more terms)
157 -- Embedded newlines are not matched by the ^ operator.
158 -- It is possible to retrieve the substring matched a parenthesis
159 -- expression. Although the depth of parenthesis is not limited in the
160 -- regexp, only the first 9 substrings can be retrieved.
162 -- The highest value possible for the arguments to the curly operator ({})
163 -- are given by the constant Max_Curly_Repeat below.
165 -- The operators '*', '+', '?' and '{}' always match the longest possible
166 -- substring. They all have a non-greedy version (with an extra ? after the
167 -- operator), which matches the shortest possible substring.
170 -- regexp="<.*>" string="<h1>title</h1>" matches="<h1>title</h1>"
171 -- regexp="<.*?>" string="<h1>title</h1>" matches="<h1>"
173 -- '{' and '}' are only considered as special characters if they appear
174 -- in a substring that looks exactly like '{n}', '{n,m}' or '{n,}', where
175 -- n and m are digits. No space is allowed. In other contexts, the curly
176 -- braces will simply be treated as normal characters.
178 -- Compiling Regular Expressions
179 -- =============================
181 -- To use this package, you first need to compile the regular expression
182 -- (a string) into a byte-code program, in a Pattern_Matcher structure.
183 -- This first step checks that the regexp is valid, and optimizes the
184 -- matching algorithms of the second step.
186 -- Two versions of the Compile subprogram are given: one in which this
187 -- package will compute itself the best possible size to allocate for the
188 -- byte code; the other where you must allocate enough memory yourself. An
189 -- exception is raised if there is not enough memory.
192 -- Regexp : String := "a|b";
194 -- Matcher : Pattern_Matcher := Compile (Regexp);
195 -- -- The size for matcher is automatically allocated
197 -- Matcher2 : Pattern_Matcher (1000);
198 -- -- Some space is allocated directly.
201 -- Compile (Matcher2, Regexp);
205 -- Note that the second version is significantly faster, since with the
206 -- first version the regular expression has in fact to be compiled twice
207 -- (first to compute the size, then to generate the byte code).
209 -- Note also that you cannot use the function version of Compile if you
210 -- specify the size of the Pattern_Matcher, since the discriminants will
211 -- most probably be different and you will get a Constraint_Error
216 -- Once the regular expression has been compiled, you can use it as often
217 -- as needed to match strings.
219 -- Several versions of the Match subprogram are provided, with different
220 -- parameters and return results.
222 -- See the description under each of these subprograms
224 -- Here is a short example showing how to get the substring matched by
225 -- the first parenthesis pair.
228 -- Matches : Match_Array (0 .. 1);
229 -- Regexp : String := "a(b|c)d";
230 -- Str : String := "gacdg";
233 -- Match (Compile (Regexp), Str, Matches);
234 -- return Str (Matches (1).First .. Matches (1).Last);
238 -- Finding all occurrences
239 -- =======================
241 -- Finding all the occurrences of a regular expression in a string cannot
242 -- be done by simply passing a slice of the string. This wouldn't work for
243 -- anchored regular expressions (the ones starting with "^" or ending with
245 -- Instead, you need to use the last parameter to Match (Data_First), as in
246 -- the following loop:
250 -- "-- first line" & ASCII.LF & "-- second line";
251 -- Matches : Match_array (0 .. 0);
252 -- Regexp : Pattern_Matcher := Compile ("^--", Multiple_Lines);
253 -- Current : Natural := Str'First;
256 -- Match (Regexp, Str, Matches, Current);
257 -- exit when Matches (0) = No_Match;
259 -- -- Process the match at position Matches (0).First
261 -- Current := Matches (0).Last + 1;
265 -- String Substitution
266 -- ===================
268 -- No subprogram is currently provided for string substitution.
269 -- However, this is easy to simulate with the parenthesis groups, as
272 -- This example swaps the first two words of the string:
275 -- Regexp : String := "([a-z]+) +([a-z]+)";
276 -- Str : String := " first second third ";
277 -- Matches : Match_Array (0 .. 2);
280 -- Match (Compile (Regexp), Str, Matches);
281 -- return Str (Str'First .. Matches (1).First - 1)
282 -- & Str (Matches (2).First .. Matches (2).Last)
284 -- & Str (Matches (1).First .. Matches (1).Last)
285 -- & Str (Matches (2).Last + 1 .. Str'Last);
286 -- -- returns " second first third "
293 Expression_Error
: exception;
294 -- This exception is raised when trying to compile an invalid regular
295 -- expression. All subprograms taking an expression as parameter may raise
298 Max_Paren_Count
: constant := 255;
299 -- Maximum number of parenthesis in a regular expression. This is limited
300 -- by the size of a Character, as found in the byte-compiled version of
301 -- regular expressions.
303 Max_Curly_Repeat
: constant := 32767;
304 -- Maximum number of repetition for the curly operator. The digits in the
305 -- {n}, {n,} and {n,m } operators cannot be higher than this constant,
306 -- since they have to fit on two characters in the byte-compiled version of
307 -- regular expressions.
309 Max_Program_Size
: constant := 2**15 - 1;
310 -- Maximum size that can be allocated for a program
312 type Program_Size
is range 0 .. Max_Program_Size
;
313 for Program_Size
'Size use 16;
314 -- Number of bytes allocated for the byte-compiled version of a regular
315 -- expression. The size required depends on the complexity of the regular
316 -- expression in a complex manner that is undocumented (other than in the
317 -- body of the Compile procedure). Normally the size is automatically set
318 -- and the programmer need not be concerned about it. There are two
319 -- exceptions to this. First in the calls to Match, it is possible to
320 -- specify a non-zero size that is known to be large enough. This can
321 -- slightly increase the efficiency by avoiding a copy. Second, in the case
322 -- of calling compile, it is possible using the procedural form of Compile
323 -- to use a single Pattern_Matcher variable for several different
324 -- expressions by setting its size sufficiently large.
326 Auto_Size
: constant := 0;
327 -- Used in calls to Match to indicate that the Size should be set to
328 -- a value appropriate to the expression being used automatically.
330 type Regexp_Flags
is mod 256;
331 for Regexp_Flags
'Size use 8;
332 -- Flags that can be given at compile time to specify default
333 -- properties for the regular expression.
335 No_Flags
: constant Regexp_Flags
;
336 Case_Insensitive
: constant Regexp_Flags
;
337 -- The automaton is optimized so that the matching is done in a case
338 -- insensitive manner (upper case characters and lower case characters
339 -- are all treated the same way).
341 Single_Line
: constant Regexp_Flags
;
342 -- Treat the Data we are matching as a single line. This means that
343 -- ^ and $ will ignore \n (unless Multiple_Lines is also specified),
344 -- and that '.' will match \n.
346 Multiple_Lines
: constant Regexp_Flags
;
347 -- Treat the Data as multiple lines. This means that ^ and $ will also
348 -- match on internal newlines (ASCII.LF), in addition to the beginning
349 -- and end of the string.
351 -- This can be combined with Single_Line.
357 subtype Match_Count
is Natural range 0 .. Max_Paren_Count
;
359 type Match_Location
is record
360 First
: Natural := 0;
364 type Match_Array
is array (Match_Count
range <>) of Match_Location
;
365 -- The substring matching a given pair of parenthesis. Index 0 is the whole
366 -- substring that matched the full regular expression.
368 -- For instance, if your regular expression is something like: "a(b*)(c+)",
369 -- then Match_Array(1) will be the indexes of the substring that matched
370 -- "b*" and Match_Array(2) will be the substring that matched "c+".
372 -- The number of parenthesis groups that can be retrieved is unlimited, and
373 -- all the Match subprograms below can use a Match_Array of any size.
374 -- Indexes that do not have any matching parenthesis are set to No_Match.
376 No_Match
: constant Match_Location
:= (First
=> 0, Last
=> 0);
377 -- The No_Match constant is (0, 0) to differentiate between matching a null
378 -- string at position 1, which uses (1, 0) and no match at all.
380 ---------------------------------
381 -- Pattern_Matcher Compilation --
382 ---------------------------------
384 -- The subprograms here are used to precompile regular expressions for use
385 -- in subsequent Match calls. Precompilation improves efficiency if the
386 -- same regular expression is to be used in more than one Match call.
388 type Pattern_Matcher
(Size
: Program_Size
) is private;
389 -- Type used to represent a regular expression compiled into byte code
391 Never_Match
: constant Pattern_Matcher
;
392 -- A regular expression that never matches anything
395 (Expression
: String;
396 Flags
: Regexp_Flags
:= No_Flags
) return Pattern_Matcher
;
397 -- Compile a regular expression into internal code
399 -- Raises Expression_Error if Expression is not a legal regular expression
401 -- The appropriate size is calculated automatically to correspond to the
402 -- provided expression. This is the normal default method of compilation.
403 -- Note that it is generally not possible to assign the result of two
404 -- different calls to this Compile function to the same Pattern_Matcher
405 -- variable, since the sizes will differ.
407 -- Flags is the default value to use to set properties for Expression
408 -- (e.g. case sensitivity,...).
411 (Matcher
: out Pattern_Matcher
;
413 Final_Code_Size
: out Program_Size
;
414 Flags
: Regexp_Flags
:= No_Flags
);
415 -- Compile a regular expression into into internal code
417 -- This procedure is significantly faster than the Compile function since
418 -- it avoids the extra step of precomputing the required size.
420 -- However, it requires the user to provide a Pattern_Matcher variable
421 -- whose size is preset to a large enough value. One advantage of this
422 -- approach, in addition to the improved efficiency, is that the same
423 -- Pattern_Matcher variable can be used to hold the compiled code for
424 -- several different regular expressions by setting a size that is large
425 -- enough to accomodate all possibilities.
427 -- In this version of the procedure call, the actual required code size is
428 -- returned. Also if Matcher.Size is zero on entry, then the resulting code
429 -- is not stored. A call with Matcher.Size set to Auto_Size can thus be
430 -- used to determine the space required for compiling the given regular
433 -- This function raises Storage_Error if Matcher is too small to hold
434 -- the resulting code (i.e. Matcher.Size has too small a value).
436 -- Expression_Error is raised if the string Expression does not contain
437 -- a valid regular expression.
439 -- Flags is the default value to use to set properties for Expression (case
443 (Matcher
: out Pattern_Matcher
;
445 Flags
: Regexp_Flags
:= No_Flags
);
446 -- -- Same procedure as above, expect it does not return the final
447 -- -- program size, and Matcher.Size cannot be Auto_Size.
449 function Paren_Count
(Regexp
: Pattern_Matcher
) return Match_Count
;
450 pragma Inline
(Paren_Count
);
451 -- Return the number of parenthesis pairs in Regexp.
453 -- This is the maximum index that will be filled if a Match_Array is
454 -- used as an argument to Match.
456 -- Thus, if you want to be sure to get all the parenthesis, you should
457 -- do something like:
460 -- Regexp : Pattern_Matcher := Compile ("a(b*)(c+)");
461 -- Matched : Match_Array (0 .. Paren_Count (Regexp));
463 -- Match (Regexp, "a string", Matched);
470 function Quote
(Str
: String) return String;
471 -- Return a version of Str so that every special character is quoted.
472 -- The resulting string can be used in a regular expression to match
473 -- exactly Str, whatever character was present in Str.
479 -- The Match subprograms are given a regular expression in string
480 -- form, and perform the corresponding match. The following parameters
481 -- are present in all forms of the Match call.
483 -- Expression contains the regular expression to be matched as a string
485 -- Data contains the string to be matched
487 -- Data_First is the lower bound for the match, i.e. Data (Data_First)
488 -- will be the first character to be examined. If Data_First is set to
489 -- the special value of -1 (the default), then the first character to
490 -- be examined is Data (Data_First). However, the regular expression
491 -- character ^ (start of string) still refers to the first character
492 -- of the full string (Data (Data'First)), which is why there is a
493 -- separate mechanism for specifying Data_First.
495 -- Data_Last is the upper bound for the match, i.e. Data (Data_Last)
496 -- will be the last character to be examined. If Data_Last is set to
497 -- the special value of Positive'Last (the default), then the last
498 -- character to be examined is Data (Data_Last). However, the regular
499 -- expression character $ (end of string) still refers to the last
500 -- character of the full string (Data (Data'Last)), which is why there
501 -- is a separate mechanism for specifying Data_Last.
503 -- Note: the use of Data_First and Data_Last is not equivalent to
504 -- simply passing a slice as Expression because of the handling of
505 -- regular expression characters ^ and $.
507 -- Size is the size allocated for the compiled byte code. Normally
508 -- this is defaulted to Auto_Size which means that the appropriate
509 -- size is allocated automatically. It is possible to specify an
510 -- explicit size, which must be sufficiently large. This slightly
511 -- increases the efficiency by avoiding the extra step of computing
512 -- the appropriate size.
514 -- The following exceptions can be raised in calls to Match
516 -- Storage_Error is raised if a non-zero value is given for Size
517 -- and it is too small to hold the compiled byte code.
519 -- Expression_Error is raised if the given expression is not a legal
520 -- regular expression.
523 (Expression
: String;
525 Matches
: out Match_Array
;
526 Size
: Program_Size
:= Auto_Size
;
527 Data_First
: Integer := -1;
528 Data_Last
: Positive := Positive'Last);
529 -- This version returns the result of the match stored in Match_Array.
530 -- At most Matches'Length parenthesis are returned.
533 (Expression
: String;
535 Size
: Program_Size
:= Auto_Size
;
536 Data_First
: Integer := -1;
537 Data_Last
: Positive := Positive'Last) return Natural;
538 -- This version returns the position where Data matches, or if there is
539 -- no match, then the value Data'First - 1.
542 (Expression
: String;
544 Size
: Program_Size
:= Auto_Size
;
545 Data_First
: Integer := -1;
546 Data_Last
: Positive := Positive'Last) return Boolean;
547 -- This version returns True if the match succeeds, False otherwise
549 ------------------------------------------------
550 -- Matching a Pre-Compiled Regular Expression --
551 ------------------------------------------------
553 -- The following functions are significantly faster if you need to reuse
554 -- the same regular expression multiple times, since you only have to
555 -- compile it once. For these functions you must first compile the
556 -- expression with a call to Compile as previously described.
558 -- The parameters Data, Data_First and Data_Last are as described
559 -- in the previous section.
562 (Self
: Pattern_Matcher
;
564 Data_First
: Integer := -1;
565 Data_Last
: Positive := Positive'Last) return Natural;
566 -- Match Data using the given pattern matcher. Returns the position
567 -- where Data matches, or (Data'First - 1) if there is no match.
570 (Self
: Pattern_Matcher
;
572 Data_First
: Integer := -1;
573 Data_Last
: Positive := Positive'Last) return Boolean;
574 -- Return True if Data matches using the given pattern matcher
576 pragma Inline
(Match
);
577 -- All except the last one below
580 (Self
: Pattern_Matcher
;
582 Matches
: out Match_Array
;
583 Data_First
: Integer := -1;
584 Data_Last
: Positive := Positive'Last);
585 -- Match Data using the given pattern matcher and store result in Matches.
586 -- The expression matches if Matches (0) /= No_Match. The lower bound of
587 -- Matches is required to be zero.
589 -- At most Matches'Length parenthesis are returned
595 procedure Dump
(Self
: Pattern_Matcher
);
596 -- Dump the compiled version of the regular expression matched by Self
598 --------------------------
599 -- Private Declarations --
600 --------------------------
604 subtype Pointer
is Program_Size
;
605 -- The Pointer type is used to point into Program_Data
607 -- Note that the pointer type is not necessarily 2 bytes
608 -- although it is stored in the program using 2 bytes
610 type Program_Data
is array (Pointer
range <>) of Character;
612 Program_First
: constant := 1;
614 -- The "internal use only" fields in regexp are present to pass info from
615 -- compile to execute that permits the execute phase to run lots faster on
616 -- simple cases. They are:
618 -- First character that must begin a match or ASCII.Nul
619 -- Anchored true iff match must start at beginning of line
620 -- Must_Have pointer to string that match must include or null
621 -- Must_Have_Length length of Must_Have string
623 -- First and Anchored permit very fast decisions on suitable starting
624 -- points for a match, cutting down the work a lot. Must_Have permits fast
625 -- rejection of lines that cannot possibly match.
627 -- The Must_Have tests are costly enough that Optimize supplies a Must_Have
628 -- only if the r.e. contains something potentially expensive (at present,
629 -- the only such thing detected is * or at the start of the r.e., which can
630 -- involve a lot of backup). The length is supplied because the test in
631 -- Execute needs it and Optimize is computing it anyway.
633 -- The initialization is meant to fail-safe in case the user of this
634 -- package tries to use an uninitialized matcher. This takes advantage
635 -- of the knowledge that ASCII.Nul translates to the end-of-program (EOP)
636 -- instruction code of the state machine.
638 No_Flags
: constant Regexp_Flags
:= 0;
639 Case_Insensitive
: constant Regexp_Flags
:= 1;
640 Single_Line
: constant Regexp_Flags
:= 2;
641 Multiple_Lines
: constant Regexp_Flags
:= 4;
643 type Pattern_Matcher
(Size
: Pointer
) is record
644 First
: Character := ASCII
.NUL
; -- internal use only
645 Anchored
: Boolean := False; -- internal use only
646 Must_Have
: Pointer
:= 0; -- internal use only
647 Must_Have_Length
: Natural := 0; -- internal use only
648 Paren_Count
: Natural := 0; -- # paren groups
649 Flags
: Regexp_Flags
:= No_Flags
;
650 Program
: Program_Data
(Program_First
.. Size
) :=
651 (others => ASCII
.NUL
);
654 Never_Match
: constant Pattern_Matcher
:=
655 (0, ASCII
.NUL
, False, 0, 0, 0, No_Flags
, (others => ASCII
.NUL
));