| blob | 3aad37adf2dd02bf250a2eb873a20e61f68be452 |
1 .\"
2 .\" Sun Microsystems, Inc. gratefully acknowledges The Open Group for
3 .\" permission to reproduce portions of its copyrighted documentation.
4 .\" Original documentation from The Open Group can be obtained online at
5 .\" http://www.opengroup.org/bookstore/.
6 .\"
7 .\" The Institute of Electrical and Electronics Engineers and The Open
8 .\" Group, have given us permission to reprint portions of their
9 .\" documentation.
10 .\"
11 .\" In the following statement, the phrase ``this text'' refers to portions
12 .\" of the system documentation.
13 .\"
14 .\" Portions of this text are reprinted and reproduced in electronic form
15 .\" in the SunOS Reference Manual, from IEEE Std 1003.1, 2004 Edition,
16 .\" Standard for Information Technology -- Portable Operating System
17 .\" Interface (POSIX), The Open Group Base Specifications Issue 6,
18 .\" Copyright (C) 2001-2004 by the Institute of Electrical and Electronics
19 .\" Engineers, Inc and The Open Group. In the event of any discrepancy
20 .\" between these versions and the original IEEE and The Open Group
21 .\" Standard, the original IEEE and The Open Group Standard is the referee
22 .\" document. The original Standard can be obtained online at
23 .\" http://www.opengroup.org/unix/online.html.
24 .\"
25 .\" This notice shall appear on any product containing this material.
26 .\"
27 .\" The contents of this file are subject to the terms of the
28 .\" Common Development and Distribution License (the "License").
29 .\" You may not use this file except in compliance with the License.
30 .\"
31 .\" You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
32 .\" or http://www.opensolaris.org/os/licensing.
33 .\" See the License for the specific language governing permissions
34 .\" and limitations under the License.
35 .\"
36 .\" When distributing Covered Code, include this CDDL HEADER in each
37 .\" file and include the License file at usr/src/OPENSOLARIS.LICENSE.
38 .\" If applicable, add the following below this CDDL HEADER, with the
39 .\" fields enclosed by brackets "[]" replaced with your own identifying
40 .\" information: Portions Copyright [yyyy] [name of copyright owner]
41 .\"
42 .\"
43 .\" Copyright (c) 1992, X/Open Company Limited. All Rights Reserved.
44 .\" Portions Copyright (c) 1997, Sun Microsystems, Inc. All Rights Reserved
45 .\" Copyright (c) 2014 Gary Mills
46 .\"
47 .TH LEX 1 "Jan 1, 2014"
48 .SH NAME
49 lex \- generate programs for lexical tasks
50 .SH SYNOPSIS
51 .LP
52 .nf
53 \fBlex\fR [\fB-cntv\fR] [\fB-e\fR | \fB-w\fR] [\fB-V\fR \fB-Q\fR [y | n]] [\fB-Y\fR \fIdirectory\fR]] [\fIfile\fR]...
54 .fi
56 .SH DESCRIPTION
57 .sp
58 .LP
59 The \fBlex\fR utility generates C programs to be used in lexical processing of
60 character input, and that can be used as an interface to \fByacc\fR. The C
61 programs are generated from \fBlex\fR source code and conform to the ISO C
62 standard. Usually, the \fBlex\fR utility writes the program it generates to the
63 file \fBlex.yy.c\fR. The state of this file is unspecified if \fBlex\fR exits
64 with a non-zero exit status. See \fBEXTENDED DESCRIPTION\fR for a complete
65 description of the \fBlex\fR input language.
66 .SH OPTIONS
67 .sp
68 .LP
69 The following options are supported:
70 .sp
71 .ne 2
72 .na
73 \fB\fB-c\fR \fR
74 .ad
75 .RS 12n
76 Indicates C-language action (default option).
77 .RE
79 .sp
80 .ne 2
81 .na
82 \fB\fB-e\fR \fR
83 .ad
84 .RS 12n
85 Generates a program that can handle \fBEUC\fR characters (cannot be used with
86 the \fB-w\fR option). \fByytext[\|]\fR is of type \fBunsigned char[\|]\fR.
87 .RE
89 .sp
90 .ne 2
91 .na
92 \fB\fB-n\fR \fR
93 .ad
94 .RS 12n
95 Suppresses the summary of statistics usually written with the \fB-v\fR option.
96 If no table sizes are specified in the \fBlex\fR source code and the \fB-v\fR
97 option is not specified, then \fB-n\fR is implied.
98 .RE
100 .sp
101 .ne 2
102 .na
103 \fB\fB-t\fR \fR
104 .ad
105 .RS 12n
106 Writes the resulting program to standard output instead of \fBlex.yy.c\fR.
107 .RE
109 .sp
110 .ne 2
111 .na
112 \fB\fB-v\fR \fR
113 .ad
114 .RS 12n
115 Writes a summary of \fBlex\fR statistics to the standard error. (See the
116 discussion of \fBlex\fR table sizes under the heading \fBDefinitions in
117 lex\fR.) If table sizes are specified in the \fBlex\fR source code, and if the
118 \fB-n\fR option is not specified, the \fB-v\fR option may be enabled.
119 .RE
121 .sp
122 .ne 2
123 .na
124 \fB\fB-w\fR \fR
125 .ad
126 .RS 12n
127 Generates a program that can handle \fBEUC\fR characters (cannot be used with
128 the \fB-e\fR option). Unlike the \fB-e\fR option, \fByytext[\|]\fR is of type
129 \fBwchar_t[\|]\fR.
130 .RE
132 .sp
133 .ne 2
134 .na
135 \fB\fB-V\fR \fR
136 .ad
137 .RS 12n
138 Prints out version information on standard error.
139 .RE
141 .sp
142 .ne 2
143 .na
144 \fB\fB\fR\fB-Q\fR\fB[y|n]\fR \fR
145 .ad
146 .RS 12n
147 Prints out version information to output file \fBlex.yy.c\fR by using
148 \fB-Qy\fR. The \fB-Qn\fR option does not print out version information and is
149 the default.
150 .RE
152 .sp
153 .ne 2
154 .na
155 \fB\fB\fR\fB-Y\fR \fBdirectory\fR \fR
156 .ad
157 .RS 12n
158 Designates an alternate directory that contains the driver files
159 used by \fBlex\fR.
160 .RE
162 .SH OPERANDS
163 .sp
164 .LP
165 The following operand is supported:
166 .sp
167 .ne 2
168 .na
169 \fB\fIfile\fR \fR
170 .ad
171 .RS 9n
172 A pathname of an input file. If more than one such \fIfile\fR is specified, all
173 files will be concatenated to produce a single \fBlex\fR program. If no
174 \fIfile\fR operands are specified, or if a \fIfile\fR operand is \fB\(mi\fR,
175 the standard input will be used.
176 .RE
178 .SH OUTPUT
179 .sp
180 .LP
181 The \fBlex\fR output files are described below.
182 .SS "Stdout"
183 .sp
184 .LP
185 If the \fB-t\fR option is specified, the text file of C source code output of
186 \fBlex\fR will be written to standard output.
187 .SS "Stderr"
188 .sp
189 .LP
190 If the \fB-t\fR option is specified informational, error and warning messages
191 concerning the contents of \fBlex\fR source code input will be written to the
192 standard error.
193 .sp
194 .LP
195 If the \fB-t\fR option is not specified:
196 .RS +4
197 .TP
198 1.
199 Informational error and warning messages concerning the contents of
200 \fBlex\fR source code input will be written to either the standard output or
201 standard error.
202 .RE
203 .RS +4
204 .TP
205 2.
206 If the \fB-v\fR option is specified and the \fB-n\fR option is not
207 specified, \fBlex\fR statistics will also be written to standard error. These
208 statistics may also be generated if table sizes are specified with a \fB%\fR
209 operator in the \fBDefinitions\fR \fBin\fR \fBlex\fR section (see \fBEXTENDED
210 DESCRIPTION\fR), as long as the \fB-n\fR option is not specified.
211 .RE
212 .SS "Output Files"
213 .sp
214 .LP
215 A text file containing C source code will be written to \fBlex.yy.c\fR, or to
216 the standard output if the \fB-t\fR option is present.
217 .SH EXTENDED DESCRIPTION
218 .sp
219 .LP
220 Each input file contains \fBlex\fR source code, which is a table of regular
221 expressions with corresponding actions in the form of C program fragments.
222 .sp
223 .LP
224 When \fBlex.yy.c\fR is compiled and linked with the \fBlex\fR library (using
225 the \fB\fR\fB-l\fR\fB l\fR operand with \fBc89\fR or \fBcc\fR), the resulting
226 program reads character input from the standard input and partitions it into
227 strings that match the given expressions.
228 .sp
229 .LP
230 When an expression is matched, these actions will occur:
231 .RS +4
232 .TP
233 .ie t \(bu
234 .el o
235 The input string that was matched is left in \fIyytext\fR as a null-terminated
236 string; \fIyytext\fR is either an external character array or a pointer to a
237 character string. As explained in \fBDefinitions in lex\fR, the type can be
238 explicitly selected using the \fB%array\fR or \fB%pointer\fR declarations, but
239 the default is \fB%array\fR.
240 .RE
241 .RS +4
242 .TP
243 .ie t \(bu
244 .el o
245 The external \fBint\fR \fIyyleng\fR is set to the length of the matching
246 string.
247 .RE
248 .RS +4
249 .TP
250 .ie t \(bu
251 .el o
252 The expression's corresponding program fragment, or action, is executed.
253 .RE
254 .sp
255 .LP
256 During pattern matching, \fBlex\fR searches the set of patterns for the single
257 longest possible match. Among rules that match the same number of characters,
258 the rule given first will be chosen.
259 .sp
260 .LP
261 The general format of \fBlex\fR source is:
262 .sp
263 .in +2
264 .nf
265 \fIDefinitions\fR
266 %%
267 \fIRules\fR
268 %%
269 \fIUser Subroutines\fR
270 .fi
271 .in -2
273 .sp
274 .LP
275 The first \fB%%\fR is required to mark the beginning of the rules (regular
276 expressions and actions); the second \fB%%\fR is required only if user
277 subroutines follow.
278 .sp
279 .LP
280 Any line in the \fBDefinitions\fR \fBin\fR \fBlex\fR section beginning with a
281 blank character will be assumed to be a C program fragment and will be copied
282 to the external definition area of the \fBlex.yy.c\fR file. Similarly, anything
283 in the \fBDefinitions\fR \fBin\fR \fBlex\fR section included between delimiter
284 lines containing only \fB%{\fR and \fB%}\fR will also be copied unchanged to
285 the external definition area of the \fBlex.yy.c\fR file.
286 .sp
287 .LP
288 Any such input (beginning with a blank character or within \fB%{\fR and
289 \fB%}\fR delimiter lines) appearing at the beginning of the \fIRules\fR section
290 before any rules are specified will be written to \fBlex.yy.c\fR after the
291 declarations of variables for the \fByylex\fR function and before the first
292 line of code in \fByylex\fR. Thus, user variables local to \fByylex\fR can be
293 declared here, as well as application code to execute upon entry to
294 \fByylex\fR.
295 .sp
296 .LP
297 The action taken by \fBlex\fR when encountering any input beginning with a
298 blank character or within \fB%{\fR and \fB%}\fR delimiter lines appearing in
299 the \fIRules\fR section but coming after one or more rules is undefined. The
300 presence of such input may result in an erroneous definition of the \fByylex\fR
301 function.
302 .SS "Definitions in lex"
303 .sp
304 .LP
305 \fBDefinitions\fR \fBin\fR \fBlex\fR appear before the first \fB%%\fR
306 delimiter. Any line in this section not contained between \fB%{\fR and \fB%}\fR
307 lines and not beginning with a blank character is assumed to define a \fBlex\fR
308 substitution string. The format of these lines is:
309 .sp
310 .in +2
311 .nf
312 \fIname substitute\fR
313 .fi
314 .in -2
315 .sp
317 .sp
318 .LP
319 If a \fIname\fR does not meet the requirements for identifiers in the ISO C
320 standard, the result is undefined. The string \fIsubstitute\fR will replace the
321 string \fI{\fR \fIname\fR \fI}\fR when it is used in a rule. The \fIname\fR
322 string is recognized in this context only when the braces are provided and when
323 it does not appear within a bracket expression or within double-quotes.
324 .sp
325 .LP
326 In the \fBDefinitions\fR \fBin\fR \fBlex\fR section, any line beginning with a
327 \fB%\fR (percent sign) character and followed by an alphanumeric word beginning
328 with either \fBs\fR or \fBS\fR defines a set of start conditions. Any line
329 beginning with a \fB%\fR followed by a word beginning with either \fBx\fR or
330 \fBX\fR defines a set of exclusive start conditions. When the generated scanner
331 is in a \fB%s\fR state, patterns with no state specified will be also active;
332 in a \fB%x\fR state, such patterns will not be active. The rest of the line,
333 after the first word, is considered to be one or more blank-character-separated
334 names of start conditions. Start condition names are constructed in the same
335 way as definition names. Start conditions can be used to restrict the matching
336 of regular expressions to one or more states as described in \fBRegular
337 expressions in lex\fR.
338 .sp
339 .LP
340 Implementations accept either of the following two mutually exclusive
341 declarations in the \fBDefinitions\fR \fBin\fR \fBlex\fR section:
342 .sp
343 .ne 2
344 .na
345 \fB\fB%array\fR \fR
346 .ad
347 .RS 13n
348 Declare the type of \fIyytext\fR to be a null-terminated character array.
349 .RE
351 .sp
352 .ne 2
353 .na
354 \fB\fB%pointer\fR \fR
355 .ad
356 .RS 13n
357 Declare the type of \fIyytext\fR to be a pointer to a null-terminated character
358 string.
359 .RE
361 .sp
362 .LP
363 \fBNote:\fR When using the \fB%pointer\fR option, you may not also use the
364 \fByyless\fR function to alter \fIyytext\fR.
365 .sp
366 .LP
367 \fB%array\fR is the default. If \fB%array\fR is specified (or neither
368 \fB%array\fR nor \fB%pointer\fR is specified), then the correct way to make an
369 external reference to \fIyyext\fR is with a declaration of the form:
370 .sp
371 .LP
372 \fBextern char\fR\fI yytext\fR\fB[\|]\fR
373 .sp
374 .LP
375 If \fB%pointer\fR is specified, then the correct external reference is of the
376 form:
377 .sp
378 .LP
379 \fBextern char *\fR\fIyytext\fR\fB;\fR
380 .sp
381 .LP
382 \fBlex\fR will accept declarations in the \fBDefinitions in lex\fR section for
383 setting certain internal table sizes. The declarations are shown in the
384 following table.
385 .sp
386 .LP
387 \fBTable\fR \fBSize\fR \fBDeclaration\fR \fBin\fR \fBlex\fR
388 .sp
390 .sp
391 .TS
392 box;
393 c c c
394 l l l .
395 \fBDeclaration\fR \fBDescription\fR \fBDefault\fR
396 _
397 \fB%p\fR\fIn\fR Number of positions 2500
398 \fB%n\fR\fIn\fR Number of states 500
399 \fB%a\fR\fI n\fR Number of transitions 2000
400 \fB%e\fR\fIn\fR Number of parse tree nodes 1000
401 \fB%k\fR\fIn\fR Number of packed character classes 10000
402 \fB%o\fR\fIn\fR Size of the output array 3000
403 .TE
405 .sp
406 .LP
407 Programs generated by \fBlex\fR need either the \fB-e\fR or \fB-w\fR option to
408 handle input that contains \fBEUC\fR characters from supplementary codesets. If
409 neither of these options is specified, \fByytext\fR is of the type
410 \fBchar[\|]\fR, and the generated program can handle only \fBASCII\fR
411 characters.
412 .sp
413 .LP
414 When the \fB-e\fR option is used, \fByytext\fR is of the type \fBunsigned\fR
415 \fBchar[\|]\fR and \fByyleng\fR gives the total number of \fIbytes\fR in the
416 matched string. With this option, the macros \fBinput()\fR,
417 \fBunput(\fIc\fR)\fR, and \fBoutput(\fIc\fR)\fR should do a byte-based
418 \fBI/O\fR in the same way as with the regular \fBASCII\fR \fBlex\fR. Two more
419 variables are available with the \fB-e\fR option, \fByywtext\fR and
420 \fByywleng\fR, which behave the same as \fByytext\fR and \fByyleng\fR would
421 under the \fB-w\fR option.
422 .sp
423 .LP
424 When the \fB-w\fR option is used, \fByytext\fR is of the type \fBwchar_t[\|]\fR
425 and \fByyleng\fR gives the total number of \fIcharacters\fR in the matched
426 string. If you supply your own \fBinput()\fR, \fBunput(\fIc\fR)\fR, or
427 \fBoutput(\fR\fIc\fR\fB)\fR macros with this option, they must return or accept
428 \fBEUC\fR characters in the form of wide character (\fBwchar_t\fR). This allows
429 a different interface between your program and the lex internals, to expedite
430 some programs.
431 .SS "Rules in lex"
432 .sp
433 .LP
434 The \fBRules\fR \fBin\fR \fBlex\fR source files are a table in which the left
435 column contains regular expressions and the right column contains actions (C
436 program fragments) to be executed when the expressions are recognized.
437 .sp
438 .in +2
439 .nf
440 \fIERE action\fR
441 \fIERE action\fR
442 \&...
443 .fi
444 .in -2
446 .sp
447 .LP
448 The extended regular expression (ERE) portion of a row will be separated from
449 \fIaction\fR by one or more blank characters. A regular expression containing
450 blank characters is recognized under one of the following conditions:
451 .RS +4
452 .TP
453 .ie t \(bu
454 .el o
455 The entire expression appears within double-quotes.
456 .RE
457 .RS +4
458 .TP
459 .ie t \(bu
460 .el o
461 The blank characters appear within double-quotes or square brackets.
462 .RE
463 .RS +4
464 .TP
465 .ie t \(bu
466 .el o
467 Each blank character is preceded by a backslash character.
468 .RE
469 .SS "User Subroutines in lex"
470 .sp
471 .LP
472 Anything in the user subroutines section will be copied to \fBlex.yy.c\fR
473 following \fByylex\fR.
474 .SS "Regular Expressions in lex"
475 .sp
476 .LP
477 The \fBlex\fR utility supports the set of Extended Regular Expressions (EREs)
478 described on \fBregex\fR(5) with the following additions and exceptions to the
479 syntax:
480 .sp
481 .ne 2
482 .na
483 \fB\fB\|.\|.\|.\fR \fR
484 .ad
485 .RS 14n
486 Any string enclosed in double-quotes will represent the characters within the
487 double-quotes as themselves, except that backslash escapes (which appear in the
488 following table) are recognized. Any backslash-escape sequence is terminated by
489 the closing quote. For example, "\|\e\|01""1" represents a single string: the
490 octal value 1 followed by the character 1.
491 .RE
493 .sp
494 .LP
495 \fI<\fR\fIstate\fR\fI>\fR\fIr\fR
496 .sp
497 .ne 2
498 .na
499 \fB<\fIstate1\fR, \fIstate2\fR, \|.\|.\|.\|>\fIr\fR\fR
500 .ad
501 .sp .6
502 .RS 4n
503 The regular expression \fIr\fR will be matched only when the program is in one
504 of the start conditions indicated by \fIstate\fR, \fIstate1\fR, and so forth.
505 For more information, see \fBActions in lex\fR. As an exception to the
506 typographical conventions of the rest of this document, in this case
507 <\fIstate\fR> does not represent a metavariable, but the literal angle-bracket
508 characters surrounding a symbol. The start condition is recognized as such only
509 at the beginning of a regular expression.
510 .RE
512 .sp
513 .ne 2
514 .na
515 \fB\fIr\fR/\fIx\fR \fR
516 .ad
517 .sp .6
518 .RS 4n
519 The regular expression \fIr\fR will be matched only if it is followed by an
520 occurrence of regular expression \fIx\fR. The token returned in \fIyytext\fR
521 will only match \fIr\fR. If the trailing portion of \fIr\fR matches the
522 beginning of \fIx\fR, the result is unspecified. The \fIr\fR expression cannot
523 include further trailing context or the \fB$\fR (match-end-of-line) operator;
524 \fIx\fR cannot include the \fB^\fR (match-beginning-of-line) operator, nor
525 trailing context, nor the \fB$\fR operator. That is, only one occurrence of
526 trailing context is allowed in a \fBlex\fR regular expression, and the \fB^\fR
527 operator only can be used at the beginning of such an expression. A further
528 restriction is that the trailing-context operator \fB/\fR (slash) cannot be
529 grouped within parentheses.
530 .RE
532 .sp
533 .ne 2
534 .na
535 \fB\fB{\fR\fIname\fR\fB}\fR \fR
536 .ad
537 .sp .6
538 .RS 4n
539 When \fIname\fR is one of the substitution symbols from the \fIDefinitions\fR
540 section, the string, including the enclosing braces, will be replaced by the
541 \fIsubstitute\fR value. The \fIsubstitute\fR value will be treated in the
542 extended regular expression as if it were enclosed in parentheses. No
543 substitution will occur if \fB{\fR\fIname\fR\fB}\fR occurs within a bracket
544 expression or within double-quotes.
545 .RE
547 .sp
548 .LP
549 Within an \fBERE,\fR a backslash character (\fB\|\e\e\fR, \fB\e\|a\fR,
550 \fB\e\|b\fR, \fB\e\|f\fR, \fB\e\|n\fR, \fB\e\|r\fR, \fB\e\|t\fR, \fB\e\|v\fR)
551 is considered to begin an escape sequence. In addition, the escape sequences in
552 the following table will be recognized.
553 .sp
554 .LP
555 A literal newline character cannot occur within an \fBERE;\fR the escape
556 sequence \fB\e\|n\fR can be used to represent a newline character. A newline
557 character cannot be matched by a period operator.
558 .sp
559 .LP
560 \fBEscape Sequences in lex\fR
561 .sp
563 .sp
564 .TS
565 box;
566 c c c
567 c c c .
568 Escape Sequences in lex
569 _
570 Escape Sequence Description Meaning
571 _
572 \e\fIdigits\fR T{
573 A backslash character followed by the longest sequence of one, two or three octal-digit characters (01234567). Ifall of the digits are 0, (that is, representation of the NUL character), the behavior is undefined.
574 T} T{
575 The character whose encoding is represented by the one-, two- or three-digit octal integer. Multi-byte characters require multiple, concatenated escape sequences of this type, including the leading \e for each byte.
576 T}
577 _
578 \e\fBx\fR\fIdigits\fR T{
579 A backslash character followed by the longest sequence of hexadecimal-digit characters (01234567abcdefABCDEF). If all of the digits are 0, (that is, representation of the NUL character), the behavior is undefined.
580 T} T{
581 The character whose encoding is represented by the hexadecimal integer.
582 T}
583 _
584 \e\fIc\fR T{
585 A backslash character followed by any character not described in this table. (\e\e, \ea, \eb, \ef, \een, \er, \et, \ev).
586 T} The character c, unchanged.
587 .TE
589 .sp
590 .LP
591 The order of precedence given to extended regular expressions for \fBlex\fR is
592 as shown in the following table, from high to low.
593 .sp
594 .ne 2
595 .na
596 \fB\fBNote\fR: \fR
597 .ad
598 .RS 10n
599 The escaped characters entry is not meant to imply that these are operators,
600 but they are included in the table to show their relationships to the true
601 operators. The start condition, trailing context and anchoring notations have
602 been omitted from the table because of the placement restrictions described in
603 this section; they can only appear at the beginning or ending of an \fBERE.\fR
604 .RE
606 .sp
608 .sp
609 .TS
610 box;
611 c c
612 l l .
613 ERE Precedence in lex
614 _
615 \fIcollation-related bracket symbols\fR \fB[= =] [: :] [. .]\fR
616 \fIescaped characters\fR \fB\e<\fR\fIspecial character\fR>
617 \fIbracket expression\fR \fB[ ]\fR
618 \fIquoting\fR \fB".\|.\|."\fR
619 \fIgrouping\fR \fB()\fR
620 \fIdefinition\fR \fB{\fR\fIname\fR}
621 \fIsingle-character RE duplication\fR \fB* + ?\fR
622 \fIconcatenation\fR
623 \fIinterval expression\fR \fB{\fR\fIm\fR,\fIn\fR}
624 \fIalternation\fR \fB|\fR
625 .TE
627 .sp
628 .LP
629 The \fBERE\fR anchoring operators (\fB\|^\fR and \fB$\fR\|) do not appear in
630 the table. With \fBlex\fR regular expressions, these operators are restricted
631 in their use: the \fB^\fR operator can only be used at the beginning of an
632 entire regular expression, and the \fB$\fR operator only at the end. The
633 operators apply to the entire regular expression. Thus, for example, the
634 pattern (\fB^abc)|(def$\fR) is undefined; it can instead be written as two
635 separate rules, one with the regular expression \fB^abc\fR and one with
636 \fBdef$\fR, which share a common action via the special \fB|\fR action (see
637 below). If the pattern were written \fB^abc|def$\fR, it would match either of
638 \fBabc\fR or \fBdef\fR on a line by itself.
639 .sp
640 .LP
641 Unlike the general \fBERE\fR rules, embedded anchoring is not allowed by most
642 historical \fBlex\fR implementations. An example of embedded anchoring would be
643 for patterns such as (^)foo($) to match \fBfoo\fR when it exists as a complete
644 word. This functionality can be obtained using existing \fBlex\fR features:
645 .sp
646 .in +2
647 .nf
648 ^foo/[ \e\|n]|
649 " foo"/[ \e\|n] /* found foo as a separate word */
650 .fi
651 .in -2
653 .sp
654 .LP
655 Notice also that \fB$\fR is a form of trailing context (it is equivalent to
656 \fB/\e\|n\fR and as such cannot be used with regular expressions containing
657 another instance of the operator (see the preceding discussion of trailing
658 context).
659 .sp
660 .LP
661 The additional regular expressions trailing-context operator \fB/\fR (slash)
662 can be used as an ordinary character if presented within double-quotes,
663 \fB"\|/\|"\fR; preceded by a backslash, \fB\e\|/\fR; or within a bracket
664 expression, \fB[\|/\|]\fR. The start-condition \fB<\fR and \fB>\fR operators
665 are special only in a start condition at the beginning of a regular expression;
666 elsewhere in the regular expression they are treated as ordinary characters.
667 .sp
668 .LP
669 The following examples clarify the differences between \fBlex\fR regular
670 expressions and regular expressions appearing elsewhere in this document. For
671 regular expressions of the form \fIr\fR/\fIx\fR, the string matching \fIr\fR is
672 always returned; confusion may arise when the beginning of \fIx\fR matches the
673 trailing portion of \fIr\fR. For example, given the regular expression a*b/cc
674 and the input \fBaaabcc\fR, \fIyytext\fR would contain the string \fBaaab\fR on
675 this match. But given the regular expression x*/xy and the input \fBxxxy\fR,
676 the token \fBxxx\fR, not \fBxx\fR, is returned by some implementations because
677 \fBxxx\fR matches x*.
678 .sp
679 .LP
680 In the rule ab*/bc, the b* at the end of \fIr\fR will extend \fIr\fR's match
681 into the beginning of the trailing context, so the result is unspecified. If
682 this rule were ab/bc, however, the rule matches the text \fBab\fR when it is
683 followed by the text \fBbc\fR. In this latter case, the matching of \fIr\fR
684 cannot extend into the beginning of \fIx\fR, so the result is specified.
685 .SS "Actions in lex"
686 .sp
687 .LP
688 The action to be taken when an \fBERE\fR is matched can be a C program fragment
689 or the special actions described below; the program fragment can contain one or
690 more C statements, and can also include special actions. The empty C statement
691 \fB;\fR is a valid action; any string in the \fBlex.yy.c\fR input that matches
692 the pattern portion of such a rule is effectively ignored or skipped. However,
693 the absence of an action is not valid, and the action \fBlex\fR takes in such a
694 condition is undefined.
695 .sp
696 .LP
697 The specification for an action, including C statements and special actions,
698 can extend across several lines if enclosed in braces:
699 .sp
700 .in +2
701 .nf
702 ERE <one or more blanks> { program statement
703 program statement }
704 .fi
705 .in -2
706 .sp
708 .sp
709 .LP
710 The default action when a string in the input to a \fBlex.yy.c\fR program is
711 not matched by any expression is to copy the string to the output. Because the
712 default behavior of a program generated by \fBlex\fR is to read the input and
713 copy it to the output, a minimal \fBlex\fR source program that has just
714 \fB%%\fR generates a C program that simply copies the input to the output
715 unchanged.
716 .sp
717 .LP
718 Four special actions are available:
719 .sp
720 .in +2
721 .nf
722 | ECHO; REJECT; BEGIN
723 .fi
724 .in -2
725 .sp
727 .sp
728 .ne 2
729 .na
730 \fB|\fR
731 .ad
732 .RS 12n
733 The action | means that the action for the next rule is the action for this
734 rule. Unlike the other three actions, | cannot be enclosed in braces or be
735 semicolon-terminated. It must be specified alone, with no other actions.
736 .RE
738 .sp
739 .ne 2
740 .na
741 \fB\fBECHO;\fR \fR
742 .ad
743 .RS 12n
744 Writes the contents of the string \fIyytext\fR on the output.
745 .RE
747 .sp
748 .ne 2
749 .na
750 \fB\fBREJECT;\fR \fR
751 .ad
752 .RS 12n
753 Usually only a single expression is matched by a given string in the input.
754 \fBREJECT\fR means "continue to the next expression that matches the current
755 input," and causes whatever rule was the second choice after the current rule
756 to be executed for the same input. Thus, multiple rules can be matched and
757 executed for one input string or overlapping input strings. For example, given
758 the regular expressions \fBxyz\fR and \fBxy\fR and the input \fBxyz\fR, usually
759 only the regular expression \fBxyz\fR would match. The next attempted match
760 would start after z. If the last action in the \fBxyz\fR rule is \fBREJECT\fR ,
761 both this rule and the \fBxy\fR rule would be executed. The \fBREJECT\fR action
762 may be implemented in such a fashion that flow of control does not continue
763 after it, as if it were equivalent to a \fBgoto\fR to another part of
764 \fByylex\fR. The use of \fBREJECT\fR may result in somewhat larger and slower
765 scanners.
766 .RE
768 .sp
769 .ne 2
770 .na
771 \fB\fBBEGIN\fR \fR
772 .ad
773 .RS 12n
774 The action:
775 .sp
776 \fBBEGIN\fR \fInewstate\fR\fB;\fR
777 .sp
778 switches the state (start condition) to \fInewstate\fR. If the string
779 \fInewstate\fR has not been declared previously as a start condition in the
780 \fBDefinitions\fR \fBin\fR \fBlex\fR section, the results are unspecified. The
781 initial state is indicated by the digit \fB0\fR or the token \fBINITIAL\fR.
782 .RE
784 .sp
785 .LP
786 The functions or macros described below are accessible to user code included in
787 the \fBlex\fR input. It is unspecified whether they appear in the C code output
788 of \fBlex\fR, or are accessible only through the \fB\fR\fB-l\fR\fB l\fR operand
789 to \fBc89\fR or \fBcc\fR (the \fBlex\fR library).
790 .sp
791 .ne 2
792 .na
793 \fB\fBint\fR \fByylex(void)\fR \fR
794 .ad
795 .RS 21n
796 Performs lexical analysis on the input; this is the primary function generated
797 by the \fBlex\fR utility. The function returns zero when the end of input is
798 reached; otherwise it returns non-zero values (tokens) determined by the
799 actions that are selected.
800 .RE
802 .sp
803 .ne 2
804 .na
805 \fB\fBint\fR \fByymore(void)\fR \fR
806 .ad
807 .RS 21n
808 When called, indicates that when the next input string is recognized, it is to
809 be appended to the current value of \fIyytext\fR rather than replacing it; the
810 value in \fIyyleng\fR is adjusted accordingly.
811 .RE
813 .sp
814 .ne 2
815 .na
816 \fB\fBint\fR\fIyyless(int\fR\fB n\fR\fI)\fR \fR
817 .ad
818 .RS 21n
819 Retains \fIn\fR initial characters in \fIyytext\fR, NUL-terminated, and treats
820 the remaining characters as if they had not been read; the value in
821 \fIyyleng\fR is adjusted accordingly.
822 .RE
824 .sp
825 .ne 2
826 .na
827 \fB\fBint\fR \fBinput(void)\fR \fR
828 .ad
829 .RS 21n
830 Returns the next character from the input, or zero on end-of-file. It obtains
831 input from the stream pointer \fIyyin\fR, although possibly via an intermediate
832 buffer. Thus, once scanning has begun, the effect of altering the value of
833 \fIyyin\fR is undefined. The character read is removed from the input stream of
834 the scanner without any processing by the scanner.
835 .RE
837 .sp
838 .ne 2
839 .na
840 \fB\fBint\fR \fBunput(int\fR \fB\fIc\fR\fR\fB)\fR \fR
841 .ad
842 .RS 21n
843 Returns the character \fIc\fR to the input; \fIyytext\fR and \fIyyleng\fR are
844 undefined until the next expression is matched. The result of using \fIunput\fR
845 for more characters than have been input is unspecified.
846 .RE
848 .sp
849 .LP
850 The following functions appear only in the \fBlex\fR library accessible through
851 the \fB\fR\fB-l\fR\fB l\fR operand; they can therefore be redefined by a
852 portable application:
853 .sp
854 .ne 2
855 .na
856 \fB\fBint\fR \fByywrap(void)\fR \fR
857 .ad
858 .sp .6
859 .RS 4n
860 Called by \fByylex\fR at end-of-file; the default \fByywrap\fR always will
861 return 1. If the application requires \fByylex\fR to continue processing with
862 another source of input, then the application can include a function
863 \fByywrap\fR, which associates another file with the external variable
864 \fBFILE\fR *\fIyyin\fR and will return a value of zero.
865 .RE
867 .sp
868 .ne 2
869 .na
870 \fB\fBint\fR \fBmain(int\fR \fB\fIargc\fR,\fR \fBchar\fR \fB*\fIargv\fR[\|])\fR
871 \fR
872 .ad
873 .sp .6
874 .RS 4n
875 Calls \fByylex\fR to perform lexical analysis, then exits. The user code can
876 contain \fBmain\fR to perform application-specific operations, calling
877 \fByylex\fR as applicable.
878 .RE
880 .sp
881 .LP
882 The reason for breaking these functions into two lists is that only those
883 functions in \fBlibl.a\fR can be reliably redefined by a portable application.
884 .sp
885 .LP
886 Except for \fBinput\fR, \fBunput\fR and \fBmain\fR, all external and static
887 names generated by \fBlex\fR begin with the prefix \fByy\fR or \fBYY\fR.
888 .SH USAGE
889 .sp
890 .LP
891 Portable applications are warned that in the \fBRules in lex\fR section, an
892 \fBERE\fR without an action is not acceptable, but need not be detected as
893 erroneous by \fBlex\fR. This may result in compilation or run-time errors.
894 .sp
895 .LP
896 The purpose of \fBinput\fR is to take characters off the input stream and
897 discard them as far as the lexical analysis is concerned. A common use is to
898 discard the body of a comment once the beginning of a comment is recognized.
899 .sp
900 .LP
901 The \fBlex\fR utility is not fully internationalized in its treatment of
902 regular expressions in the \fBlex\fR source code or generated lexical analyzer.
903 It would seem desirable to have the lexical analyzer interpret the regular
904 expressions given in the \fBlex\fR source according to the environment
905 specified when the lexical analyzer is executed, but this is not possible with
906 the current \fBlex\fR technology. Furthermore, the very nature of the lexical
907 analyzers produced by \fBlex\fR must be closely tied to the lexical
908 requirements of the input language being described, which will frequently be
909 locale-specific anyway. (For example, writing an analyzer that is used for
910 French text will not automatically be useful for processing other languages.)
911 .SH EXAMPLES
912 .LP
913 \fBExample 1 \fRUsing lex
914 .sp
915 .LP
916 The following is an example of a \fBlex\fR program that implements a
917 rudimentary scanner for a Pascal-like syntax:
919 .sp
920 .in +2
921 .nf
922 %{
923 /* need this for the call to atof() below */
924 #include <math.h>
925 /* need this for printf(), fopen() and stdin below */
926 #include <stdio.h>
927 %}
929 DIGIT [0-9]
930 ID [a-z][a-z0-9]*
931 %%
933 {DIGIT}+ {
934 printf("An integer: %s (%d)\en", yytext,
935 atoi(yytext));
936 }
938 {DIGIT}+"."{DIGIT}* {
939 printf("A float: %s (%g)\en", yytext,
940 atof(yytext));
941 }
943 if|then|begin|end|procedure|function {
944 printf("A keyword: %s\en", yytext);
945 }
947 {ID} printf("An identifier: %s\en", yytext);
949 "+"|"-"|"*"|"/" printf("An operator: %s\en", yytext);
951 "{"[^}\en]*"}" /* eat up one-line comments */
953 [ \et\en]+ /* eat up white space */
955 \&. printf("Unrecognized character: %s\en", yytext);
957 %%
959 int main(int argc, char *argv[\|])
960 {
961 ++argv, --argc; /* skip over program name */
962 if (argc > 0)
963 yyin = fopen(argv[0], "r");
964 else
965 yyin = stdin;
967 yylex();
968 }
969 .fi
970 .in -2
971 .sp
973 .SH ENVIRONMENT VARIABLES
974 .sp
975 .LP
976 See \fBenviron\fR(5) for descriptions of the following environment variables
977 that affect the execution of \fBlex\fR: \fBLANG\fR, \fBLC_ALL\fR,
978 \fBLC_COLLATE\fR, \fBLC_CTYPE\fR, \fBLC_MESSAGES\fR, and \fBNLSPATH\fR.
979 .SH EXIT STATUS
980 .sp
981 .LP
982 The following exit values are returned:
983 .sp
984 .ne 2
985 .na
986 \fB\fB0\fR \fR
987 .ad
988 .RS 7n
989 Successful completion.
990 .RE
992 .sp
993 .ne 2
994 .na
995 \fB\fB>0\fR \fR
996 .ad
997 .RS 7n
998 An error occurred.
999 .RE
1001 .SH ATTRIBUTES
1002 .sp
1003 .LP
1004 See \fBattributes\fR(5) for descriptions of the following attributes:
1005 .sp
1007 .sp
1008 .TS
1009 box;
1010 c | c
1011 l | l .
1012 ATTRIBUTE TYPE ATTRIBUTE VALUE
1013 _
1014 Interface Stability Standard
1015 .TE
1017 .SH SEE ALSO
1018 .sp
1019 .LP
1020 \fByacc\fR(1), \fBattributes\fR(5), \fBenviron\fR(5), \fBregex\fR(5),
1021 \fBstandards\fR(5)
1022 .SH NOTES
1023 .sp
1024 .LP
1025 If routines such as \fByyback()\fR, \fByywrap()\fR, and \fByylock()\fR
1026 in \fB\|.l\fR (ell) files are to be external C functions, the command line to
1027 compile a C++ program must define the \fB__EXTERN_C__\fR macro. For example:
1028 .sp
1029 .in +2
1030 .nf
1031 example% \fBCC -D__EXTERN_C__ ... file\fR
1032 .fi
1033 .in -2
1034 .sp