2 * This file and its contents are supplied under the terms of the
3 * Common Development and Distribution License ("CDDL"), version 1.0.
4 * You may only use this file in accordance with the terms of version
7 * A full copy of the text of the CDDL should have accompanied this
8 * source. A copy of the CDDL is also available via the Internet at
9 * http://www.illumos.org/license/CDDL.
13 * Copyright 2010 Nexenta Systems, Inc. All rights reserved.
17 * This file contains the "scanner", which tokenizes the input files
18 * for localedef for processing by the higher level grammar processor.
27 #include <sys/types.h>
29 #include "localedef.h"
30 #include "parser.tab.h"
39 static FILE *input
= stdin
;
40 static const char *filename
= "<stdin>";
41 static int instring
= 0;
42 static int escaped
= 0;
45 * Token space ... grows on demand.
47 static char *token
= NULL
;
50 static int hadtok
= 0;
53 * Wide string space ... grows on demand.
55 static wchar_t *widestr
= NULL
;
56 static int wideidx
= 0;
57 static int widesz
= 0;
60 * The last keyword seen. This is useful to trigger the special lexer rules
61 * for "copy" and also collating symbols and elements.
64 static int category
= T_END
;
70 { T_COM_CHAR
, "comment_char" },
71 { T_ESC_CHAR
, "escape_char" },
74 { T_MESSAGES
, "LC_MESSAGES" },
75 { T_YESSTR
, "yesstr" },
76 { T_YESEXPR
, "yesexpr" },
78 { T_NOEXPR
, "noexpr" },
79 { T_MONETARY
, "LC_MONETARY" },
80 { T_INT_CURR_SYMBOL
, "int_curr_symbol" },
81 { T_CURRENCY_SYMBOL
, "currency_symbol" },
82 { T_MON_DECIMAL_POINT
, "mon_decimal_point" },
83 { T_MON_THOUSANDS_SEP
, "mon_thousands_sep" },
84 { T_POSITIVE_SIGN
, "positive_sign" },
85 { T_NEGATIVE_SIGN
, "negative_sign" },
86 { T_MON_GROUPING
, "mon_grouping" },
87 { T_INT_FRAC_DIGITS
, "int_frac_digits" },
88 { T_FRAC_DIGITS
, "frac_digits" },
89 { T_P_CS_PRECEDES
, "p_cs_precedes" },
90 { T_P_SEP_BY_SPACE
, "p_sep_by_space" },
91 { T_N_CS_PRECEDES
, "n_cs_precedes" },
92 { T_N_SEP_BY_SPACE
, "n_sep_by_space" },
93 { T_P_SIGN_POSN
, "p_sign_posn" },
94 { T_N_SIGN_POSN
, "n_sign_posn" },
95 { T_INT_P_CS_PRECEDES
, "int_p_cs_precedes" },
96 { T_INT_N_CS_PRECEDES
, "int_n_cs_precedes" },
97 { T_INT_P_SEP_BY_SPACE
, "int_p_sep_by_space" },
98 { T_INT_N_SEP_BY_SPACE
, "int_n_sep_by_space" },
99 { T_INT_P_SIGN_POSN
, "int_p_sign_posn" },
100 { T_INT_N_SIGN_POSN
, "int_n_sign_posn" },
101 { T_COLLATE
, "LC_COLLATE" },
102 { T_COLLATING_SYMBOL
, "collating-symbol" },
103 { T_COLLATING_ELEMENT
, "collating-element" },
105 { T_ORDER_START
, "order_start" },
106 { T_ORDER_END
, "order_end" },
107 { T_FORWARD
, "forward" },
108 { T_BACKWARD
, "backward" },
109 { T_POSITION
, "position" },
110 { T_IGNORE
, "IGNORE" },
111 { T_UNDEFINED
, "UNDEFINED" },
112 { T_NUMERIC
, "LC_NUMERIC" },
113 { T_DECIMAL_POINT
, "decimal_point" },
114 { T_THOUSANDS_SEP
, "thousands_sep" },
115 { T_GROUPING
, "grouping" },
116 { T_TIME
, "LC_TIME" },
117 { T_ABDAY
, "abday" },
119 { T_ABMON
, "abmon" },
121 { T_D_T_FMT
, "d_t_fmt" },
122 { T_D_FMT
, "d_fmt" },
123 { T_T_FMT
, "t_fmt" },
124 { T_AM_PM
, "am_pm" },
125 { T_T_FMT_AMPM
, "t_fmt_ampm" },
127 { T_ERA_D_FMT
, "era_d_fmt" },
128 { T_ERA_T_FMT
, "era_t_fmt" },
129 { T_ERA_D_T_FMT
, "era_d_t_fmt" },
130 { T_ALT_DIGITS
, "alt_digits" },
131 { T_CTYPE
, "LC_CTYPE" },
132 { T_ISUPPER
, "upper" },
133 { T_ISLOWER
, "lower" },
134 { T_ISALPHA
, "alpha" },
135 { T_ISDIGIT
, "digit" },
136 { T_ISPUNCT
, "punct" },
137 { T_ISXDIGIT
, "xdigit" },
138 { T_ISSPACE
, "space" },
139 { T_ISPRINT
, "print" },
140 { T_ISGRAPH
, "graph" },
141 { T_ISBLANK
, "blank" },
142 { T_ISCNTRL
, "cntrl" },
144 * These entries are local additions, and not specified by
145 * TOG. Note that they are not guaranteed to be accurate for
146 * all locales, and so applications should not depend on them.
148 { T_ISSPECIAL
, "special" },
149 { T_ISENGLISH
, "english" },
150 { T_ISPHONOGRAM
, "phonogram" },
151 { T_ISIDEOGRAM
, "ideogram" },
152 { T_ISNUMBER
, "number" },
154 * We have to support this in the grammar, but it would be a
155 * syntax error to define a character as one of these without
156 * also defining it as an alpha or digit. We ignore it in our
159 { T_ISALNUM
, "alnum" },
160 { T_TOUPPER
, "toupper" },
161 { T_TOLOWER
, "tolower" },
164 * These are keywords used in the charmap file. Note that
165 * Solaris orginally used angle brackets to wrap some of them,
166 * but we removed that to simplify our parser. The first of these
167 * items are "global items."
169 { T_CHARMAP
, "CHARMAP" },
170 { T_WIDTH
, "WIDTH" },
171 { T_WIDTH_DEFAULT
, "WIDTH_DEFAULT" },
177 * These special words are only used in a charmap file, enclosed in <>.
179 static struct token symwords
[] = {
180 { T_COM_CHAR
, "comment_char" },
181 { T_ESC_CHAR
, "escape_char" },
182 { T_CODE_SET
, "code_set_name" },
183 { T_MB_CUR_MAX
, "mb_cur_max" },
184 { T_MB_CUR_MIN
, "mb_cur_min" },
188 static int categories
[] = {
200 reset_scanner(const char *fname
)
203 filename
= "<stdin>";
207 (void) fclose(input
);
208 if ((input
= fopen(fname
, "r")) == NULL
) {
225 (isdigit(x) ? (x - '0') : ((islower(x) ? (x - 'a') : (x - 'A')) + 10))
226 #define isodigit(x) ((x >= '0') && (x <= '7'))
247 if (ungetc(c
, input
) < 0) {
248 yyerror(_("ungetc failed"));
260 yyerror(_("malformed hex digit"));
265 yyerror(_("malformed hex digit"));
268 v
= ((hex(c1
) << 4) | hex(c2
));
280 yyerror(_("malformed decimal digit"));
286 yyerror(_("malformed decimal digit"));
311 yyerror(_("malformed octal digit"));
317 yyerror(_("malformed octal digit"));
335 if ((tokidx
+ 1) >= toksz
) {
337 if ((token
= realloc(token
, toksz
)) == NULL
) {
338 yyerror(_("out of memory"));
345 token
[tokidx
++] = (char)c
;
351 if ((wideidx
+ 1) >= widesz
) {
353 widestr
= realloc(widestr
, (widesz
* sizeof (wchar_t)));
354 if (widestr
== NULL
) {
355 yyerror(_("out of memory"));
362 widestr
[wideidx
++] = c
;
363 widestr
[wideidx
] = 0;
369 wchar_t *ws
= widestr
;
374 if ((ws
= wsdup(L
"")) == NULL
) {
375 yyerror(_("out of memory"));
386 if ((c
= scanc()) != esc_char
) {
395 return (scan_dec_byte());
398 return (scan_hex_byte());
407 /* put the character back so we can get it */
409 return (scan_oct_byte());
443 static char mbs
[MB_LEN_MAX
+ 1] = "";
448 if (mb_cur_max
>= sizeof (mbs
)) {
449 yyerror(_("max multibyte character size too big"));
454 if ((mbi
== mb_cur_max
) || ((c
= get_byte()) == EOF
)) {
456 * end of the byte sequence reached, but no
457 * valid wide decoding. fatal error.
460 yyerror(_("not a valid character encoding"));
466 /* does it decode? */
467 if (to_wide(&wc
, mbs
) >= 0) {
473 if (category
!= T_CHARMAP
) {
474 if (check_charmap(wc
) < 0) {
475 yyerror(_("no symbolic name for character"));
489 while ((c
= scanc()) != EOF
) {
494 add_tok(get_escaped(c
));
501 if (c
== '\n') { /* well that's strange! */
502 yyerror(_("unterminated symbolic name"));
505 if (c
== '>') { /* end of symbol */
508 * This restarts the token from the beginning
509 * the next time we scan a character. (This
510 * token is complete.)
514 yyerror(_("missing symbolic name"));
520 * A few symbols are handled as keywords outside
521 * of the normal categories.
523 if (category
== T_END
) {
525 for (i
= 0; symwords
[i
].name
!= 0; i
++) {
526 if (strcmp(token
, symwords
[i
].name
) ==
528 last_kw
= symwords
[i
].id
;
534 * Contextual rule: Only literal characters are
535 * permitted in CHARMAP. Anywhere else the symbolic
538 if ((category
!= T_CHARMAP
) &&
539 (lookup_charmap(token
, &yylval
.wc
)) != -1) {
542 if ((yylval
.collsym
= lookup_collsym(token
)) != NULL
) {
545 if ((yylval
.collelem
= lookup_collelem(token
)) !=
549 /* its an undefined symbol */
550 yylval
.token
= strdup(token
);
559 yyerror(_("unterminated symbolic name"));
580 * this one is special, because we don't want it to alter the
583 if (strcmp(token
, "...") == 0) {
587 /* search for reserved words first */
588 for (i
= 0; keywords
[i
].name
; i
++) {
590 if (strcmp(keywords
[i
].name
, token
) != 0) {
594 last_kw
= keywords
[i
].id
;
596 /* clear the top level category if we're done with it */
597 if (last_kw
== T_END
) {
601 /* set the top level category if we're changing */
602 for (j
= 0; categories
[j
]; j
++) {
603 if (categories
[j
] != last_kw
)
608 return (keywords
[i
].id
);
611 /* maybe its a numeric constant? */
612 if (isdigit(*token
) || (*token
== '-' && isdigit(token
[1]))) {
614 yylval
.num
= strtol(token
, &eptr
, 10);
616 yyerror(_("malformed number"));
621 * A single lone character is treated as a character literal.
622 * To avoid duplication of effort, we stick in the charmap.
625 yylval
.wc
= token
[0];
629 /* anything else is treated as a symbolic name */
630 yylval
.token
= strdup(token
);
641 while ((c
= scanc()) != '\n') {
643 /* end of file without newline! */
644 errf(_("missing newline"));
656 while ((c
= scanc()) != EOF
) {
658 /* special handling for quoted string */
663 /* if newline, just eat and forget it */
667 if (strchr("xXd01234567", c
)) {
672 yylval
.wc
= get_escaped(c
);
681 return (get_symbol());
683 /* oops! should generate syntax error */
694 /* escaped characters first */
698 /* eat the newline */
703 /* an escape mid-token is nonsense */
707 /* numeric escapes are treated as wide characters */
708 if (strchr("xXd01234567", c
)) {
714 add_tok(get_escaped(c
));
718 /* if it is the escape charter itself note it */
724 /* remove from the comment char to end of line */
727 if ((c
= scanc()) == EOF
) {
728 /* end of file without newline! */
735 * If there were no tokens on this line,
736 * then just pretend it didn't exist at all.
744 if (strchr(" \t\n;()<>,\"", c
) && (tokidx
!= 0)) {
746 * These are all token delimiters. If there
747 * is a token already in progress, we need to
751 return (consume_token());
758 * If the line was completely devoid of tokens,
759 * then just ignore it.
763 /* we're starting a new line, reset the token state */
784 return (get_symbol());
787 /* whitespace, just ignore it */
803 yyerror(const char *msg
)
805 (void) fprintf(stderr
, _("%s: %d: error: %s\n"),
806 filename
, lineno
, msg
);
811 errf(const char *fmt
, ...)
817 (void) vasprintf(&msg
, fmt
, va
);
820 (void) fprintf(stderr
, _("%s: %d: error: %s\n"),
821 filename
, lineno
, msg
);
827 warn(const char *fmt
, ...)
833 (void) vasprintf(&msg
, fmt
, va
);
836 (void) fprintf(stderr
, _("%s: %d: warning: %s\n"),
837 filename
, lineno
, msg
);