| blob | d9bac2e8f130b313f6d3c98d1c36dd2cf6228f95 |
1 From: Tony Balinski <ajbj@free.fr>
2 Subject: Unlimited macro string literal length and single-quoted strings
4 Available as a patch:
6 http://sourceforge.net/tracker/?func=detail&atid=311005&aid=1598271&group_id=11005
7 [ 1598271 ] Unlimited macro string literal length, single-quoted strings
8 macroStringLiterals.diff 2006-11-21
10 String literals are scanned twice, firstly to calculate their space
11 requirements, secondly to read their contents into allocated memory.
13 Separate string literals that follow one another are combined into one,
14 avoiding run-time concatenation of the pieces. Also single-quoted string
15 literals are allowed, within which backslash ('\') has no special meaning
16 (so you can't include a single-quote in a single-quoted string).
18 Note that a double-quoted string can be continued over multiple lines by
19 ending each line but the last with a backslash, like in C.
21 2006-11-21:
23 Fixed adjacent string literal merging to allow concatenation with "".
25 ---
27 doc/help.etx | 41 +++++---
28 source/parse.y | 291 ++++++++++++++++++++++++++++++++++-----------------------
29 2 files changed, 206 insertions(+), 126 deletions(-)
31 diff --quilt old/doc/help.etx new/doc/help.etx
32 --- old/doc/help.etx
33 +++ new/doc/help.etx
34 @@ -1947,11 +1947,12 @@ Macro Language
36 Blank lines and comments are also allowed. Comments begin with a "#" and end
37 with a newline, and can appear either on a line by themselves, or at the end
38 - of a statement.
39 + of a statement line.
41 Statements which are too long to fit on a single line may be split across
42 several lines, by placing a backslash "\" character at the end of each line
43 - to be continued.
44 + to be continued. Note that a comment with a backslash at the end is treated
45 + as a continuation in this way too.
48 3>Data Types
49 @@ -1973,12 +1974,14 @@ Macro Language
51 4>Character String Constants
53 - Character string constants are enclosed in double quotes. For example:
54 + Character string constants are enclosed in single or double quotes, but the
55 + start and end quotes must be the same character. For example:
57 a = "a string"
58 - dialog("Hi there!", "OK")
59 + dialog('Hi there!', "OK")
61 - Strings may also include C-language style escape sequences:
62 + A double-quoted string may also include C-language style escape
63 + sequences:
65 \\ Backslash \t Tab \f Form feed
66 \" Double quote \b Backspace \a Alert
67 @@ -1996,11 +1999,11 @@ Macro Language
68 t_print("a = " a "\n")
70 Other characters can be expressed as backslash-escape sequences in macro
71 - strings. The format is the same as for regular expressions, described in the
72 - paragraphs headed "Octal and Hex Escape Sequences" of the section
73 - "Metacharacters_", except that an octal escape sequence can start with any
74 - octal digit, not just 0, so the single character string "\0033" is the same
75 - as "\33", "\x1B" and "\e" (for an ASCII version of NEdit).
76 + double-quoted strings. The format is the same as for regular expressions,
77 + described in the paragraphs headed "Octal and Hex Escape Sequences" of the
78 + section "Metacharacters_", except that an octal escape sequence can start with
79 + any octal digit, not just 0, so the single character string "\0033" is the
80 + same as "\33", "\x1B" and "\e" (for an ASCII version of NEdit).
82 Note that if you want to define a regular expression in a macro string,
83 you need to "double-up" the backslashes for the metacharacters with
84 @@ -2009,7 +2012,7 @@ Macro Language
85 (?N(\s|/\*(?n(?:(?!\*/).)*)\*/|//.*\n|\n)+)
87 which matches whitespace or C/C++/Java-style comments, should be written as
88 - a macro string as
89 + a macro double-quoted string as
91 "(?N(\\s|/\\*(?n(?:(?!\\*/).)*)\\*/|//.*\n|\n)+)"
93 @@ -2018,6 +2021,22 @@ Macro Language
94 also interpret the sequence "\\n" as a newline, although the macro string here
95 would then contain a literal backslash followed by a lowercase `N'.)
97 + Alternatively, if you don't need special escapes or a single quote
98 + (apostrophe) in your string (true for this example), just turn the expression
99 + into a single-quoted string, as
100 +
101 + '(?N(\s|/\*(?n(?:(?!\*/).)*)\*/|//.*\n|\n)+)'
102 +
103 + Neighboring string literals (separated by whitespace or line continuations)
104 + are combined, as if by the concatenation operation before use. For example
105 +
106 + "The backslash '" '\' "' is an " \
107 + 'escape only in "double-quoted" strings' "\n"
108 +
109 + is treated as a single string ending with a newline character, looking like
110 +
111 + The backslash '\' is an escape only in "double-quoted" strings
112 +
114 3>Variables
116 diff --quilt old/source/parse.y new/source/parse.y
117 --- old/source/parse.y
118 +++ new/source/parse.y
119 @@ -69,6 +69,7 @@ static int follow(char expect, int yes,
120 static int follow2(char expect1, int yes1, char expect2, int yes2, int no);
121 static int follow_non_whitespace(char expect, int yes, int no);
122 static Symbol *matchesActionRoutine(char **inPtr);
123 +static int scanString(void);
125 static char *ErrMsg;
126 static char *InPtr;
127 @@ -675,12 +676,6 @@ static int yylex(void)
128 int i, len;
129 Symbol *s;
130 static DataValue value = {NO_TAG, {0}};
131 - static char escape[] = "\\\"ntbrfave";
132 -#ifdef EBCDIC_CHARSET
133 - static char replace[] = "\\\"\n\t\b\r\f\a\v\x27"; /* EBCDIC escape */
134 -#else
135 - static char replace[] = "\\\"\n\t\b\r\f\a\v\x1B"; /* ASCII escape */
136 -#endif
137 int result;
139 skipWhitespace();
140 @@ -739,115 +734,10 @@ static int yylex(void)
141 return SYMBOL;
142 }
144 - /* Process quoted strings with embedded escape sequences:
145 - For backslashes we recognise hexadecimal values with initial 'x' such
146 - as "\x1B"; octal value (upto 3 oct digits with a possible leading zero)
147 - such as "\33", "\033" or "\0033", and the C escapes: \", \', \n, \t, \b,
148 - \r, \f, \a, \v, and the added \e for the escape character, as for REs.
149 - Disallow hex/octal zero values (NUL): instead ignore the introductory
150 - backslash, eg "\x0xyz" becomes "x0xyz" and "\0000hello" becomes
151 - "0000hello". */
152 -
153 - if (*InPtr == '\"') {
154 - char string[MAX_STRING_CONST_LEN], *p = string;
155 - char *backslash;
156 - InPtr++;
157 - while (*InPtr != '\0' && *InPtr != '\"' && *InPtr != '\n') {
158 - if (p >= string + MAX_STRING_CONST_LEN) {
159 - InPtr++;
160 - continue;
161 - }
162 - if (*InPtr == '\\') {
163 - backslash = InPtr;
164 - InPtr++;
165 - if (*InPtr == '\n') {
166 - InPtr++;
167 - continue;
168 - }
169 - if (*InPtr == 'x') {
170 - /* a hex introducer */
171 - int hexValue = 0;
172 - const char *hexDigits = "0123456789abcdef";
173 - const char *hexD;
174 - InPtr++;
175 - if (*InPtr == '\0' ||
176 - (hexD = strchr(hexDigits, tolower(*InPtr))) == NULL) {
177 - *p++ = 'x';
178 - }
179 - else {
180 - hexValue = hexD - hexDigits;
181 - InPtr++;
182 - /* now do we have another digit? only accept one more */
183 - if (*InPtr != '\0' &&
184 - (hexD = strchr(hexDigits,tolower(*InPtr))) != NULL){
185 - hexValue = hexD - hexDigits + (hexValue << 4);
186 - InPtr++;
187 - }
188 - if (hexValue != 0) {
189 - *p++ = (char)hexValue;
190 - }
191 - else {
192 - InPtr = backslash + 1; /* just skip the backslash */
193 - }
194 - }
195 - continue;
196 - }
197 - /* the RE documentation requires \0 as the octal introducer;
198 - here you can start with any octal digit, but you are only
199 - allowed up to three (or four if the first is '0'). */
200 - if ('0' <= *InPtr && *InPtr <= '7') {
201 - if (*InPtr == '0') {
202 - InPtr++; /* octal introducer: don't count this digit */
203 - }
204 - if ('0' <= *InPtr && *InPtr <= '7') {
205 - /* treat as octal - first digit */
206 - char octD = *InPtr++;
207 - int octValue = octD - '0';
208 - if ('0' <= *InPtr && *InPtr <= '7') {
209 - /* second digit */
210 - octD = *InPtr++;
211 - octValue = (octValue << 3) + octD - '0';
212 - /* now do we have another digit? can we add it?
213 - if value is going to be too big for char (greater
214 - than 0377), stop converting now before adding the
215 - third digit */
216 - if ('0' <= *InPtr && *InPtr <= '7' &&
217 - octValue <= 037) {
218 - /* third digit is acceptable */
219 - octD = *InPtr++;
220 - octValue = (octValue << 3) + octD - '0';
221 - }
222 - }
223 - if (octValue != 0) {
224 - *p++ = (char)octValue;
225 - }
226 - else {
227 - InPtr = backslash + 1; /* just skip the backslash */
228 - }
229 - }
230 - else { /* \0 followed by non-digits: go back to 0 */
231 - InPtr = backslash + 1; /* just skip the backslash */
232 - }
233 - continue;
234 - }
235 - for (i=0; escape[i]!='\0'; i++) {
236 - if (escape[i] == *InPtr) {
237 - *p++ = replace[i];
238 - InPtr++;
239 - break;
240 - }
241 - }
242 - /* if we get here, we didn't recognise the character after
243 - the backslash: just copy it next time round the loop */
244 - }
245 - else {
246 - *p++= *InPtr++;
247 - }
248 - }
249 - *p = '\0';
250 - InPtr++;
251 - yylval.sym = InstallStringConstSymbol(string);
252 - return STRING;
253 + /* Process quoted strings */
254 +
255 + if (*InPtr == '\"' || *InPtr == '\'') {
256 + return scanString();
257 }
259 /* process remaining two character tokens or return single char as token */
260 @@ -951,6 +841,177 @@ static Symbol *matchesActionRoutine(char
261 }
263 /*
264 +** Process quoted string literals. These can be in single or double quotes.
265 +** A sequence of string literals separated by whitespace (see skipWhitespace())
266 +** are read as a single string.
267 +**
268 +** Double-quoted string literals allow embedded escape sequences:
269 +** For backslashes we recognise hexadecimal values with initial 'x' such
270 +** as "\x1B"; octal value (upto 3 oct digits with a possible leading zero)
271 +** such as "\33", "\033" or "\0033", and the C escapes: \", \', \n, \t, \b,
272 +** \r, \f, \a, \v, and the added \e for the escape character, as for REs.
273 +** We disallow hex/octal zero values (NUL): instead ignore the introductory
274 +** backslash, eg "\x0xyz" becomes "x0xyz" and "\0000hello" becomes "0000hello".
275 +** An escaped newline is elided, and the string content continues on the next
276 +** source line.
277 +*/
278 +static int scanString(void)
279 +{
280 +# define SCANSTRING_WRITE_TO_STRING(p, len, val) \
281 + do { char mc = (val); if (p) { *p++ = mc; } else { ++len; } } while (0)
282 +
283 + /* scan the string twice: once to get its size, then again to build it */
284 + char *startPtr = InPtr;
285 + char *p = NULL, *string = NULL;
286 + int len, scan, i;
287 + char stopper, first_stopper = *startPtr;
288 + char *backslash;
289 + int handleBackslash;
290 +
291 + static char escape[] = "\\\"ntbrfave";
292 +#ifdef EBCDIC_CHARSET
293 + static char replace[] = "\\\"\n\t\b\r\f\a\v\x27"; /* EBCDIC escape */
294 +#else
295 + static char replace[] = "\\\"\n\t\b\r\f\a\v\x1B"; /* ASCII escape */
296 +#endif
297 +
298 + if (first_stopper != '\"' && first_stopper != '\'')
299 + return yyerror("expected a string");
300 +
301 + for (scan = 0; scan < 2; ++scan)
302 + {
303 + InPtr = startPtr;
304 + stopper = first_stopper;
305 + handleBackslash = (stopper == '\"');
306 + len = 0;
307 + InPtr++;
308 + while (*InPtr != '\0' && *InPtr != '\n') {
309 + if (*InPtr == stopper) {
310 + char *endPtr = InPtr++;
311 + skipWhitespace();
312 + /* is this followed by another string literal? */
313 + if (*InPtr == '\"' || *InPtr == '\'') {
314 + stopper = *InPtr++; /* add it to the end of the first */
315 + handleBackslash = (stopper == '\"');
316 + }
317 + else {
318 + InPtr = endPtr; /* no further string: restore position */
319 + break;
320 + }
321 + }
322 + else if (handleBackslash && *InPtr == '\\') {
323 + backslash = InPtr;
324 + InPtr++;
325 + if (*InPtr == '\n') { /* allows newline to be skipped */
326 + InPtr++;
327 + continue;
328 + }
329 + if (*InPtr == 'x') {
330 + /* a hex introducer */
331 + int hexValue = 0;
332 + const char *hexDigits = "0123456789abcdef";
333 + const char *hexD;
334 + InPtr++;
335 + if (*InPtr == '\0')
336 + break;
337 + if ((hexD = strchr(hexDigits, tolower(*InPtr))) == NULL) {
338 + SCANSTRING_WRITE_TO_STRING(p, len, 'x');
339 + }
340 + else {
341 + hexValue = hexD - hexDigits;
342 + InPtr++;
343 + if (*InPtr == '\0')
344 + break;
345 + /* now do we have another digit? only accept one more */
346 + if ((hexD = strchr(hexDigits,tolower(*InPtr))) != NULL){
347 + hexValue = hexD - hexDigits + (hexValue << 4);
348 + InPtr++;
349 + }
350 + if (hexValue != 0) {
351 + SCANSTRING_WRITE_TO_STRING(p, len, (char)hexValue);
352 + }
353 + else {
354 + InPtr = backslash + 1; /* just skip the backslash */
355 + }
356 + }
357 + continue;
358 + }
359 + /* the RE documentation requires \0 as the octal introducer;
360 + here you can start with any octal digit, but you are only
361 + allowed up to three (or four if the first is '0'). */
362 + if ('0' <= *InPtr && *InPtr <= '7') {
363 + if (*InPtr == '0') {
364 + InPtr++; /* octal introducer: don't count this digit */
365 + }
366 + if ('0' <= *InPtr && *InPtr <= '7') {
367 + /* treat as octal - first digit */
368 + char octD = *InPtr++;
369 + int octValue = octD - '0';
370 + if ('0' <= *InPtr && *InPtr <= '7') {
371 + /* second digit */
372 + octD = *InPtr++;
373 + octValue = (octValue << 3) + octD - '0';
374 + /* now do we have another digit? can we add it?
375 + if value is going to be too big for char (greater
376 + than 0377), stop converting now before adding the
377 + third digit */
378 + if ('0' <= *InPtr && *InPtr <= '7' &&
379 + octValue <= 037) {
380 + /* third digit is acceptable */
381 + octD = *InPtr++;
382 + octValue = (octValue << 3) + octD - '0';
383 + }
384 + }
385 + if (octValue != 0) {
386 + SCANSTRING_WRITE_TO_STRING(p, len, (char)octValue);
387 + }
388 + else {
389 + InPtr = backslash + 1; /* just skip the backslash */
390 + }
391 + }
392 + else { /* \0 followed by non-digits: go back to 0 */
393 + InPtr = backslash + 1; /* just skip the backslash */
394 + }
395 + continue;
396 + }
397 + /* check for a valid c-style escape character */
398 + for (i = 0; escape[i] != '\0'; i++) {
399 + if (escape[i] == *InPtr) {
400 + SCANSTRING_WRITE_TO_STRING(p, len, replace[i]);
401 + InPtr++;
402 + break;
403 + }
404 + }
405 + /* if we get here, we didn't recognise the character after
406 + the backslash: just copy it next time round the loop */
407 + }
408 + else {
409 + SCANSTRING_WRITE_TO_STRING(p, len, *InPtr++);
410 + }
411 + }
412 + /* terminate the string content */
413 + SCANSTRING_WRITE_TO_STRING(p, len, '\0');
414 + if (*InPtr == stopper) {
415 + if (!p) {
416 + /* this was the size measurement and validation */
417 + p = string = AllocString(len);
418 + }
419 + else {
420 + /* OK: string now contains our string text */
421 + InPtr++; /* skip past stopper */
422 + yylval.sym = InstallStringConstSymbol(string);
423 + return STRING;
424 + }
425 + }
426 + else {
427 + /* failure: end quote doesn't match start quote */
428 + break;
429 + }
430 + }
431 + return yyerror("unterminated string");
432 +}
433 +
434 +/*
435 ** Called by yacc to report errors (just stores for returning when
436 ** parsing is aborted. The error token action is to immediate abort
437 ** parsing, so this message is immediately reported to the caller