1 /* $NetBSD: tokenizer.c,v 1.21 2011/08/16 16:25:15 christos Exp $ */
4 * Copyright (c) 1992, 1993
5 * The Regents of the University of California. All rights reserved.
7 * This code is derived from software contributed to Berkeley by
8 * Christos Zoulas of Cornell University.
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 * 3. Neither the name of the University nor the names of its contributors
19 * may be used to endorse or promote products derived from this software
20 * without specific prior written permission.
22 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
23 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
24 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
25 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
26 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
27 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
28 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
29 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
30 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
31 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
39 #if !defined(lint) && !defined(SCCSID)
41 static char sccsid
[] = "@(#)tokenizer.c 8.1 (Berkeley) 6/4/93";
43 __RCSID("$NetBSD: tokenizer.c,v 1.21 2011/08/16 16:25:15 christos Exp $");
45 #endif /* not lint && not SCCSID */
47 /* We build this file twice, once as NARROW, once as WIDE. */
49 * tokenize.c: Bourne shell like tokenizer
57 Q_none
, Q_single
, Q_double
, Q_one
, Q_doubleone
66 #define IFS STR("\t \n")
68 #define tok_malloc(a) malloc(a)
69 #define tok_free(a) free(a)
70 #define tok_realloc(a, b) realloc(a, b)
71 #define tok_strdup(a) Strdup(a)
74 struct TYPE(tokenizer
) {
75 Char
*ifs
; /* In field separator */
76 size_t argc
, amax
; /* Current and maximum number of args */
77 Char
**argv
; /* Argument list */
78 Char
*wptr
, *wmax
; /* Space and limit on the word buffer */
79 Char
*wstart
; /* Beginning of next word */
80 Char
*wspace
; /* Space of word buffer */
81 quote_t quote
; /* Quoting state */
82 int flags
; /* flags; */
86 private void FUN(tok
,finish
)(TYPE(Tokenizer
) *);
90 * Finish a word in the tokenizer.
93 FUN(tok
,finish
)(TYPE(Tokenizer
) *tok
)
97 if ((tok
->flags
& TOK_KEEP
) || tok
->wptr
!= tok
->wstart
) {
98 tok
->argv
[tok
->argc
++] = tok
->wstart
;
99 tok
->argv
[tok
->argc
] = NULL
;
100 tok
->wstart
= ++tok
->wptr
;
102 tok
->flags
&= ~TOK_KEEP
;
107 * Initialize the tokenizer
109 public TYPE(Tokenizer
) *
110 FUN(tok
,init
)(const Char
*ifs
)
112 TYPE(Tokenizer
) *tok
= tok_malloc(sizeof(*tok
));
116 tok
->ifs
= tok_strdup(ifs
? ifs
: IFS
);
117 if (tok
->ifs
== NULL
) {
123 tok
->argv
= tok_malloc(sizeof(*tok
->argv
) * tok
->amax
);
124 if (tok
->argv
== NULL
) {
130 tok
->wspace
= tok_malloc(WINCR
* sizeof(*tok
->wspace
));
131 if (tok
->wspace
== NULL
) {
137 tok
->wmax
= tok
->wspace
+ WINCR
;
138 tok
->wstart
= tok
->wspace
;
139 tok
->wptr
= tok
->wspace
;
148 * Reset the tokenizer
151 FUN(tok
,reset
)(TYPE(Tokenizer
) *tok
)
155 tok
->wstart
= tok
->wspace
;
156 tok
->wptr
= tok
->wspace
;
166 FUN(tok
,end
)(TYPE(Tokenizer
) *tok
)
170 tok_free(tok
->wspace
);
178 * Bourne shell (sh(1)) like tokenizing
180 * tok current tokenizer state (setup with FUN(tok,init)())
185 * 2 Unmatched double quote
186 * 1 Unmatched single quote
188 * Modifies (if return value is 0):
189 * argc number of arguments
190 * argv argument array
191 * cursorc if !NULL, argv element containing cursor
192 * cursorv if !NULL, offset in argv[cursorc] of cursor
195 FUN(tok
,line
)(TYPE(Tokenizer
) *tok
, const TYPE(LineInfo
) *line
,
196 int *argc
, const Char
***argv
, int *cursorc
, int *cursoro
)
203 for (ptr
= line
->buffer
; ;ptr
++) {
204 if (ptr
>= line
->lastchar
)
206 if (ptr
== line
->cursor
) {
208 co
= (int)(tok
->wptr
- tok
->wstart
);
212 tok
->flags
|= TOK_KEEP
;
213 tok
->flags
&= ~TOK_EAT
;
214 switch (tok
->quote
) {
216 tok
->quote
= Q_single
; /* Enter single quote
220 case Q_single
: /* Exit single quote mode */
224 case Q_one
: /* Quote this ' */
229 case Q_double
: /* Stay in double quote mode */
233 case Q_doubleone
: /* Quote this ' */
234 tok
->quote
= Q_double
;
244 tok
->flags
&= ~TOK_EAT
;
245 tok
->flags
|= TOK_KEEP
;
246 switch (tok
->quote
) {
247 case Q_none
: /* Enter double quote mode */
248 tok
->quote
= Q_double
;
251 case Q_double
: /* Exit double quote mode */
255 case Q_one
: /* Quote this " */
260 case Q_single
: /* Stay in single quote mode */
264 case Q_doubleone
: /* Quote this " */
265 tok
->quote
= Q_double
;
275 tok
->flags
|= TOK_KEEP
;
276 tok
->flags
&= ~TOK_EAT
;
277 switch (tok
->quote
) {
278 case Q_none
: /* Quote next character */
282 case Q_double
: /* Quote next character */
283 tok
->quote
= Q_doubleone
;
286 case Q_one
: /* Quote this, restore state */
291 case Q_single
: /* Stay in single quote mode */
295 case Q_doubleone
: /* Quote this \ */
296 tok
->quote
= Q_double
;
306 tok
->flags
&= ~TOK_EAT
;
307 switch (tok
->quote
) {
313 *tok
->wptr
++ = *ptr
; /* Add the return */
316 case Q_doubleone
: /* Back to double, eat the '\n' */
317 tok
->flags
|= TOK_EAT
;
318 tok
->quote
= Q_double
;
321 case Q_one
: /* No quote, more eat the '\n' */
322 tok
->flags
|= TOK_EAT
;
332 switch (tok
->quote
) {
334 /* Finish word and return */
335 if (tok
->flags
& TOK_EAT
) {
336 tok
->flags
&= ~TOK_EAT
;
348 tok
->quote
= Q_double
;
363 tok
->flags
&= ~TOK_EAT
;
364 switch (tok
->quote
) {
366 if (Strchr(tok
->ifs
, *ptr
) != NULL
)
367 FUN(tok
,finish
)(tok
);
380 tok
->quote
= Q_double
;
396 if (tok
->wptr
>= tok
->wmax
- 4) {
397 size_t size
= (size_t)(tok
->wmax
- tok
->wspace
+ WINCR
);
398 Char
*s
= tok_realloc(tok
->wspace
,
403 if (s
!= tok
->wspace
) {
405 for (i
= 0; i
< tok
->argc
; i
++) {
407 (tok
->argv
[i
] - tok
->wspace
) + s
;
409 tok
->wptr
= (tok
->wptr
- tok
->wspace
) + s
;
410 tok
->wstart
= (tok
->wstart
- tok
->wspace
) + s
;
413 tok
->wmax
= s
+ size
;
415 if (tok
->argc
>= tok
->amax
- 4) {
418 p
= tok_realloc(tok
->argv
, tok
->amax
* sizeof(*p
));
425 if (cc
== -1 && co
== -1) {
427 co
= (int)(tok
->wptr
- tok
->wstart
);
433 FUN(tok
,finish
)(tok
);
434 *argv
= (const Char
**)tok
->argv
;
435 *argc
= (int)tok
->argc
;
440 * Simpler version of tok_line, taking a NUL terminated line
441 * and splitting into words, ignoring cursor state.
444 FUN(tok
,str
)(TYPE(Tokenizer
) *tok
, const Char
*line
, int *argc
,
449 memset(&li
, 0, sizeof(li
));
451 li
.cursor
= li
.lastchar
= Strchr(line
, '\0');
452 return FUN(tok
,line(tok
, &li
, argc
, argv
, NULL
, NULL
));