2 * Copyright (c) 2003, Darren Hiebert
4 * This source code is released for free distribution under the terms of the
5 * GNU General Public License version 2 or (at your option) any later version.
7 * This module contains functions for generating tags for JavaScript language
10 * Reference: http://www.ecma-international.org/publications/files/ECMA-ST/Ecma-262.pdf
12 * This is a good reference for different forms of the function statement:
13 * http://www.permadi.com/tutorial/jsFunc/
14 * Another good reference:
15 * http://developer.mozilla.org/en/docs/Core_JavaScript_1.5_Guide
21 #include "general.h" /* must always come first */
22 #include <ctype.h> /* to define isalpha () */
30 # ifdef WORDS_BIGENDIAN
31 # define INTERNAL_ENCODING "UTF-32BE"
33 # define INTERNAL_ENCODING "UTF-32LE"
34 # endif /* WORDS_BIGENDIAN */
54 #define isType(token,t) (bool) ((token)->type == (t))
55 #define isKeyword(token,k) (bool) ((token)->keyword == (k))
56 #define isIdentChar(c) \
57 (isalpha (c) || isdigit (c) || (c) == '$' || \
58 (c) == '@' || (c) == '_' || (c) == '#' || \
60 #define newToken() (objPoolGet (TokenPool))
61 #define deleteToken(t) (objPoolPut (TokenPool, (t)))
66 * Uncomment this to enable extensive debugging to stderr in jscript code.
67 * Please note that TRACING_ENABLED should be #defined in main/trace.h
71 //#define JSCRIPT_DEBUGGING_ENABLED 1
73 #if defined(DO_TRACING) && defined(JSCRIPT_DEBUGGING_ENABLED)
74 #define JSCRIPT_DO_DEBUGGING
77 #ifdef JSCRIPT_DO_DEBUGGING
79 #define JSCRIPT_DEBUG_ENTER() TRACE_ENTER()
80 #define JSCRIPT_DEBUG_LEAVE() TRACE_LEAVE()
82 #define JSCRIPT_DEBUG_ENTER_TEXT(_szFormat,...) \
83 TRACE_ENTER_TEXT(_szFormat,## __VA_ARGS__)
85 #define JSCRIPT_DEBUG_LEAVE_TEXT(_szFormat,...) \
86 TRACE_LEAVE_TEXT(_szFormat,## __VA_ARGS__)
88 #define JSCRIPT_DEBUG_PRINT(_szFormat,...) \
89 TRACE_PRINT(_szFormat,## __VA_ARGS__)
91 #define JSCRIPT_DEBUG_ASSERT(_condition,_szFormat,...) \
92 TRACE_ASSERT(_condition,_szFormat,## __VA_ARGS__)
94 #else //!JSCRIPT_DO_DEBUGGING
96 #define JSCRIPT_DEBUG_ENTER() do { } while(0)
97 #define JSCRIPT_DEBUG_LEAVE() do { } while(0)
99 #define JSCRIPT_DEBUG_ENTER_TEXT(_szFormat,...) do { } while(0)
100 #define JSCRIPT_DEBUG_LEAVE_TEXT(_szFormat,...) do { } while(0)
102 #define JSCRIPT_DEBUG_PRINT(_szFormat,...) do { } while(0)
104 #define JSCRIPT_DEBUG_ASSERT(_condition,_szFormat,...) do { } while(0)
106 #endif //!JSCRIPT_DO_DEBUGGING
114 * Tracks class and function names already created
116 static stringList
*ClassNames
;
117 static stringList
*FunctionNames
;
119 /* Used to specify type of keyword.
123 KEYWORD_capital_function
,
124 KEYWORD_capital_object
,
149 typedef int keywordId
; /* to allow KEYWORD_NONE */
151 typedef enum eTokenType
{
163 TOKEN_TEMPLATE_STRING
,
171 TOKEN_POSTFIX_OPERATOR
,
173 TOKEN_BINARY_OPERATOR
176 typedef struct sTokenInfo
{
181 unsigned long lineNumber
;
191 static tokenType LastTokenType
;
192 static tokenInfo
*NextToken
;
194 static langType Lang_js
;
196 static objPool
*TokenPool
= NULL
;
199 static iconv_t JSUnicodeConverter
= (iconv_t
) -2;
213 static kindDefinition JsKinds
[] = {
214 { true, 'f', "function", "functions" },
215 { true, 'c', "class", "classes" },
216 { true, 'm', "method", "methods" },
217 { true, 'p', "property", "properties" },
218 { true, 'C', "constant", "constants" },
219 { true, 'v', "variable", "global variables" },
220 { true, 'g', "generator", "generators" }
223 static const keywordTable JsKeywordTable
[] = {
224 /* keyword keyword ID */
225 { "function", KEYWORD_function
},
226 { "Function", KEYWORD_capital_function
},
227 { "Object", KEYWORD_capital_object
},
228 { "prototype", KEYWORD_prototype
},
229 { "var", KEYWORD_var
},
230 { "let", KEYWORD_let
},
231 { "const", KEYWORD_const
},
232 { "new", KEYWORD_new
},
233 { "this", KEYWORD_this
},
234 { "for", KEYWORD_for
},
235 { "while", KEYWORD_while
},
236 { "do", KEYWORD_do
},
237 { "if", KEYWORD_if
},
238 { "else", KEYWORD_else
},
239 { "switch", KEYWORD_switch
},
240 { "try", KEYWORD_try
},
241 { "catch", KEYWORD_catch
},
242 { "finally", KEYWORD_finally
},
243 { "sap", KEYWORD_sap
},
244 { "return", KEYWORD_return
},
245 { "class", KEYWORD_class
},
246 { "extends", KEYWORD_extends
},
247 { "static", KEYWORD_static
},
248 { "default", KEYWORD_default
},
249 { "export", KEYWORD_export
},
250 { "async", KEYWORD_async
},
254 * FUNCTION DEFINITIONS
257 /* Recursive functions */
258 static void readTokenFull (tokenInfo
*const token
, bool include_newlines
, vString
*const repr
);
259 static void parseFunction (tokenInfo
*const token
);
260 static bool parseBlock (tokenInfo
*const token
, const vString
*const parentScope
);
261 static bool parseLine (tokenInfo
*const token
, bool is_inside_class
);
262 static void parseUI5 (tokenInfo
*const token
);
264 static void *newPoolToken (void *createArg CTAGS_ATTR_UNUSED
)
266 tokenInfo
*token
= xMalloc (1, tokenInfo
);
268 token
->string
= vStringNew ();
269 token
->scope
= vStringNew ();
274 static void clearPoolToken (void *data
)
276 tokenInfo
*token
= data
;
278 token
->type
= TOKEN_UNDEFINED
;
279 token
->keyword
= KEYWORD_NONE
;
280 token
->nestLevel
= 0;
281 token
->ignoreTag
= false;
282 token
->lineNumber
= getInputLineNumber ();
283 token
->filePosition
= getInputFilePosition ();
284 vStringClear (token
->string
);
285 vStringClear (token
->scope
);
288 static void deletePoolToken (void *data
)
290 tokenInfo
*token
= data
;
291 vStringDelete (token
->string
);
292 vStringDelete (token
->scope
);
296 static void copyToken (tokenInfo
*const dest
, const tokenInfo
*const src
,
297 bool const include_non_read_info
)
299 dest
->lineNumber
= src
->lineNumber
;
300 dest
->filePosition
= src
->filePosition
;
301 dest
->type
= src
->type
;
302 dest
->keyword
= src
->keyword
;
303 vStringCopy(dest
->string
, src
->string
);
304 if (include_non_read_info
)
306 dest
->nestLevel
= src
->nestLevel
;
307 vStringCopy(dest
->scope
, src
->scope
);
312 * Tag generation functions
315 static void makeJsTagCommon (const tokenInfo
*const token
, const jsKind kind
,
316 vString
*const signature
, vString
*const inheritance
,
319 if (JsKinds
[kind
].enabled
&& ! token
->ignoreTag
)
321 const char *name
= vStringValue (token
->string
);
322 vString
*fullscope
= vStringNewCopy (token
->scope
);
326 if (kind
!= JSTAG_PROPERTY
&& (p
= strrchr (name
, '.')) != NULL
)
328 if (vStringLength (fullscope
) > 0)
329 vStringPut (fullscope
, '.');
330 vStringNCatS (fullscope
, name
, (size_t) (p
- name
));
334 initTagEntry (&e
, name
, kind
);
336 JSCRIPT_DEBUG_PRINT("Emitting tag for symbol '%s' of kind %02x with scope '%s'",name
,kind
,vStringValue(fullscope
));
338 e
.lineNumber
= token
->lineNumber
;
339 e
.filePosition
= token
->filePosition
;
341 if ( vStringLength(fullscope
) > 0 )
343 /* FIXME: proper parent type */
344 jsKind parent_kind
= JSTAG_CLASS
;
347 * If we're creating a function (and not a method),
348 * guess we're inside another function
350 if (kind
== JSTAG_FUNCTION
)
351 parent_kind
= JSTAG_FUNCTION
;
353 e
.extensionFields
.scopeKindIndex
= parent_kind
;
354 e
.extensionFields
.scopeName
= vStringValue (fullscope
);
357 if (signature
&& vStringLength(signature
))
360 /* sanitize signature by replacing all control characters with a
361 * space (because it's simple).
362 * there should never be any junk in a valid signature, but who
363 * knows what the user wrote and CTags doesn't cope well with weird
365 for (i
= 0; i
< signature
->length
; i
++)
367 unsigned char c
= (unsigned char) signature
->buffer
[i
];
368 if (c
< 0x20 /* below space */ || c
== 0x7F /* DEL */)
369 signature
->buffer
[i
] = ' ';
371 e
.extensionFields
.signature
= vStringValue(signature
);
375 e
.extensionFields
.inheritance
= vStringValue(inheritance
);
378 markTagExtraBit (&e
, XTAG_ANONYMOUS
);
381 vStringDelete (fullscope
);
385 static void makeJsTag (const tokenInfo
*const token
, const jsKind kind
,
386 vString
*const signature
, vString
*const inheritance
)
388 makeJsTagCommon (token
, kind
, signature
, inheritance
, false);
391 static void makeClassTagCommon (tokenInfo
*const token
, vString
*const signature
,
392 vString
*const inheritance
, bool anonymous
)
396 if ( ! token
->ignoreTag
)
398 fulltag
= vStringNew ();
399 if (vStringLength (token
->scope
) > 0)
401 vStringCopy(fulltag
, token
->scope
);
402 vStringPut (fulltag
, '.');
403 vStringCat (fulltag
, token
->string
);
407 vStringCopy(fulltag
, token
->string
);
409 if ( ! stringListHas(ClassNames
, vStringValue (fulltag
)) )
411 stringListAdd (ClassNames
, vStringNewCopy (fulltag
));
412 makeJsTagCommon (token
, JSTAG_CLASS
, signature
, inheritance
,
415 vStringDelete (fulltag
);
419 static void makeClassTag (tokenInfo
*const token
, vString
*const signature
,
420 vString
*const inheritance
)
422 makeClassTagCommon (token
, signature
, inheritance
, false);
425 static void makeFunctionTagCommon (tokenInfo
*const token
, vString
*const signature
, bool generator
,
430 if ( ! token
->ignoreTag
)
432 fulltag
= vStringNew ();
433 if (vStringLength (token
->scope
) > 0)
435 vStringCopy(fulltag
, token
->scope
);
436 vStringPut (fulltag
, '.');
437 vStringCat (fulltag
, token
->string
);
441 vStringCopy(fulltag
, token
->string
);
443 if ( ! stringListHas(FunctionNames
, vStringValue (fulltag
)) )
445 stringListAdd (FunctionNames
, vStringNewCopy (fulltag
));
446 makeJsTagCommon (token
, generator
? JSTAG_GENERATOR
: JSTAG_FUNCTION
, signature
, NULL
,
449 vStringDelete (fulltag
);
453 static void makeFunctionTag (tokenInfo
*const token
, vString
*const signature
, bool generator
)
455 makeFunctionTagCommon (token
, signature
, generator
, false);
462 /* given @p point, returns the first byte of the encoded output sequence, and
463 * make sure the next ones will be returned by calls to getcFromInputFile()
464 * as if the code point was simply written in the input file. */
465 static int handleUnicodeCodePoint (uint32_t point
)
469 Assert (point
< 0x110000);
472 /* if we do have iconv and the encodings are specified, use this */
473 if (isConverting () && JSUnicodeConverter
== (iconv_t
) -2)
475 /* if we didn't try creating the converter yet, try and do so */
476 JSUnicodeConverter
= iconv_open (getLanguageEncoding (Lang_js
), INTERNAL_ENCODING
);
478 if (isConverting () && JSUnicodeConverter
!= (iconv_t
) -1)
480 char *input_ptr
= (char *) &point
;
481 size_t input_left
= sizeof point
;
482 /* 4 bytes should be enough for any encoding (it's how much UTF-32
484 /* FIXME: actually iconv has a tendency to output a BOM for Unicode
485 * encodings where it matters when the endianess is not specified in
486 * the target encoding name. E.g., if the target encoding is "UTF-32"
487 * or "UTF-16" it will output 2 code points, the BOM (U+FEFF) and the
488 * one we expect. This does not happen if the endianess is specified
489 * explicitly, e.g. with "UTF-32LE", or "UTF-16BE".
490 * However, it's not very relevant for the moment as nothing in CTags
491 * cope well (if at all) with non-ASCII-compatible encodings like
492 * UTF-32 or UTF-16 anyway. */
493 char output
[4] = { 0 };
494 char *output_ptr
= output
;
495 size_t output_left
= ARRAY_SIZE (output
);
497 if (iconv (JSUnicodeConverter
, &input_ptr
, &input_left
, &output_ptr
, &output_left
) == (size_t) -1)
499 /* something went wrong, which probably means the output encoding
500 * cannot represent the character. Use a placeholder likely to be
501 * supported instead, that's also valid in an identifier */
502 verbose ("JavaScript: Encoding: %s\n", strerror (errno
));
507 const size_t output_len
= ARRAY_SIZE (output
) - output_left
;
509 /* put all but the first byte back so that getcFromInputFile() will
510 * return them in the right order */
511 for (unsigned int i
= 1; i
< output_len
; i
++)
512 ungetcToInputFile ((unsigned char) output
[output_len
- i
]);
513 c
= (unsigned char) output
[0];
516 iconv (JSUnicodeConverter
, NULL
, NULL
, NULL
, NULL
);
521 /* when no encoding is specified (or no iconv), assume UTF-8 is good.
522 * Why UTF-8? Because it's an ASCII-compatible common Unicode encoding. */
524 c
= (unsigned char) point
;
525 else if (point
< 0x800)
527 c
= (unsigned char) (0xc0 | ((point
>> 6) & 0x1f));
528 ungetcToInputFile ((unsigned char) (0x80 | (point
& 0x3f)));
530 else if (point
< 0x10000)
532 c
= (unsigned char) (0xe0 | ((point
>> 12) & 0x0f));
533 ungetcToInputFile ((unsigned char) (0x80 | ((point
>> 0) & 0x3f)));
534 ungetcToInputFile ((unsigned char) (0x80 | ((point
>> 6) & 0x3f)));
536 else if (point
< 0x110000)
538 c
= (unsigned char) (0xf0 | ((point
>> 18) & 0x07));
539 ungetcToInputFile ((unsigned char) (0x80 | ((point
>> 0) & 0x3f)));
540 ungetcToInputFile ((unsigned char) (0x80 | ((point
>> 6) & 0x3f)));
541 ungetcToInputFile ((unsigned char) (0x80 | ((point
>> 12) & 0x3f)));
548 /* reads a Unicode escape sequence after the "\" prefix.
549 * @param value Location to store the escape sequence value.
550 * @param isUTF16 Location to store whether @param value is an UTF-16 word.
551 * @returns Whether a valid sequence was read. */
552 static bool readUnicodeEscapeSequenceValue (uint32_t *const value
,
556 int d
= getcFromInputFile ();
559 ungetcToInputFile (d
);
562 int e
= getcFromInputFile ();
563 char cp
[6 + 1]; /* up to 6 hex + possible closing '}' or invalid char */
564 unsigned int cp_len
= 0;
566 *isUTF16
= (e
!= '{');
568 { /* Handles Unicode code point escapes: \u{ HexDigits }
569 * We skip the leading 0s because there can be any number of them
570 * and they don't change any meaning. */
571 bool has_leading_zero
= false;
573 while ((cp
[cp_len
] = (char) getcFromInputFile ()) == '0')
574 has_leading_zero
= true;
576 while (isxdigit (cp
[cp_len
]) && ++cp_len
< ARRAY_SIZE (cp
))
577 cp
[cp_len
] = (char) getcFromInputFile ();
578 valid
= ((cp_len
> 0 || has_leading_zero
) &&
579 cp_len
< ARRAY_SIZE (cp
) && cp
[cp_len
] == '}' &&
580 /* also check if it's a valid Unicode code point */
582 (cp_len
== 6 && strncmp (cp
, "110000", 6) < 0)));
583 if (! valid
) /* put back the last (likely invalid) character */
584 ungetcToInputFile (cp
[cp_len
]);
587 { /* Handles Unicode escape sequences: \u Hex4Digits */
589 cp
[cp_len
] = (char) ((cp_len
== 0) ? e
: getcFromInputFile ());
590 while (isxdigit (cp
[cp_len
]) && ++cp_len
< 4);
591 valid
= (cp_len
== 4);
596 /* we don't get every character back, but it would require to
597 * be able to put up to 9 characters back (in the worst case
598 * for handling invalid \u{10FFFFx}), and here we're recovering
599 * from invalid syntax anyway. */
600 ungetcToInputFile (e
);
601 ungetcToInputFile (d
);
606 for (unsigned int i
= 0; i
< cp_len
; i
++)
610 /* we know it's a hex digit, no need to double check */
612 *value
+= (unsigned int) cp
[i
] - '0';
613 else if (cp
[i
] < 'a')
614 *value
+= 10 + (unsigned int) cp
[i
] - 'A';
616 *value
+= 10 + (unsigned int) cp
[i
] - 'a';
624 static int valueToXDigit (unsigned char v
)
629 return 'A' + (v
- 0xA);
634 /* Reads and expands a Unicode escape sequence after the "\" prefix. If the
635 * escape sequence is a UTF16 high surrogate, also try and read the low
636 * surrogate to emit the proper code point.
637 * @param fallback The character to return if the sequence is invalid. Usually
638 * this would be the '\' character starting the sequence.
639 * @returns The first byte of the sequence, or @param fallback if the sequence
640 * is invalid. On success, next calls to getcFromInputFile() will
641 * return subsequent bytes (if any). */
642 static int readUnicodeEscapeSequence (const int fallback
)
648 if (! readUnicodeEscapeSequenceValue (&value
, &isUTF16
))
652 if (isUTF16
&& (value
& 0xfc00) == 0xd800)
653 { /* this is a high surrogate, try and read its low surrogate and
654 * emit the resulting code point */
656 int d
= getcFromInputFile ();
658 if (d
!= '\\' || ! readUnicodeEscapeSequenceValue (&low
, &isUTF16
))
659 ungetcToInputFile (d
);
661 { /* not UTF-16 low surrogate but a plain code point */
662 d
= handleUnicodeCodePoint (low
);
663 ungetcToInputFile (d
);
665 else if ((low
& 0xfc00) != 0xdc00)
666 { /* not a low surrogate, so put back the escaped representation
667 * in case it was another high surrogate we should read as part
668 * of another pair. */
669 ungetcToInputFile (valueToXDigit ((unsigned char) ((low
& 0x000f) >> 0)));
670 ungetcToInputFile (valueToXDigit ((unsigned char) ((low
& 0x00f0) >> 4)));
671 ungetcToInputFile (valueToXDigit ((unsigned char) ((low
& 0x0f00) >> 8)));
672 ungetcToInputFile (valueToXDigit ((unsigned char) ((low
& 0xf000) >> 12)));
673 ungetcToInputFile ('u');
674 ungetcToInputFile ('\\');
677 value
= 0x010000 + ((value
& 0x03ff) << 10) + (low
& 0x03ff);
679 c
= handleUnicodeCodePoint (value
);
685 static void parseString (vString
*const string
, const int delimiter
)
690 int c
= getcFromInputFile ();
695 /* Eat the escape sequence (\", \', etc). We properly handle
696 * <LineContinuation> by eating a whole \<CR><LF> not to see <LF>
697 * as an unescaped character, which is invalid and handled below.
698 * Also, handle the fact that <LineContinuation> produces an empty
700 * See ECMA-262 7.8.4 */
701 c
= getcFromInputFile ();
704 ungetcToInputFile (c
);
705 c
= readUnicodeEscapeSequence ('\\');
706 vStringPut (string
, c
);
708 else if (c
!= '\r' && c
!= '\n')
709 vStringPut(string
, c
);
712 c
= getcFromInputFile();
714 ungetcToInputFile (c
);
717 else if (c
== delimiter
)
719 else if (c
== '\r' || c
== '\n')
721 /* those are invalid when not escaped */
723 /* we don't want to eat the newline itself to let the automatic
724 * semicolon insertion code kick in */
725 ungetcToInputFile (c
);
728 vStringPut (string
, c
);
732 static void parseRegExp (void)
735 bool in_range
= false;
739 c
= getcFromInputFile ();
740 if (! in_range
&& c
== '/')
744 c
= getcFromInputFile ();
745 } while (isalpha (c
));
746 ungetcToInputFile (c
);
749 else if (c
== '\n' || c
== '\r')
751 /* invalid in a regex */
752 ungetcToInputFile (c
);
756 c
= getcFromInputFile (); /* skip next character */
764 /* Read a C identifier beginning with "firstChar" and places it into
767 static void parseIdentifier (vString
*const string
, const int firstChar
)
770 Assert (isIdentChar (c
));
773 vStringPut (string
, c
);
774 c
= getcFromInputFile ();
776 c
= readUnicodeEscapeSequence (c
);
777 } while (isIdentChar (c
));
778 /* if readUnicodeEscapeSequence() read an escape sequence this is incorrect,
779 * as we should actually put back the whole escape sequence and not the
780 * decoded character. However, it's not really worth the hassle as it can
781 * only happen if the input has an invalid escape sequence. */
782 ungetcToInputFile (c
); /* unget non-identifier character */
785 static void parseTemplateString (vString
*const string
)
790 c
= getcFromInputFile ();
791 if (c
== '`' || c
== EOF
)
794 vStringPut (string
, c
);
798 c
= getcFromInputFile();
800 vStringPut(string
, c
);
804 c
= getcFromInputFile ();
806 ungetcToInputFile (c
);
810 /* we need to use the real token machinery to handle strings,
811 * comments, regexes and whatnot */
812 tokenInfo
*token
= newToken ();
813 LastTokenType
= TOKEN_UNDEFINED
;
814 vStringPut(string
, c
);
817 readTokenFull (token
, false, string
);
818 if (isType (token
, TOKEN_OPEN_CURLY
))
820 else if (isType (token
, TOKEN_CLOSE_CURLY
))
823 while (! isType (token
, TOKEN_EOF
) && depth
> 0);
831 static void readTokenFull (tokenInfo
*const token
, bool include_newlines
, vString
*const repr
)
835 bool newline_encountered
= false;
837 /* if we've got a token held back, emit it */
840 copyToken (token
, NextToken
, false);
841 deleteToken (NextToken
);
846 token
->type
= TOKEN_UNDEFINED
;
847 token
->keyword
= KEYWORD_NONE
;
848 vStringClear (token
->string
);
854 c
= getcFromInputFile ();
855 if (include_newlines
&& (c
== '\r' || c
== '\n'))
856 newline_encountered
= true;
859 while (c
== '\t' || c
== ' ' || c
== '\r' || c
== '\n');
861 token
->lineNumber
= getInputLineNumber ();
862 token
->filePosition
= getInputFilePosition ();
864 if (repr
&& c
!= EOF
)
867 vStringPut (repr
, ' ');
868 vStringPut (repr
, c
);
873 case EOF
: token
->type
= TOKEN_EOF
; break;
874 case '(': token
->type
= TOKEN_OPEN_PAREN
; break;
875 case ')': token
->type
= TOKEN_CLOSE_PAREN
; break;
876 case ';': token
->type
= TOKEN_SEMICOLON
; break;
877 case ',': token
->type
= TOKEN_COMMA
; break;
878 case '.': token
->type
= TOKEN_PERIOD
; break;
879 case ':': token
->type
= TOKEN_COLON
; break;
880 case '{': token
->type
= TOKEN_OPEN_CURLY
; break;
881 case '}': token
->type
= TOKEN_CLOSE_CURLY
; break;
882 case '=': token
->type
= TOKEN_EQUAL_SIGN
; break;
883 case '[': token
->type
= TOKEN_OPEN_SQUARE
; break;
884 case ']': token
->type
= TOKEN_CLOSE_SQUARE
; break;
889 int d
= getcFromInputFile ();
890 if (d
== c
) /* ++ or -- */
891 token
->type
= TOKEN_POSTFIX_OPERATOR
;
894 ungetcToInputFile (d
);
895 token
->type
= TOKEN_BINARY_OPERATOR
;
901 token
->type
= TOKEN_STAR
;
910 token
->type
= TOKEN_BINARY_OPERATOR
;
915 token
->type
= TOKEN_STRING
;
916 parseString (token
->string
, c
);
917 token
->lineNumber
= getInputLineNumber ();
918 token
->filePosition
= getInputFilePosition ();
921 vStringCat (repr
, token
->string
);
922 vStringPut (repr
, c
);
927 token
->type
= TOKEN_TEMPLATE_STRING
;
928 parseTemplateString (token
->string
);
929 token
->lineNumber
= getInputLineNumber ();
930 token
->filePosition
= getInputFilePosition ();
933 vStringCat (repr
, token
->string
);
934 vStringPut (repr
, c
);
940 int d
= getcFromInputFile ();
941 if ( (d
!= '*') && /* is this the start of a comment? */
942 (d
!= '/') ) /* is a one line comment? */
944 ungetcToInputFile (d
);
945 switch (LastTokenType
)
947 case TOKEN_CHARACTER
:
948 case TOKEN_IDENTIFIER
:
950 case TOKEN_TEMPLATE_STRING
:
951 case TOKEN_CLOSE_CURLY
:
952 case TOKEN_CLOSE_PAREN
:
953 case TOKEN_CLOSE_SQUARE
:
954 token
->type
= TOKEN_BINARY_OPERATOR
;
958 token
->type
= TOKEN_REGEXP
;
960 token
->lineNumber
= getInputLineNumber ();
961 token
->filePosition
= getInputFilePosition ();
967 if (repr
) /* remove the / we added */
968 repr
->buffer
[--repr
->length
] = 0;
973 skipToCharacterInInputFile ('*');
974 c
= getcFromInputFile ();
978 ungetcToInputFile (c
);
979 } while (c
!= EOF
&& c
!= '\0');
982 else if (d
== '/') /* is this the start of a comment? */
984 skipToCharacterInInputFile ('\n');
985 /* if we care about newlines, put it back so it is seen */
986 if (include_newlines
)
987 ungetcToInputFile ('\n');
995 /* skip shebang in case of e.g. Node.js scripts */
996 if (token
->lineNumber
> 1)
997 token
->type
= TOKEN_UNDEFINED
;
998 else if ((c
= getcFromInputFile ()) != '!')
1000 ungetcToInputFile (c
);
1001 token
->type
= TOKEN_UNDEFINED
;
1005 skipToCharacterInInputFile ('\n');
1011 c
= readUnicodeEscapeSequence (c
);
1014 if (! isIdentChar (c
))
1015 token
->type
= TOKEN_UNDEFINED
;
1018 parseIdentifier (token
->string
, c
);
1019 token
->lineNumber
= getInputLineNumber ();
1020 token
->filePosition
= getInputFilePosition ();
1021 token
->keyword
= lookupKeyword (vStringValue (token
->string
), Lang_js
);
1022 if (isKeyword (token
, KEYWORD_NONE
))
1023 token
->type
= TOKEN_IDENTIFIER
;
1025 token
->type
= TOKEN_KEYWORD
;
1026 if (repr
&& vStringLength (token
->string
) > 1)
1027 vStringCatS (repr
, vStringValue (token
->string
) + 1);
1032 if (include_newlines
&& newline_encountered
)
1034 /* This isn't strictly correct per the standard, but following the
1035 * real rules means understanding all statements, and that's not
1036 * what the parser currently does. What we do here is a guess, by
1037 * avoiding inserting semicolons that would make the statement on
1038 * the left or right obviously invalid. Hopefully this should not
1039 * have false negatives (e.g. should not miss insertion of a semicolon)
1040 * but might have false positives (e.g. it will wrongfully emit a
1041 * semicolon sometimes, i.e. for the newline in "foo\n(bar)").
1042 * This should however be mostly harmless as we only deal with
1043 * newlines in specific situations where we know a false positive
1044 * wouldn't hurt too bad. */
1046 /* these already end a statement, so no need to duplicate it */
1047 #define IS_STMT_SEPARATOR(t) ((t) == TOKEN_SEMICOLON || \
1048 (t) == TOKEN_EOF || \
1049 (t) == TOKEN_COMMA || \
1050 (t) == TOKEN_OPEN_CURLY)
1051 /* these cannot be the start or end of a statement */
1052 #define IS_BINARY_OPERATOR(t) ((t) == TOKEN_EQUAL_SIGN || \
1053 (t) == TOKEN_COLON || \
1054 (t) == TOKEN_PERIOD || \
1055 (t) == TOKEN_STAR || \
1056 (t) == TOKEN_BINARY_OPERATOR)
1058 if (! IS_STMT_SEPARATOR(LastTokenType
) &&
1059 ! IS_STMT_SEPARATOR(token
->type
) &&
1060 ! IS_BINARY_OPERATOR(LastTokenType
) &&
1061 ! IS_BINARY_OPERATOR(token
->type
) &&
1062 /* these cannot be followed by a semicolon */
1063 ! (LastTokenType
== TOKEN_OPEN_PAREN
||
1064 LastTokenType
== TOKEN_OPEN_SQUARE
))
1066 /* hold the token... */
1067 Assert (NextToken
== NULL
);
1068 NextToken
= newToken ();
1069 copyToken (NextToken
, token
, false);
1071 /* ...and emit a semicolon instead */
1072 token
->type
= TOKEN_SEMICOLON
;
1073 token
->keyword
= KEYWORD_NONE
;
1074 vStringClear (token
->string
);
1076 vStringPut (token
->string
, '\n');
1079 #undef IS_STMT_SEPARATOR
1080 #undef IS_BINARY_OPERATOR
1083 LastTokenType
= token
->type
;
1086 #ifdef JSCRIPT_DO_DEBUGGING
1087 /* trace readTokenFull() */
1088 static void readTokenFullDebug (tokenInfo
*const token
, bool include_newlines
, vString
*const repr
)
1090 readTokenFull (token
, include_newlines
, repr
);
1091 JSCRIPT_DEBUG_PRINT("token '%s' of type %02x with scope '%s'",vStringValue(token
->string
),token
->type
, vStringValue(token
->scope
));
1093 # define readTokenFull readTokenFullDebug
1096 static void readToken (tokenInfo
*const token
)
1098 readTokenFull (token
, false, NULL
);
1102 * Token parsing functions
1105 static void skipArgumentList (tokenInfo
*const token
, bool include_newlines
, vString
*const repr
)
1109 if (isType (token
, TOKEN_OPEN_PAREN
)) /* arguments? */
1113 vStringPut (repr
, '(');
1114 while (nest_level
> 0 && ! isType (token
, TOKEN_EOF
))
1116 readTokenFull (token
, false, repr
);
1117 if (isType (token
, TOKEN_OPEN_PAREN
))
1119 else if (isType (token
, TOKEN_CLOSE_PAREN
))
1121 else if (isKeyword (token
, KEYWORD_function
))
1122 parseFunction (token
);
1124 readTokenFull (token
, include_newlines
, NULL
);
1128 static void skipArrayList (tokenInfo
*const token
, bool include_newlines
)
1133 * Handle square brackets
1135 * So we must check for nested open and closing square brackets
1138 if (isType (token
, TOKEN_OPEN_SQUARE
)) /* arguments? */
1141 while (nest_level
> 0 && ! isType (token
, TOKEN_EOF
))
1144 if (isType (token
, TOKEN_OPEN_SQUARE
))
1146 else if (isType (token
, TOKEN_CLOSE_SQUARE
))
1149 readTokenFull (token
, include_newlines
, NULL
);
1153 static void addContext (tokenInfo
* const parent
, const tokenInfo
* const child
)
1155 if (vStringLength (parent
->string
) > 0)
1157 vStringPut (parent
->string
, '.');
1159 vStringCat (parent
->string
, child
->string
);
1162 static void addToScope (tokenInfo
* const token
, const vString
* const extra
)
1164 if (vStringLength (token
->scope
) > 0)
1166 vStringPut (token
->scope
, '.');
1168 vStringCat (token
->scope
, extra
);
1172 * Scanning functions
1175 static bool findCmdTerm (tokenInfo
*const token
, bool include_newlines
,
1176 bool include_commas
)
1179 * Read until we find either a semicolon or closing brace.
1180 * Any nested braces will be handled within.
1182 while (! isType (token
, TOKEN_SEMICOLON
) &&
1183 ! isType (token
, TOKEN_CLOSE_CURLY
) &&
1184 ! (include_commas
&& isType (token
, TOKEN_COMMA
)) &&
1185 ! isType (token
, TOKEN_EOF
))
1187 /* Handle nested blocks */
1188 if ( isType (token
, TOKEN_OPEN_CURLY
))
1190 parseBlock (token
, NULL
);
1191 readTokenFull (token
, include_newlines
, NULL
);
1193 else if ( isType (token
, TOKEN_OPEN_PAREN
) )
1195 skipArgumentList(token
, include_newlines
, NULL
);
1197 else if ( isType (token
, TOKEN_OPEN_SQUARE
) )
1199 skipArrayList(token
, include_newlines
);
1203 readTokenFull (token
, include_newlines
, NULL
);
1207 return isType (token
, TOKEN_SEMICOLON
);
1210 static void parseSwitch (tokenInfo
*const token
)
1213 * switch (expression) {
1220 * default : statement;
1226 if (isType (token
, TOKEN_OPEN_PAREN
))
1228 skipArgumentList(token
, false, NULL
);
1231 if (isType (token
, TOKEN_OPEN_CURLY
))
1233 parseBlock (token
, NULL
);
1237 static bool parseLoop (tokenInfo
*const token
)
1240 * Handles these statements
1241 * for (x=0; x<3; x++)
1242 * document.write("This text is repeated three times<br>");
1244 * for (x=0; x<3; x++)
1246 * document.write("This text is repeated three times<br>");
1250 * document.write(number+"<br>");
1255 * document.write(number+"<br>");
1260 bool is_terminated
= true;
1262 if (isKeyword (token
, KEYWORD_for
) || isKeyword (token
, KEYWORD_while
))
1266 if (isType (token
, TOKEN_OPEN_PAREN
))
1268 skipArgumentList(token
, false, NULL
);
1271 if (isType (token
, TOKEN_OPEN_CURLY
))
1273 parseBlock (token
, NULL
);
1277 is_terminated
= parseLine(token
, false);
1280 else if (isKeyword (token
, KEYWORD_do
))
1284 if (isType (token
, TOKEN_OPEN_CURLY
))
1286 parseBlock (token
, NULL
);
1290 is_terminated
= parseLine(token
, false);
1296 if (isKeyword (token
, KEYWORD_while
))
1300 if (isType (token
, TOKEN_OPEN_PAREN
))
1302 skipArgumentList(token
, true, NULL
);
1304 if (! isType (token
, TOKEN_SEMICOLON
))
1306 /* oddly enough, `do {} while (0) var foo = 42` is perfectly
1307 * valid JS, so explicitly handle the remaining of the line
1308 * for the sake of the root scope handling (as parseJsFile()
1309 * always advances a token not to ever get stuck) */
1310 is_terminated
= parseLine(token
, false);
1315 return is_terminated
;
1318 static bool parseIf (tokenInfo
*const token
)
1320 bool read_next_token
= true;
1322 * If statements have two forms
1341 * This example if correctly written, but the
1342 * else contains only 1 statement without a terminator
1343 * since the function finishes with the closing brace.
1352 * TODO: Deal with statements that can optional end
1353 * without a semi-colon. Currently this messes up
1354 * the parsing of blocks.
1355 * Need to somehow detect this has happened, and either
1356 * backup a token, or skip reading the next token if
1357 * that is possible from all code locations.
1363 if (isKeyword (token
, KEYWORD_if
))
1366 * Check for an "else if" and consume the "if"
1371 if (isType (token
, TOKEN_OPEN_PAREN
))
1373 skipArgumentList(token
, false, NULL
);
1376 if (isType (token
, TOKEN_OPEN_CURLY
))
1378 parseBlock (token
, NULL
);
1382 /* The next token should only be read if this statement had its own
1384 read_next_token
= findCmdTerm (token
, true, false);
1386 return read_next_token
;
1389 static void parseFunction (tokenInfo
*const token
)
1391 tokenInfo
*const name
= newToken ();
1392 vString
*const signature
= vStringNew ();
1393 bool is_class
= false;
1394 bool is_generator
= false;
1395 bool is_anonymous
= false;
1397 * This deals with these formats
1398 * function validFunctionTwo(a,b) {}
1399 * function * generator(a,b) {}
1402 copyToken (name
, token
, true);
1404 if (isType (name
, TOKEN_STAR
))
1406 is_generator
= true;
1409 if (isType (name
, TOKEN_OPEN_PAREN
))
1411 /* anonymous function */
1412 copyToken (token
, name
, false);
1413 anonGenerate (name
->string
, "AnonymousFunction", JSTAG_FUNCTION
);
1414 is_anonymous
= true;
1416 else if (!isType (name
, TOKEN_IDENTIFIER
))
1421 while (isType (token
, TOKEN_PERIOD
))
1424 if (! isType(token
, TOKEN_KEYWORD
))
1426 addContext (name
, token
);
1431 if ( isType (token
, TOKEN_OPEN_PAREN
) )
1432 skipArgumentList(token
, false, signature
);
1434 if ( isType (token
, TOKEN_OPEN_CURLY
) )
1436 is_class
= parseBlock (token
, name
->string
);
1438 makeClassTagCommon (name
, signature
, NULL
, is_anonymous
);
1440 makeFunctionTagCommon (name
, signature
, is_generator
, is_anonymous
);
1443 findCmdTerm (token
, false, false);
1446 vStringDelete (signature
);
1450 /* Parses a block surrounded by curly braces.
1451 * @p parentScope is the scope name for this block, or NULL for unnamed scopes */
1452 static bool parseBlock (tokenInfo
*const token
, const vString
*const parentScope
)
1454 JSCRIPT_DEBUG_ENTER();
1456 bool is_class
= false;
1457 bool read_next_token
= true;
1458 vString
* saveScope
= vStringNew ();
1460 vStringCopy(saveScope
, token
->scope
);
1463 addToScope (token
, parentScope
);
1468 * Make this routine a bit more forgiving.
1469 * If called on an open_curly advance it
1471 if (isType (token
, TOKEN_OPEN_CURLY
))
1474 if (! isType (token
, TOKEN_CLOSE_CURLY
))
1477 * Read until we find the closing brace,
1478 * any nested braces will be handled within
1482 read_next_token
= true;
1483 if (isKeyword (token
, KEYWORD_this
))
1486 * Means we are inside a class and have found
1487 * a class, not a function
1492 * Ignore the remainder of the line
1493 * findCmdTerm(token);
1495 read_next_token
= parseLine (token
, is_class
);
1497 else if (isKeyword (token
, KEYWORD_var
) ||
1498 isKeyword (token
, KEYWORD_let
) ||
1499 isKeyword (token
, KEYWORD_const
))
1502 * Potentially we have found an inner function.
1503 * Set something to indicate the scope
1505 read_next_token
= parseLine (token
, is_class
);
1507 else if (isType (token
, TOKEN_OPEN_CURLY
))
1509 /* Handle nested blocks */
1510 parseBlock (token
, NULL
);
1515 * It is possible for a line to have no terminator
1516 * if the following line is a closing brace.
1517 * parseLine will detect this case and indicate
1518 * whether we should read an additional token.
1520 read_next_token
= parseLine (token
, is_class
);
1524 * Always read a new token unless we find a statement without
1525 * a ending terminator
1527 if( read_next_token
)
1531 * If we find a statement without a terminator consider the
1532 * block finished, otherwise the stack will be off by one.
1534 } while (! isType (token
, TOKEN_EOF
) &&
1535 ! isType (token
, TOKEN_CLOSE_CURLY
) && read_next_token
);
1538 vStringCopy(token
->scope
, saveScope
);
1539 vStringDelete(saveScope
);
1543 JSCRIPT_DEBUG_LEAVE();
1548 static bool parseMethods (tokenInfo
*const token
, const tokenInfo
*const class,
1549 const bool is_es6_class
)
1551 JSCRIPT_DEBUG_ENTER();
1553 tokenInfo
*const name
= newToken ();
1554 bool has_methods
= false;
1555 vString
*saveScope
= vStringNew ();
1557 vStringCopy (saveScope
, token
->scope
);
1558 addToScope (token
, class->string
);
1561 * This deals with these formats
1562 * validProperty : 2,
1563 * validMethod : function(a,b) {}
1564 * 'validMethod2' : function(a,b) {}
1565 * container.dirtyTab = {'url': false, 'title':false, 'snapshot':false, '*': false}
1570 * FIXME: what to do with computed names?
1578 if (isType (token
, TOKEN_CLOSE_CURLY
))
1583 if (isKeyword (token
, KEYWORD_async
))
1586 if (! isType (token
, TOKEN_KEYWORD
) &&
1587 ! isType (token
, TOKEN_SEMICOLON
))
1589 bool is_generator
= false;
1590 bool is_shorthand
= false; /* ES6 shorthand syntax */
1592 if (isType (token
, TOKEN_STAR
)) /* shorthand generator */
1594 is_generator
= true;
1598 copyToken(name
, token
, true);
1601 is_shorthand
= isType (token
, TOKEN_OPEN_PAREN
);
1602 if ( isType (token
, TOKEN_COLON
) || is_shorthand
)
1607 if (isKeyword (token
, KEYWORD_async
))
1610 if ( is_shorthand
|| isKeyword (token
, KEYWORD_function
) )
1612 JSCRIPT_DEBUG_PRINT("Seems to be a function or shorthand");
1613 vString
*const signature
= vStringNew ();
1618 if (isType (token
, TOKEN_STAR
))
1620 /* generator: 'function' '*' '(' ... ')' '{' ... '}' */
1621 is_generator
= true;
1625 if ( isType (token
, TOKEN_OPEN_PAREN
) )
1627 skipArgumentList(token
, false, signature
);
1630 if (isType (token
, TOKEN_OPEN_CURLY
))
1633 makeJsTag (name
, is_generator
? JSTAG_GENERATOR
: JSTAG_METHOD
, signature
, NULL
);
1634 parseBlock (token
, name
->string
);
1637 * If we aren't parsing an ES6 class (for which there
1638 * is no mandatory separators), read to the closing
1639 * curly, check next token, if a comma, we must loop
1646 vStringDelete (signature
);
1648 else if (! is_es6_class
)
1650 bool has_child_methods
= false;
1652 /* skip whatever is the value */
1653 while (! isType (token
, TOKEN_COMMA
) &&
1654 ! isType (token
, TOKEN_CLOSE_CURLY
) &&
1655 ! isType (token
, TOKEN_EOF
))
1657 if (isType (token
, TOKEN_OPEN_CURLY
))
1659 /* Recurse to find child properties/methods */
1660 has_child_methods
= parseMethods (token
, name
, false);
1663 else if (isType (token
, TOKEN_OPEN_PAREN
))
1665 skipArgumentList (token
, false, NULL
);
1667 else if (isType (token
, TOKEN_OPEN_SQUARE
))
1669 skipArrayList (token
, false);
1678 if (has_child_methods
)
1679 makeJsTag (name
, JSTAG_CLASS
, NULL
, NULL
);
1681 makeJsTag (name
, JSTAG_PROPERTY
, NULL
, NULL
);
1685 } while ( isType(token
, TOKEN_COMMA
) ||
1686 ( is_es6_class
&& ! isType(token
, TOKEN_EOF
) ) );
1688 JSCRIPT_DEBUG_PRINT("Finished parsing methods");
1690 findCmdTerm (token
, false, false);
1693 vStringCopy (token
->scope
, saveScope
);
1694 vStringDelete (saveScope
);
1697 JSCRIPT_DEBUG_LEAVE();
1702 static bool parseES6Class (tokenInfo
*const token
, const tokenInfo
*targetName
)
1704 JSCRIPT_DEBUG_ENTER();
1706 tokenInfo
* className
= newToken ();
1707 vString
*inheritance
= NULL
;
1708 bool is_anonymous
= true;
1710 copyToken (className
, token
, true);
1711 readToken (className
);
1714 if (isType (className
, TOKEN_IDENTIFIER
))
1717 is_anonymous
= false;
1721 copyToken (token
, className
, true);
1722 /* We create a fake name so we have a scope for the members */
1724 anonGenerate (className
->string
, "AnonymousClass", JSTAG_CLASS
);
1728 targetName
= className
;
1730 if (isKeyword (token
, KEYWORD_extends
))
1731 inheritance
= vStringNew ();
1733 /* skip inheritance info */
1734 while (! isType (token
, TOKEN_OPEN_CURLY
) &&
1735 ! isType (token
, TOKEN_EOF
) &&
1736 ! isType (token
, TOKEN_SEMICOLON
))
1737 readTokenFull (token
, false, inheritance
);
1739 /* remove the last added token (here we assume it's one char, "{" or ";" */
1740 if (inheritance
&& vStringLength (inheritance
) > 0 &&
1741 ! isType (token
, TOKEN_EOF
))
1743 vStringChop (inheritance
);
1744 vStringStripTrailing (inheritance
);
1745 vStringStripLeading (inheritance
);
1748 JSCRIPT_DEBUG_PRINT("Emitting tag for class '%s'", vStringValue(targetName
->string
));
1750 makeJsTagCommon (targetName
, JSTAG_CLASS
, NULL
, inheritance
,
1751 (is_anonymous
&& (targetName
== className
)));
1753 if (! is_anonymous
&& targetName
!= className
)
1755 /* FIXME: what to do with the secondary name? It's local to the
1756 * class itself, so not very useful... let's hope people
1757 * don't give it another name than the target in case of
1758 * var MyClass = class MyClassSecondaryName { ... }
1759 * I guess it could be an alias to MyClass, or duplicate it
1760 * altogether, not sure. */
1761 makeJsTag (className
, JSTAG_CLASS
, NULL
, inheritance
);
1765 vStringDelete (inheritance
);
1767 if (isType (token
, TOKEN_OPEN_CURLY
))
1768 parseMethods (token
, targetName
, true);
1770 deleteToken (className
);
1772 JSCRIPT_DEBUG_LEAVE();
1776 static bool parseStatement (tokenInfo
*const token
, bool is_inside_class
)
1778 JSCRIPT_DEBUG_ENTER();
1780 tokenInfo
*const name
= newToken ();
1781 tokenInfo
*const secondary_name
= newToken ();
1782 tokenInfo
*const method_body_token
= newToken ();
1783 vString
* saveScope
= vStringNew ();
1784 bool is_class
= false;
1785 bool is_var
= false;
1786 bool is_const
= false;
1787 bool is_terminated
= true;
1788 bool is_global
= false;
1789 bool has_methods
= false;
1792 vStringCopy (saveScope
, token
->scope
);
1794 * Functions can be named or unnamed.
1795 * This deals with these formats:
1797 * validFunctionOne = function(a,b) {}
1798 * testlib.validFunctionFive = function(a,b) {}
1799 * var innerThree = function(a,b) {}
1800 * var innerFour = (a,b) {}
1801 * var D2 = secondary_fcn_name(a,b) {}
1802 * var D3 = new Function("a", "b", "return a+b;");
1804 * testlib.extras.ValidClassOne = function(a,b) {
1808 * testlib.extras.ValidClassOne.prototype = {
1809 * 'validMethodOne' : function(a,b) {},
1810 * 'validMethodTwo' : function(a,b) {}
1812 * ValidClassTwo = function ()
1814 * this.validMethodThree = function() {}
1816 * this.validMethodFour = () {}
1818 * Database.prototype.validMethodThree = Database_getTodaysDate;
1821 if ( is_inside_class
)
1824 * var can precede an inner function
1826 if ( isKeyword(token
, KEYWORD_var
) ||
1827 isKeyword(token
, KEYWORD_let
) ||
1828 isKeyword(token
, KEYWORD_const
) )
1830 JSCRIPT_DEBUG_PRINT("var/let/const case");
1831 is_const
= isKeyword(token
, KEYWORD_const
);
1833 * Only create variables for global scope
1835 if ( token
->nestLevel
== 0 )
1843 if ( isKeyword(token
, KEYWORD_this
) )
1845 JSCRIPT_DEBUG_PRINT("found 'this' keyword");
1848 if (isType (token
, TOKEN_PERIOD
))
1854 copyToken(name
, token
, true);
1855 JSCRIPT_DEBUG_PRINT("name becomes '%s'",vStringValue(name
->string
));
1857 while (! isType (token
, TOKEN_CLOSE_CURLY
) &&
1858 ! isType (token
, TOKEN_SEMICOLON
) &&
1859 ! isType (token
, TOKEN_EQUAL_SIGN
) &&
1860 ! isType (token
, TOKEN_COMMA
) &&
1861 ! isType (token
, TOKEN_EOF
))
1863 if (isType (token
, TOKEN_OPEN_CURLY
))
1864 parseBlock (token
, NULL
);
1866 /* Potentially the name of the function */
1867 if (isType (token
, TOKEN_PERIOD
))
1870 * Cannot be a global variable is it has dot references in the name
1873 /* Assume it's an assignment to a global name (e.g. a class) using
1874 * its fully qualified name, so strip the scope.
1875 * FIXME: resolve the scope so we can make more than an assumption. */
1876 vStringClear (token
->scope
);
1877 vStringClear (name
->scope
);
1881 if (! isType(token
, TOKEN_KEYWORD
))
1885 addToScope(token
, name
->string
);
1888 addContext (name
, token
);
1892 else if ( isKeyword(token
, KEYWORD_prototype
) )
1895 * When we reach the "prototype" tag, we infer:
1896 * "BindAgent" is a class
1897 * "build" is a method
1899 * function BindAgent( repeatableIdName, newParentIdName ) {
1903 * Specified function name: "build"
1904 * BindAgent.prototype.build = function( mode ) {
1905 * maybe parse nested functions
1910 * ValidClassOne.prototype = {
1911 * 'validMethodOne' : function(a,b) {},
1912 * 'validMethodTwo' : function(a,b) {}
1916 if (! ( isType (name
, TOKEN_IDENTIFIER
)
1917 || isType (name
, TOKEN_STRING
) ) )
1919 * Unexpected input. Try to reset the parsing.
1921 * TOKEN_STRING is acceptable. e.g.:
1922 * -----------------------------------
1923 * "a".prototype = function( mode ) {}
1927 makeClassTag (name
, NULL
, NULL
);
1931 * There should a ".function_name" next.
1934 if (isType (token
, TOKEN_PERIOD
))
1940 if (! isType(token
, TOKEN_KEYWORD
))
1942 vString
*const signature
= vStringNew ();
1944 addToScope(token
, name
->string
);
1946 copyToken (method_body_token
, token
, true);
1947 readToken (method_body_token
);
1949 while (! isType (method_body_token
, TOKEN_SEMICOLON
) &&
1950 ! isType (method_body_token
, TOKEN_CLOSE_CURLY
) &&
1951 ! isType (method_body_token
, TOKEN_OPEN_CURLY
) &&
1952 ! isType (method_body_token
, TOKEN_EOF
))
1954 if ( isType (method_body_token
, TOKEN_OPEN_PAREN
) )
1955 skipArgumentList(method_body_token
, false,
1956 vStringLength (signature
) == 0 ? signature
: NULL
);
1958 readToken (method_body_token
);
1961 makeJsTag (token
, JSTAG_METHOD
, signature
, NULL
);
1962 vStringDelete (signature
);
1964 if ( isType (method_body_token
, TOKEN_OPEN_CURLY
))
1966 parseBlock (method_body_token
, token
->string
);
1967 is_terminated
= true;
1970 is_terminated
= isType (method_body_token
, TOKEN_SEMICOLON
);
1974 else if (isType (token
, TOKEN_EQUAL_SIGN
))
1977 if (isType (token
, TOKEN_OPEN_CURLY
))
1982 * Creates tags for each of these class methods
1983 * ValidClassOne.prototype = {
1984 * 'validMethodOne' : function(a,b) {},
1985 * 'validMethodTwo' : function(a,b) {}
1988 parseMethods(token
, name
, false);
1990 * Find to the end of the statement
1992 findCmdTerm (token
, false, false);
1993 token
->ignoreTag
= false;
1994 is_terminated
= true;
2001 } while (isType (token
, TOKEN_PERIOD
));
2004 readTokenFull (token
, true, NULL
);
2006 if ( isType (token
, TOKEN_OPEN_PAREN
) )
2007 skipArgumentList(token
, false, NULL
);
2009 if ( isType (token
, TOKEN_OPEN_SQUARE
) )
2010 skipArrayList(token
, false);
2013 if ( isType (token, TOKEN_OPEN_CURLY) )
2015 is_class = parseBlock (token, name->string);
2020 if ( isType (token
, TOKEN_CLOSE_CURLY
) )
2023 * Reaching this section without having
2024 * processed an open curly brace indicates
2025 * the statement is most likely not terminated.
2027 is_terminated
= false;
2031 if ( isType (token
, TOKEN_SEMICOLON
) ||
2032 isType (token
, TOKEN_EOF
) ||
2033 isType (token
, TOKEN_COMMA
) )
2036 * Only create variables for global scope
2038 if ( token
->nestLevel
== 0 && is_global
)
2041 * Handles this syntax:
2044 makeJsTag (name
, is_const
? JSTAG_CONSTANT
: JSTAG_VARIABLE
, NULL
, NULL
);
2047 * Statement has ended.
2048 * This deals with calls to functions, like:
2051 if (isType (token
, TOKEN_COMMA
))
2059 if ( isType (token
, TOKEN_EQUAL_SIGN
) )
2065 /* rvalue might be surrounded with parentheses */
2066 while (isType (token
, TOKEN_OPEN_PAREN
))
2072 if (isKeyword (token
, KEYWORD_async
))
2075 if ( isKeyword (token
, KEYWORD_function
) )
2077 vString
*const signature
= vStringNew ();
2078 bool is_generator
= false;
2081 if (isType (token
, TOKEN_STAR
))
2083 is_generator
= true;
2087 if (! isType (token
, TOKEN_KEYWORD
) &&
2088 ! isType (token
, TOKEN_OPEN_PAREN
))
2091 * Functions of this format:
2092 * var D2A = function theAdd(a, b)
2096 * Are really two separate defined functions and
2097 * can be referenced in two ways:
2098 * alert( D2A(1,2) ); // produces 3
2099 * alert( theAdd(1,2) ); // also produces 3
2100 * So it must have two tags:
2103 * Save the reference to the name for later use, once
2104 * we have established this is a valid function we will
2105 * create the secondary reference to it.
2107 copyToken(secondary_name
, token
, true);
2111 if ( isType (token
, TOKEN_OPEN_PAREN
) )
2112 skipArgumentList(token
, false, signature
);
2114 if (isType (token
, TOKEN_OPEN_CURLY
))
2117 * This will be either a function or a class.
2118 * We can only determine this by checking the body
2119 * of the function. If we find a "this." we know
2120 * it is a class, otherwise it is a function.
2122 if ( is_inside_class
)
2124 makeJsTag (name
, is_generator
? JSTAG_GENERATOR
: JSTAG_METHOD
, signature
, NULL
);
2125 if ( vStringLength(secondary_name
->string
) > 0 )
2126 makeFunctionTag (secondary_name
, signature
, is_generator
);
2127 parseBlock (token
, name
->string
);
2131 if (! ( isType (name
, TOKEN_IDENTIFIER
)
2132 || isType (name
, TOKEN_STRING
)
2133 || isType (name
, TOKEN_KEYWORD
) ) )
2135 /* Unexpected input. Try to reset the parsing. */
2136 JSCRIPT_DEBUG_PRINT("Unexpected input, trying to reset");
2137 vStringDelete (signature
);
2141 is_class
= parseBlock (token
, name
->string
);
2143 makeClassTag (name
, signature
, NULL
);
2145 makeFunctionTag (name
, signature
, is_generator
);
2147 if ( vStringLength(secondary_name
->string
) > 0 )
2148 makeFunctionTag (secondary_name
, signature
, is_generator
);
2152 vStringDelete (signature
);
2154 else if (isKeyword (token
, KEYWORD_class
))
2156 is_terminated
= parseES6Class (token
, name
);
2158 else if (isType (token
, TOKEN_OPEN_CURLY
))
2161 * Creates tags for each of these class methods
2162 * ValidClassOne.prototype = {
2163 * 'validMethodOne' : function(a,b) {},
2164 * 'validMethodTwo' : function(a,b) {}
2166 * Or checks if this is a hash variable.
2169 has_methods
= parseMethods(token
, name
, false);
2171 makeJsTag (name
, JSTAG_CLASS
, NULL
, NULL
);
2175 * Only create variables for global scope
2177 if ( token
->nestLevel
== 0 && is_global
)
2180 * A pointer can be created to the function.
2181 * If we recognize the function/class name ignore the variable.
2182 * This format looks identical to a variable definition.
2183 * A variable defined outside of a block is considered
2184 * a global variable:
2187 * This is not a global variable:
2188 * var g_var = function;
2189 * This is a global variable:
2190 * var g_var = different_var_name;
2192 fulltag
= vStringNew ();
2193 if (vStringLength (token
->scope
) > 0)
2195 vStringCopy(fulltag
, token
->scope
);
2196 vStringPut (fulltag
, '.');
2197 vStringCat (fulltag
, token
->string
);
2201 vStringCopy(fulltag
, token
->string
);
2203 if ( ! stringListHas(FunctionNames
, vStringValue (fulltag
)) &&
2204 ! stringListHas(ClassNames
, vStringValue (fulltag
)) )
2206 makeJsTag (name
, is_const
? JSTAG_CONSTANT
: JSTAG_VARIABLE
, NULL
, NULL
);
2208 vStringDelete (fulltag
);
2211 /* Here we should be at the end of the block, on the close curly.
2212 * If so, read the next token not to confuse that close curly with
2213 * the end of the current statement. */
2214 if (isType (token
, TOKEN_CLOSE_CURLY
))
2216 readTokenFull(token
, true, NULL
);
2217 is_terminated
= isType (token
, TOKEN_SEMICOLON
);
2220 else if (isKeyword (token
, KEYWORD_new
))
2223 is_var
= isType (token
, TOKEN_IDENTIFIER
);
2224 if ( isKeyword (token
, KEYWORD_function
) ||
2225 isKeyword (token
, KEYWORD_capital_function
) ||
2226 isKeyword (token
, KEYWORD_capital_object
) ||
2229 if ( isKeyword (token
, KEYWORD_capital_object
) )
2233 if ( isType (token
, TOKEN_OPEN_PAREN
) )
2234 skipArgumentList(token
, true, NULL
);
2236 if (isType (token
, TOKEN_SEMICOLON
))
2238 if ( token
->nestLevel
== 0 )
2242 makeJsTag (name
, is_const
? JSTAG_CONSTANT
: JSTAG_VARIABLE
, NULL
, NULL
);
2248 makeClassTag (name
, NULL
, NULL
);
2250 /* FIXME: we cannot really get a meaningful
2251 * signature from a `new Function()` call,
2252 * so for now just don't set any */
2253 makeFunctionTag (name
, NULL
, false);
2258 else if (isType (token
, TOKEN_CLOSE_CURLY
))
2259 is_terminated
= false;
2262 else if (! isType (token
, TOKEN_KEYWORD
))
2265 * Only create variables for global scope
2267 if ( token
->nestLevel
== 0 && is_global
)
2270 * A pointer can be created to the function.
2271 * If we recognize the function/class name ignore the variable.
2272 * This format looks identical to a variable definition.
2273 * A variable defined outside of a block is considered
2274 * a global variable:
2277 * This is not a global variable:
2278 * var g_var = function;
2279 * This is a global variable:
2280 * var g_var = different_var_name;
2282 fulltag
= vStringNew ();
2283 if (vStringLength (token
->scope
) > 0)
2285 vStringCopy(fulltag
, token
->scope
);
2286 vStringPut (fulltag
, '.');
2287 vStringCat (fulltag
, token
->string
);
2291 vStringCopy(fulltag
, token
->string
);
2293 if ( ! stringListHas(FunctionNames
, vStringValue (fulltag
)) &&
2294 ! stringListHas(ClassNames
, vStringValue (fulltag
)) )
2296 makeJsTag (name
, is_const
? JSTAG_CONSTANT
: JSTAG_VARIABLE
, NULL
, NULL
);
2298 vStringDelete (fulltag
);
2304 while (parenDepth
> 0 && ! isType (token
, TOKEN_EOF
))
2306 if (isType (token
, TOKEN_OPEN_PAREN
))
2308 else if (isType (token
, TOKEN_CLOSE_PAREN
))
2310 readTokenFull (token
, true, NULL
);
2312 if (isType (token
, TOKEN_CLOSE_CURLY
))
2313 is_terminated
= false;
2316 /* if we aren't already at the cmd end, advance to it and check whether
2317 * the statement was terminated */
2318 if (! isType (token
, TOKEN_CLOSE_CURLY
) &&
2319 ! isType (token
, TOKEN_SEMICOLON
))
2322 * Statements can be optionally terminated in the case of
2323 * statement prior to a close curly brace as in the
2324 * document.write line below:
2326 * function checkForUpdate() {
2328 * document.write("hello from checkForUpdate<br>")
2333 is_terminated
= findCmdTerm (token
, true, true);
2334 /* if we're at a comma, try and read a second var */
2335 if (isType (token
, TOKEN_COMMA
))
2343 vStringCopy(token
->scope
, saveScope
);
2345 deleteToken (secondary_name
);
2346 deleteToken (method_body_token
);
2347 vStringDelete(saveScope
);
2349 JSCRIPT_DEBUG_LEAVE();
2351 return is_terminated
;
2354 static void parseUI5 (tokenInfo
*const token
)
2356 tokenInfo
*const name
= newToken ();
2358 * SAPUI5 is built on top of jQuery.
2359 * It follows a standard format:
2360 * sap.ui.controller("id.of.controller", {
2361 * method_name : function... {
2364 * method_name : function ... {
2368 * Handle the parsing of the initial controller (and the
2369 * same for "view") and then allow the methods to be
2375 if (isType (token
, TOKEN_PERIOD
))
2378 while (! isType (token
, TOKEN_OPEN_PAREN
) &&
2379 ! isType (token
, TOKEN_EOF
))
2385 if (isType (token
, TOKEN_STRING
))
2387 copyToken(name
, token
, true);
2391 if (isType (token
, TOKEN_COMMA
))
2396 parseMethods (token
, name
, false);
2397 } while (! isType (token
, TOKEN_CLOSE_CURLY
) &&
2398 ! isType (token
, TOKEN_EOF
));
2404 static bool parseLine (tokenInfo
*const token
, bool is_inside_class
)
2406 JSCRIPT_DEBUG_ENTER_TEXT("token is '%s' of type %02x",vStringValue(token
->string
),token
->type
);
2408 bool is_terminated
= true;
2410 * Detect the common statements, if, while, for, do, ...
2411 * This is necessary since the last statement within a block "{}"
2412 * can be optionally terminated.
2414 * If the statement is not terminated, we need to tell
2415 * the calling routine to prevent reading an additional token
2416 * looking for the end of the statement.
2419 if (isType(token
, TOKEN_KEYWORD
))
2421 switch (token
->keyword
)
2426 is_terminated
= parseLoop (token
);
2432 case KEYWORD_finally
:
2433 /* Common semantics */
2434 is_terminated
= parseIf (token
);
2436 case KEYWORD_switch
:
2437 parseSwitch (token
);
2439 case KEYWORD_return
:
2442 is_terminated
= parseLine (token
, is_inside_class
);
2444 case KEYWORD_function
:
2445 parseFunction (token
);
2448 is_terminated
= parseES6Class (token
, NULL
);
2451 is_terminated
= parseStatement (token
, is_inside_class
);
2458 * Special case where single line statements may not be
2459 * SEMICOLON terminated. parseBlock needs to know this
2460 * so that it does not read the next token.
2462 is_terminated
= parseStatement (token
, is_inside_class
);
2465 JSCRIPT_DEBUG_LEAVE();
2467 return is_terminated
;
2470 static void parseJsFile (tokenInfo
*const token
)
2472 JSCRIPT_DEBUG_ENTER();
2478 if (isType (token
, TOKEN_KEYWORD
) && token
->keyword
== KEYWORD_sap
)
2480 else if (isType (token
, TOKEN_KEYWORD
) && (token
->keyword
== KEYWORD_export
||
2481 token
->keyword
== KEYWORD_default
))
2482 /* skip those at top-level */;
2484 parseLine (token
, false);
2485 } while (! isType (token
, TOKEN_EOF
));
2487 JSCRIPT_DEBUG_LEAVE();
2490 static void initialize (const langType language
)
2492 Assert (ARRAY_SIZE (JsKinds
) == JSTAG_COUNT
);
2495 TokenPool
= objPoolNew (16, newPoolToken
, deletePoolToken
, clearPoolToken
, NULL
);
2498 static void finalize (langType language CTAGS_ATTR_UNUSED
, bool initialized
)
2503 objPoolDelete (TokenPool
);
2506 static void findJsTags (void)
2508 tokenInfo
*const token
= newToken ();
2511 ClassNames
= stringListNew ();
2512 FunctionNames
= stringListNew ();
2513 LastTokenType
= TOKEN_UNDEFINED
;
2515 parseJsFile (token
);
2517 stringListDelete (ClassNames
);
2518 stringListDelete (FunctionNames
);
2520 FunctionNames
= NULL
;
2521 deleteToken (token
);
2524 if (JSUnicodeConverter
!= (iconv_t
) -2 && /* not created */
2525 JSUnicodeConverter
!= (iconv_t
) -1 /* creation failed */)
2527 iconv_close (JSUnicodeConverter
);
2528 JSUnicodeConverter
= (iconv_t
) -2;
2532 Assert (NextToken
== NULL
);
2535 /* Create parser definition structure */
2536 extern parserDefinition
* JavaScriptParser (void)
2538 // .jsx files are JSX: https://facebook.github.io/jsx/
2539 // which have JS function definitions, so we just use the JS parser
2540 static const char *const extensions
[] = { "js", "jsx", NULL
};
2541 static const char *const aliases
[] = { "js", "node", "nodejs",
2542 "seed", "gjs", NULL
};
2543 parserDefinition
*const def
= parserNew ("JavaScript");
2544 def
->extensions
= extensions
;
2545 def
->aliases
= aliases
;
2547 * New definitions for parsing instead of regex
2549 def
->kindTable
= JsKinds
;
2550 def
->kindCount
= ARRAY_SIZE (JsKinds
);
2551 def
->parser
= findJsTags
;
2552 def
->initialize
= initialize
;
2553 def
->finalize
= finalize
;
2554 def
->keywordTable
= JsKeywordTable
;
2555 def
->keywordCount
= ARRAY_SIZE (JsKeywordTable
);