ctags/parsers/gdscript.c

   1 /*
   2 *   Copyright (c) 2000-2003, Darren Hiebert
   3 *   Copyright (c) 2014-2016, Colomban Wendling <ban@herbesfolles.org>
   4 *       Copyright (c) 2021, David Yang <davidyang6us@gmail.com>
   5 *
   6 *   This source code is released for free distribution under the terms of the
   7 *   GNU General Public License version 2 or (at your option) any later version.
   8 *
   9 *   This module contains functions for generating tags for GDScript language
  10 *   files. This module is derived from the Python module.
  11 *
  12 *       GDScript language reference:
  13 *       https://docs.godotengine.org/en/latest/tutorials/scripting/gdscript/gdscript_basics.html
  14 *       https://docs.godotengine.org/en/stable/development/file_formats/gdscript_grammar.html#doc-gdscript-grammar
  15 *       https://godotengine.org/article/gdscript-progress-report-new-gdscript-now-merged
  16 *
  17 */
  18
  19 #include "general.h"  /* must always come first */
  20
  21 #include <string.h>
  22
  23 #include "entry.h"
  24 #include "nestlevel.h"
  25 #include "read.h"
  26 #include "parse.h"
  27 #include "vstring.h"
  28 #include "keyword.h"
  29 #include "routines.h"
  30 #include "debug.h"
  31 #include "xtag.h"
  32 #include "objpool.h"
  33 #include "strlist.h"
  34
  35 #define isIdentifierChar(c) \
  36         (isalnum (c) || (c) == '_' || (c) >= 0x80)
  37 #define newToken() (objPoolGet (TokenPool))
  38 #define deleteToken(t) (objPoolPut (TokenPool, (t)))
  39
  40 enum {
  41         KEYWORD_class,
  42         KEYWORD_func,
  43         KEYWORD_extends,
  44         KEYWORD_pass,
  45         KEYWORD_return,
  46         KEYWORD_lambda,
  47         KEYWORD_variable,
  48         KEYWORD_const,
  49         KEYWORD_enum,
  50         KEYWORD_class_name,
  51         KEYWORD_signal,
  52         KEYWORD_modifier,
  53 };
  54 typedef int keywordId; /* to allow KEYWORD_NONE */
  55
  56 typedef enum {
  57         ACCESS_PRIVATE,
  58         ACCESS_PROTECTED,
  59         ACCESS_PUBLIC,
  60         COUNT_ACCESS
  61 } accessType;
  62
  63 static const char *const GDScriptAccesses[COUNT_ACCESS] = {
  64         "private",
  65         "protected",
  66         "public"
  67 };
  68
  69 typedef enum {
  70         F_ANNOTATIONS,
  71         COUNT_FIELD
  72 } gdscriptField;
  73
  74 static fieldDefinition GDScriptFields[COUNT_FIELD] = {
  75         { .name = "annotations",
  76           .description = "annotations on functions and variables",
  77           .enabled = true },
  78 };
  79
  80 typedef enum {
  81         K_CLASS,
  82         K_METHOD,
  83         K_VARIABLE,
  84         K_CONST,
  85         K_ENUM,
  86         K_ENUMERATOR,
  87         K_PARAMETER,
  88         K_LOCAL_VARIABLE,
  89         K_SIGNAL,
  90         COUNT_KIND
  91 } gdscriptKind;
  92
  93 typedef enum {
  94         GDSCRIPT_CLASS_EXTENDED,
  95 } gdscriptClassRole;
  96
  97 static roleDefinition GDScriptClassRoles [] = {
  98         { true, "extended",   "used as a base class for extending" },
  99 };
 100
 101 static kindDefinition GDScriptKinds[COUNT_KIND] = {
 102         {true, 'c', "class",    "classes",
 103          .referenceOnly = false, ATTACH_ROLES(GDScriptClassRoles)},
 104         {true, 'm', "method",   "methods"},
 105         {true, 'v', "variable", "variables"},
 106         {true, 'C', "const", "constants"},
 107         {true, 'g', "enum",     "enumeration names"},
 108         {true, 'e', "enumerator",       "enumerated values"},
 109         {false,'z', "parameter",        "function parameters"},
 110         {false,'l', "local",    "local variables"},
 111         {true, 's', "signal",   "signals"},
 112 };
 113
 114 typedef enum {
 115         X_IMPLICIT_CLASS,
 116 } gdscriptXtag;
 117
 118 static xtagDefinition GDScriptXtagTable [] = {
 119         {
 120                 .enabled     = false,
 121                 .name        = "implicitClass",
 122                 .description = "Include tag for the implicitly defined unnamed class",
 123         },
 124 };
 125
 126 static const keywordTable GDScriptKeywordTable[] = {
 127         /* keyword                      keyword ID */
 128         { "class",                      KEYWORD_class                   },
 129         { "func",                       KEYWORD_func                    },
 130         { "extends",            KEYWORD_extends                 },
 131         { "lambda",                     KEYWORD_lambda                  }, // Future GDScript lambda currently uses func, may change
 132         { "pass",                       KEYWORD_pass                    },
 133         { "return",                     KEYWORD_return                  },
 134         { "var",                        KEYWORD_variable                },
 135         { "const",                      KEYWORD_const                   },
 136         { "enum",                       KEYWORD_enum                    },
 137         { "class_name",         KEYWORD_class_name              },
 138         { "signal",                     KEYWORD_signal                  },
 139
 140 };
 141
 142 const static struct keywordGroup modifierKeywords = {
 143         .value = KEYWORD_modifier,
 144         .addingUnlessExisting = false,
 145         .keywords = {
 146                 "static",
 147                 "remote", "remotesync",
 148                 "master", "mastersycn",
 149                 "puppet", "puppetsync",
 150                 NULL,
 151         },
 152 };
 153
 154 typedef enum eTokenType {
 155         /* 0..255 are the byte's value */
 156         TOKEN_EOF = 256,
 157         TOKEN_UNDEFINED,
 158         TOKEN_INDENT,
 159         TOKEN_KEYWORD,
 160         TOKEN_OPERATOR,
 161         TOKEN_IDENTIFIER,
 162         TOKEN_STRING,
 163         TOKEN_ARROW,                            /* -> */
 164         TOKEN_WHITESPACE,
 165 } tokenType;
 166
 167 typedef struct {
 168         int                             type;
 169         keywordId               keyword;
 170         vString *               string;
 171         int                             indent;
 172         unsigned long   lineNumber;
 173         MIOPos                  filePosition;
 174 } tokenInfo;
 175
 176 struct gdscriptNestingLevelUserData {
 177         int indentation;
 178 };
 179 #define GDS_NL(nl) ((struct gdscriptNestingLevelUserData *) nestingLevelGetUserData (nl))
 180
 181 static langType Lang_gdscript;
 182 static unsigned int TokenContinuationDepth = 0;
 183 static tokenInfo *NextToken = NULL;
 184 static NestingLevels *GDScriptNestingLevels = NULL;
 185 static objPool *TokenPool = NULL;
 186
 187
 188 // Always reports single-underscores as protected
 189 static accessType accessFromIdentifier (const vString *const ident, int parentKind)
 190 {
 191         const char *const p = vStringValue (ident);
 192         const size_t len = vStringLength (ident);
 193
 194         /* inside a function/method, private */
 195         if (parentKind != -1 && parentKind != K_CLASS)
 196                 return ACCESS_PRIVATE;
 197         /* not starting with "_", public */
 198         else if (len < 1 || p[0] != '_')
 199                 return ACCESS_PUBLIC;
 200         /* "_...": suggested as non-public, but easily accessible */
 201         else
 202                 return ACCESS_PROTECTED;
 203 }
 204
 205 static void initGDScriptEntry (tagEntryInfo *const e, const tokenInfo *const token,
 206                                                          const gdscriptKind kind)
 207 {
 208         accessType access;
 209         int parentKind = -1;
 210         NestingLevel *nl;
 211
 212         initTagEntry (e, vStringValue (token->string), kind);
 213
 214         e->lineNumber   = token->lineNumber;
 215         e->filePosition = token->filePosition;
 216
 217         nl = nestingLevelsGetCurrent (GDScriptNestingLevels);
 218         if (nl)
 219         {
 220                 tagEntryInfo *nlEntry = getEntryOfNestingLevel (nl);
 221
 222                 e->extensionFields.scopeIndex = nl->corkIndex;
 223
 224                 /* nlEntry can be NULL if a kind was disabled.  But what can we do
 225                  * here?  Even disabled kinds should count for the hierarchy I
 226                  * guess -- as it'd otherwise be wrong -- but with cork we're
 227                  * fucked up as there's nothing to look up.  Damn. */
 228                 if (nlEntry)
 229                         parentKind = nlEntry->kindIndex;
 230         }
 231
 232         access = accessFromIdentifier (token->string, parentKind);
 233         e->extensionFields.access = GDScriptAccesses[access];
 234         /* FIXME: should we really set isFileScope in addition to access? */
 235         if (access == ACCESS_PRIVATE)
 236                 e->isFileScope = true;
 237 }
 238
 239 static int makeClassTag (const tokenInfo *const token,
 240                                                  const vString *const inheritance)
 241 {
 242         if (GDScriptKinds[K_CLASS].enabled)
 243         {
 244                 tagEntryInfo e;
 245
 246                 initGDScriptEntry (&e, token, K_CLASS);
 247
 248                 e.extensionFields.inheritance = inheritance ? vStringValue (inheritance) : "";
 249
 250                 return makeTagEntry (&e);
 251         }
 252
 253         return CORK_NIL;
 254 }
 255
 256 static vString *makeDecoratorString (const stringList *const strlist)
 257 {
 258         vString *vstr = vStringNew ();
 259
 260         for (unsigned int i = 0; i < stringListCount (strlist); i++)
 261         {
 262                 vString *elt = stringListItem (strlist, i);
 263                 if (i != 0 && (vStringValue (elt) > 0
 264                                            && vStringValue (elt)[0] != '('))
 265                         vStringPut (vstr, ',');
 266                 vStringCat (vstr, elt);
 267         }
 268         return vstr;
 269 }
 270
 271 static int makeFunctionTag (const tokenInfo *const token,
 272                                                         int kind,
 273                                                         const vString *const arglist,
 274                                                         const stringList *const decorators)
 275 {
 276         if (GDScriptKinds[kind].enabled)
 277         {
 278                 tagEntryInfo e;
 279                 vString *vstr = NULL;
 280                 int r;
 281
 282                 initGDScriptEntry (&e, token, kind);
 283
 284                 if (arglist)
 285                         e.extensionFields.signature = vStringValue (arglist);
 286                 if (decorators && stringListCount (decorators) > 0)
 287                 {
 288                         vstr = makeDecoratorString (decorators);
 289                         attachParserField (&e, false, GDScriptFields[F_ANNOTATIONS].ftype,
 290                                                            vStringValue (vstr));
 291                 }
 292
 293                 r = makeTagEntry (&e);
 294                 vStringDelete (vstr);   /* NULL is ignored. */
 295
 296                 return r;
 297         }
 298
 299         return CORK_NIL;
 300 }
 301
 302 static int makeSimpleGDScriptTag (const tokenInfo *const token, gdscriptKind const kind)
 303 {
 304         if (GDScriptKinds[kind].enabled)
 305         {
 306                 tagEntryInfo e;
 307
 308                 initGDScriptEntry (&e, token, kind);
 309                 return makeTagEntry (&e);
 310         }
 311
 312         return CORK_NIL;
 313 }
 314
 315 static int makeSimpleGDScriptRefTag (const tokenInfo *const token,
 316                                                                          gdscriptKind const kind,
 317                                                                          int roleIndex, xtagType xtag)
 318 {
 319         if (isXtagEnabled (XTAG_REFERENCE_TAGS))
 320         {
 321                 tagEntryInfo e;
 322
 323                 initRefTagEntry (&e, vStringValue (token->string),
 324                                                  kind, roleIndex);
 325
 326                 e.lineNumber    = token->lineNumber;
 327                 e.filePosition  = token->filePosition;
 328
 329                 if (xtag != XTAG_UNKNOWN)
 330                         markTagExtraBit (&e, xtag);
 331
 332                 return makeTagEntry (&e);
 333         }
 334
 335         return CORK_NIL;
 336 }
 337
 338 static void *newPoolToken (void *createArg CTAGS_ATTR_UNUSED)
 339 {
 340         tokenInfo *token = xMalloc (1, tokenInfo);
 341         token->string = vStringNew ();
 342         return token;
 343 }
 344
 345 static void deletePoolToken (void *data)
 346 {
 347         tokenInfo *token = data;
 348         vStringDelete (token->string);
 349         eFree (token);
 350 }
 351
 352 static void clearPoolToken (void *data)
 353 {
 354         tokenInfo *token = data;
 355
 356         token->type                     = TOKEN_UNDEFINED;
 357         token->keyword          = KEYWORD_NONE;
 358         token->indent           = 0;
 359         token->lineNumber   = getInputLineNumber ();
 360         token->filePosition = getInputFilePosition ();
 361         vStringClear (token->string);
 362 }
 363
 364 static void copyToken (tokenInfo *const dest, const tokenInfo *const src)
 365 {
 366         dest->lineNumber = src->lineNumber;
 367         dest->filePosition = src->filePosition;
 368         dest->type = src->type;
 369         dest->keyword = src->keyword;
 370         dest->indent = src->indent;
 371         vStringCopy(dest->string, src->string);
 372 }
 373
 374 /* Skip a single or double quoted string. */
 375 static void readString (vString *const string, const int delimiter)
 376 {
 377         int escaped = 0;
 378         int c;
 379
 380         while ((c = getcFromInputFile ()) != EOF)
 381         {
 382                 if (escaped)
 383                 {
 384                         vStringPut (string, c);
 385                         escaped--;
 386                 }
 387                 else if (c == '\\')
 388                         escaped++;
 389                 else if (c == delimiter || c == '\n' || c == '\r')
 390                 {
 391                         if (c != delimiter)
 392                                 ungetcToInputFile (c);
 393                         break;
 394                 }
 395                 else
 396                         vStringPut (string, c);
 397         }
 398 }
 399
 400 /* Skip a single or double triple quoted string. */
 401 static void readTripleString (vString *const string, const int delimiter)
 402 {
 403         int c;
 404         int escaped = 0;
 405         int n = 0;
 406         while ((c = getcFromInputFile ()) != EOF)
 407         {
 408                 if (c == delimiter && ! escaped)
 409                 {
 410                         if (++n >= 3)
 411                                 break;
 412                 }
 413                 else
 414                 {
 415                         for (; n > 0; n--)
 416                                 vStringPut (string, delimiter);
 417                         if (c != '\\' || escaped)
 418                                 vStringPut (string, c);
 419                         n = 0;
 420                 }
 421
 422                 if (escaped)
 423                         escaped--;
 424                 else if (c == '\\')
 425                         escaped++;
 426         }
 427 }
 428
 429 static void readIdentifier (vString *const string, const int firstChar)
 430 {
 431         int c = firstChar;
 432         do
 433         {
 434                 vStringPut (string, c);
 435                 c = getcFromInputFile ();
 436         }
 437         while (isIdentifierChar (c));
 438         ungetcToInputFile (c);
 439 }
 440
 441 static void ungetToken (tokenInfo *const token)
 442 {
 443         Assert (NextToken == NULL);
 444         NextToken = newToken ();
 445         copyToken (NextToken, token);
 446 }
 447
 448 static void readTokenFull (tokenInfo *const token, bool inclWhitespaces)
 449 {
 450         int c;
 451         int n;
 452
 453         /* if we've got a token held back, emit it */
 454         if (NextToken)
 455         {
 456                 copyToken (token, NextToken);
 457                 deleteToken (NextToken);
 458                 NextToken = NULL;
 459                 return;
 460         }
 461
 462         token->type             = TOKEN_UNDEFINED;
 463         token->keyword  = KEYWORD_NONE;
 464         vStringClear (token->string);
 465
 466 getNextChar:
 467
 468         n = 0;
 469         do
 470         {
 471                 c = getcFromInputFile ();
 472                 n++;
 473         }
 474         while (c == ' ' || c == '\t' || c == '\f');
 475
 476         token->lineNumber   = getInputLineNumber ();
 477         token->filePosition = getInputFilePosition ();
 478
 479         if (inclWhitespaces && n > 1 && c != '\r' && c != '\n')
 480         {
 481                 ungetcToInputFile (c);
 482                 vStringPut (token->string, ' ');
 483                 token->type = TOKEN_WHITESPACE;
 484                 return;
 485         }
 486
 487         switch (c)
 488         {
 489                 case EOF:
 490                         token->type = TOKEN_EOF;
 491                         break;
 492
 493                 case '\'':
 494                 case '"':
 495                 {
 496                         int d = getcFromInputFile ();
 497                         token->type = TOKEN_STRING;
 498                         vStringPut (token->string, c);
 499                         if (d != c)
 500                         {
 501                                 ungetcToInputFile (d);
 502                                 readString (token->string, c);
 503                         }
 504                         else if ((d = getcFromInputFile ()) == c)
 505                                 readTripleString (token->string, c);
 506                         else /* empty string */
 507                                 ungetcToInputFile (d);
 508                         vStringPut (token->string, c);
 509                         token->lineNumber = getInputLineNumber ();
 510                         token->filePosition = getInputFilePosition ();
 511                         break;
 512                 }
 513
 514                 case '=':
 515                 {
 516                         int d = getcFromInputFile ();
 517                         vStringPut (token->string, c);
 518                         if (d == c)
 519                         {
 520                                 vStringPut (token->string, d);
 521                                 token->type = TOKEN_OPERATOR;
 522                         }
 523                         else
 524                         {
 525                                 ungetcToInputFile (d);
 526                                 token->type = c;
 527                         }
 528                         break;
 529                 }
 530
 531                 case '-':
 532                 {
 533                         int d = getcFromInputFile ();
 534                         if (d == '>')
 535                         {
 536                                 vStringPut (token->string, c);
 537                                 vStringPut (token->string, d);
 538                                 token->type = TOKEN_ARROW;
 539                                 break;
 540                         }
 541                         ungetcToInputFile (d);
 542                         /* fall through */
 543                 }
 544                 case '+':
 545                 case '*':
 546                 case '%':
 547                 case '<':
 548                 case '>':
 549                 case '/':
 550                 {
 551                         int d = getcFromInputFile ();
 552                         vStringPut (token->string, c);
 553                         if (d != '=')
 554                         {
 555                                 ungetcToInputFile (d);
 556                                 token->type = c;
 557                         }
 558                         else
 559                         {
 560                                 vStringPut (token->string, d);
 561                                 token->type = TOKEN_OPERATOR;
 562                         }
 563                         break;
 564                 }
 565
 566                 /* eats newline to implement line continuation  */
 567                 case '\\':
 568                 {
 569                         int d = getcFromInputFile ();
 570                         if (d == '\r')
 571                                 d = getcFromInputFile ();
 572                         if (d != '\n')
 573                                 ungetcToInputFile (d);
 574                         goto getNextChar;
 575                 }
 576
 577                 case '#': /* comment */
 578                 case '\r': /* newlines for indent */
 579                 case '\n':
 580                 {
 581                         int indent = 0;
 582                         do
 583                         {
 584                                 if (c == '#')
 585                                 {
 586                                         do
 587                                                 c = getcFromInputFile ();
 588                                         while (c != EOF && c != '\r' && c != '\n');
 589                                 }
 590                                 if (c == '\r')
 591                                 {
 592                                         int d = getcFromInputFile ();
 593                                         if (d != '\n')
 594                                                 ungetcToInputFile (d);
 595                                 }
 596                                 indent = 0;
 597                                 while ((c = getcFromInputFile ()) == ' ' || c == '\t' || c == '\f')
 598                                 {
 599                                         if (c == '\t')
 600                                                 indent += 8 - (indent % 8);
 601                                         else if (c == '\f') /* yeah, it's weird */
 602                                                 indent = 0;
 603                                         else
 604                                                 indent++;
 605                                 }
 606                         } /* skip completely empty lines, so retry */
 607                         while (c == '\r' || c == '\n' || c == '#');
 608                         ungetcToInputFile (c);
 609                         if (TokenContinuationDepth > 0)
 610                         {
 611                                 if (inclWhitespaces)
 612                                 {
 613                                         vStringPut (token->string, ' ');
 614                                         token->type = TOKEN_WHITESPACE;
 615                                 }
 616                                 else
 617                                         goto getNextChar;
 618                         }
 619                         else
 620                         {
 621                                 token->type = TOKEN_INDENT;
 622                                 token->indent = indent;
 623                         }
 624                         break;
 625                 }
 626
 627                 default:
 628                         if (! isIdentifierChar (c))
 629                         {
 630                                 vStringPut (token->string, c);
 631                                 token->type = c;
 632                         }
 633                         else
 634                         {
 635                                 /* FIXME: handle U, B, R and F string prefixes? */
 636                                 readIdentifier (token->string, c);
 637                                 token->keyword = lookupKeyword (vStringValue (token->string), Lang_gdscript);
 638                                 if (token->keyword == KEYWORD_NONE)
 639                                         token->type = TOKEN_IDENTIFIER;
 640                                 else
 641                                         token->type = TOKEN_KEYWORD;
 642                         }
 643                         break;
 644         }
 645
 646         // handle implicit continuation lines not to emit INDENT inside brackets
 647         if (token->type == '(' ||
 648                 token->type == '{' ||
 649                 token->type == '[')
 650         {
 651                 TokenContinuationDepth ++;
 652         }
 653         else if (TokenContinuationDepth > 0 &&
 654                          (token->type == ')' ||
 655                           token->type == '}' ||
 656                           token->type == ']'))
 657         {
 658                 TokenContinuationDepth --;
 659         }
 660 }
 661
 662 static void readToken (tokenInfo *const token)
 663 {
 664         readTokenFull (token, false);
 665 }
 666
 667 /*================================= parsing =================================*/
 668
 669
 670 static void reprCat (vString *const repr, const tokenInfo *const token)
 671 {
 672         if (token->type != TOKEN_INDENT &&
 673                 token->type != TOKEN_WHITESPACE)
 674         {
 675                 vStringCat (repr, token->string);
 676         }
 677         else if (vStringLength (repr) > 0 && vStringLast (repr) != ' ')
 678         {
 679                 vStringPut (repr, ' ');
 680         }
 681 }
 682
 683 static bool skipOverPair (tokenInfo *const token, int tOpen, int tClose,
 684                                                   vString *const repr, bool reprOuterPair)
 685 {
 686         if (token->type == tOpen)
 687         {
 688                 int depth = 1;
 689
 690                 if (repr && reprOuterPair)
 691                         reprCat (repr, token);
 692                 do
 693                 {
 694                         readTokenFull (token, true);
 695                         if (repr && (reprOuterPair || token->type != tClose || depth > 1))
 696                         {
 697                                 reprCat (repr, token);
 698                         }
 699                         if (token->type == tOpen)
 700                                 depth ++;
 701                         else if (token->type == tClose)
 702                                 depth --;
 703                 }
 704                 while (token->type != TOKEN_EOF && depth > 0);
 705         }
 706
 707         return token->type == tClose;
 708 }
 709
 710 static void readQualifiedName (tokenInfo *const nameToken)
 711 {
 712         readToken (nameToken);
 713
 714         if (nameToken->type == TOKEN_IDENTIFIER ||
 715                 nameToken->type == '.')
 716         {
 717                 vString *qualifiedName = vStringNew ();
 718                 tokenInfo *token = newToken ();
 719
 720                 while (nameToken->type == TOKEN_IDENTIFIER ||
 721                            nameToken->type == '.')
 722                 {
 723                         vStringCat (qualifiedName, nameToken->string);
 724                         copyToken (token, nameToken);
 725
 726                         readToken (nameToken);
 727                 }
 728                 /* put the last, non-matching, token back */
 729                 ungetToken (nameToken);
 730
 731                 copyToken (nameToken, token);
 732                 nameToken->type = TOKEN_IDENTIFIER;
 733                 vStringCopy (nameToken->string, qualifiedName);
 734
 735                 deleteToken (token);
 736                 vStringDelete (qualifiedName);
 737         }
 738 }
 739
 740 static vString *parseParamTypeAnnotation (tokenInfo *const token,
 741                                                                                   vString *arglist)
 742 {
 743         readToken (token);
 744         if (token->type != ':')
 745         {
 746                 ungetToken (token);
 747                 return NULL;
 748         }
 749
 750         reprCat (arglist, token);
 751         int depth = 0;
 752         vString *t = vStringNew ();
 753         while (true)
 754         {
 755                 readTokenFull (token, true);
 756                 if (token->type == TOKEN_WHITESPACE)
 757                 {
 758                         reprCat (arglist, token);
 759                         continue;
 760                 }
 761                 else if (token->type == TOKEN_EOF)
 762                         break;
 763
 764                 if (token->type == '(' ||
 765                         token->type == '[' ||
 766                         token->type == '{')
 767                         depth ++;
 768                 else if (token->type == ')' ||
 769                                  token->type == ']' ||
 770                                  token->type == '}')
 771                         depth --;
 772
 773                 if (depth < 0
 774                         || (depth == 0 && (token->type == '='
 775                                                            || token->type == ',')))
 776                 {
 777                         ungetToken (token);
 778                         return t;
 779                 }
 780                 reprCat (arglist, token);
 781                 reprCat (t, token);
 782         }
 783         vStringDelete (t);
 784         return NULL;
 785 }
 786
 787 static vString *parseReturnTypeAnnotation (tokenInfo *const token)
 788 {
 789         readToken (token);
 790         if (token->type != TOKEN_ARROW)
 791         {
 792                 return NULL;
 793         }
 794
 795         int depth = 0;
 796         vString *t = vStringNew ();
 797         while (true)
 798         {
 799                 readToken (token);
 800                 if (token->type == TOKEN_EOF)
 801                         break;
 802
 803                 if (token->type == '(' ||
 804                         token->type == '[' ||
 805                         token->type == '{')
 806                         depth ++;
 807                 else if (token->type == ')' ||
 808                                  token->type == ']' ||
 809                                  token->type == '}')
 810                         depth --;
 811                 if (depth == 0 && token->type == ':')
 812                 {
 813                         ungetToken (token);
 814                         return t;
 815                 }
 816                 else
 817                         reprCat (t, token);
 818         }
 819         vStringDelete (t);
 820         return NULL;
 821 }
 822
 823 static bool parseClassOrDef (tokenInfo *const token,
 824                                                          const stringList *const decorators,
 825                                                          gdscriptKind kind, bool isCDef)
 826 {
 827         vString *arglist = NULL;
 828         tokenInfo *name = NULL;
 829         tokenInfo *parameterTokens[16] = { NULL };
 830         vString   *parameterTypes [ARRAY_SIZE(parameterTokens)] = { NULL };
 831         unsigned int parameterCount = 0;
 832         NestingLevel *lv;
 833         int corkIndex;
 834
 835         readToken (token);
 836         if (token->type != TOKEN_IDENTIFIER)
 837                 return false;
 838
 839         name = newToken ();
 840         copyToken (name, token);
 841
 842         readToken (token);
 843         /* collect parameters or inheritance */
 844         if (token->type == '(')
 845         {
 846                 int prevTokenType = token->type;
 847                 int depth = 1;
 848
 849                 arglist = vStringNew ();
 850                 if (kind != K_CLASS)
 851                         reprCat (arglist, token);
 852
 853                 do
 854                 {
 855                         if (token->type != TOKEN_WHITESPACE &&
 856                                 token->type != '*')
 857                         {
 858                                 prevTokenType = token->type;
 859                         }
 860
 861                         readTokenFull (token, true);
 862                         if (kind != K_CLASS || token->type != ')' || depth > 1)
 863                                 reprCat (arglist, token);
 864
 865                         if (token->type == '(' ||
 866                                 token->type == '[' ||
 867                                 token->type == '{')
 868                                 depth ++;
 869                         else if (token->type == ')' ||
 870                                          token->type == ']' ||
 871                                          token->type == '}')
 872                                 depth --;
 873                         else if (kind != K_CLASS && depth == 1 &&
 874                                          token->type == TOKEN_IDENTIFIER &&
 875                                          (prevTokenType == '(' || prevTokenType == ',') &&
 876                                          parameterCount < ARRAY_SIZE (parameterTokens) &&
 877                                          GDScriptKinds[K_PARAMETER].enabled)
 878                         {
 879                                 tokenInfo *parameterName = newToken ();
 880
 881                                 copyToken (parameterName, token);
 882                                 parameterTokens[parameterCount] = parameterName;
 883                                 parameterTypes [parameterCount++] = parseParamTypeAnnotation (token, arglist);
 884                         }
 885                 }
 886                 while (token->type != TOKEN_EOF && depth > 0);
 887         }
 888         else if (token->type == TOKEN_KEYWORD && token->keyword == KEYWORD_extends)
 889         {
 890                 readToken (token);
 891                 if (token->type == TOKEN_IDENTIFIER)
 892                 {
 893                         makeSimpleGDScriptRefTag (token, K_CLASS, GDSCRIPT_CLASS_EXTENDED, XTAG_UNKNOWN);
 894                         arglist = vStringNewCopy (token->string);
 895                 }
 896         }
 897         else if (kind == K_SIGNAL)
 898         {
 899                 /* signal can be defined with no parameter list. */
 900                 ungetToken (token);
 901         }
 902
 903         if (kind == K_CLASS)
 904                 corkIndex = makeClassTag (name, arglist);
 905         else
 906                 corkIndex = makeFunctionTag (name, kind, arglist, decorators);
 907
 908         lv = nestingLevelsPush (GDScriptNestingLevels, corkIndex);
 909         GDS_NL (lv)->indentation = token->indent;
 910
 911         deleteToken (name);
 912         vStringDelete (arglist);
 913
 914         if (parameterCount > 0)
 915         {
 916                 unsigned int i;
 917
 918                 for (i = 0; i < parameterCount; i++)
 919                 {
 920                         int paramCorkIndex = makeSimpleGDScriptTag (parameterTokens[i], K_PARAMETER);
 921                         deleteToken (parameterTokens[i]);
 922                         tagEntryInfo *e = getEntryInCorkQueue (paramCorkIndex);
 923                         if (e && parameterTypes[i])
 924                         {
 925                                 e->extensionFields.typeRef [0] = eStrdup ("typename");
 926                                 e->extensionFields.typeRef [1] = vStringDeleteUnwrap (parameterTypes[i]);
 927                                 parameterTypes[i] = NULL;
 928                         }
 929                         vStringDelete (parameterTypes[i]); /* NULL is acceptable. */
 930                 }
 931         }
 932
 933         tagEntryInfo *e;
 934         vString *t;
 935         if (kind != K_CLASS
 936                 && (e = getEntryInCorkQueue (corkIndex))
 937                 && (t = parseReturnTypeAnnotation (token)))
 938         {
 939                 e->extensionFields.typeRef [0] = eStrdup ("typename");
 940                 e->extensionFields.typeRef [1] = vStringDeleteUnwrap (t);
 941         }
 942
 943         if (kind == K_SIGNAL)
 944                 nestingLevelsPop (GDScriptNestingLevels);
 945
 946         return true;
 947 }
 948
 949 static bool parseEnum (tokenInfo *const token)
 950 {
 951         int corkIndex;
 952
 953         readToken (token);
 954
 955         if (token->type == '{')
 956         {
 957                 tokenInfo *name = newToken ();
 958                 copyToken (name, token);
 959                 vStringClear (name->string);
 960                 anonGenerate (name->string, "anon_enum_", K_ENUM);
 961                 name->type = TOKEN_IDENTIFIER;
 962                 corkIndex = makeSimpleGDScriptTag (name, K_ENUM);
 963                 deleteToken (name);
 964                 tagEntryInfo *e = getEntryInCorkQueue (corkIndex);
 965                 if (e)
 966                         markTagExtraBit (e, XTAG_ANONYMOUS);
 967         }
 968         else if (token->type == TOKEN_IDENTIFIER)
 969         {
 970                 corkIndex = makeSimpleGDScriptTag(token, K_ENUM);
 971                 readToken (token);
 972         }
 973         else
 974                 return false;
 975
 976         if (token->type != '{')
 977                 return false;
 978
 979         readToken (token);
 980         nestingLevelsPush (GDScriptNestingLevels, corkIndex);
 981
 982         while (token->type != '}' && token->type != TOKEN_EOF)
 983         {
 984                 if (token->type == TOKEN_IDENTIFIER)
 985                         makeSimpleGDScriptTag(token, K_ENUMERATOR);
 986                 else if (token->type == '=')
 987                 {
 988                         /* Skip the right value. */
 989                         do
 990                                 readToken (token);
 991                         while (token->type != ','
 992                                    && token->type != '}'
 993                                    && token->type != TOKEN_EOF);
 994                         if (token->type != ',')
 995                                 continue;
 996                 }
 997                 readToken (token);
 998         }
 999
1000         tagEntryInfo *e;
1001         vString *t;
1002         if ((e = getEntryInCorkQueue (corkIndex))
1003                 && (t = parseReturnTypeAnnotation (token)))
1004         {
1005                 e->extensionFields.typeRef [0] = eStrdup ("typename");
1006                 e->extensionFields.typeRef [1] = vStringDeleteUnwrap (t);
1007         }
1008
1009         nestingLevelsPop (GDScriptNestingLevels);
1010         return true;
1011 }
1012
1013 static bool parseClassName (tokenInfo *const token)
1014 {
1015         readToken (token);
1016         if (token->type == TOKEN_IDENTIFIER)
1017         {
1018                 /* A class name is explicitly given with "class_name" keyword.
1019                  * Let's overwrite the anonymous tag for the class
1020                  */
1021                 NestingLevel *nl = nestingLevelsGetNthFromRoot (GDScriptNestingLevels, 0);
1022                 tagEntryInfo *klass = nl? getEntryInCorkQueue (nl->corkIndex): NULL;
1023
1024                 tagEntryInfo e;
1025                 char *name = vStringStrdup (token->string);
1026                 initTagEntry (&e, klass? "UNUSED": name, K_CLASS);
1027
1028                 if (klass)
1029                 {
1030                         eFree ((void *)klass->name);
1031                         klass->name = name;
1032                         name = NULL;
1033                         unmarkTagExtraBit(klass, XTAG_ANONYMOUS);
1034
1035                         /* Adjust the position. */
1036                         setTagPositionFromTag (klass, &e);
1037                 }
1038
1039                 /* Extract B in class_name C extends B */
1040                 readToken (token);
1041                 if (token->type == TOKEN_KEYWORD
1042                         && token->keyword == KEYWORD_extends)
1043                 {
1044                         readToken (token);
1045                         if (token->type == TOKEN_IDENTIFIER)
1046                         {
1047                                 makeSimpleGDScriptRefTag (token, K_CLASS,
1048                                                                                   GDSCRIPT_CLASS_EXTENDED,
1049                                                                                   XTAG_UNKNOWN);
1050                                 if (klass)
1051                                 {
1052                                         if (klass->extensionFields.inheritance)
1053                                                 eFree ((void *)klass->extensionFields.inheritance);
1054                                         klass->extensionFields.inheritance = vStringStrdup (token->string);
1055                                 }
1056                                 else
1057                                         e.extensionFields.inheritance = vStringValue(token->string);
1058                         }
1059                 }
1060
1061                 if (!klass)
1062                         makeTagEntry (&e);
1063
1064                 if (name)
1065                         eFree (name);
1066         }
1067
1068         while (token->type != TOKEN_EOF &&
1069                    token->type != ';' &&
1070                    token->type != TOKEN_INDENT)
1071                 readToken (token);
1072
1073         return false;
1074 }
1075
1076 static bool parseExtends (tokenInfo *const token)
1077 {
1078         if (token->keyword == KEYWORD_extends)
1079         {
1080                 readQualifiedName (token);
1081                 if (token->type == TOKEN_IDENTIFIER)
1082                 {
1083                         makeSimpleGDScriptRefTag (token, K_CLASS, GDSCRIPT_CLASS_EXTENDED, XTAG_UNKNOWN);
1084                         NestingLevel *nl = nestingLevelsGetCurrent (GDScriptNestingLevels);
1085                         if (nl)
1086                         {
1087                                 tagEntryInfo *klass = getEntryInCorkQueue (nl->corkIndex);
1088                                 if (klass)
1089                                 {
1090                                         if (klass->extensionFields.inheritance)
1091                                                 eFree ((void *)klass->extensionFields.inheritance);
1092                                         klass->extensionFields.inheritance = vStringStrdup(token->string);
1093                                 }
1094                         }
1095                 }
1096         }
1097         readToken (token);
1098         return false;
1099 }
1100
1101 /* this only handles the most common cases, but an annotation can be any
1102  * expression in theory.
1103  * this function assumes there must be an annotation, and doesn't do any check
1104  * on the token on which it is called: the caller should do that part. */
1105 static bool skipVariableTypeAnnotation (tokenInfo *const token, vString *const repr)
1106 {
1107         bool readNext = true;
1108
1109         readToken (token);
1110         switch (token->type)
1111         {
1112                 case '[': readNext = skipOverPair (token, '[', ']', repr, true); break;
1113                 case '(': readNext = skipOverPair (token, '(', ')', repr, true); break;
1114                 case '{': readNext = skipOverPair (token, '{', '}', repr, true); break;
1115                 default: reprCat (repr, token);
1116         }
1117         if (readNext)
1118                 readToken (token);
1119         /* skip subscripts and calls */
1120         while (token->type == '[' || token->type == '(' || token->type == '.' || token->type == '|')
1121         {
1122                 switch (token->type)
1123                 {
1124                         case '[': readNext = skipOverPair (token, '[', ']', repr, true); break;
1125                         case '(': readNext = skipOverPair (token, '(', ')', repr, true); break;
1126                         case '|':
1127                                 reprCat (repr, token);
1128                                 skipVariableTypeAnnotation (token, repr);
1129                                 readNext = false;
1130                                 break;
1131                         case '.':
1132                                 reprCat (repr, token);
1133                                 readToken (token);
1134                                 readNext = token->type == TOKEN_IDENTIFIER;
1135                                 if (readNext)
1136                                         reprCat (repr, token);
1137                                 break;
1138                         default:  readNext = false; break;
1139                 }
1140                 if (readNext)
1141                         readToken (token);
1142         }
1143
1144         return false;
1145 }
1146
1147 static bool parseVariable (tokenInfo *const token, const gdscriptKind kind,
1148                                                         const stringList *const decorators,
1149                                                         const int keyword)
1150 {
1151         readToken(token);
1152         vString *type = vStringNew();
1153         tokenInfo *name = newToken ();
1154         copyToken (name, token);
1155         if (!name)
1156                 return false;
1157
1158         readToken (token);
1159         // Variable declarations with dotted names are illegal
1160         if (token->type == '.')
1161                 return false;
1162
1163         /* (parse and) skip annotations.  we need not to be too permissive because we
1164          * aren't yet sure we're actually parsing a variable. */
1165         if (token->type == ':' && skipVariableTypeAnnotation (token, type))
1166                 readToken (token);
1167
1168         int index = makeSimpleGDScriptTag (name, kind);
1169         deleteToken(name);
1170         tagEntryInfo *e = getEntryInCorkQueue (index);
1171
1172         if (e && decorators && stringListCount (decorators) > 0)
1173         {
1174                 vString *vstr = makeDecoratorString (decorators);
1175                 attachParserField (e, true, GDScriptFields[F_ANNOTATIONS].ftype,
1176                                                    vStringValue (vstr));
1177                 vStringDelete (vstr);
1178         }
1179
1180         vString *vtype = vStringNew();
1181         char * stype = vStringValue (type);
1182         if (strcmp(stype, "=") && strcmp(stype, ""))
1183         {
1184                         vStringCatS(vtype, stype);
1185         }
1186         vStringDelete(type);
1187
1188         if (e && vStringLength(vtype) > 0) /// TODO: Fix types away
1189         {
1190                 e->extensionFields.typeRef [0] = eStrdup ("typename");
1191                 e->extensionFields.typeRef [1] = vStringDeleteUnwrap (vtype);
1192         }
1193         else
1194         {
1195                 vStringDelete(vtype);
1196         }
1197
1198
1199         while ((TokenContinuationDepth > 0 || token->type != ',') &&
1200                    token->type != TOKEN_EOF &&
1201                    token->type != ';' &&
1202                    token->type != TOKEN_INDENT)
1203         {
1204                 readToken (token);
1205         }
1206
1207
1208         return false;
1209 }
1210
1211 /* pops any level >= to indent */
1212 static void setIndent (tokenInfo *const token)
1213 {
1214         NestingLevel *lv = nestingLevelsGetCurrent (GDScriptNestingLevels);
1215
1216         while (lv && GDS_NL (lv)->indentation >= token->indent)
1217         {
1218                 tagEntryInfo *e = getEntryInCorkQueue (lv->corkIndex);
1219                 if (e)
1220                         e->extensionFields.endLine = token->lineNumber;
1221
1222                 nestingLevelsPop (GDScriptNestingLevels);
1223                 lv = nestingLevelsGetCurrent (GDScriptNestingLevels);
1224         }
1225 }
1226
1227 static int prepareUnnamedClass (struct NestingLevels *nls)
1228 {
1229         {
1230                 /* Ugly: we need a "position" on the input stream for making a tag.
1231                  * At the begining of parsing, the position is undefined.
1232                  * By reading a byte, the position is defined.
1233                  */
1234                 int c = getcFromInputFile ();
1235                 if (c == EOF)
1236                         return CORK_NIL;
1237                 ungetcToInputFile (c);
1238         }
1239
1240         vString * tmp_class = anonGenerateNew ("anon_class_", K_CLASS);
1241         int corkIndex = makeSimpleTag (tmp_class, K_CLASS);
1242         vStringDelete (tmp_class);
1243
1244         tagEntryInfo *e = getEntryInCorkQueue (corkIndex);
1245         if (e)
1246                 markTagExtraBit (e, XTAG_ANONYMOUS);
1247
1248         /* This virtual scope should not be poped. */
1249         NestingLevel *lv = nestingLevelsPush (nls, corkIndex);
1250         GDS_NL (lv)->indentation = -1;
1251
1252         return corkIndex;
1253 }
1254
1255 static void findGDScriptTags (void)
1256 {
1257         tokenInfo *const token = newToken ();
1258         stringList *decorators = stringListNew();
1259         bool atStatementStart = true;
1260
1261         TokenContinuationDepth = 0;
1262         NextToken = NULL;
1263         GDScriptNestingLevels = nestingLevelsNew (sizeof (struct gdscriptNestingLevelUserData));
1264
1265         if (isXtagEnabled (GDScriptXtagTable[X_IMPLICIT_CLASS].xtype))
1266                 prepareUnnamedClass (GDScriptNestingLevels);
1267
1268         readToken (token);
1269         while (token->type != TOKEN_EOF)
1270         {
1271                 tokenType iterationTokenType = token->type;
1272                 int iterationTokenKeyword = token->keyword;
1273                 bool readNext = true;
1274
1275                 if (token->type == TOKEN_INDENT)
1276                         setIndent (token);
1277                 else if (token->keyword == KEYWORD_class ||
1278                                  token->keyword == KEYWORD_func  ||
1279                                  token->keyword == KEYWORD_signal)
1280                 {
1281                         gdscriptKind kind = K_METHOD;
1282                         switch (token->keyword)
1283                         {
1284                         case KEYWORD_class:  kind = K_CLASS;  break;
1285                         case KEYWORD_func:   kind = K_METHOD; break;
1286                         case KEYWORD_signal: kind = K_SIGNAL; break;
1287                         default:
1288                                 AssertNotReached();
1289                         }
1290                         readNext = parseClassOrDef (token, decorators, kind, false);
1291                 }
1292                 else if (token->keyword == KEYWORD_extends)
1293                 {
1294                         readNext = parseExtends (token);
1295                 }
1296                 else if (token->type == '(')
1297                 { /* skip parentheses to avoid finding stuff inside them */
1298                         readNext = skipOverPair (token, '(', ')', NULL, false);
1299                 }
1300                 else if (token->keyword == KEYWORD_variable || token->keyword == KEYWORD_const)
1301                 {
1302                         NestingLevel *lv = nestingLevelsGetCurrent (GDScriptNestingLevels);
1303                         tagEntryInfo *lvEntry = NULL;
1304                         gdscriptKind kind = K_VARIABLE;
1305
1306                         if (lv)
1307                                 lvEntry = getEntryOfNestingLevel (lv);
1308
1309                         if (lvEntry && lvEntry->kindIndex != K_CLASS)
1310                                 kind = K_LOCAL_VARIABLE;
1311
1312                         if (token->keyword == KEYWORD_const)
1313                                 kind = K_CONST;
1314
1315                         readNext = parseVariable (token, kind, decorators, token->keyword);
1316                 }
1317                 else if (token->keyword == KEYWORD_enum)
1318                 {
1319                         readNext = parseEnum (token);
1320                 }
1321                 else if (token->keyword == KEYWORD_class_name)
1322                 {
1323                         readNext = parseClassName (token);
1324                 }
1325                 else if (token->type == TOKEN_KEYWORD
1326                                  && token->keyword == KEYWORD_modifier)
1327                 {
1328                         stringListAdd (decorators, vStringNewCopy(token->string));
1329                 }
1330                 else if (token->type == '@' && atStatementStart &&
1331                                  GDScriptFields[F_ANNOTATIONS].enabled)
1332                 {
1333                         /* collect decorators */
1334                         readQualifiedName (token);
1335                         if (token->type != TOKEN_IDENTIFIER
1336                                 && (token->keyword != KEYWORD_modifier))
1337                                 readNext = false;
1338                         else
1339                         {
1340                                 stringListAdd (decorators, vStringNewCopy(token->string));
1341                                 readToken (token);
1342
1343                                 vString *d = vStringNew ();
1344                                 readNext = skipOverPair (token, '(', ')', d, true);
1345                                 if (vStringLength (d) > 0)
1346                                         stringListAdd (decorators, d);
1347                                 else
1348                                         vStringDelete (d);
1349                         }
1350                 }
1351
1352                 /* clear collected decorators for any non-decorator tokens non-indent
1353                  * token.  decorator collection takes care of skipping the possible
1354                  * argument list, so we should never hit here parsing a decorator */
1355                 if (iterationTokenType != TOKEN_INDENT &&
1356                         iterationTokenType != '@' &&
1357                         iterationTokenKeyword != KEYWORD_modifier &&
1358                         GDScriptFields[F_ANNOTATIONS].enabled)
1359                 {
1360                         stringListClear (decorators);
1361                 }
1362
1363                 atStatementStart = (token->type == TOKEN_INDENT || token->type == ';');
1364
1365                 if (readNext)
1366                         readToken (token);
1367         }
1368
1369         nestingLevelsFree (GDScriptNestingLevels);
1370         stringListDelete (decorators);
1371         deleteToken (token);
1372         Assert (NextToken == NULL);
1373 }
1374
1375 static void initialize (const langType language)
1376 {
1377         Lang_gdscript = language;
1378
1379         TokenPool = objPoolNew (16, newPoolToken, deletePoolToken, clearPoolToken, NULL);
1380         addKeywordGroup (&modifierKeywords, language);
1381 }
1382
1383 static void finalize (langType language CTAGS_ATTR_UNUSED, bool initialized)
1384 {
1385         if (!initialized)
1386                 return;
1387
1388         objPoolDelete (TokenPool);
1389 }
1390
1391 extern parserDefinition* GDScriptParser (void)
1392 {
1393         static const char *const extensions[] = { "gd", NULL };
1394         parserDefinition *def = parserNew ("GDScript");
1395         def->kindTable = GDScriptKinds;
1396         def->kindCount = ARRAY_SIZE (GDScriptKinds);
1397         def->extensions = extensions;
1398         def->parser = findGDScriptTags;
1399         def->initialize = initialize;
1400         def->finalize = finalize;
1401         def->keywordTable = GDScriptKeywordTable;
1402         def->keywordCount = ARRAY_SIZE (GDScriptKeywordTable);
1403         def->fieldTable = GDScriptFields;
1404         def->fieldCount = ARRAY_SIZE (GDScriptFields);
1405         def->xtagTable     = GDScriptXtagTable;
1406         def->xtagCount     = ARRAY_SIZE(GDScriptXtagTable);
1407         def->useCork = CORK_QUEUE;
1408         def->requestAutomaticFQTag = true;
1409         return def;
1410 }