ctags/parsers/gdscript.c

   1 /*
   2 *   Copyright (c) 2000-2003, Darren Hiebert
   3 *   Copyright (c) 2014-2016, Colomban Wendling <ban@herbesfolles.org>
   4 *       Copyright (c) 2021, David Yang <davidyang6us@gmail.com>
   5 *
   6 *   This source code is released for free distribution under the terms of the
   7 *   GNU General Public License version 2 or (at your option) any later version.
   8 *
   9 *   This module contains functions for generating tags for GDScript language
  10 *   files. This module is derived from the Python module.
  11 *
  12 *       GDScript language reference:
  13 *       https://docs.godotengine.org/en/latest/tutorials/scripting/gdscript/gdscript_basics.html
  14 *       https://docs.godotengine.org/en/stable/development/file_formats/gdscript_grammar.html#doc-gdscript-grammar
  15 *       https://godotengine.org/article/gdscript-progress-report-new-gdscript-now-merged
  16 *
  17 */
  18
  19 #include "general.h"  /* must always come first */
  20
  21 #include <string.h>
  22
  23 #include "entry.h"
  24 #include "nestlevel.h"
  25 #include "read.h"
  26 #include "parse.h"
  27 #include "vstring.h"
  28 #include "keyword.h"
  29 #include "routines.h"
  30 #include "debug.h"
  31 #include "xtag.h"
  32 #include "objpool.h"
  33 #include "strlist.h"
  34
  35 #define isIdentifierChar(c) \
  36         (isalnum (c) || (c) == '_' || (c) >= 0x80)
  37 #define newToken() (objPoolGet (TokenPool))
  38 #define deleteToken(t) (objPoolPut (TokenPool, (t)))
  39
  40 enum {
  41         KEYWORD_class,
  42         KEYWORD_func,
  43         KEYWORD_extends,
  44         KEYWORD_pass,
  45         KEYWORD_return,
  46         KEYWORD_lambda,
  47         KEYWORD_variable,
  48         KEYWORD_const,
  49         KEYWORD_enum,
  50         KEYWORD_class_name,
  51         KEYWORD_signal,
  52         KEYWORD_modifier,
  53 };
  54 typedef int keywordId; /* to allow KEYWORD_NONE */
  55
  56 typedef enum {
  57         ACCESS_PRIVATE,
  58         ACCESS_PROTECTED,
  59         ACCESS_PUBLIC,
  60         COUNT_ACCESS
  61 } accessType;
  62
  63 static const char *const GDScriptAccesses[COUNT_ACCESS] = {
  64         "private",
  65         "protected",
  66         "public"
  67 };
  68
  69 typedef enum {
  70         F_ANNOTATIONS,
  71         COUNT_FIELD
  72 } gdscriptField;
  73
  74 static fieldDefinition GDScriptFields[COUNT_FIELD] = {
  75         { .name = "annotations",
  76           .description = "annotations on functions and variables",
  77           .enabled = true },
  78 };
  79
  80 typedef enum {
  81         K_CLASS,
  82         K_METHOD,
  83         K_VARIABLE,
  84         K_CONST,
  85         K_ENUM,
  86         K_ENUMERATOR,
  87         K_PARAMETER,
  88         K_LOCAL_VARIABLE,
  89         K_SIGNAL,
  90         COUNT_KIND
  91 } gdscriptKind;
  92
  93 typedef enum {
  94         GDSCRIPT_CLASS_EXTENDED,
  95 } gdscriptClassRole;
  96
  97 static roleDefinition GDScriptClassRoles [] = {
  98         { true, "extended",   "used as a base class for extending" },
  99 };
 100
 101 static kindDefinition GDScriptKinds[COUNT_KIND] = {
 102         {true, 'c', "class",    "classes",
 103          .referenceOnly = false, ATTACH_ROLES(GDScriptClassRoles)},
 104         {true, 'm', "method",   "methods"},
 105         {true, 'v', "variable", "variables"},
 106         {true, 'C', "const", "constants"},
 107         {true, 'g', "enum",     "enumeration names"},
 108         {true, 'e', "enumerator",       "enumerated values"},
 109         {false,'z', "parameter",        "function parameters"},
 110         {false,'l', "local",    "local variables"},
 111         {true, 's', "signal",   "signals"},
 112 };
 113
 114 typedef enum {
 115         X_IMPLICIT_CLASS,
 116 } gdscriptXtag;
 117
 118 static xtagDefinition GDScriptXtagTable [] = {
 119         {
 120                 .enabled     = false,
 121                 .name        = "implicitClass",
 122                 .description = "Include tag for the implicitly defined unnamed class",
 123         },
 124 };
 125
 126 static const keywordTable GDScriptKeywordTable[] = {
 127         /* keyword                      keyword ID */
 128         { "class",                      KEYWORD_class                   },
 129         { "func",                       KEYWORD_func                    },
 130         { "extends",            KEYWORD_extends                 },
 131         { "lambda",                     KEYWORD_lambda                  }, // Future GDScript lambda currently uses func, may change
 132         { "pass",                       KEYWORD_pass                    },
 133         { "return",                     KEYWORD_return                  },
 134         { "var",                        KEYWORD_variable                },
 135         { "const",                      KEYWORD_const                   },
 136         { "enum",                       KEYWORD_enum                    },
 137         { "class_name",         KEYWORD_class_name              },
 138         { "signal",                     KEYWORD_signal                  },
 139
 140 };
 141
 142 static const struct keywordGroup modifierKeywords = {
 143         .value = KEYWORD_modifier,
 144         .addingUnlessExisting = false,
 145         .keywords = {
 146                 "static",
 147                 "remote", "remotesync",
 148                 "master", "mastersycn",
 149                 "puppet", "puppetsync",
 150                 NULL,
 151         },
 152 };
 153
 154 typedef enum eTokenType {
 155         /* 0..255 are the byte's value */
 156         TOKEN_EOF = 256,
 157         TOKEN_UNDEFINED,
 158         TOKEN_INDENT,
 159         TOKEN_KEYWORD,
 160         TOKEN_OPERATOR,
 161         TOKEN_IDENTIFIER,
 162         TOKEN_STRING,
 163         TOKEN_ARROW,                            /* -> */
 164         TOKEN_WHITESPACE,
 165 } tokenType;
 166
 167 typedef struct {
 168         int                             type;
 169         keywordId               keyword;
 170         vString *               string;
 171         int                             indent;
 172         unsigned long   lineNumber;
 173         MIOPos                  filePosition;
 174 } tokenInfo;
 175
 176 struct gdscriptNestingLevelUserData {
 177         int indentation;
 178 };
 179 #define GDS_NL(nl) ((struct gdscriptNestingLevelUserData *) nestingLevelGetUserData (nl))
 180
 181 static langType Lang_gdscript;
 182 static unsigned int TokenContinuationDepth = 0;
 183 static tokenInfo *NextToken = NULL;
 184 static NestingLevels *GDScriptNestingLevels = NULL;
 185 static objPool *TokenPool = NULL;
 186
 187
 188 // Always reports single-underscores as protected
 189 static accessType accessFromIdentifier (const vString *const ident, int parentKind)
 190 {
 191         const char *const p = vStringValue (ident);
 192         const size_t len = vStringLength (ident);
 193
 194         /* inside a function/method, private */
 195         if (parentKind != -1 && parentKind != K_CLASS)
 196                 return ACCESS_PRIVATE;
 197         /* not starting with "_", public */
 198         else if (len < 1 || p[0] != '_')
 199                 return ACCESS_PUBLIC;
 200         /* "_...": suggested as non-public, but easily accessible */
 201         else
 202                 return ACCESS_PROTECTED;
 203 }
 204
 205 static void initGDScriptEntry (tagEntryInfo *const e, const tokenInfo *const token,
 206                                                          const gdscriptKind kind)
 207 {
 208         accessType access;
 209         int parentKind = -1;
 210         NestingLevel *nl;
 211
 212         initTagEntry (e, vStringValue (token->string), kind);
 213         updateTagLine (e, token->lineNumber, token->filePosition);
 214
 215         nl = nestingLevelsGetCurrent (GDScriptNestingLevels);
 216         if (nl)
 217         {
 218                 tagEntryInfo *nlEntry = getEntryOfNestingLevel (nl);
 219
 220                 e->extensionFields.scopeIndex = nl->corkIndex;
 221
 222                 /* nlEntry can be NULL if a kind was disabled.  But what can we do
 223                  * here?  Even disabled kinds should count for the hierarchy I
 224                  * guess -- as it'd otherwise be wrong -- but with cork we're
 225                  * fucked up as there's nothing to look up.  Damn. */
 226                 if (nlEntry)
 227                         parentKind = nlEntry->kindIndex;
 228         }
 229
 230         access = accessFromIdentifier (token->string, parentKind);
 231         e->extensionFields.access = GDScriptAccesses[access];
 232         /* FIXME: should we really set isFileScope in addition to access? */
 233         if (access == ACCESS_PRIVATE)
 234                 e->isFileScope = true;
 235 }
 236
 237 static int makeClassTag (const tokenInfo *const token,
 238                                                  const vString *const inheritance)
 239 {
 240         if (GDScriptKinds[K_CLASS].enabled)
 241         {
 242                 tagEntryInfo e;
 243
 244                 initGDScriptEntry (&e, token, K_CLASS);
 245
 246                 e.extensionFields.inheritance = inheritance ? vStringValue (inheritance) : "";
 247
 248                 return makeTagEntry (&e);
 249         }
 250
 251         return CORK_NIL;
 252 }
 253
 254 static vString *makeDecoratorString (const stringList *const strlist)
 255 {
 256         vString *vstr = vStringNew ();
 257
 258         for (unsigned int i = 0; i < stringListCount (strlist); i++)
 259         {
 260                 vString *elt = stringListItem (strlist, i);
 261                 if (i != 0 && (vStringValue (elt) > 0
 262                                            && vStringValue (elt)[0] != '('))
 263                         vStringPut (vstr, ',');
 264                 vStringCat (vstr, elt);
 265         }
 266         return vstr;
 267 }
 268
 269 static int makeFunctionTag (const tokenInfo *const token,
 270                                                         int kind,
 271                                                         const vString *const arglist,
 272                                                         const stringList *const decorators)
 273 {
 274         if (GDScriptKinds[kind].enabled)
 275         {
 276                 tagEntryInfo e;
 277                 vString *vstr = NULL;
 278                 int r;
 279
 280                 initGDScriptEntry (&e, token, kind);
 281
 282                 if (arglist)
 283                         e.extensionFields.signature = vStringValue (arglist);
 284                 if (decorators && stringListCount (decorators) > 0)
 285                 {
 286                         vstr = makeDecoratorString (decorators);
 287                         attachParserField (&e, GDScriptFields[F_ANNOTATIONS].ftype,
 288                                                            vStringValue (vstr));
 289                 }
 290
 291                 r = makeTagEntry (&e);
 292                 vStringDelete (vstr);   /* NULL is ignored. */
 293
 294                 return r;
 295         }
 296
 297         return CORK_NIL;
 298 }
 299
 300 static int makeSimpleGDScriptTag (const tokenInfo *const token, gdscriptKind const kind)
 301 {
 302         if (GDScriptKinds[kind].enabled)
 303         {
 304                 tagEntryInfo e;
 305
 306                 initGDScriptEntry (&e, token, kind);
 307                 return makeTagEntry (&e);
 308         }
 309
 310         return CORK_NIL;
 311 }
 312
 313 static int makeSimpleGDScriptRefTag (const tokenInfo *const token,
 314                                                                          gdscriptKind const kind,
 315                                                                          int roleIndex, xtagType xtag)
 316 {
 317         if (isXtagEnabled (XTAG_REFERENCE_TAGS))
 318         {
 319                 tagEntryInfo e;
 320
 321                 initRefTagEntry (&e, vStringValue (token->string),
 322                                                  kind, roleIndex);
 323                 updateTagLine (&e, token->lineNumber, token->filePosition);
 324
 325                 if (xtag != XTAG_UNKNOWN)
 326                         markTagExtraBit (&e, xtag);
 327
 328                 return makeTagEntry (&e);
 329         }
 330
 331         return CORK_NIL;
 332 }
 333
 334 static void *newPoolToken (void *createArg CTAGS_ATTR_UNUSED)
 335 {
 336         tokenInfo *token = xMalloc (1, tokenInfo);
 337         token->string = vStringNew ();
 338         return token;
 339 }
 340
 341 static void deletePoolToken (void *data)
 342 {
 343         tokenInfo *token = data;
 344         vStringDelete (token->string);
 345         eFree (token);
 346 }
 347
 348 static void clearPoolToken (void *data)
 349 {
 350         tokenInfo *token = data;
 351
 352         token->type                     = TOKEN_UNDEFINED;
 353         token->keyword          = KEYWORD_NONE;
 354         token->indent           = 0;
 355         token->lineNumber   = getInputLineNumber ();
 356         token->filePosition = getInputFilePosition ();
 357         vStringClear (token->string);
 358 }
 359
 360 static void copyToken (tokenInfo *const dest, const tokenInfo *const src)
 361 {
 362         dest->lineNumber = src->lineNumber;
 363         dest->filePosition = src->filePosition;
 364         dest->type = src->type;
 365         dest->keyword = src->keyword;
 366         dest->indent = src->indent;
 367         vStringCopy (dest->string, src->string);
 368 }
 369
 370 /* Skip a single or double quoted string. */
 371 static void readString (vString *const string, const int delimiter)
 372 {
 373         int escaped = 0;
 374         int c;
 375
 376         while ((c = getcFromInputFile ()) != EOF)
 377         {
 378                 if (escaped)
 379                 {
 380                         vStringPut (string, c);
 381                         escaped--;
 382                 }
 383                 else if (c == '\\')
 384                         escaped++;
 385                 else if (c == delimiter || c == '\n' || c == '\r')
 386                 {
 387                         if (c != delimiter)
 388                                 ungetcToInputFile (c);
 389                         break;
 390                 }
 391                 else
 392                         vStringPut (string, c);
 393         }
 394 }
 395
 396 /* Skip a single or double triple quoted string. */
 397 static void readTripleString (vString *const string, const int delimiter)
 398 {
 399         int c;
 400         int escaped = 0;
 401         int n = 0;
 402         while ((c = getcFromInputFile ()) != EOF)
 403         {
 404                 if (c == delimiter && ! escaped)
 405                 {
 406                         if (++n >= 3)
 407                                 break;
 408                 }
 409                 else
 410                 {
 411                         for (; n > 0; n--)
 412                                 vStringPut (string, delimiter);
 413                         if (c != '\\' || escaped)
 414                                 vStringPut (string, c);
 415                         n = 0;
 416                 }
 417
 418                 if (escaped)
 419                         escaped--;
 420                 else if (c == '\\')
 421                         escaped++;
 422         }
 423 }
 424
 425 static void readIdentifier (vString *const string, const int firstChar)
 426 {
 427         int c = firstChar;
 428         do
 429         {
 430                 vStringPut (string, c);
 431                 c = getcFromInputFile ();
 432         }
 433         while (isIdentifierChar (c));
 434         ungetcToInputFile (c);
 435 }
 436
 437 static void ungetToken (tokenInfo *const token)
 438 {
 439         Assert (NextToken == NULL);
 440         NextToken = newToken ();
 441         copyToken (NextToken, token);
 442 }
 443
 444 static void readTokenFull (tokenInfo *const token, bool inclWhitespaces)
 445 {
 446         int c;
 447         int n;
 448
 449         /* if we've got a token held back, emit it */
 450         if (NextToken)
 451         {
 452                 copyToken (token, NextToken);
 453                 deleteToken (NextToken);
 454                 NextToken = NULL;
 455                 return;
 456         }
 457
 458         token->type             = TOKEN_UNDEFINED;
 459         token->keyword  = KEYWORD_NONE;
 460         vStringClear (token->string);
 461
 462 getNextChar:
 463
 464         n = 0;
 465         do
 466         {
 467                 c = getcFromInputFile ();
 468                 n++;
 469         }
 470         while (c == ' ' || c == '\t' || c == '\f');
 471
 472         token->lineNumber   = getInputLineNumber ();
 473         token->filePosition = getInputFilePosition ();
 474
 475         if (inclWhitespaces && n > 1 && c != '\r' && c != '\n')
 476         {
 477                 ungetcToInputFile (c);
 478                 vStringPut (token->string, ' ');
 479                 token->type = TOKEN_WHITESPACE;
 480                 return;
 481         }
 482
 483         switch (c)
 484         {
 485                 case EOF:
 486                         token->type = TOKEN_EOF;
 487                         break;
 488
 489                 case '\'':
 490                 case '"':
 491                 {
 492                         int d = getcFromInputFile ();
 493                         token->type = TOKEN_STRING;
 494                         vStringPut (token->string, c);
 495                         if (d != c)
 496                         {
 497                                 ungetcToInputFile (d);
 498                                 readString (token->string, c);
 499                         }
 500                         else if ((d = getcFromInputFile ()) == c)
 501                                 readTripleString (token->string, c);
 502                         else /* empty string */
 503                                 ungetcToInputFile (d);
 504                         vStringPut (token->string, c);
 505                         token->lineNumber = getInputLineNumber ();
 506                         token->filePosition = getInputFilePosition ();
 507                         break;
 508                 }
 509
 510                 case '=':
 511                 {
 512                         int d = getcFromInputFile ();
 513                         vStringPut (token->string, c);
 514                         if (d == c)
 515                         {
 516                                 vStringPut (token->string, d);
 517                                 token->type = TOKEN_OPERATOR;
 518                         }
 519                         else
 520                         {
 521                                 ungetcToInputFile (d);
 522                                 token->type = c;
 523                         }
 524                         break;
 525                 }
 526
 527                 case '-':
 528                 {
 529                         int d = getcFromInputFile ();
 530                         if (d == '>')
 531                         {
 532                                 vStringPut (token->string, c);
 533                                 vStringPut (token->string, d);
 534                                 token->type = TOKEN_ARROW;
 535                                 break;
 536                         }
 537                         ungetcToInputFile (d);
 538                         /* fall through */
 539                 }
 540                 case '+':
 541                 case '*':
 542                 case '%':
 543                 case '<':
 544                 case '>':
 545                 case '/':
 546                 {
 547                         int d = getcFromInputFile ();
 548                         vStringPut (token->string, c);
 549                         if (d != '=')
 550                         {
 551                                 ungetcToInputFile (d);
 552                                 token->type = c;
 553                         }
 554                         else
 555                         {
 556                                 vStringPut (token->string, d);
 557                                 token->type = TOKEN_OPERATOR;
 558                         }
 559                         break;
 560                 }
 561
 562                 /* eats newline to implement line continuation  */
 563                 case '\\':
 564                 {
 565                         int d = getcFromInputFile ();
 566                         if (d == '\r')
 567                                 d = getcFromInputFile ();
 568                         if (d != '\n')
 569                                 ungetcToInputFile (d);
 570                         goto getNextChar;
 571                 }
 572
 573                 case '#': /* comment */
 574                 case '\r': /* newlines for indent */
 575                 case '\n':
 576                 {
 577                         int indent = 0;
 578                         do
 579                         {
 580                                 if (c == '#')
 581                                 {
 582                                         do
 583                                                 c = getcFromInputFile ();
 584                                         while (c != EOF && c != '\r' && c != '\n');
 585                                 }
 586                                 if (c == '\r')
 587                                 {
 588                                         int d = getcFromInputFile ();
 589                                         if (d != '\n')
 590                                                 ungetcToInputFile (d);
 591                                 }
 592                                 indent = 0;
 593                                 while ((c = getcFromInputFile ()) == ' ' || c == '\t' || c == '\f')
 594                                 {
 595                                         if (c == '\t')
 596                                                 indent += 8 - (indent % 8);
 597                                         else if (c == '\f') /* yeah, it's weird */
 598                                                 indent = 0;
 599                                         else
 600                                                 indent++;
 601                                 }
 602                         } /* skip completely empty lines, so retry */
 603                         while (c == '\r' || c == '\n' || c == '#');
 604                         ungetcToInputFile (c);
 605                         if (TokenContinuationDepth > 0)
 606                         {
 607                                 if (inclWhitespaces)
 608                                 {
 609                                         vStringPut (token->string, ' ');
 610                                         token->type = TOKEN_WHITESPACE;
 611                                 }
 612                                 else
 613                                         goto getNextChar;
 614                         }
 615                         else
 616                         {
 617                                 token->type = TOKEN_INDENT;
 618                                 token->indent = indent;
 619                         }
 620                         break;
 621                 }
 622
 623                 default:
 624                         if (! isIdentifierChar (c))
 625                         {
 626                                 vStringPut (token->string, c);
 627                                 token->type = c;
 628                         }
 629                         else
 630                         {
 631                                 /* FIXME: handle U, B, R and F string prefixes? */
 632                                 readIdentifier (token->string, c);
 633                                 token->keyword = lookupKeyword (vStringValue (token->string), Lang_gdscript);
 634                                 if (token->keyword == KEYWORD_NONE)
 635                                         token->type = TOKEN_IDENTIFIER;
 636                                 else
 637                                         token->type = TOKEN_KEYWORD;
 638                         }
 639                         break;
 640         }
 641
 642         // handle implicit continuation lines not to emit INDENT inside brackets
 643         if (token->type == '(' ||
 644                 token->type == '{' ||
 645                 token->type == '[')
 646         {
 647                 TokenContinuationDepth ++;
 648         }
 649         else if (TokenContinuationDepth > 0 &&
 650                          (token->type == ')' ||
 651                           token->type == '}' ||
 652                           token->type == ']'))
 653         {
 654                 TokenContinuationDepth --;
 655         }
 656 }
 657
 658 static void readToken (tokenInfo *const token)
 659 {
 660         readTokenFull (token, false);
 661 }
 662
 663 /*================================= parsing =================================*/
 664
 665
 666 static void reprCat (vString *const repr, const tokenInfo *const token)
 667 {
 668         if (token->type != TOKEN_INDENT &&
 669                 token->type != TOKEN_WHITESPACE)
 670         {
 671                 vStringCat (repr, token->string);
 672         }
 673         else if (vStringLength (repr) > 0 && vStringLast (repr) != ' ')
 674         {
 675                 vStringPut (repr, ' ');
 676         }
 677 }
 678
 679 static bool skipOverPair (tokenInfo *const token, int tOpen, int tClose,
 680                                                   vString *const repr, bool reprOuterPair)
 681 {
 682         if (token->type == tOpen)
 683         {
 684                 int depth = 1;
 685
 686                 if (repr && reprOuterPair)
 687                         reprCat (repr, token);
 688                 do
 689                 {
 690                         readTokenFull (token, true);
 691                         if (repr && (reprOuterPair || token->type != tClose || depth > 1))
 692                         {
 693                                 reprCat (repr, token);
 694                         }
 695                         if (token->type == tOpen)
 696                                 depth ++;
 697                         else if (token->type == tClose)
 698                                 depth --;
 699                 }
 700                 while (token->type != TOKEN_EOF && depth > 0);
 701         }
 702
 703         return token->type == tClose;
 704 }
 705
 706 static void readQualifiedName (tokenInfo *const nameToken)
 707 {
 708         readToken (nameToken);
 709
 710         if (nameToken->type == TOKEN_IDENTIFIER ||
 711                 nameToken->type == '.')
 712         {
 713                 vString *qualifiedName = vStringNew ();
 714                 tokenInfo *token = newToken ();
 715
 716                 while (nameToken->type == TOKEN_IDENTIFIER ||
 717                            nameToken->type == '.')
 718                 {
 719                         vStringCat (qualifiedName, nameToken->string);
 720                         copyToken (token, nameToken);
 721
 722                         readToken (nameToken);
 723                 }
 724                 /* put the last, non-matching, token back */
 725                 ungetToken (nameToken);
 726
 727                 copyToken (nameToken, token);
 728                 nameToken->type = TOKEN_IDENTIFIER;
 729                 vStringCopy (nameToken->string, qualifiedName);
 730
 731                 deleteToken (token);
 732                 vStringDelete (qualifiedName);
 733         }
 734 }
 735
 736 static vString *parseParamTypeAnnotation (tokenInfo *const token,
 737                                                                                   vString *arglist)
 738 {
 739         readToken (token);
 740         if (token->type != ':')
 741         {
 742                 ungetToken (token);
 743                 return NULL;
 744         }
 745
 746         reprCat (arglist, token);
 747         int depth = 0;
 748         vString *t = vStringNew ();
 749         while (true)
 750         {
 751                 readTokenFull (token, true);
 752                 if (token->type == TOKEN_WHITESPACE)
 753                 {
 754                         reprCat (arglist, token);
 755                         continue;
 756                 }
 757                 else if (token->type == TOKEN_EOF)
 758                         break;
 759
 760                 if (token->type == '(' ||
 761                         token->type == '[' ||
 762                         token->type == '{')
 763                         depth ++;
 764                 else if (token->type == ')' ||
 765                                  token->type == ']' ||
 766                                  token->type == '}')
 767                         depth --;
 768
 769                 if (depth < 0
 770                         || (depth == 0 && (token->type == '='
 771                                                            || token->type == ',')))
 772                 {
 773                         ungetToken (token);
 774                         return t;
 775                 }
 776                 reprCat (arglist, token);
 777                 reprCat (t, token);
 778         }
 779         vStringDelete (t);
 780         return NULL;
 781 }
 782
 783 static vString *parseReturnTypeAnnotation (tokenInfo *const token)
 784 {
 785         readToken (token);
 786         if (token->type != TOKEN_ARROW)
 787         {
 788                 return NULL;
 789         }
 790
 791         int depth = 0;
 792         vString *t = vStringNew ();
 793         while (true)
 794         {
 795                 readToken (token);
 796                 if (token->type == TOKEN_EOF)
 797                         break;
 798
 799                 if (token->type == '(' ||
 800                         token->type == '[' ||
 801                         token->type == '{')
 802                         depth ++;
 803                 else if (token->type == ')' ||
 804                                  token->type == ']' ||
 805                                  token->type == '}')
 806                         depth --;
 807                 if (depth == 0 && token->type == ':')
 808                 {
 809                         ungetToken (token);
 810                         return t;
 811                 }
 812                 else
 813                         reprCat (t, token);
 814         }
 815         vStringDelete (t);
 816         return NULL;
 817 }
 818
 819 static bool parseClassOrDef (tokenInfo *const token,
 820                                                          const stringList *const decorators,
 821                                                          gdscriptKind kind)
 822 {
 823         vString *arglist = NULL;
 824         tokenInfo *name = NULL;
 825         tokenInfo *parameterTokens[16] = { NULL };
 826         vString   *parameterTypes [ARRAY_SIZE (parameterTokens)] = { NULL };
 827         unsigned int parameterCount = 0;
 828         NestingLevel *lv;
 829         int corkIndex;
 830
 831         readToken (token);
 832         if (token->type != TOKEN_IDENTIFIER)
 833                 return false;
 834
 835         name = newToken ();
 836         copyToken (name, token);
 837
 838         readToken (token);
 839         /* collect parameters or inheritance */
 840         if (token->type == '(')
 841         {
 842                 int prevTokenType = token->type;
 843                 int depth = 1;
 844
 845                 arglist = vStringNew ();
 846                 if (kind != K_CLASS)
 847                         reprCat (arglist, token);
 848
 849                 do
 850                 {
 851                         if (token->type != TOKEN_WHITESPACE &&
 852                                 token->type != '*')
 853                         {
 854                                 prevTokenType = token->type;
 855                         }
 856
 857                         readTokenFull (token, true);
 858                         if (kind != K_CLASS || token->type != ')' || depth > 1)
 859                                 reprCat (arglist, token);
 860
 861                         if (token->type == '(' ||
 862                                 token->type == '[' ||
 863                                 token->type == '{')
 864                                 depth ++;
 865                         else if (token->type == ')' ||
 866                                          token->type == ']' ||
 867                                          token->type == '}')
 868                                 depth --;
 869                         else if (kind != K_CLASS && depth == 1 &&
 870                                          token->type == TOKEN_IDENTIFIER &&
 871                                          (prevTokenType == '(' || prevTokenType == ',') &&
 872                                          parameterCount < ARRAY_SIZE (parameterTokens) &&
 873                                          GDScriptKinds[K_PARAMETER].enabled)
 874                         {
 875                                 tokenInfo *parameterName = newToken ();
 876
 877                                 copyToken (parameterName, token);
 878                                 parameterTokens[parameterCount] = parameterName;
 879                                 parameterTypes [parameterCount++] = parseParamTypeAnnotation (token, arglist);
 880                         }
 881                 }
 882                 while (token->type != TOKEN_EOF && depth > 0);
 883         }
 884         else if (token->type == TOKEN_KEYWORD && token->keyword == KEYWORD_extends)
 885         {
 886                 readToken (token);
 887                 if (token->type == TOKEN_IDENTIFIER)
 888                 {
 889                         makeSimpleGDScriptRefTag (token, K_CLASS, GDSCRIPT_CLASS_EXTENDED, XTAG_UNKNOWN);
 890                         arglist = vStringNewCopy (token->string);
 891                 }
 892         }
 893         else if (kind == K_SIGNAL)
 894         {
 895                 /* signal can be defined with no parameter list. */
 896                 ungetToken (token);
 897         }
 898
 899         if (kind == K_CLASS)
 900                 corkIndex = makeClassTag (name, arglist);
 901         else
 902                 corkIndex = makeFunctionTag (name, kind, arglist, decorators);
 903
 904         lv = nestingLevelsPush (GDScriptNestingLevels, corkIndex);
 905         GDS_NL (lv)->indentation = token->indent;
 906
 907         deleteToken (name);
 908         vStringDelete (arglist);
 909
 910         if (parameterCount > 0)
 911         {
 912                 unsigned int i;
 913
 914                 for (i = 0; i < parameterCount; i++)
 915                 {
 916                         int paramCorkIndex = makeSimpleGDScriptTag (parameterTokens[i], K_PARAMETER);
 917                         deleteToken (parameterTokens[i]);
 918                         tagEntryInfo *e = getEntryInCorkQueue (paramCorkIndex);
 919                         if (e && parameterTypes[i])
 920                         {
 921                                 e->extensionFields.typeRef [0] = eStrdup ("typename");
 922                                 e->extensionFields.typeRef [1] = vStringDeleteUnwrap (parameterTypes[i]);
 923                                 parameterTypes[i] = NULL;
 924                         }
 925                         vStringDelete (parameterTypes[i]); /* NULL is acceptable. */
 926                 }
 927         }
 928
 929         tagEntryInfo *e;
 930         vString *t;
 931         if (kind != K_CLASS
 932                 && (e = getEntryInCorkQueue (corkIndex))
 933                 && (t = parseReturnTypeAnnotation (token)))
 934         {
 935                 e->extensionFields.typeRef [0] = eStrdup ("typename");
 936                 e->extensionFields.typeRef [1] = vStringDeleteUnwrap (t);
 937         }
 938
 939         if (kind == K_SIGNAL)
 940                 nestingLevelsPop (GDScriptNestingLevels);
 941
 942         return true;
 943 }
 944
 945 static bool parseEnum (tokenInfo *const token)
 946 {
 947         int corkIndex;
 948
 949         readToken (token);
 950
 951         if (token->type == '{')
 952         {
 953                 tokenInfo *name = newToken ();
 954                 copyToken (name, token);
 955                 vStringClear (name->string);
 956                 anonGenerate (name->string, "anon_enum_", K_ENUM);
 957                 name->type = TOKEN_IDENTIFIER;
 958                 corkIndex = makeSimpleGDScriptTag (name, K_ENUM);
 959                 deleteToken (name);
 960                 tagEntryInfo *e = getEntryInCorkQueue (corkIndex);
 961                 if (e)
 962                         markTagExtraBit (e, XTAG_ANONYMOUS);
 963         }
 964         else if (token->type == TOKEN_IDENTIFIER)
 965         {
 966                 corkIndex = makeSimpleGDScriptTag (token, K_ENUM);
 967                 readToken (token);
 968         }
 969         else
 970                 return false;
 971
 972         if (token->type != '{')
 973                 return false;
 974
 975         readToken (token);
 976         nestingLevelsPush (GDScriptNestingLevels, corkIndex);
 977
 978         while (token->type != '}' && token->type != TOKEN_EOF)
 979         {
 980                 if (token->type == TOKEN_IDENTIFIER)
 981                         makeSimpleGDScriptTag (token, K_ENUMERATOR);
 982                 else if (token->type == '=')
 983                 {
 984                         /* Skip the right value. */
 985                         do
 986                                 readToken (token);
 987                         while (token->type != ','
 988                                    && token->type != '}'
 989                                    && token->type != TOKEN_EOF);
 990                         if (token->type != ',')
 991                                 continue;
 992                 }
 993                 readToken (token);
 994         }
 995
 996         tagEntryInfo *e;
 997         vString *t;
 998         if ((e = getEntryInCorkQueue (corkIndex))
 999                 && (t = parseReturnTypeAnnotation (token)))
1000         {
1001                 e->extensionFields.typeRef [0] = eStrdup ("typename");
1002                 e->extensionFields.typeRef [1] = vStringDeleteUnwrap (t);
1003         }
1004
1005         nestingLevelsPop (GDScriptNestingLevels);
1006         return true;
1007 }
1008
1009 static bool parseClassName (tokenInfo *const token)
1010 {
1011         readToken (token);
1012         if (token->type == TOKEN_IDENTIFIER)
1013         {
1014                 /* A class name is explicitly given with "class_name" keyword.
1015                  * Let's overwrite the anonymous tag for the class
1016                  */
1017                 NestingLevel *nl = nestingLevelsGetNthFromRoot (GDScriptNestingLevels, 0);
1018                 tagEntryInfo *klass = nl? getEntryInCorkQueue (nl->corkIndex): NULL;
1019
1020                 tagEntryInfo e;
1021                 char *name = vStringStrdup (token->string);
1022                 initTagEntry (&e, klass? "UNUSED": name, K_CLASS);
1023
1024                 if (klass)
1025                 {
1026                         eFree ((void *)klass->name);
1027                         klass->name = name;
1028                         name = NULL;
1029                         unmarkTagExtraBit (klass, XTAG_ANONYMOUS);
1030                         unmarkTagExtraBit (klass, GDScriptXtagTable[X_IMPLICIT_CLASS].xtype);
1031
1032                         /* Adjust the position. */
1033                         setTagPositionFromTag (klass, &e);
1034                 }
1035
1036                 /* Extract B in class_name C extends B */
1037                 readToken (token);
1038                 if (token->type == TOKEN_KEYWORD
1039                         && token->keyword == KEYWORD_extends)
1040                 {
1041                         readToken (token);
1042                         if (token->type == TOKEN_IDENTIFIER)
1043                         {
1044                                 makeSimpleGDScriptRefTag (token, K_CLASS,
1045                                                                                   GDSCRIPT_CLASS_EXTENDED,
1046                                                                                   XTAG_UNKNOWN);
1047                                 if (klass)
1048                                 {
1049                                         if (klass->extensionFields.inheritance)
1050                                                 eFree ((void *)klass->extensionFields.inheritance);
1051                                         klass->extensionFields.inheritance = vStringStrdup (token->string);
1052                                 }
1053                                 else
1054                                         e.extensionFields.inheritance = vStringValue (token->string);
1055                         }
1056                 }
1057
1058                 if (!klass)
1059                         makeTagEntry (&e);
1060
1061                 if (name)
1062                         eFree (name);
1063         }
1064
1065         while (token->type != TOKEN_EOF &&
1066                    token->type != ';' &&
1067                    token->type != TOKEN_INDENT)
1068                 readToken (token);
1069
1070         return false;
1071 }
1072
1073 static bool parseExtends (tokenInfo *const token)
1074 {
1075         if (token->keyword == KEYWORD_extends)
1076         {
1077                 readQualifiedName (token);
1078                 if (token->type == TOKEN_IDENTIFIER)
1079                 {
1080                         makeSimpleGDScriptRefTag (token, K_CLASS, GDSCRIPT_CLASS_EXTENDED, XTAG_UNKNOWN);
1081                         NestingLevel *nl = nestingLevelsGetCurrent (GDScriptNestingLevels);
1082                         if (nl)
1083                         {
1084                                 tagEntryInfo *klass = getEntryInCorkQueue (nl->corkIndex);
1085                                 if (klass)
1086                                 {
1087                                         if (klass->extensionFields.inheritance)
1088                                                 eFree ((void *)klass->extensionFields.inheritance);
1089                                         klass->extensionFields.inheritance = vStringStrdup (token->string);
1090                                 }
1091                         }
1092                 }
1093         }
1094         readToken (token);
1095         return false;
1096 }
1097
1098 /* this only handles the most common cases, but an annotation can be any
1099  * expression in theory.
1100  * this function assumes there must be an annotation, and doesn't do any check
1101  * on the token on which it is called: the caller should do that part. */
1102 static bool skipVariableTypeAnnotation (tokenInfo *const token, vString *const repr)
1103 {
1104         bool readNext = true;
1105
1106         readToken (token);
1107         switch (token->type)
1108         {
1109                 case '[': readNext = skipOverPair (token, '[', ']', repr, true); break;
1110                 case '(': readNext = skipOverPair (token, '(', ')', repr, true); break;
1111                 case '{': readNext = skipOverPair (token, '{', '}', repr, true); break;
1112                 default: reprCat (repr, token);
1113         }
1114         if (readNext)
1115                 readToken (token);
1116         /* skip subscripts and calls */
1117         while (token->type == '[' || token->type == '(' || token->type == '.' || token->type == '|')
1118         {
1119                 switch (token->type)
1120                 {
1121                         case '[': readNext = skipOverPair (token, '[', ']', repr, true); break;
1122                         case '(': readNext = skipOverPair (token, '(', ')', repr, true); break;
1123                         case '|':
1124                                 reprCat (repr, token);
1125                                 skipVariableTypeAnnotation (token, repr);
1126                                 readNext = false;
1127                                 break;
1128                         case '.':
1129                                 reprCat (repr, token);
1130                                 readToken (token);
1131                                 readNext = token->type == TOKEN_IDENTIFIER;
1132                                 if (readNext)
1133                                         reprCat (repr, token);
1134                                 break;
1135                         default:  readNext = false; break;
1136                 }
1137                 if (readNext)
1138                         readToken (token);
1139         }
1140
1141         return false;
1142 }
1143
1144 static bool parseVariable (tokenInfo *const token, const gdscriptKind kind,
1145                                                         const stringList *const decorators,
1146                                                         const int keyword)
1147 {
1148         readToken (token);
1149         vString *type = vStringNew ();
1150         tokenInfo *name = newToken ();
1151         copyToken (name, token);
1152         if (!name)
1153                 return false;
1154
1155         readToken (token);
1156         // Variable declarations with dotted names are illegal
1157         if (token->type == '.')
1158                 return false;
1159
1160         /* (parse and) skip annotations.  we need not to be too permissive because we
1161          * aren't yet sure we're actually parsing a variable. */
1162         if (token->type == ':' && skipVariableTypeAnnotation (token, type))
1163                 readToken (token);
1164
1165         int index = makeSimpleGDScriptTag (name, kind);
1166         deleteToken (name);
1167         tagEntryInfo *e = getEntryInCorkQueue (index);
1168
1169         if (e && decorators && stringListCount (decorators) > 0)
1170         {
1171                 vString *vstr = makeDecoratorString (decorators);
1172                 attachParserField (e, GDScriptFields[F_ANNOTATIONS].ftype,
1173                                                    vStringValue (vstr));
1174                 vStringDelete (vstr);
1175         }
1176
1177         vString *vtype = vStringNew ();
1178         char * stype = vStringValue (type);
1179         if (strcmp (stype, "=") && strcmp (stype, ""))
1180         {
1181                         vStringCatS (vtype, stype);
1182         }
1183         vStringDelete (type);
1184
1185         if (e && vStringLength (vtype) > 0) /// TODO: Fix types away
1186         {
1187                 e->extensionFields.typeRef [0] = eStrdup ("typename");
1188                 e->extensionFields.typeRef [1] = vStringDeleteUnwrap (vtype);
1189         }
1190         else
1191         {
1192                 vStringDelete (vtype);
1193         }
1194
1195
1196         while ((TokenContinuationDepth > 0 || token->type != ',') &&
1197                    token->type != TOKEN_EOF &&
1198                    token->type != ';' &&
1199                    token->type != TOKEN_INDENT)
1200         {
1201                 readToken (token);
1202         }
1203
1204
1205         return false;
1206 }
1207
1208 /* pops any level >= to indent */
1209 static void setIndent (tokenInfo *const token)
1210 {
1211         NestingLevel *lv = nestingLevelsGetCurrent (GDScriptNestingLevels);
1212
1213         while (lv && GDS_NL (lv)->indentation >= token->indent)
1214         {
1215                 setTagEndLineToCorkEntry (lv->corkIndex, token->lineNumber);
1216
1217                 nestingLevelsPop (GDScriptNestingLevels);
1218                 lv = nestingLevelsGetCurrent (GDScriptNestingLevels);
1219         }
1220 }
1221
1222 static int prepareUnnamedClass (struct NestingLevels *nls)
1223 {
1224         {
1225                 /* Ugly: we need a "position" on the input stream for making a tag.
1226                  * At the begining of parsing, the position is undefined.
1227                  * By reading a byte, the position is defined.
1228                  */
1229                 int c = getcFromInputFile ();
1230                 if (c == EOF)
1231                         return CORK_NIL;
1232                 ungetcToInputFile (c);
1233         }
1234
1235         vString * tmp_class = anonGenerateNew ("anon_class_", K_CLASS);
1236         int corkIndex = makeSimpleTag (tmp_class, K_CLASS);
1237         vStringDelete (tmp_class);
1238
1239         tagEntryInfo *e = getEntryInCorkQueue (corkIndex);
1240         if (e)
1241                 markTagExtraBit (e, XTAG_ANONYMOUS);
1242
1243         /* This virtual scope should not be poped. */
1244         NestingLevel *lv = nestingLevelsPush (nls, corkIndex);
1245         GDS_NL (lv)->indentation = -1;
1246
1247         return corkIndex;
1248 }
1249
1250 static void findGDScriptTags (void)
1251 {
1252         tokenInfo *const token = newToken ();
1253         stringList *decorators = stringListNew ();
1254         bool atStatementStart = true;
1255
1256         TokenContinuationDepth = 0;
1257         NextToken = NULL;
1258         GDScriptNestingLevels = nestingLevelsNew (sizeof (struct gdscriptNestingLevelUserData));
1259
1260         if (isXtagEnabled (GDScriptXtagTable[X_IMPLICIT_CLASS].xtype))
1261         {
1262                 int index = prepareUnnamedClass (GDScriptNestingLevels);
1263                 tagEntryInfo *e = getEntryInCorkQueue (index);
1264                 if (e)
1265                         markTagExtraBit (e, GDScriptXtagTable[X_IMPLICIT_CLASS].xtype);
1266         }
1267
1268         readToken (token);
1269         while (token->type != TOKEN_EOF)
1270         {
1271                 tokenType iterationTokenType = token->type;
1272                 int iterationTokenKeyword = token->keyword;
1273                 bool readNext = true;
1274
1275                 if (token->type == TOKEN_INDENT)
1276                         setIndent (token);
1277                 else if (token->keyword == KEYWORD_class ||
1278                                  token->keyword == KEYWORD_func  ||
1279                                  token->keyword == KEYWORD_signal)
1280                 {
1281                         gdscriptKind kind = K_METHOD;
1282                         switch (token->keyword)
1283                         {
1284                         case KEYWORD_class:  kind = K_CLASS;  break;
1285                         case KEYWORD_func:   kind = K_METHOD; break;
1286                         case KEYWORD_signal: kind = K_SIGNAL; break;
1287                         default:
1288                                 AssertNotReached ();
1289                         }
1290                         readNext = parseClassOrDef (token, decorators, kind);
1291                 }
1292                 else if (token->keyword == KEYWORD_extends)
1293                 {
1294                         readNext = parseExtends (token);
1295                 }
1296                 else if (token->type == '(')
1297                 { /* skip parentheses to avoid finding stuff inside them */
1298                         readNext = skipOverPair (token, '(', ')', NULL, false);
1299                 }
1300                 else if (token->keyword == KEYWORD_variable || token->keyword == KEYWORD_const)
1301                 {
1302                         NestingLevel *lv = nestingLevelsGetCurrent (GDScriptNestingLevels);
1303                         tagEntryInfo *lvEntry = NULL;
1304                         gdscriptKind kind = K_VARIABLE;
1305
1306                         if (lv)
1307                                 lvEntry = getEntryOfNestingLevel (lv);
1308
1309                         if (lvEntry && lvEntry->kindIndex != K_CLASS)
1310                                 kind = K_LOCAL_VARIABLE;
1311
1312                         if (token->keyword == KEYWORD_const)
1313                                 kind = K_CONST;
1314
1315                         readNext = parseVariable (token, kind, decorators, token->keyword);
1316                 }
1317                 else if (token->keyword == KEYWORD_enum)
1318                 {
1319                         readNext = parseEnum (token);
1320                 }
1321                 else if (token->keyword == KEYWORD_class_name)
1322                 {
1323                         readNext = parseClassName (token);
1324                 }
1325                 else if (token->type == TOKEN_KEYWORD
1326                                  && token->keyword == KEYWORD_modifier)
1327                 {
1328                         stringListAdd (decorators, vStringNewCopy (token->string));
1329                 }
1330                 else if (token->type == '@' && atStatementStart &&
1331                                  GDScriptFields[F_ANNOTATIONS].enabled)
1332                 {
1333                         /* collect decorators */
1334                         readQualifiedName (token);
1335                         if (token->type != TOKEN_IDENTIFIER
1336                                 && (token->keyword != KEYWORD_modifier))
1337                                 readNext = false;
1338                         else
1339                         {
1340                                 stringListAdd (decorators, vStringNewCopy (token->string));
1341                                 readToken (token);
1342
1343                                 vString *d = vStringNew ();
1344                                 readNext = skipOverPair (token, '(', ')', d, true);
1345                                 if (vStringLength (d) > 0)
1346                                         stringListAdd (decorators, d);
1347                                 else
1348                                         vStringDelete (d);
1349                         }
1350                 }
1351
1352                 /* clear collected decorators for any non-decorator tokens non-indent
1353                  * token.  decorator collection takes care of skipping the possible
1354                  * argument list, so we should never hit here parsing a decorator */
1355                 if (iterationTokenType != TOKEN_INDENT &&
1356                         iterationTokenType != '@' &&
1357                         iterationTokenKeyword != KEYWORD_modifier &&
1358                         GDScriptFields[F_ANNOTATIONS].enabled)
1359                 {
1360                         stringListClear (decorators);
1361                 }
1362
1363                 atStatementStart = (token->type == TOKEN_INDENT || token->type == ';');
1364
1365                 if (readNext)
1366                         readToken (token);
1367         }
1368
1369         nestingLevelsFree (GDScriptNestingLevels);
1370         stringListDelete (decorators);
1371         deleteToken (token);
1372         Assert (NextToken == NULL);
1373 }
1374
1375 static void initialize (const langType language)
1376 {
1377         Lang_gdscript = language;
1378
1379         TokenPool = objPoolNew (16, newPoolToken, deletePoolToken, clearPoolToken, NULL);
1380         addKeywordGroup (&modifierKeywords, language);
1381 }
1382
1383 static void finalize (langType language CTAGS_ATTR_UNUSED, bool initialized)
1384 {
1385         if (!initialized)
1386                 return;
1387
1388         objPoolDelete (TokenPool);
1389 }
1390
1391 extern parserDefinition* GDScriptParser (void)
1392 {
1393         static const char *const extensions[] = { "gd", NULL };
1394         parserDefinition *def = parserNew ("GDScript");
1395         def->kindTable = GDScriptKinds;
1396         def->kindCount = ARRAY_SIZE (GDScriptKinds);
1397         def->extensions = extensions;
1398         def->parser = findGDScriptTags;
1399         def->initialize = initialize;
1400         def->finalize = finalize;
1401         def->keywordTable = GDScriptKeywordTable;
1402         def->keywordCount = ARRAY_SIZE (GDScriptKeywordTable);
1403         def->fieldTable = GDScriptFields;
1404         def->fieldCount = ARRAY_SIZE (GDScriptFields);
1405         def->xtagTable = GDScriptXtagTable;
1406         def->xtagCount = ARRAY_SIZE (GDScriptXtagTable);
1407         def->useCork = CORK_QUEUE;
1408         def->requestAutomaticFQTag = true;
1409         return def;
1410 }