ctags/parsers/php.c

   1 /*
   2 *   Copyright (c) 2013, Colomban Wendling <ban@herbesfolles.org>
   3 *
   4 *   This source code is released for free distribution under the terms of the
   5 *   GNU General Public License version 2 or (at your option) any later version.
   6 *
   7 *   This module contains code for generating tags for the PHP scripting
   8 *   language.
   9 *
  10 *   The language reference: http://php.net/manual/en/langref.php
  11 */
  12
  13 /*
  14 *   INCLUDE FILES
  15 */
  16 #include "general.h"  /* must always come first */
  17
  18 #include <string.h>
  19
  20 #include "parse.h"
  21 #include "read.h"
  22 #include "vstring.h"
  23 #include "keyword.h"
  24 #include "entry.h"
  25 #include "routines.h"
  26 #include "debug.h"
  27 #include "objpool.h"
  28
  29 #define isIdentChar(c) (isalnum (c) || (c) == '_' || (c) >= 0x80)
  30 #define newToken() (objPoolGet (TokenPool))
  31 #define deleteToken(t) (objPoolPut (TokenPool, (t)))
  32
  33 enum {
  34         KEYWORD_abstract,
  35         KEYWORD_and,
  36         KEYWORD_as,
  37         KEYWORD_break,
  38         KEYWORD_callable,
  39         KEYWORD_case,
  40         KEYWORD_catch,
  41         KEYWORD_class,
  42         KEYWORD_clone,
  43         KEYWORD_const,
  44         KEYWORD_continue,
  45         KEYWORD_declare,
  46         KEYWORD_define,
  47         KEYWORD_default,
  48         KEYWORD_do,
  49         KEYWORD_echo,
  50         KEYWORD_else,
  51         KEYWORD_elif,
  52         KEYWORD_enddeclare,
  53         KEYWORD_endfor,
  54         KEYWORD_endforeach,
  55         KEYWORD_endif,
  56         KEYWORD_endswitch,
  57         KEYWORD_endwhile,
  58         KEYWORD_extends,
  59         KEYWORD_final,
  60         KEYWORD_finally,
  61         KEYWORD_for,
  62         KEYWORD_foreach,
  63         KEYWORD_function,
  64         KEYWORD_global,
  65         KEYWORD_goto,
  66         KEYWORD_if,
  67         KEYWORD_implements,
  68         KEYWORD_include,
  69         KEYWORD_include_once,
  70         KEYWORD_instanceof,
  71         KEYWORD_insteadof,
  72         KEYWORD_interface,
  73         KEYWORD_namespace,
  74         KEYWORD_new,
  75         KEYWORD_or,
  76         KEYWORD_print,
  77         KEYWORD_private,
  78         KEYWORD_protected,
  79         KEYWORD_public,
  80         KEYWORD_require,
  81         KEYWORD_require_once,
  82         KEYWORD_return,
  83         KEYWORD_static,
  84         KEYWORD_switch,
  85         KEYWORD_throw,
  86         KEYWORD_trait,
  87         KEYWORD_try,
  88         KEYWORD_use,
  89         KEYWORD_var,
  90         KEYWORD_while,
  91         KEYWORD_xor,
  92         KEYWORD_yield
  93 };
  94 typedef int keywordId; /* to allow KEYWORD_NONE */
  95
  96 typedef enum {
  97         ACCESS_UNDEFINED,
  98         ACCESS_PRIVATE,
  99         ACCESS_PROTECTED,
 100         ACCESS_PUBLIC,
 101         COUNT_ACCESS
 102 } accessType;
 103
 104 typedef enum {
 105         IMPL_UNDEFINED,
 106         IMPL_ABSTRACT,
 107         COUNT_IMPL
 108 } implType;
 109
 110 typedef enum {
 111         K_CLASS,
 112         K_DEFINE,
 113         K_FUNCTION,
 114         K_INTERFACE,
 115         K_LOCAL_VARIABLE,
 116         K_NAMESPACE,
 117         K_TRAIT,
 118         K_VARIABLE,
 119         K_ALIAS,
 120         COUNT_KIND
 121 } phpKind;
 122
 123 #define NAMESPACE_SEPARATOR "\\"
 124 static scopeSeparator PhpGenericSeparators [] = {
 125         { K_NAMESPACE        , NAMESPACE_SEPARATOR },
 126         { KIND_WILDCARD_INDEX, "::" },
 127 };
 128
 129 static kindDefinition PhpKinds[COUNT_KIND] = {
 130         { true, 'c', "class",           "classes",
 131           ATTACH_SEPARATORS(PhpGenericSeparators) },
 132         { true, 'd', "define",          "constant definitions",
 133           ATTACH_SEPARATORS(PhpGenericSeparators)},
 134         { true, 'f', "function",        "functions",
 135           ATTACH_SEPARATORS(PhpGenericSeparators)},
 136         { true, 'i', "interface",       "interfaces",
 137           ATTACH_SEPARATORS(PhpGenericSeparators)},
 138         { false, 'l', "local",          "local variables",
 139           ATTACH_SEPARATORS(PhpGenericSeparators)},
 140         { true, 'n', "namespace",       "namespaces",
 141           ATTACH_SEPARATORS(PhpGenericSeparators)},
 142         { true, 't', "trait",           "traits",
 143           ATTACH_SEPARATORS(PhpGenericSeparators)},
 144         { true, 'v', "variable",        "variables",
 145           ATTACH_SEPARATORS(PhpGenericSeparators)},
 146         { true, 'a', "alias",           "aliases",
 147           ATTACH_SEPARATORS(PhpGenericSeparators)},
 148 };
 149
 150 static const keywordTable PhpKeywordTable[] = {
 151         /* keyword                      keyword ID */
 152         { "abstract",           KEYWORD_abstract                },
 153         { "and",                        KEYWORD_and                             },
 154         { "as",                         KEYWORD_as                              },
 155         { "break",                      KEYWORD_break                   },
 156         { "callable",           KEYWORD_callable                },
 157         { "case",                       KEYWORD_case                    },
 158         { "catch",                      KEYWORD_catch                   },
 159         { "cfunction",          KEYWORD_function                }, /* nobody knows what the hell this is, but it seems to behave much like "function" so bind it to it */
 160         { "class",                      KEYWORD_class                   },
 161         { "clone",                      KEYWORD_clone                   },
 162         { "const",                      KEYWORD_const                   },
 163         { "continue",           KEYWORD_continue                },
 164         { "declare",            KEYWORD_declare                 },
 165         { "define",                     KEYWORD_define                  }, /* this isn't really a keyword but we handle it so it's easier this way */
 166         { "default",            KEYWORD_default                 },
 167         { "do",                         KEYWORD_do                              },
 168         { "echo",                       KEYWORD_echo                    },
 169         { "else",                       KEYWORD_else                    },
 170         { "elseif",                     KEYWORD_elif                    },
 171         { "enddeclare",         KEYWORD_enddeclare              },
 172         { "endfor",                     KEYWORD_endfor                  },
 173         { "endforeach",         KEYWORD_endforeach              },
 174         { "endif",                      KEYWORD_endif                   },
 175         { "endswitch",          KEYWORD_endswitch               },
 176         { "endwhile",           KEYWORD_endwhile                },
 177         { "extends",            KEYWORD_extends                 },
 178         { "final",                      KEYWORD_final                   },
 179         { "finally",            KEYWORD_finally                 },
 180         { "for",                        KEYWORD_for                             },
 181         { "foreach",            KEYWORD_foreach                 },
 182         { "function",           KEYWORD_function                },
 183         { "global",                     KEYWORD_global                  },
 184         { "goto",                       KEYWORD_goto                    },
 185         { "if",                         KEYWORD_if                              },
 186         { "implements",         KEYWORD_implements              },
 187         { "include",            KEYWORD_include                 },
 188         { "include_once",       KEYWORD_include_once    },
 189         { "instanceof",         KEYWORD_instanceof              },
 190         { "insteadof",          KEYWORD_insteadof               },
 191         { "interface",          KEYWORD_interface               },
 192         { "namespace",          KEYWORD_namespace               },
 193         { "new",                        KEYWORD_new                             },
 194         { "or",                         KEYWORD_or                              },
 195         { "print",                      KEYWORD_print                   },
 196         { "private",            KEYWORD_private                 },
 197         { "protected",          KEYWORD_protected               },
 198         { "public",                     KEYWORD_public                  },
 199         { "require",            KEYWORD_require                 },
 200         { "require_once",       KEYWORD_require_once    },
 201         { "return",                     KEYWORD_return                  },
 202         { "static",                     KEYWORD_static                  },
 203         { "switch",                     KEYWORD_switch                  },
 204         { "throw",                      KEYWORD_throw                   },
 205         { "trait",                      KEYWORD_trait                   },
 206         { "try",                        KEYWORD_try                             },
 207         { "use",                        KEYWORD_use                             },
 208         { "var",                        KEYWORD_var                             },
 209         { "while",                      KEYWORD_while                   },
 210         { "xor",                        KEYWORD_xor                             },
 211         { "yield",                      KEYWORD_yield                   }
 212 };
 213
 214
 215 typedef enum eTokenType {
 216         TOKEN_UNDEFINED,
 217         TOKEN_EOF,
 218         TOKEN_CHARACTER,
 219         TOKEN_CLOSE_PAREN,
 220         TOKEN_SEMICOLON,
 221         TOKEN_COLON,
 222         TOKEN_COMMA,
 223         TOKEN_KEYWORD,
 224         TOKEN_OPEN_PAREN,
 225         TOKEN_OPERATOR,
 226         TOKEN_IDENTIFIER,
 227         TOKEN_STRING,
 228         TOKEN_PERIOD,
 229         TOKEN_OPEN_CURLY,
 230         TOKEN_CLOSE_CURLY,
 231         TOKEN_EQUAL_SIGN,
 232         TOKEN_OPEN_SQUARE,
 233         TOKEN_CLOSE_SQUARE,
 234         TOKEN_VARIABLE,
 235         TOKEN_AMPERSAND,
 236         TOKEN_BACKSLASH,
 237         TOKEN_QMARK,
 238 } tokenType;
 239
 240 typedef struct {
 241         tokenType               type;
 242         keywordId               keyword;
 243         vString *               string;
 244         vString *               scope;
 245         unsigned long   lineNumber;
 246         MIOPos                  filePosition;
 247         int                     parentKind; /* -1 if none */
 248         bool                    anonymous;      /* true if token specifies
 249                                                                  * an anonymous class */
 250 } tokenInfo;
 251
 252 static langType Lang_php;
 253 static langType Lang_zephir;
 254
 255 static bool InPhp = false; /* whether we are between <? ?> */
 256 /* whether the next token may be a keyword, e.g. not after "::" or "->" */
 257 static bool MayBeKeyword = true;
 258
 259 /* current statement details */
 260 static struct {
 261         accessType access;
 262         implType impl;
 263 } CurrentStatement;
 264
 265 /* Current namespace */
 266 static vString *CurrentNamesapce;
 267 /* Cache variable to build the tag's scope.  It has no real meaning outside
 268  * of initPhpEntry()'s scope. */
 269 static vString *FullScope;
 270 /* The class name specified at "extends" keyword in the current class
 271  * definition. Used to resolve "parent" in return type. */
 272 static vString *ParentClass;
 273
 274 static objPool *TokenPool = NULL;
 275
 276 static const char *phpScopeSeparatorFor (int kind, int upperScopeKind)
 277 {
 278         return scopeSeparatorFor (getInputLanguage(), kind, upperScopeKind);
 279 }
 280
 281 static const char *accessToString (const accessType access)
 282 {
 283         static const char *const names[COUNT_ACCESS] = {
 284                 "undefined",
 285                 "private",
 286                 "protected",
 287                 "public"
 288         };
 289
 290         Assert (access < COUNT_ACCESS);
 291
 292         return names[access];
 293 }
 294
 295 static const char *implToString (const implType impl)
 296 {
 297         static const char *const names[COUNT_IMPL] = {
 298                 "undefined",
 299                 "abstract"
 300         };
 301
 302         Assert (impl < COUNT_IMPL);
 303
 304         return names[impl];
 305 }
 306
 307 static void initPhpEntry (tagEntryInfo *const e, const tokenInfo *const token,
 308                                                   const phpKind kind, const accessType access)
 309 {
 310         int parentKind = -1;
 311
 312         vStringClear (FullScope);
 313
 314         if (vStringLength (CurrentNamesapce) > 0)
 315         {
 316                 parentKind = K_NAMESPACE;
 317                 vStringCat (FullScope, CurrentNamesapce);
 318
 319         }
 320
 321         initTagEntry (e, vStringValue (token->string), kind);
 322
 323         e->lineNumber   = token->lineNumber;
 324         e->filePosition = token->filePosition;
 325
 326         if (access != ACCESS_UNDEFINED)
 327                 e->extensionFields.access = accessToString (access);
 328         if (vStringLength (token->scope) > 0)
 329         {
 330                 parentKind = token->parentKind;
 331
 332                 if (vStringLength (FullScope) > 0)
 333                 {
 334                         const char* sep;
 335
 336                         sep = phpScopeSeparatorFor (parentKind,
 337                                                     K_NAMESPACE);
 338                         vStringCatS (FullScope, sep);
 339                 }
 340                         vStringCat (FullScope, token->scope);
 341         }
 342         if (vStringLength (FullScope) > 0)
 343         {
 344                 Assert (parentKind >= 0);
 345
 346                 e->extensionFields.scopeKindIndex = parentKind;
 347                 e->extensionFields.scopeName = vStringValue (FullScope);
 348         }
 349
 350         if (token->anonymous)
 351                 markTagExtraBit (e, XTAG_ANONYMOUS);
 352 }
 353
 354 static void  makePhpTagEntry  (tagEntryInfo *const e)
 355 {
 356         makeTagEntry (e);
 357         makeQualifiedTagEntry (e);
 358 }
 359
 360 static void fillTypeRefField (tagEntryInfo *const e,
 361                                                           const vString *const rtype, const tokenInfo *const token)
 362 {
 363         if ((vStringLength (rtype) == 4)
 364                 && (strcmp (vStringValue (rtype), "self") == 0)
 365                 && vStringLength (token->scope) > 0)
 366         {
 367                 if (token->parentKind == -1)
 368                         e->extensionFields.typeRef [0] = "unknown";
 369                 else
 370                         e->extensionFields.typeRef [0] = PhpKinds [token->parentKind].name;
 371                 e->extensionFields.typeRef [1] = vStringValue (token->scope);
 372         }
 373         else if ((vStringLength (rtype) == 6)
 374                          && (strcmp (vStringValue (rtype), "parent") == 0)
 375                          && (ParentClass && vStringLength (ParentClass) > 0))
 376         {
 377                 e->extensionFields.typeRef [0] = "class";
 378                 e->extensionFields.typeRef [1] = vStringValue (ParentClass);
 379         }
 380         else
 381         {
 382                 e->extensionFields.typeRef [0] = "unknown";
 383                 e->extensionFields.typeRef [1] = vStringValue (rtype);
 384         }
 385 }
 386
 387 static void makeTypedPhpTag (const tokenInfo *const token, const phpKind kind,
 388                                                          const accessType access, vString* typeName)
 389 {
 390         if (PhpKinds[kind].enabled)
 391         {
 392                 tagEntryInfo e;
 393
 394                 initPhpEntry (&e, token, kind, access);
 395                 if (typeName)
 396                         fillTypeRefField (&e, typeName, token);
 397                 makePhpTagEntry (&e);
 398         }
 399 }
 400
 401 static void makeSimplePhpTag (const tokenInfo *const token, const phpKind kind,
 402                                                           const accessType access)
 403 {
 404         makeTypedPhpTag (token, kind, access, NULL);
 405 }
 406
 407 static void makeNamespacePhpTag (const tokenInfo *const token, const vString *const name)
 408 {
 409         if (PhpKinds[K_NAMESPACE].enabled)
 410         {
 411                 tagEntryInfo e;
 412
 413                 initTagEntry (&e, vStringValue (name), K_NAMESPACE);
 414
 415                 e.lineNumber    = token->lineNumber;
 416                 e.filePosition  = token->filePosition;
 417
 418                 makePhpTagEntry (&e);
 419         }
 420 }
 421
 422 static void makeClassOrIfaceTag (const phpKind kind, const tokenInfo *const token,
 423                                                                  vString *const inheritance, const implType impl)
 424 {
 425         if (PhpKinds[kind].enabled)
 426         {
 427                 tagEntryInfo e;
 428
 429                 initPhpEntry (&e, token, kind, ACCESS_UNDEFINED);
 430
 431                 if (impl != IMPL_UNDEFINED)
 432                         e.extensionFields.implementation = implToString (impl);
 433                 if (vStringLength (inheritance) > 0)
 434                         e.extensionFields.inheritance = vStringValue (inheritance);
 435
 436                 makePhpTagEntry (&e);
 437         }
 438 }
 439
 440 static void makeFunctionTag (const tokenInfo *const token,
 441                                                          const vString *const arglist,
 442                                                          const vString *const rtype,
 443                                                          const accessType access, const implType impl)
 444 {
 445         if (PhpKinds[K_FUNCTION].enabled)
 446         {
 447                 tagEntryInfo e;
 448
 449                 initPhpEntry (&e, token, K_FUNCTION, access);
 450
 451                 if (impl != IMPL_UNDEFINED)
 452                         e.extensionFields.implementation = implToString (impl);
 453                 if (arglist)
 454                         e.extensionFields.signature = vStringValue (arglist);
 455                 if (rtype)
 456                         fillTypeRefField (&e, rtype, token);
 457
 458                 makePhpTagEntry (&e);
 459         }
 460 }
 461
 462 static void *newPoolToken (void *createArg CTAGS_ATTR_UNUSED)
 463 {
 464         tokenInfo *token = xMalloc (1, tokenInfo);
 465
 466         token->string = vStringNew ();
 467         token->scope  = vStringNew ();
 468         return token;
 469 }
 470
 471 static void clearPoolToken (void *data)
 472 {
 473         tokenInfo *token = data;
 474
 475         token->type                     = TOKEN_UNDEFINED;
 476         token->keyword          = KEYWORD_NONE;
 477         token->lineNumber   = getInputLineNumber ();
 478         token->filePosition = getInputFilePosition ();
 479         token->parentKind       = -1;
 480         token->anonymous        = false;
 481         vStringClear (token->string);
 482         vStringClear (token->scope);
 483 }
 484
 485 static void deletePoolToken (void *data)
 486 {
 487         tokenInfo *token = data;
 488         vStringDelete (token->string);
 489         vStringDelete (token->scope);
 490         eFree (token);
 491 }
 492
 493 static void copyToken (tokenInfo *const dest, const tokenInfo *const src,
 494                                            bool scope)
 495 {
 496         dest->lineNumber = src->lineNumber;
 497         dest->filePosition = src->filePosition;
 498         dest->type = src->type;
 499         dest->keyword = src->keyword;
 500         vStringCopy(dest->string, src->string);
 501         dest->parentKind = src->parentKind;
 502         if (scope)
 503                 vStringCopy(dest->scope, src->scope);
 504         dest->anonymous = src->anonymous;
 505 }
 506
 507 #if 0
 508 #include <stdio.h>
 509
 510 static const char *tokenTypeName (const tokenType type)
 511 {
 512         switch (type)
 513         {
 514                 case TOKEN_UNDEFINED:           return "undefined";
 515                 case TOKEN_EOF:                         return "EOF";
 516                 case TOKEN_CHARACTER:           return "character";
 517                 case TOKEN_CLOSE_PAREN:         return "')'";
 518                 case TOKEN_SEMICOLON:           return "';'";
 519                 case TOKEN_COLON:                       return "':'";
 520                 case TOKEN_COMMA:                       return "','";
 521                 case TOKEN_OPEN_PAREN:          return "'('";
 522                 case TOKEN_OPERATOR:            return "operator";
 523                 case TOKEN_IDENTIFIER:          return "identifier";
 524                 case TOKEN_KEYWORD:                     return "keyword";
 525                 case TOKEN_STRING:                      return "string";
 526                 case TOKEN_PERIOD:                      return "'.'";
 527                 case TOKEN_OPEN_CURLY:          return "'{'";
 528                 case TOKEN_CLOSE_CURLY:         return "'}'";
 529                 case TOKEN_EQUAL_SIGN:          return "'='";
 530                 case TOKEN_OPEN_SQUARE:         return "'['";
 531                 case TOKEN_CLOSE_SQUARE:        return "']'";
 532                 case TOKEN_VARIABLE:            return "variable";
 533         }
 534         return NULL;
 535 }
 536
 537 static void printToken (const tokenInfo *const token)
 538 {
 539         fprintf (stderr, "%p:\n\ttype:\t%s\n\tline:\t%lu\n\tscope:\t%s\n", (void *) token,
 540                          tokenTypeName (token->type),
 541                          token->lineNumber,
 542                          vStringValue (token->scope));
 543         switch (token->type)
 544         {
 545                 case TOKEN_IDENTIFIER:
 546                 case TOKEN_STRING:
 547                 case TOKEN_VARIABLE:
 548                         fprintf (stderr, "\tcontent:\t%s\n", vStringValue (token->string));
 549                         break;
 550
 551                 case TOKEN_KEYWORD:
 552                 {
 553                         size_t n = ARRAY_SIZE (PhpKeywordTable);
 554                         size_t i;
 555
 556                         fprintf (stderr, "\tkeyword:\t");
 557                         for (i = 0; i < n; i++)
 558                         {
 559                                 if (PhpKeywordTable[i].id == token->keyword)
 560                                 {
 561                                         fprintf (stderr, "%s\n", PhpKeywordTable[i].name);
 562                                         break;
 563                                 }
 564                         }
 565                         if (i >= n)
 566                                 fprintf (stderr, "(unknown)\n");
 567                 }
 568
 569                 default: break;
 570         }
 571 }
 572 #endif
 573
 574 static void addToScope (tokenInfo *const token, const vString *const extra,
 575                         int kindOfUpperScope)
 576 {
 577         if (vStringLength (token->scope) > 0)
 578         {
 579                 const char* sep;
 580
 581                 sep = phpScopeSeparatorFor(token->parentKind,
 582                                            kindOfUpperScope);
 583                 vStringCatS (token->scope, sep);
 584         }
 585         vStringCat (token->scope, extra);
 586 }
 587
 588 static int skipToCharacter (const int c)
 589 {
 590         int d;
 591         do
 592         {
 593                 d = getcFromInputFile ();
 594         } while (d != EOF  &&  d != c);
 595         return d;
 596 }
 597
 598 static void parseString (vString *const string, const int delimiter)
 599 {
 600         while (true)
 601         {
 602                 int c = getcFromInputFile ();
 603
 604                 if (c == '\\' && (c = getcFromInputFile ()) != EOF)
 605                         vStringPut (string, (char) c);
 606                 else if (c == EOF || c == delimiter)
 607                         break;
 608                 else
 609                         vStringPut (string, (char) c);
 610         }
 611 }
 612
 613 /* Strips @indent_len characters from lines in @string to get the correct
 614  * string value for an indented heredoc (PHP 7.3+).
 615  * This doesn't handle invalid values specially and might yield surprising
 616  * results with them, but it doesn't really matter as it's invalid anyway. */
 617 static void stripHeredocIndent (vString *const string, size_t indent_len)
 618 {
 619         char *str = vStringValue (string);
 620         size_t str_len = vStringLength (string);
 621         char *p = str;
 622         size_t new_len = str_len;
 623         bool at_line_start = true;
 624
 625         while (*p)
 626         {
 627                 if (at_line_start)
 628                 {
 629                         size_t p_len;
 630                         size_t strip_len;
 631
 632                         p_len = str_len - (p - str);
 633                         strip_len = p_len < indent_len ? p_len : indent_len;
 634                         memmove (p, p + strip_len, p_len - strip_len);
 635                         p += strip_len;
 636                         new_len -= strip_len;
 637                 }
 638                 /* CRLF is already normalized as LF */
 639                 at_line_start = (*p == '\r' || *p == '\n');
 640                 p++;
 641         }
 642         vStringTruncate (string, new_len);
 643 }
 644
 645 /* reads a PHP >= 7.3 HereDoc or a NowDoc (the part after the <<<).
 646  *      <<<[ \t]*(ID|'ID'|"ID")
 647  *      ...
 648  *      [ \t]*ID[^:indent-char:];?
 649  *
 650  * note that:
 651  *  1) starting ID must be immediately followed by a newline;
 652  *  2) closing ID is the same as opening one;
 653  *  3) closing ID must not be immediately followed by an identifier character;
 654  *  4) optional indentation of the closing ID is stripped from body lines,
 655  *     which lines must have the exact same prefix indentation.
 656  *
 657  * This is slightly relaxed from PHP < 7.3, where the closing ID had to be the
 658  * only thing on its line, with the only exception of a semicolon right after
 659  * the ID.
 660  *
 661  * Example of a single valid heredoc:
 662  *      <<< FOO
 663  *      something
 664  *      something else
 665  *      FOO_this is not an end
 666  *      FOO;
 667  *      # previous line was the end, but the semicolon wasn't required
 668  *
 669  * Another example using indentation and more code after the heredoc:
 670  *      <<<FOO
 671  *              something
 672  *              something else
 673  *              FOO . 'hello';
 674  *      # the heredoc ends at FOO, and leading tabs are stripped from the body.
 675  *      # ". 'hello'" is a normal concatenation operator and the string "hello".
 676  */
 677 static void parseHeredoc (vString *const string)
 678 {
 679         int c;
 680         unsigned int len;
 681         char delimiter[64]; /* arbitrary limit, but more is crazy anyway */
 682         int quote = 0;
 683
 684         do
 685         {
 686                 c = getcFromInputFile ();
 687         }
 688         while (c == ' ' || c == '\t');
 689
 690         if (c == '\'' || c == '"')
 691         {
 692                 quote = c;
 693                 c = getcFromInputFile ();
 694         }
 695         for (len = 0; len < ARRAY_SIZE (delimiter) - 1; len++)
 696         {
 697                 if (! isIdentChar (c))
 698                         break;
 699                 delimiter[len] = (char) c;
 700                 c = getcFromInputFile ();
 701         }
 702         delimiter[len] = 0;
 703
 704         if (len == 0) /* no delimiter, give up */
 705                 goto error;
 706         if (quote)
 707         {
 708                 if (c != quote) /* no closing quote for quoted identifier, give up */
 709                         goto error;
 710                 c = getcFromInputFile ();
 711         }
 712         if (c != '\r' && c != '\n') /* missing newline, give up */
 713                 goto error;
 714
 715         do
 716         {
 717                 c = getcFromInputFile ();
 718
 719                 vStringPut (string, (char) c);
 720                 if (c == '\r' || c == '\n')
 721                 {
 722                         /* new line, check for a delimiter right after.  No need to handle
 723                          * CRLF, getcFromInputFile() normalizes it to LF already. */
 724                         const size_t prev_string_len = vStringLength (string) - 1;
 725                         size_t indent_len = 0;
 726
 727                         c = getcFromInputFile ();
 728                         while (c == ' ' || c == '\t')
 729                         {
 730                                 vStringPut (string, (char) c);
 731                                 c = getcFromInputFile ();
 732                                 indent_len++;
 733                         }
 734
 735                         for (len = 0; c != 0 && (c - delimiter[len]) == 0; len++)
 736                                 c = getcFromInputFile ();
 737
 738                         if (delimiter[len] != 0)
 739                                 ungetcToInputFile (c);
 740                         else if (! isIdentChar (c))
 741                         {
 742                                 /* line start matched the delimiter and has a separator, we're done */
 743                                 ungetcToInputFile (c);
 744
 745                                 /* strip trailing newline and indent of the end delimiter */
 746                                 vStringTruncate (string, prev_string_len);
 747
 748                                 /* strip indent from the value if needed */
 749                                 if (indent_len > 0)
 750                                         stripHeredocIndent (string, indent_len);
 751                                 break;
 752                         }
 753                         /* if we are here it wasn't a delimiter, so put everything in the
 754                          * string */
 755                         vStringNCatS (string, delimiter, len);
 756                 }
 757         }
 758         while (c != EOF);
 759
 760         return;
 761
 762 error:
 763         ungetcToInputFile (c);
 764 }
 765
 766 static void parseIdentifier (vString *const string, const int firstChar)
 767 {
 768         int c = firstChar;
 769         do
 770         {
 771                 vStringPut (string, (char) c);
 772                 c = getcFromInputFile ();
 773         } while (isIdentChar (c));
 774         ungetcToInputFile (c);
 775 }
 776
 777 static bool isSpace (int c)
 778 {
 779         return (c == '\t' || c == ' ' || c == '\v' ||
 780                         c == '\n' || c == '\r' || c == '\f');
 781 }
 782
 783 static int skipWhitespaces (int c)
 784 {
 785         while (isSpace (c))
 786                 c = getcFromInputFile ();
 787         return c;
 788 }
 789
 790 /* <script[:white:]+language[:white:]*=[:white:]*(php|'php'|"php")[:white:]*>
 791  *
 792  * This is ugly, but the whole "<script language=php>" tag is and we can't
 793  * really do better without adding a lot of code only for this */
 794 static bool isOpenScriptLanguagePhp (int c)
 795 {
 796         int quote = 0;
 797
 798         /* <script[:white:]+language[:white:]*= */
 799         if (c                                   != '<' ||
 800                 tolower ((c = getcFromInputFile ()))         != 's' ||
 801                 tolower ((c = getcFromInputFile ()))         != 'c' ||
 802                 tolower ((c = getcFromInputFile ()))         != 'r' ||
 803                 tolower ((c = getcFromInputFile ()))         != 'i' ||
 804                 tolower ((c = getcFromInputFile ()))         != 'p' ||
 805                 tolower ((c = getcFromInputFile ()))         != 't' ||
 806                 ! isSpace ((c = getcFromInputFile ()))              ||
 807                 tolower ((c = skipWhitespaces (c))) != 'l' ||
 808                 tolower ((c = getcFromInputFile ()))         != 'a' ||
 809                 tolower ((c = getcFromInputFile ()))         != 'n' ||
 810                 tolower ((c = getcFromInputFile ()))         != 'g' ||
 811                 tolower ((c = getcFromInputFile ()))         != 'u' ||
 812                 tolower ((c = getcFromInputFile ()))         != 'a' ||
 813                 tolower ((c = getcFromInputFile ()))         != 'g' ||
 814                 tolower ((c = getcFromInputFile ()))         != 'e' ||
 815                 (c = skipWhitespaces (getcFromInputFile ())) != '=')
 816                 return false;
 817
 818         /* (php|'php'|"php")> */
 819         c = skipWhitespaces (getcFromInputFile ());
 820         if (c == '"' || c == '\'')
 821         {
 822                 quote = c;
 823                 c = getcFromInputFile ();
 824         }
 825         if (tolower (c)                         != 'p' ||
 826                 tolower ((c = getcFromInputFile ()))         != 'h' ||
 827                 tolower ((c = getcFromInputFile ()))         != 'p' ||
 828                 (quote != 0 && (c = getcFromInputFile ()) != quote) ||
 829                 (c = skipWhitespaces (getcFromInputFile ())) != '>')
 830                 return false;
 831
 832         return true;
 833 }
 834
 835 static int findPhpStart (void)
 836 {
 837         int c;
 838         do
 839         {
 840                 if ((c = getcFromInputFile ()) == '<')
 841                 {
 842                         c = getcFromInputFile ();
 843                         /* <?, <?= and <?php, but not <?xml */
 844                         if (c == '?')
 845                         {
 846                                 c = getcFromInputFile ();
 847                                 /* echo tag */
 848                                 if (c == '=')
 849                                         c = getcFromInputFile ();
 850                                 /* don't enter PHP mode on "<?xml", yet still support short open tags (<?) */
 851                                 else if (tolower (c)                          != 'x' ||
 852                                          tolower ((c = getcFromInputFile ())) != 'm' ||
 853                                          tolower ((c = getcFromInputFile ())) != 'l')
 854                                 {
 855                                         break;
 856                                 }
 857                         }
 858                         /* <script language="php"> */
 859                         else
 860                         {
 861                                 ungetcToInputFile (c);
 862                                 if (isOpenScriptLanguagePhp ('<'))
 863                                         break;
 864                         }
 865                 }
 866         }
 867         while (c != EOF);
 868
 869         return c;
 870 }
 871
 872 static int skipSingleComment (void)
 873 {
 874         int c;
 875         do
 876         {
 877                 c = getcFromInputFile ();
 878                 /* ?> in single-line comments leaves PHP mode */
 879                 if (c == '?')
 880                 {
 881                         int next = getcFromInputFile ();
 882                         if (next == '>')
 883                                 InPhp = false;
 884                         else
 885                                 ungetcToInputFile (next);
 886                 }
 887         } while (InPhp && c != EOF && c != '\n' && c != '\r');
 888         return c;
 889 }
 890
 891 static void readToken (tokenInfo *const token)
 892 {
 893         int c;
 894         bool nextMayBeKeyword = true;
 895
 896         token->type             = TOKEN_UNDEFINED;
 897         token->keyword  = KEYWORD_NONE;
 898         vStringClear (token->string);
 899
 900 getNextChar:
 901
 902         if (! InPhp)
 903         {
 904                 c = findPhpStart ();
 905                 if (c != EOF)
 906                         InPhp = true;
 907         }
 908         else
 909                 c = getcFromInputFile ();
 910
 911         c = skipWhitespaces (c);
 912
 913         token->lineNumber   = getInputLineNumber ();
 914         token->filePosition = getInputFilePosition ();
 915
 916         switch (c)
 917         {
 918                 case EOF: token->type = TOKEN_EOF;                                      break;
 919                 case '(': token->type = TOKEN_OPEN_PAREN;                       break;
 920                 case ')': token->type = TOKEN_CLOSE_PAREN;                      break;
 921                 case ';': token->type = TOKEN_SEMICOLON;                        break;
 922                 case ',': token->type = TOKEN_COMMA;                            break;
 923                 case '.': token->type = TOKEN_PERIOD;                           break;
 924                 case '{': token->type = TOKEN_OPEN_CURLY;                       break;
 925                 case '}': token->type = TOKEN_CLOSE_CURLY;                      break;
 926                 case '[': token->type = TOKEN_OPEN_SQUARE;                      break;
 927                 case ']': token->type = TOKEN_CLOSE_SQUARE;                     break;
 928                 case '&': token->type = TOKEN_AMPERSAND;                        break;
 929                 case '\\': token->type = TOKEN_BACKSLASH;                       break;
 930
 931                 case ':':
 932                 {
 933                         int d = getcFromInputFile ();
 934                         if (d == c) /* :: */
 935                         {
 936                                 nextMayBeKeyword = false;
 937                                 token->type = TOKEN_OPERATOR;
 938                         }
 939                         else
 940                         {
 941                                 ungetcToInputFile (d);
 942                                 token->type = TOKEN_COLON;
 943                         }
 944                         break;
 945                 }
 946
 947                 case '=':
 948                 {
 949                         int d = getcFromInputFile ();
 950                         if (d == '=' || d == '>')
 951                                 token->type = TOKEN_OPERATOR;
 952                         else
 953                         {
 954                                 ungetcToInputFile (d);
 955                                 token->type = TOKEN_EQUAL_SIGN;
 956                         }
 957                         break;
 958                 }
 959
 960                 case '\'':
 961                 case '"':
 962                         token->type = TOKEN_STRING;
 963                         parseString (token->string, c);
 964                         token->lineNumber = getInputLineNumber ();
 965                         token->filePosition = getInputFilePosition ();
 966                         break;
 967
 968                 case '<':
 969                 {
 970                         int d = getcFromInputFile ();
 971                         if (d == '/')
 972                         {
 973                                 /* </script[:white:]*> */
 974                                 if (tolower ((d = getcFromInputFile ())) == 's' &&
 975                                         tolower ((d = getcFromInputFile ())) == 'c' &&
 976                                         tolower ((d = getcFromInputFile ())) == 'r' &&
 977                                         tolower ((d = getcFromInputFile ())) == 'i' &&
 978                                         tolower ((d = getcFromInputFile ())) == 'p' &&
 979                                         tolower ((d = getcFromInputFile ())) == 't' &&
 980                                         (d = skipWhitespaces (getcFromInputFile ())) == '>')
 981                                 {
 982                                         InPhp = false;
 983                                         goto getNextChar;
 984                                 }
 985                                 else
 986                                 {
 987                                         ungetcToInputFile (d);
 988                                         token->type = TOKEN_UNDEFINED;
 989                                 }
 990                         }
 991                         else if (d == '<' && (d = getcFromInputFile ()) == '<')
 992                         {
 993                                 token->type = TOKEN_STRING;
 994                                 parseHeredoc (token->string);
 995                         }
 996                         else
 997                         {
 998                                 ungetcToInputFile (d);
 999                                 token->type = TOKEN_UNDEFINED;
1000                         }
1001                         break;
1002                 }
1003
1004                 case '#': /* comment */
1005                         skipSingleComment ();
1006                         goto getNextChar;
1007                         break;
1008
1009                 case '+':
1010                 case '-':
1011                 case '*':
1012                 case '%':
1013                 {
1014                         int d = getcFromInputFile ();
1015                         if (c == '-' && d == '>')
1016                                 nextMayBeKeyword = false;
1017                         else if (d != '=')
1018                                 ungetcToInputFile (d);
1019                         token->type = TOKEN_OPERATOR;
1020                         break;
1021                 }
1022
1023                 case '/': /* division or comment start */
1024                 {
1025                         int d = getcFromInputFile ();
1026                         if (d == '/') /* single-line comment */
1027                         {
1028                                 skipSingleComment ();
1029                                 goto getNextChar;
1030                         }
1031                         else if (d == '*')
1032                         {
1033                                 do
1034                                 {
1035                                         c = skipToCharacter ('*');
1036                                         if (c != EOF)
1037                                         {
1038                                                 c = getcFromInputFile ();
1039                                                 if (c == '/')
1040                                                         break;
1041                                                 else
1042                                                         ungetcToInputFile (c);
1043                                         }
1044                                 } while (c != EOF && c != '\0');
1045                                 goto getNextChar;
1046                         }
1047                         else
1048                         {
1049                                 if (d != '=')
1050                                         ungetcToInputFile (d);
1051                                 token->type = TOKEN_OPERATOR;
1052                         }
1053                         break;
1054                 }
1055
1056                 case '$': /* variable start */
1057                 {
1058                         int d = getcFromInputFile ();
1059                         if (! isIdentChar (d))
1060                         {
1061                                 ungetcToInputFile (d);
1062                                 token->type = TOKEN_UNDEFINED;
1063                         }
1064                         else
1065                         {
1066                                 parseIdentifier (token->string, d);
1067                                 token->type = TOKEN_VARIABLE;
1068                         }
1069                         break;
1070                 }
1071
1072                 case '?': /* maybe the end of the PHP chunk */
1073                 {
1074                         int d = getcFromInputFile ();
1075                         if (d == '>')
1076                         {
1077                                 InPhp = false;
1078                                 goto getNextChar;
1079                         }
1080                         else
1081                         {
1082                                 ungetcToInputFile (d);
1083                                 token->type = TOKEN_QMARK;
1084                         }
1085                         break;
1086                 }
1087
1088                 default:
1089                         if (! isIdentChar (c))
1090                                 token->type = TOKEN_UNDEFINED;
1091                         else
1092                         {
1093                                 parseIdentifier (token->string, c);
1094                                 if (MayBeKeyword)
1095                                         token->keyword = lookupCaseKeyword (vStringValue (token->string), getInputLanguage ());
1096                                 else
1097                                         token->keyword = KEYWORD_NONE;
1098
1099                                 if (token->keyword == KEYWORD_NONE)
1100                                         token->type = TOKEN_IDENTIFIER;
1101                                 else
1102                                         token->type = TOKEN_KEYWORD;
1103                         }
1104                         break;
1105         }
1106
1107         if (token->type == TOKEN_SEMICOLON ||
1108                 token->type == TOKEN_OPEN_CURLY ||
1109                 token->type == TOKEN_CLOSE_CURLY)
1110         {
1111                 /* reset current statement details on statement end, and when entering
1112                  * a deeper scope.
1113                  * it is a bit ugly to do this in readToken(), but it makes everything
1114                  * a lot simpler. */
1115                 CurrentStatement.access = ACCESS_UNDEFINED;
1116                 CurrentStatement.impl = IMPL_UNDEFINED;
1117         }
1118
1119         MayBeKeyword = nextMayBeKeyword;
1120 }
1121
1122 static void readQualifiedName (tokenInfo *const token, vString *name,
1123                                tokenInfo *const lastToken)
1124 {
1125         while (token->type == TOKEN_IDENTIFIER || token->type == TOKEN_BACKSLASH)
1126         {
1127                 if (name)
1128                 {
1129                         if (token->type == TOKEN_BACKSLASH)
1130                                 vStringPut (name, '\\');
1131                         else
1132                                 vStringCat (name, token->string);
1133                 }
1134                 if (lastToken)
1135                         copyToken (lastToken, token, true);
1136                 readToken (token);
1137         }
1138 }
1139
1140 static void enterScope (tokenInfo *const parentToken,
1141                                                 const vString *const extraScope,
1142                                                 const int parentKind);
1143
1144 static void skipOverParens (tokenInfo *token)
1145 {
1146         if (token->type == TOKEN_OPEN_PAREN)
1147         {
1148                 int depth = 1;
1149
1150                 do
1151                 {
1152                         readToken (token);
1153                         switch (token->type)
1154                         {
1155                                 case TOKEN_OPEN_PAREN:  depth++; break;
1156                                 case TOKEN_CLOSE_PAREN: depth--; break;
1157                                 default: break;
1158                         }
1159                 }
1160                 while (token->type != TOKEN_EOF && depth > 0);
1161
1162                 readToken (token);
1163         }
1164 }
1165
1166 /* parses a class or an interface:
1167  *      class Foo {}
1168  *      class Foo extends Bar {}
1169  *      class Foo extends Bar implements iFoo, iBar {}
1170  *      interface iFoo {}
1171  *      interface iBar extends iFoo {}
1172  *
1173  * if @name is not NULL, parses an anonymous class with name @name
1174  *      new class {}
1175  *      new class(1, 2) {}
1176  *      new class(1, 2) extends Foo implements iFoo, iBar {} */
1177 static bool parseClassOrIface (tokenInfo *const token, const phpKind kind,
1178                                   const tokenInfo *name)
1179 {
1180         bool readNext = true;
1181         implType impl = CurrentStatement.impl;
1182         tokenInfo *nameFree = NULL;
1183         vString *inheritance = NULL;
1184         vString *parent = NULL;
1185
1186         readToken (token);
1187         if (name) /* anonymous class */
1188         {
1189                 /* skip possible construction arguments */
1190                 skipOverParens (token);
1191         }
1192         else /* normal, named class */
1193         {
1194                 if (token->type != TOKEN_IDENTIFIER)
1195                         return false;
1196
1197                 name = nameFree = newToken ();
1198                 copyToken (nameFree, token, true);
1199
1200                 readToken (token);
1201         }
1202
1203         inheritance = vStringNew ();
1204         /* read every identifiers, keywords and commas, and assume each
1205          *  identifier (not keyword) is an inheritance
1206          * (like in "class Foo extends Bar implements iA, iB") */
1207         enum { inheritance_initial,
1208                    inheritance_extends,
1209                    inheritance_implements
1210         } istat = inheritance_initial;
1211         while (token->type == TOKEN_IDENTIFIER ||
1212                token->type == TOKEN_BACKSLASH ||
1213                token->type == TOKEN_KEYWORD ||
1214                token->type == TOKEN_COMMA)
1215         {
1216                 if (token->type == TOKEN_IDENTIFIER || token->type == TOKEN_BACKSLASH)
1217                 {
1218                         vString *qualifiedName = vStringNew ();
1219
1220                         readQualifiedName (token, qualifiedName, NULL);
1221                         if (vStringLength (inheritance) > 0)
1222                                 vStringPut (inheritance, ',');
1223                         vStringCat (inheritance, qualifiedName);
1224                         if (istat == inheritance_extends && !parent)
1225                                 parent = qualifiedName;
1226                         else
1227                                 vStringDelete (qualifiedName);
1228                 }
1229                 else
1230                 {
1231                         if (token->type == TOKEN_KEYWORD)
1232                         {
1233                                 if (token->keyword == KEYWORD_extends)
1234                                         istat = inheritance_extends;
1235                                 else if (token->keyword == KEYWORD_implements)
1236                                         istat = inheritance_implements;
1237                         }
1238                         readToken (token);
1239                 }
1240         }
1241
1242         makeClassOrIfaceTag (kind, name, inheritance, impl);
1243
1244         if (token->type == TOKEN_OPEN_CURLY)
1245         {
1246                 vString *backup = ParentClass;
1247                 ParentClass = parent;
1248                 enterScope (token, name->string, kind);
1249                 ParentClass = backup;
1250         }
1251         else
1252                 readNext = false;
1253
1254         if (nameFree)
1255                 deleteToken (nameFree);
1256         vStringDelete (parent);
1257         vStringDelete (inheritance);
1258
1259         return readNext;
1260 }
1261
1262 /* parses a trait:
1263  *      trait Foo {} */
1264 static bool parseTrait (tokenInfo *const token)
1265 {
1266         bool readNext = true;
1267         tokenInfo *name;
1268
1269         readToken (token);
1270         if (token->type != TOKEN_IDENTIFIER)
1271                 return false;
1272
1273         name = newToken ();
1274         copyToken (name, token, true);
1275
1276         makeSimplePhpTag (name, K_TRAIT, ACCESS_UNDEFINED);
1277
1278         readToken (token);
1279         if (token->type == TOKEN_OPEN_CURLY)
1280                 enterScope (token, name->string, K_TRAIT);
1281         else
1282                 readNext = false;
1283
1284         deleteToken (name);
1285
1286         return readNext;
1287 }
1288
1289 /* parse a function
1290  *
1291  * if @name is NULL, parses a normal function
1292  *      function myfunc($foo, $bar) {}
1293  *      function &myfunc($foo, $bar) {}
1294  *      function myfunc($foo, $bar) : type {}
1295  *      function myfunc($foo, $bar) : ?type {}
1296  *
1297  * if @name is not NULL, parses an anonymous function with name @name
1298  *      $foo = function($foo, $bar) {}
1299  *      $foo = function&($foo, $bar) {}
1300  *      $foo = function($foo, $bar) use ($x, &$y) {}
1301  *      $foo = function($foo, $bar) use ($x, &$y) : type {}
1302  *      $foo = function($foo, $bar) use ($x, &$y) : ?type {} */
1303 static bool parseFunction (tokenInfo *const token, const tokenInfo *name)
1304 {
1305         bool readNext = true;
1306         accessType access = CurrentStatement.access;
1307         implType impl = CurrentStatement.impl;
1308         tokenInfo *nameFree = NULL;
1309         vString *rtype = NULL;
1310         vString *arglist = NULL;
1311
1312         readToken (token);
1313         /* skip a possible leading ampersand (return by reference) */
1314         if (token->type == TOKEN_AMPERSAND)
1315                 readToken (token);
1316
1317         if (! name)
1318         {
1319                 if (token->type != TOKEN_IDENTIFIER && token->type != TOKEN_KEYWORD)
1320                         return false;
1321
1322                 name = nameFree = newToken ();
1323                 copyToken (nameFree, token, true);
1324                 readToken (token);
1325         }
1326
1327         if (token->type == TOKEN_OPEN_PAREN)
1328         {
1329                 int depth = 1;
1330
1331                 arglist = vStringNew ();
1332                 vStringPut (arglist, '(');
1333                 do
1334                 {
1335                         readToken (token);
1336
1337                         switch (token->type)
1338                         {
1339                                 case TOKEN_OPEN_PAREN:  depth++; break;
1340                                 case TOKEN_CLOSE_PAREN: depth--; break;
1341                                 default: break;
1342                         }
1343                         /* display part */
1344                         switch (token->type)
1345                         {
1346                                 case TOKEN_AMPERSAND:           vStringPut (arglist, '&');              break;
1347                                 case TOKEN_CLOSE_CURLY:         vStringPut (arglist, '}');              break;
1348                                 case TOKEN_CLOSE_PAREN:         vStringPut (arglist, ')');              break;
1349                                 case TOKEN_CLOSE_SQUARE:        vStringPut (arglist, ']');              break;
1350                                 case TOKEN_COLON:                       vStringPut (arglist, ':');              break;
1351                                 case TOKEN_COMMA:                       vStringCatS (arglist, ", ");    break;
1352                                 case TOKEN_EQUAL_SIGN:          vStringCatS (arglist, " = ");   break;
1353                                 case TOKEN_OPEN_CURLY:          vStringPut (arglist, '{');              break;
1354                                 case TOKEN_OPEN_PAREN:          vStringPut (arglist, '(');              break;
1355                                 case TOKEN_OPEN_SQUARE:         vStringPut (arglist, '[');              break;
1356                                 case TOKEN_PERIOD:                      vStringPut (arglist, '.');              break;
1357                                 case TOKEN_SEMICOLON:           vStringPut (arglist, ';');              break;
1358                                 case TOKEN_BACKSLASH:           vStringPut (arglist, '\\');             break;
1359                                 case TOKEN_STRING:
1360                                 {
1361                                         vStringPut (arglist, '\'');
1362                                         vStringCat  (arglist, token->string);
1363                                         vStringPut (arglist, '\'');
1364                                         break;
1365                                 }
1366
1367                                 case TOKEN_IDENTIFIER:
1368                                 case TOKEN_KEYWORD:
1369                                 case TOKEN_VARIABLE:
1370                                 {
1371                                         switch (vStringLast (arglist))
1372                                         {
1373                                                 case 0:
1374                                                 case ' ':
1375                                                 case '{':
1376                                                 case '(':
1377                                                 case '[':
1378                                                 case '.':
1379                                                 case '\\':
1380                                                         /* no need for a space between those and the identifier */
1381                                                         break;
1382
1383                                                 default:
1384                                                         vStringPut (arglist, ' ');
1385                                                         break;
1386                                         }
1387                                         if (token->type == TOKEN_VARIABLE)
1388                                                 vStringPut (arglist, '$');
1389                                         vStringCat (arglist, token->string);
1390                                         break;
1391                                 }
1392
1393                                 default: break;
1394                         }
1395                 }
1396                 while (token->type != TOKEN_EOF && depth > 0);
1397
1398                 readToken (token); /* normally it's an open brace or "use" keyword */
1399         }
1400
1401         /* skip use(...) */
1402         if (token->type == TOKEN_KEYWORD && token->keyword == KEYWORD_use)
1403         {
1404                 readToken (token);
1405                 skipOverParens (token);
1406         }
1407
1408         /* PHP7 return type declaration or if parsing Zephir, gather function return
1409          * type hint to fill typeRef. */
1410         if ((getInputLanguage () == Lang_php && token->type == TOKEN_COLON) ||
1411             (getInputLanguage () == Lang_zephir && token->type == TOKEN_OPERATOR))
1412         {
1413                 if (arglist)
1414                         rtype = vStringNew ();
1415
1416                 readToken (token);
1417                 if (token->type == TOKEN_QMARK)
1418                 {
1419                         if (rtype)
1420                                 vStringPut (rtype, '?');
1421                         readToken (token);
1422                 }
1423                 readQualifiedName (token, rtype, NULL);
1424
1425                 if (rtype && vStringIsEmpty (rtype))
1426                 {
1427                         vStringDelete (rtype);
1428                         rtype = NULL;
1429                 }
1430         }
1431
1432         if (arglist)
1433                 makeFunctionTag (name, arglist, rtype, access, impl);
1434
1435         if (token->type == TOKEN_OPEN_CURLY)
1436                 enterScope (token, name->string, K_FUNCTION);
1437         else
1438                 readNext = false;
1439
1440         vStringDelete (rtype);
1441         vStringDelete (arglist);
1442         if (nameFree)
1443                 deleteToken (nameFree);
1444
1445         return readNext;
1446 }
1447
1448 /* parses declarations of the form
1449  *      const NAME = VALUE */
1450 static bool parseConstant (tokenInfo *const token)
1451 {
1452         tokenInfo *name;
1453
1454         readToken (token); /* skip const keyword */
1455         if (token->type != TOKEN_IDENTIFIER && token->type != TOKEN_KEYWORD)
1456                 return false;
1457
1458         name = newToken ();
1459         copyToken (name, token, true);
1460
1461         readToken (token);
1462         if (token->type == TOKEN_EQUAL_SIGN)
1463                 makeSimplePhpTag (name, K_DEFINE, ACCESS_UNDEFINED);
1464
1465         deleteToken (name);
1466
1467         return token->type == TOKEN_EQUAL_SIGN;
1468 }
1469
1470 /* parses declarations of the form
1471  *      define('NAME', 'VALUE')
1472  *      define(NAME, 'VALUE) */
1473 static bool parseDefine (tokenInfo *const token)
1474 {
1475         int depth = 1;
1476
1477         readToken (token); /* skip "define" identifier */
1478         if (token->type != TOKEN_OPEN_PAREN)
1479                 return false;
1480
1481         readToken (token);
1482         if (token->type == TOKEN_STRING ||
1483                 token->type == TOKEN_IDENTIFIER)
1484         {
1485                 makeSimplePhpTag (token, K_DEFINE, ACCESS_UNDEFINED);
1486                 readToken (token);
1487         }
1488
1489         /* skip until the close parenthesis.
1490          * no need to handle nested blocks since they would be invalid
1491          * in this context anyway (the VALUE may only be a scalar, like
1492          *      42
1493          *      (42)
1494          * and alike) */
1495         while (token->type != TOKEN_EOF && depth > 0)
1496         {
1497                 switch (token->type)
1498                 {
1499                         case TOKEN_OPEN_PAREN:  depth++; break;
1500                         case TOKEN_CLOSE_PAREN: depth--; break;
1501                         default: break;
1502                 }
1503                 readToken (token);
1504         }
1505
1506         return false;
1507 }
1508
1509 /* parses declarations of the form
1510  *      use Foo
1511  *      use Foo\Bar\Class
1512  *      use Foo\Bar\Class as FooBarClass
1513  *      use function Foo\Bar\func
1514  *      use function Foo\Bar\func as foobarfunc
1515  *      use const Foo\Bar\CONST
1516  *      use const Foo\Bar\CONST as FOOBARCONST
1517  *      use Foo, Bar
1518  *      use Foo, Bar as Baz
1519  *      use Foo as Test, Bar as Baz
1520  *      use Foo\{Bar, Baz as Child, Nested\Other, Even\More as Something} */
1521 static bool parseUse (tokenInfo *const token)
1522 {
1523         bool readNext = false;
1524         /* we can't know the use type, because class, interface and namespaces
1525          * aliases are the same, and the only difference is the referenced name's
1526          * type */
1527         const char *refType = "unknown";
1528         vString *refName = vStringNew ();
1529         tokenInfo *nameToken = newToken ();
1530         bool grouped = false;
1531
1532         readToken (token); /* skip use keyword itself */
1533         if (token->type == TOKEN_KEYWORD && (token->keyword == KEYWORD_function ||
1534                                              token->keyword == KEYWORD_const))
1535         {
1536                 switch (token->keyword)
1537                 {
1538                         case KEYWORD_function:  refType = PhpKinds[K_FUNCTION].name;    break;
1539                         case KEYWORD_const:             refType = PhpKinds[K_DEFINE].name;              break;
1540                         default: break; /* silence compilers */
1541                 }
1542                 readNext = true;
1543         }
1544
1545         if (readNext)
1546                 readToken (token);
1547
1548         readQualifiedName (token, refName, nameToken);
1549         grouped = readNext = (token->type == TOKEN_OPEN_CURLY);
1550
1551         do
1552         {
1553                 size_t refNamePrefixLength = grouped ? vStringLength (refName) : 0;
1554
1555                 /* if it's either not the first name in a comma-separated list, or we
1556                  * are in a grouped alias and need to read the leaf name */
1557                 if (readNext)
1558                 {
1559                         readToken (token);
1560                         /* in case of a trailing comma (or an empty group) */
1561                         if (token->type == TOKEN_CLOSE_CURLY)
1562                                 break;
1563                         readQualifiedName (token, refName, nameToken);
1564                 }
1565
1566                 if (token->type == TOKEN_KEYWORD && token->keyword == KEYWORD_as)
1567                 {
1568                         readToken (token);
1569                         copyToken (nameToken, token, true);
1570                         readToken (token);
1571                 }
1572
1573                 if (nameToken->type == TOKEN_IDENTIFIER && PhpKinds[K_ALIAS].enabled)
1574                 {
1575                         tagEntryInfo entry;
1576
1577                         initPhpEntry (&entry, nameToken, K_ALIAS, ACCESS_UNDEFINED);
1578
1579                         entry.extensionFields.typeRef[0] = refType;
1580                         entry.extensionFields.typeRef[1] = vStringValue (refName);
1581
1582                         makePhpTagEntry (&entry);
1583                 }
1584
1585                 vStringTruncate (refName, refNamePrefixLength);
1586
1587                 readNext = true;
1588         }
1589         while (token->type == TOKEN_COMMA);
1590
1591         if (grouped && token->type == TOKEN_CLOSE_CURLY)
1592                 readToken (token);
1593
1594         vStringDelete (refName);
1595         deleteToken (nameToken);
1596
1597         return (token->type == TOKEN_SEMICOLON);
1598 }
1599
1600 /* parses declarations of the form
1601  *      $var = VALUE
1602  *      $var; */
1603 static bool parseVariable (tokenInfo *const token, vString * typeName)
1604 {
1605         tokenInfo *name;
1606         bool readNext = true;
1607         accessType access = CurrentStatement.access;
1608
1609         name = newToken ();
1610         copyToken (name, token, true);
1611
1612         readToken (token);
1613         if (token->type == TOKEN_EQUAL_SIGN)
1614         {
1615                 phpKind kind = K_VARIABLE;
1616
1617                 if (token->parentKind == K_FUNCTION)
1618                         kind = K_LOCAL_VARIABLE;
1619
1620                 readToken (token);
1621                 if (token->type == TOKEN_KEYWORD &&
1622                         token->keyword == KEYWORD_function &&
1623                         PhpKinds[kind].enabled)
1624                 {
1625                         if (parseFunction (token, name))
1626                                 readToken (token);
1627                         readNext = (bool) (token->type == TOKEN_SEMICOLON);
1628                 }
1629                 else
1630                 {
1631                         makeSimplePhpTag (name, kind, access);
1632                         readNext = false;
1633                 }
1634         }
1635         else if (token->type == TOKEN_SEMICOLON)
1636         {
1637                 /* generate tags for variable declarations in classes
1638                  *      class Foo {
1639                  *              protected $foo;
1640                  *      }
1641                  * but don't get fooled by stuff like $foo = $bar; */
1642                 if (token->parentKind == K_CLASS ||
1643                     token->parentKind == K_INTERFACE ||
1644                     token->parentKind == K_TRAIT)
1645                         makeTypedPhpTag (name, K_VARIABLE, access, typeName);
1646         }
1647         else
1648                 readNext = false;
1649
1650         deleteToken (name);
1651
1652         return readNext;
1653 }
1654
1655 /* parses namespace declarations
1656  *      namespace Foo {}
1657  *      namespace Foo\Bar {}
1658  *      namespace Foo;
1659  *      namespace Foo\Bar;
1660  *      namespace;
1661  *      namespace {} */
1662 static bool parseNamespace (tokenInfo *const token)
1663 {
1664         tokenInfo *nsToken = newToken ();
1665
1666         vStringClear (CurrentNamesapce);
1667         copyToken (nsToken, token, false);
1668
1669         do
1670         {
1671                 readToken (token);
1672                 if (token->type == TOKEN_IDENTIFIER)
1673                 {
1674                         if (vStringLength (CurrentNamesapce) > 0)
1675                         {
1676                                 const char *sep;
1677
1678                                 sep = phpScopeSeparatorFor(K_NAMESPACE,
1679                                                            K_NAMESPACE);
1680                                 vStringCatS (CurrentNamesapce, sep);
1681                         }
1682                         vStringCat (CurrentNamesapce, token->string);
1683                 }
1684         }
1685         while (token->type != TOKEN_EOF &&
1686                    token->type != TOKEN_SEMICOLON &&
1687                    token->type != TOKEN_OPEN_CURLY);
1688
1689         if (vStringLength (CurrentNamesapce) > 0)
1690                 makeNamespacePhpTag (nsToken, CurrentNamesapce);
1691
1692         if (token->type == TOKEN_OPEN_CURLY)
1693                 enterScope (token, NULL, -1);
1694
1695         deleteToken (nsToken);
1696
1697         return true;
1698 }
1699
1700 static void enterScope (tokenInfo *const parentToken,
1701                                                 const vString *const extraScope,
1702                                                 const int parentKind)
1703 {
1704         tokenInfo *token = newToken ();
1705         vString *typeName = vStringNew ();
1706         int origParentKind = parentToken->parentKind;
1707
1708         copyToken (token, parentToken, true);
1709
1710         if (extraScope)
1711         {
1712                 token->parentKind = parentKind;
1713                 addToScope (token, extraScope, origParentKind);
1714         }
1715
1716         readToken (token);
1717         while (token->type != TOKEN_EOF &&
1718                    token->type != TOKEN_CLOSE_CURLY)
1719         {
1720                 bool readNext = true;
1721
1722                 switch (token->type)
1723                 {
1724                         case TOKEN_OPEN_CURLY:
1725                                 enterScope (token, NULL, -1);
1726                                 break;
1727
1728                         case TOKEN_KEYWORD:
1729                                 switch (token->keyword)
1730                                 {
1731                                         /* handle anonymous classes */
1732                                         case KEYWORD_new:
1733                                                 readToken (token);
1734                                                 if (token->keyword != KEYWORD_class)
1735                                                         readNext = false;
1736                                                 else
1737                                                 {
1738                                                         tokenInfo *name = newToken ();
1739
1740                                                         copyToken (name, token, true);
1741                                                         anonGenerate (name->string, "AnonymousClass", K_CLASS);
1742                                                         name->anonymous = true;
1743                                                         readNext = parseClassOrIface (token, K_CLASS, name);
1744                                                         deleteToken (name);
1745                                                 }
1746                                                 break;
1747
1748                                         case KEYWORD_class:             readNext = parseClassOrIface (token, K_CLASS, NULL);            break;
1749                                         case KEYWORD_interface: readNext = parseClassOrIface (token, K_INTERFACE, NULL);        break;
1750                                         case KEYWORD_trait:             readNext = parseTrait (token);                                                          break;
1751                                         case KEYWORD_function:  readNext = parseFunction (token, NULL);                                         break;
1752                                         case KEYWORD_const:             readNext = parseConstant (token);                                                       break;
1753                                         case KEYWORD_define:    readNext = parseDefine (token);                                                         break;
1754
1755                                         case KEYWORD_use:
1756                                                 /* aliases are only allowed at root scope, but the keyword
1757                                                  * is also used to i.e. "import" traits into a class */
1758                                                 if (vStringLength (token->scope) == 0)
1759                                                         readNext = parseUse (token);
1760                                                 break;
1761
1762                                         case KEYWORD_namespace: readNext = parseNamespace (token);      break;
1763
1764                                         case KEYWORD_private:   CurrentStatement.access = ACCESS_PRIVATE;       break;
1765                                         case KEYWORD_protected: CurrentStatement.access = ACCESS_PROTECTED;     break;
1766                                         case KEYWORD_public:    CurrentStatement.access = ACCESS_PUBLIC;        break;
1767                                         case KEYWORD_var:               CurrentStatement.access = ACCESS_PUBLIC;        break;
1768
1769                                         case KEYWORD_abstract:  CurrentStatement.impl = IMPL_ABSTRACT;          break;
1770
1771                                         default: break;
1772                                 }
1773                                 break;
1774
1775                         case TOKEN_QMARK:
1776                                 vStringClear (typeName);
1777                                 vStringPut (typeName, '?');
1778                                 readNext = true;
1779                                 break;
1780                         case TOKEN_IDENTIFIER:
1781                                 vStringCat (typeName, token->string);
1782                                 readNext = true;
1783                                 break;
1784                         case TOKEN_VARIABLE:
1785                                 readNext = parseVariable (token,
1786                                                                                   vStringIsEmpty(typeName)
1787                                                                                   ? NULL
1788                                                                                   : typeName);
1789                                 vStringClear (typeName);
1790                                 break;
1791
1792                         default: break;
1793                 }
1794
1795                 if (readNext)
1796                         readToken (token);
1797         }
1798
1799         copyToken (parentToken, token, false);
1800         parentToken->parentKind = origParentKind;
1801         vStringDelete (typeName);
1802         deleteToken (token);
1803 }
1804
1805 static void findTags (bool startsInPhpMode)
1806 {
1807         tokenInfo *const token = newToken ();
1808
1809         InPhp = startsInPhpMode;
1810         MayBeKeyword = true;
1811         CurrentStatement.access = ACCESS_UNDEFINED;
1812         CurrentStatement.impl = IMPL_UNDEFINED;
1813         CurrentNamesapce = vStringNew ();
1814         FullScope = vStringNew ();
1815         Assert (ParentClass == NULL);
1816
1817         do
1818         {
1819                 enterScope (token, NULL, -1);
1820         }
1821         while (token->type != TOKEN_EOF); /* keep going even with unmatched braces */
1822
1823         vStringDelete (FullScope);
1824         vStringDelete (CurrentNamesapce);
1825         deleteToken (token);
1826 }
1827
1828 static void findPhpTags (void)
1829 {
1830         findTags (false);
1831 }
1832
1833 static void findZephirTags (void)
1834 {
1835         findTags (true);
1836 }
1837
1838 static void initializePool (void)
1839 {
1840         if (TokenPool == NULL)
1841                 TokenPool = objPoolNew (16, newPoolToken, deletePoolToken, clearPoolToken, NULL);
1842 }
1843
1844 static void initializePhpParser (const langType language)
1845 {
1846         Lang_php = language;
1847         initializePool ();
1848 }
1849
1850 static void initializeZephirParser (const langType language)
1851 {
1852         Lang_zephir = language;
1853         initializePool ();
1854 }
1855
1856 static void finalize (langType language CTAGS_ATTR_UNUSED, bool initialized)
1857 {
1858         if (!initialized)
1859                 return;
1860
1861         if (TokenPool != NULL)
1862         {
1863                 objPoolDelete (TokenPool);
1864                 TokenPool = NULL;
1865         }
1866 }
1867
1868 extern parserDefinition* PhpParser (void)
1869 {
1870         static const char *const extensions [] = { "php", "php3", "php4", "php5", "php7", "phtml", NULL };
1871         parserDefinition* def = parserNew ("PHP");
1872         def->kindTable      = PhpKinds;
1873         def->kindCount  = ARRAY_SIZE (PhpKinds);
1874         def->extensions = extensions;
1875         def->parser     = findPhpTags;
1876         def->initialize = initializePhpParser;
1877         def->finalize   = finalize;
1878         def->keywordTable = PhpKeywordTable;
1879         def->keywordCount = ARRAY_SIZE (PhpKeywordTable);
1880         return def;
1881 }
1882
1883 extern parserDefinition* ZephirParser (void)
1884 {
1885         static const char *const extensions [] = { "zep", NULL };
1886         parserDefinition* def = parserNew ("Zephir");
1887         def->kindTable      = PhpKinds;
1888         def->kindCount  = ARRAY_SIZE (PhpKinds);
1889         def->extensions = extensions;
1890         def->parser     = findZephirTags;
1891         def->initialize = initializeZephirParser;
1892         def->finalize   = finalize;
1893         def->keywordTable = PhpKeywordTable;
1894         def->keywordCount = ARRAY_SIZE (PhpKeywordTable);
1895         return def;
1896 }