ctags/parsers/cxx/cxx_parser_tokenizer.c

   1 /*
   2 *   Copyright (c) 2016, Szymon Tomasz Stefanek
   3 *
   4 *   This source code is released for free distribution under the terms of the
   5 *   GNU General Public License version 2 or (at your option) any later version.
   6 *
   7 *   This module contains functions for parsing and scanning C++ source files
   8 */
   9 #include "cxx_parser.h"
  10 #include "cxx_parser_internal.h"
  11
  12 #include "cxx_debug.h"
  13 #include "cxx_keyword.h"
  14 #include "cxx_token.h"
  15 #include "cxx_token_chain.h"
  16
  17 #include "parse.h"
  18 #include "vstring.h"
  19 #include "../cpreprocessor.h"
  20 #include "debug.h"
  21 #include "keyword.h"
  22 #include "read.h"
  23 #include "options.h"
  24
  25 #include <string.h>
  26
  27 #define UINFO(c) (((c) < 0x80 && (c) >= 0) ? g_aCharTable[c].uType : 0)
  28
  29 static void cxxParserSkipToNonWhiteSpace(void)
  30 {
  31         while(cppIsspace(g_cxx.iChar))
  32                 g_cxx.iChar = cppGetc();
  33 }
  34
  35 enum CXXCharType
  36 {
  37         // Start of an identifier a-z A-Z _ and ~ since
  38         // it's part of the destructor name
  39         CXXCharTypeStartOfIdentifier = 1,
  40         // Part of identifier a-z a-Z 0-9 _
  41         CXXCharTypePartOfIdentifier = (1 << 1),
  42         // A decimal digit
  43         CXXCharTypeDecimalDigit = (1 << 2),
  44         // A hexadecimal digit
  45         CXXCharTypeHexadecimalDigit = (1 << 3),
  46         // Hex digits x X u U l L and .
  47         CXXCharTypeValidInNumber = (1 << 4),
  48         // A named single char token.
  49         CXXCharTypeNamedSingleCharToken = (1 << 5),
  50         // A named single or repeated char token.
  51         CXXCharTypeNamedSingleOrRepeatedCharToken = (1 << 6),
  52         // An operator (we merge them)
  53         CXXCharTypeOperator = (1 << 7),
  54         // Full custom handling. Mostly operators or brackets.
  55         CXXCharTypeCustomHandling = (1 << 8)
  56 };
  57
  58 typedef struct _CXXCharTypeData
  59 {
  60         unsigned int uType;
  61         unsigned int uSingleTokenType;
  62         unsigned int uMultiTokenType;
  63 } CXXCharTypeData;
  64
  65
  66 static CXXCharTypeData g_aCharTable[128] =
  67 {
  68         // 000 (0x00) NUL
  69         {
  70                 0,
  71                 0,
  72                 0
  73         },
  74         // 001 (0x01) SOH
  75         {
  76                 0,
  77                 0,
  78                 0
  79         },
  80         // 002 (0x02) STX
  81         {
  82                 0,
  83                 0,
  84                 0
  85         },
  86         // 003 (0x03) ETX
  87         {
  88                 0,
  89                 0,
  90                 0
  91         },
  92         // 004 (0x04) EOT
  93         {
  94                 0,
  95                 0,
  96                 0
  97         },
  98         // 005 (0x05) ENQ
  99         {
 100                 0,
 101                 0,
 102                 0
 103         },
 104         // 006 (0x06) ACK
 105         {
 106                 0,
 107                 0,
 108                 0
 109         },
 110         // 007 (0x07) BEL
 111         {
 112                 0,
 113                 0,
 114                 0
 115         },
 116         // 008 (0x08) BS
 117         {
 118                 0,
 119                 0,
 120                 0
 121         },
 122         // 009 (0x09) '\t' HT
 123         {
 124                 0,
 125                 0,
 126                 0
 127         },
 128         // 010 (0x0a) '\n' LF
 129         {
 130                 0,
 131                 0,
 132                 0
 133         },
 134         // 011 (0x0b) '\v' VT
 135         {
 136                 0,
 137                 0,
 138                 0
 139         },
 140         // 012 (0x0c) FF
 141         {
 142                 0,
 143                 0,
 144                 0
 145         },
 146         // 013 (0x0d) '\r' CR
 147         {
 148                 0,
 149                 0,
 150                 0
 151         },
 152         // 014 (0x0e) 'SO'
 153         {
 154                 0,
 155                 0,
 156                 0
 157         },
 158         // 015 (0x0f) 'SI'
 159         {
 160                 0,
 161                 0,
 162                 0
 163         },
 164         // 016 (0x10) DLE
 165         {
 166                 0,
 167                 0,
 168                 0
 169         },
 170         // 017 (0x11) DC1
 171         {
 172                 0,
 173                 0,
 174                 0
 175         },
 176         // 018 (0x12) DC2
 177         {
 178                 0,
 179                 0,
 180                 0
 181         },
 182         // 019 (0x13) DC3
 183         {
 184                 0,
 185                 0,
 186                 0
 187         },
 188         // 020 (0x14) DC4
 189         {
 190                 0,
 191                 0,
 192                 0
 193         },
 194         // 021 (0x15) NAK
 195         {
 196                 0,
 197                 0,
 198                 0
 199         },
 200         // 022 (0x16) SYN
 201         {
 202                 0,
 203                 0,
 204                 0
 205         },
 206         // 023 (0x17) ETB
 207         {
 208                 0,
 209                 0,
 210                 0
 211         },
 212         // 024 (0x18) CAN
 213         {
 214                 0,
 215                 0,
 216                 0
 217         },
 218         // 025 (0x19) EM
 219         {
 220                 0,
 221                 0,
 222                 0
 223         },
 224         // 026 (0x1a) SUB
 225         {
 226                 0,
 227                 0,
 228                 0
 229         },
 230         // 027 (0x1b) ESC
 231         {
 232                 0,
 233                 0,
 234                 0
 235         },
 236         // 028 (0x1c) FS
 237         {
 238                 0,
 239                 0,
 240                 0
 241         },
 242         // 029 (0x1d) GS
 243         {
 244                 0,
 245                 0,
 246                 0
 247         },
 248         // 030 (0x1e) RS
 249         {
 250                 0,
 251                 0,
 252                 0
 253         },
 254         // 031 (0x1f) US
 255         {
 256                 0,
 257                 0,
 258                 0
 259         },
 260         // 032 (0x20) ' '
 261         {
 262                 0,
 263                 0,
 264                 0
 265         },
 266         // 033 (0x21) '!'
 267         {
 268                 CXXCharTypeOperator,
 269                 0 ,
 270                 0
 271         },
 272         // 034 (0x22) '"'
 273         {
 274                 0,
 275                 0,
 276                 0
 277         },
 278         // 035 (0x23) '#'
 279         {
 280                 0,
 281                 0,
 282                 0
 283         },
 284         // 036 (0x24) '$'
 285         {
 286                 CXXCharTypeStartOfIdentifier | CXXCharTypePartOfIdentifier,
 287                 0,
 288                 0
 289         },
 290         // 037 (0x25) '%'
 291         {
 292                 CXXCharTypeOperator,
 293                 0 ,
 294                 0
 295         },
 296         // 038 (0x26) '&'
 297         {
 298                 CXXCharTypeNamedSingleOrRepeatedCharToken,
 299                 CXXTokenTypeAnd,
 300                 CXXTokenTypeMultipleAnds
 301         },
 302         // 039 (0x27) '''
 303         {
 304                 0,
 305                 0,
 306                 0
 307         },
 308         // 040 (0x28) '('
 309         {
 310                 CXXCharTypeNamedSingleCharToken,
 311                 CXXTokenTypeOpeningParenthesis,
 312                 0
 313         },
 314         // 041 (0x29) ')'
 315         {
 316                 CXXCharTypeNamedSingleCharToken,
 317                 CXXTokenTypeClosingParenthesis,
 318                 0
 319         },
 320         // 042 (0x2a) '*'
 321         {
 322                 CXXCharTypeNamedSingleCharToken,
 323                 CXXTokenTypeStar,
 324                 0
 325         },
 326         // 043 (0x2b) '+'
 327         {
 328                 CXXCharTypeOperator,
 329                 0 ,
 330                 0
 331         },
 332         // 044 (0x2c) ','
 333         {
 334                 CXXCharTypeNamedSingleCharToken,
 335                 CXXTokenTypeComma,
 336                 0
 337         },
 338         // 045 (0x2d) '-'
 339         {
 340                 CXXCharTypeOperator,
 341                 0 ,
 342                 0
 343         },
 344         // 046 (0x2e) '.'
 345         {
 346                 CXXCharTypeValidInNumber | CXXCharTypeNamedSingleOrRepeatedCharToken,
 347                 CXXTokenTypeDotOperator,
 348                 CXXTokenTypeMultipleDots
 349         },
 350         // 047 (0x2f) '/'
 351         {
 352                 CXXCharTypeOperator,
 353                 0 ,
 354                 0
 355         },
 356         // 048 (0x30) '0'
 357         {
 358                 CXXCharTypePartOfIdentifier | CXXCharTypeDecimalDigit |
 359                         CXXCharTypeHexadecimalDigit | CXXCharTypeValidInNumber,
 360                 0,
 361                 0
 362         },
 363         // 049 (0x31) '1'
 364         {
 365                 CXXCharTypePartOfIdentifier | CXXCharTypeDecimalDigit |
 366                         CXXCharTypeHexadecimalDigit | CXXCharTypeValidInNumber,
 367                 0,
 368                 0
 369         },
 370         // 050 (0x32) '2'
 371         {
 372                 CXXCharTypePartOfIdentifier | CXXCharTypeDecimalDigit |
 373                         CXXCharTypeHexadecimalDigit | CXXCharTypeValidInNumber,
 374                 0,
 375                 0
 376         },
 377         // 051 (0x33) '3'
 378         {
 379                 CXXCharTypePartOfIdentifier | CXXCharTypeDecimalDigit |
 380                         CXXCharTypeHexadecimalDigit | CXXCharTypeValidInNumber,
 381                 0,
 382                 0
 383         },
 384         // 052 (0x34) '4'
 385         {
 386                 CXXCharTypePartOfIdentifier | CXXCharTypeDecimalDigit |
 387                         CXXCharTypeHexadecimalDigit | CXXCharTypeValidInNumber,
 388                 0,
 389                 0
 390         },
 391         // 053 (0x35) '5'
 392         {
 393                 CXXCharTypePartOfIdentifier | CXXCharTypeDecimalDigit |
 394                         CXXCharTypeHexadecimalDigit | CXXCharTypeValidInNumber,
 395                 0,
 396                 0
 397         },
 398         // 054 (0x36) '6'
 399         {
 400                 CXXCharTypePartOfIdentifier | CXXCharTypeDecimalDigit |
 401                         CXXCharTypeHexadecimalDigit | CXXCharTypeValidInNumber,
 402                 0,
 403                 0
 404         },
 405         // 055 (0x37) '7'
 406         {
 407                 CXXCharTypePartOfIdentifier | CXXCharTypeDecimalDigit |
 408                         CXXCharTypeHexadecimalDigit | CXXCharTypeValidInNumber,
 409                 0,
 410                 0
 411         },
 412         // 056 (0x38) '8'
 413         {
 414                 CXXCharTypePartOfIdentifier | CXXCharTypeDecimalDigit |
 415                         CXXCharTypeHexadecimalDigit | CXXCharTypeValidInNumber,
 416                 0,
 417                 0
 418         },
 419         // 057 (0x39) '9'
 420         {
 421                 CXXCharTypePartOfIdentifier | CXXCharTypeDecimalDigit |
 422                         CXXCharTypeHexadecimalDigit | CXXCharTypeValidInNumber,
 423                 0,
 424                 0
 425         },
 426         // 058 (0x3a) ':'
 427         {
 428                 CXXCharTypeNamedSingleOrRepeatedCharToken,
 429                 CXXTokenTypeSingleColon,
 430                 CXXTokenTypeMultipleColons
 431         },
 432         // 059 (0x3b) ';'
 433         {
 434                 CXXCharTypeNamedSingleCharToken,
 435                 CXXTokenTypeSemicolon,
 436                 0
 437         },
 438         // 060 (0x3c) '<'
 439         {
 440                 CXXCharTypeCustomHandling,
 441                 CXXTokenTypeSmallerThanSign,
 442                 0
 443         },
 444         // 061 (0x3d) '='
 445         {
 446                 CXXCharTypeOperator | CXXCharTypeNamedSingleOrRepeatedCharToken,
 447                 CXXTokenTypeAssignment,
 448                 CXXTokenTypeOperator
 449         },
 450         // 062 (0x3e) '>' // We never merge two >>
 451         {
 452                 CXXCharTypeNamedSingleCharToken,
 453                 CXXTokenTypeGreaterThanSign,
 454                 0
 455         },
 456         // 063 (0x3f) '?'
 457         {
 458                 CXXCharTypeOperator,
 459                 0 ,
 460                 0
 461         },
 462         // 064 (0x40) '@'
 463         {
 464                 0,
 465                 0,
 466                 0
 467         },
 468         // 065 (0x41) 'A'
 469         {
 470                 CXXCharTypeStartOfIdentifier | CXXCharTypePartOfIdentifier |
 471                         CXXCharTypeHexadecimalDigit | CXXCharTypeValidInNumber,
 472                 0,
 473                 0
 474         },
 475         // 066 (0x42) 'B'
 476         {
 477                 CXXCharTypeStartOfIdentifier | CXXCharTypePartOfIdentifier |
 478                         CXXCharTypeHexadecimalDigit | CXXCharTypeValidInNumber,
 479                 0,
 480                 0
 481         },
 482         // 067 (0x43) 'C'
 483         {
 484                 CXXCharTypeStartOfIdentifier | CXXCharTypePartOfIdentifier |
 485                         CXXCharTypeHexadecimalDigit | CXXCharTypeValidInNumber,
 486                 0,
 487                 0
 488         },
 489         // 068 (0x44) 'D'
 490         {
 491                 CXXCharTypeStartOfIdentifier | CXXCharTypePartOfIdentifier |
 492                         CXXCharTypeHexadecimalDigit | CXXCharTypeValidInNumber,
 493                 0,
 494                 0
 495         },
 496         // 069 (0x45) 'E'
 497         {
 498                 CXXCharTypeStartOfIdentifier | CXXCharTypePartOfIdentifier |
 499                         CXXCharTypeHexadecimalDigit | CXXCharTypeValidInNumber,
 500                 0,
 501                 0
 502         },
 503         // 070 (0x46) 'F'
 504         {
 505                 CXXCharTypeStartOfIdentifier | CXXCharTypePartOfIdentifier |
 506                         CXXCharTypeHexadecimalDigit | CXXCharTypeValidInNumber,
 507                 0,
 508                 0
 509         },
 510         // 071 (0x47) 'G'
 511         {
 512                 CXXCharTypeStartOfIdentifier | CXXCharTypePartOfIdentifier,
 513                 0 ,
 514                 0
 515         },
 516         // 072 (0x48) 'H'
 517         {
 518                 CXXCharTypeStartOfIdentifier | CXXCharTypePartOfIdentifier |
 519                         CXXCharTypeValidInNumber,
 520                 0,
 521                 0
 522         },
 523         // 073 (0x49) 'I'
 524         {
 525                 CXXCharTypeStartOfIdentifier | CXXCharTypePartOfIdentifier,
 526                 0 ,
 527                 0
 528         },
 529         // 074 (0x4a) 'J'
 530         {
 531                 CXXCharTypeStartOfIdentifier | CXXCharTypePartOfIdentifier,
 532                 0 ,
 533                 0
 534         },
 535         // 075 (0x4b) 'K'
 536         {
 537                 CXXCharTypeStartOfIdentifier | CXXCharTypePartOfIdentifier,
 538                 0 ,
 539                 0
 540         },
 541         // 076 (0x4c) 'L'
 542         {
 543                 CXXCharTypeStartOfIdentifier | CXXCharTypePartOfIdentifier |
 544                         CXXCharTypeValidInNumber,
 545                 0,
 546                 0
 547         },
 548         // 077 (0x4d) 'M'
 549         {
 550                 CXXCharTypeStartOfIdentifier | CXXCharTypePartOfIdentifier,
 551                 0 ,
 552                 0
 553         },
 554         // 078 (0x4e) 'N'
 555         {
 556                 CXXCharTypeStartOfIdentifier | CXXCharTypePartOfIdentifier,
 557                 0 ,
 558                 0
 559         },
 560         // 079 (0x4f) 'O'
 561         {
 562                 CXXCharTypeStartOfIdentifier | CXXCharTypePartOfIdentifier,
 563                 0 ,
 564                 0
 565         },
 566         // 080 (0x50) 'P'
 567         {
 568                 CXXCharTypeStartOfIdentifier | CXXCharTypePartOfIdentifier,
 569                 0 ,
 570                 0
 571         },
 572         // 081 (0x51) 'Q'
 573         {
 574                 CXXCharTypeStartOfIdentifier | CXXCharTypePartOfIdentifier,
 575                 0,
 576                 0
 577         },
 578         // 082 (0x52) 'R'
 579         {
 580                 CXXCharTypeStartOfIdentifier | CXXCharTypePartOfIdentifier,
 581                 0 ,
 582                 0
 583         },
 584         // 083 (0x53) 'S'
 585         {
 586                 CXXCharTypeStartOfIdentifier | CXXCharTypePartOfIdentifier,
 587                 0 ,
 588                 0
 589         },
 590         // 084 (0x54) 'T'
 591         {
 592                 CXXCharTypeStartOfIdentifier | CXXCharTypePartOfIdentifier,
 593                 0 ,
 594                 0
 595         },
 596         // 085 (0x55) 'U'
 597         {
 598                 CXXCharTypeStartOfIdentifier | CXXCharTypePartOfIdentifier |
 599                         CXXCharTypeValidInNumber,
 600                 0 ,
 601                 0
 602         },
 603         // 086 (0x56) 'V'
 604         {
 605                 CXXCharTypeStartOfIdentifier | CXXCharTypePartOfIdentifier,
 606                 0 ,
 607                 0
 608         },
 609         // 087 (0x57) 'W'
 610         {
 611                 CXXCharTypeStartOfIdentifier | CXXCharTypePartOfIdentifier,
 612                 0 ,
 613                 0
 614         },
 615         // 088 (0x58) 'X'
 616         {
 617                 CXXCharTypeStartOfIdentifier | CXXCharTypePartOfIdentifier |
 618                         CXXCharTypeValidInNumber,
 619                 0 ,
 620                 0
 621         },
 622         // 089 (0x59) 'Y'
 623         {
 624                 CXXCharTypeStartOfIdentifier | CXXCharTypePartOfIdentifier,
 625                 0 ,
 626                 0
 627         },
 628         // 090 (0x5a) 'Z'
 629         {
 630                 CXXCharTypeStartOfIdentifier | CXXCharTypePartOfIdentifier,
 631                 0 ,
 632                 0
 633         },
 634         // 091 (0x5b) '['
 635         {
 636                 CXXCharTypeCustomHandling,
 637                 CXXTokenTypeOpeningSquareParenthesis,
 638                 0
 639         },
 640         // 092 (0x5c) '\'
 641         {
 642                 0,
 643                 0,
 644                 0
 645         },
 646         // 093 (0x5d) ']'
 647         {
 648                 CXXCharTypeNamedSingleCharToken,
 649                 CXXTokenTypeClosingSquareParenthesis,
 650                 0
 651         },
 652         // 094 (0x5e) '^'
 653         {
 654                 CXXCharTypeOperator,
 655                 0,
 656                 0
 657         },
 658         // 095 (0x5f) '_'
 659         {
 660                 CXXCharTypeStartOfIdentifier | CXXCharTypePartOfIdentifier,
 661                 0 ,
 662                 0
 663         },
 664         // 096 (0x60) '`'
 665         {
 666                 0,
 667                 0,
 668                 0
 669         },
 670         // 097 (0x61) 'a'
 671         {
 672                 CXXCharTypeStartOfIdentifier | CXXCharTypePartOfIdentifier |
 673                         CXXCharTypeHexadecimalDigit | CXXCharTypeValidInNumber,
 674                 0,
 675                 0
 676         },
 677         // 098 (0x62) 'b'
 678         {
 679                 CXXCharTypeStartOfIdentifier | CXXCharTypePartOfIdentifier |
 680                         CXXCharTypeHexadecimalDigit | CXXCharTypeValidInNumber,
 681                 0 ,
 682                 0
 683         },
 684         // 099 (0x63) 'c'
 685         {
 686                 CXXCharTypeStartOfIdentifier | CXXCharTypePartOfIdentifier |
 687                         CXXCharTypeHexadecimalDigit | CXXCharTypeValidInNumber,
 688                 0 ,
 689                 0
 690         },
 691         // 100 (0x64) 'd'
 692         {
 693                 CXXCharTypeStartOfIdentifier | CXXCharTypePartOfIdentifier |
 694                         CXXCharTypeHexadecimalDigit | CXXCharTypeValidInNumber,
 695                 0 ,
 696                 0
 697         },
 698         // 101 (0x65) 'e'
 699         {
 700                 CXXCharTypeStartOfIdentifier | CXXCharTypePartOfIdentifier |
 701                         CXXCharTypeHexadecimalDigit | CXXCharTypeValidInNumber,
 702                 0 ,
 703                 0
 704         },
 705         // 102 (0x66) 'f'
 706         {
 707                 CXXCharTypeStartOfIdentifier | CXXCharTypePartOfIdentifier |
 708                         CXXCharTypeHexadecimalDigit | CXXCharTypeValidInNumber,
 709                 0 ,
 710                 0
 711         },
 712         // 103 (0x67) 'g'
 713         {
 714                 CXXCharTypeStartOfIdentifier | CXXCharTypePartOfIdentifier,
 715                 0 ,
 716                 0
 717         },
 718         // 104 (0x68) 'h'
 719         {
 720                 CXXCharTypeStartOfIdentifier | CXXCharTypePartOfIdentifier |
 721                         CXXCharTypeValidInNumber,
 722                 0 ,
 723                 0
 724         },
 725         // 105 (0x69) 'i'
 726         {
 727                 CXXCharTypeStartOfIdentifier | CXXCharTypePartOfIdentifier,
 728                 0 ,
 729                 0
 730         },
 731         // 106 (0x6a) 'j'
 732         {
 733                 CXXCharTypeStartOfIdentifier | CXXCharTypePartOfIdentifier,
 734                 0 ,
 735                 0
 736         },
 737         // 107 (0x6b) 'k'
 738         {
 739                 CXXCharTypeStartOfIdentifier | CXXCharTypePartOfIdentifier,
 740                 0 ,
 741                 0
 742         },
 743         // 108 (0x6c) 'l'
 744         {
 745                 CXXCharTypeStartOfIdentifier | CXXCharTypePartOfIdentifier |
 746                         CXXCharTypeValidInNumber,
 747                 0 ,
 748                 0
 749         },
 750         // 109 (0x6d) 'm'
 751         {
 752                 CXXCharTypeStartOfIdentifier | CXXCharTypePartOfIdentifier,
 753                 0 ,
 754                 0
 755         },
 756         // 110 (0x6e) 'n'
 757         {
 758                 CXXCharTypeStartOfIdentifier | CXXCharTypePartOfIdentifier,
 759                 0 ,
 760                 0
 761         },
 762         // 111 (0x6f) 'o'
 763         {
 764                 CXXCharTypeStartOfIdentifier | CXXCharTypePartOfIdentifier,
 765                 0 ,
 766                 0
 767         },
 768         // 112 (0x70) 'p'
 769         {
 770                 CXXCharTypeStartOfIdentifier | CXXCharTypePartOfIdentifier,
 771                 0 ,
 772                 0
 773         },
 774         // 113 (0x71) 'q'
 775         {
 776                 CXXCharTypeStartOfIdentifier | CXXCharTypePartOfIdentifier,
 777                 0 ,
 778                 0
 779         },
 780         // 114 (0x72) 'r'
 781         {
 782                 CXXCharTypeStartOfIdentifier | CXXCharTypePartOfIdentifier,
 783                 0 ,
 784                 0
 785         },
 786         // 115 (0x73) 's'
 787         {
 788                 CXXCharTypeStartOfIdentifier | CXXCharTypePartOfIdentifier,
 789                 0 ,
 790                 0
 791         },
 792         // 116 (0x74) 't'
 793         {
 794                 CXXCharTypeStartOfIdentifier | CXXCharTypePartOfIdentifier,
 795                 0 ,
 796                 0
 797         },
 798         // 117 (0x75) 'u'
 799         {
 800                 CXXCharTypeStartOfIdentifier | CXXCharTypePartOfIdentifier |
 801                         CXXCharTypeValidInNumber,
 802                 0 ,
 803                 0
 804         },
 805         // 118 (0x76) 'v'
 806         {
 807                 CXXCharTypeStartOfIdentifier | CXXCharTypePartOfIdentifier,
 808                 0 ,
 809                 0
 810         },
 811         // 119 (0x77) 'w'
 812         {
 813                 CXXCharTypeStartOfIdentifier | CXXCharTypePartOfIdentifier,
 814                 0 ,
 815                 0
 816         },
 817         // 120 (0x78) 'x'
 818         {
 819                 CXXCharTypeStartOfIdentifier | CXXCharTypePartOfIdentifier |
 820                         CXXCharTypeValidInNumber,
 821                 0 ,
 822                 0
 823         },
 824         // 121 (0x79) 'y'
 825         {
 826                 CXXCharTypeStartOfIdentifier | CXXCharTypePartOfIdentifier,
 827                 0 ,
 828                 0
 829         },
 830         // 122 (0x7a) 'z'
 831         {
 832                 CXXCharTypeStartOfIdentifier | CXXCharTypePartOfIdentifier,
 833                 0 ,
 834                 0
 835         },
 836         // 123 (0x7b) '{'
 837         {
 838                 CXXCharTypeNamedSingleCharToken,
 839                 CXXTokenTypeOpeningBracket,
 840                 0
 841         },
 842         // 124 (0x7c) '|'
 843         {
 844                 CXXCharTypeOperator,
 845                 0 ,
 846                 0
 847         },
 848         // 125 (0x7d) '}'
 849         {
 850                 CXXCharTypeNamedSingleCharToken,
 851                 CXXTokenTypeClosingBracket,
 852                 0
 853         },
 854         // 126 (0x7e) '~'
 855         {
 856                 CXXCharTypeStartOfIdentifier,
 857                 0 ,
 858                 0
 859         },
 860         // 127 (0x7f)
 861         { 0, 0, 0 }
 862 };
 863
 864 // Parse the contents of an attribute chain.
 865 // The input is the innermost chain of __attribute__((...)) or [[...]]
 866 static void cxxParserAnalyzeAttributeChain(CXXTokenChain * pChain)
 867 {
 868         CXXToken * pToken = cxxTokenChainFirst(pChain);
 869
 870         while(pToken)
 871         {
 872                 if(cxxTokenTypeIs(pToken,CXXTokenTypeIdentifier))
 873                 {
 874                         CXX_DEBUG_PRINT("Analyzing attribute %s",vStringValue(pToken->pszWord));
 875                         if(
 876                                         (strcmp(vStringValue(pToken->pszWord),"always_inline") == 0) ||
 877                                         (strcmp(vStringValue(pToken->pszWord),"__always_inline__") == 0)
 878                                 )
 879                         {
 880                                 CXX_DEBUG_PRINT("Found attribute 'always_inline'");
 881                                 // assume "inline" has been seen.
 882                                 g_cxx.uKeywordState |= CXXParserKeywordStateSeenInline;
 883                         } else if(
 884                                         (strcmp(vStringValue(pToken->pszWord),"deprecated") == 0) ||
 885                                         (strcmp(vStringValue(pToken->pszWord),"__deprecated__") == 0)
 886                                 )
 887                         {
 888                                 CXX_DEBUG_PRINT("Found attribute 'deprecated'");
 889                                 // assume "inline" has been seen.
 890                                 g_cxx.uKeywordState |= CXXParserKeywordStateSeenAttributeDeprecated;
 891                         }
 892                 }
 893
 894                 pToken = pToken->pNext;
 895         }
 896 }
 897
 898 //
 899 // The __attribute__((...)) sequence complicates parsing quite a lot.
 900 // For this reason we attempt to "hide" it from the rest of the parser
 901 // at tokenizer level. However, we will not kill it. For extracting interesting
 902 // information from the sequence in upper layers, attach the token chain
 903 // built from the sequence to the token AROUND the sequence.
 904 // In this function, we call the token "attributes owner" token.
 905 // CXXToken::pSideChain is the member for attaching.
 906 //
 907 // Returns false if it finds an EOF. This is an important invariant required by
 908 // cxxParserParseNextToken(), the only caller.
 909 //
 910 static bool cxxParserParseNextTokenCondenseAttribute(void)
 911 {
 912         // Since cxxParserParseNextToken() returns false only when it has found
 913         // an EOF, this function must do the same.
 914         // This means that any broken input must be discarded here.
 915
 916         CXX_DEBUG_ENTER();
 917
 918         CXX_DEBUG_ASSERT(
 919                         cxxTokenTypeIs(g_cxx.pToken,CXXTokenTypeKeyword) &&
 920                         (cxxKeywordMayDropInTokenizer(g_cxx.pToken->eKeyword)),
 921                         "This function should be called only after we have parsed __attribute__ or __declspec"
 922                 );
 923
 924         CXXToken * pAttrHead = cxxTokenChainTakeLast(g_cxx.pTokenChain);
 925
 926         // And go ahead.
 927
 928         if(!cxxParserParseNextToken())
 929         {
 930                 cxxTokenDestroy(pAttrHead);
 931                 CXX_DEBUG_LEAVE_TEXT("No next token after __attribute__");
 932                 return false;
 933         }
 934
 935         if(!cxxTokenTypeIs(g_cxx.pToken,CXXTokenTypeOpeningParenthesis))
 936         {
 937                 cxxTokenDestroy(pAttrHead);
 938                 CXX_DEBUG_LEAVE_TEXT("Something that is not an opening parenthesis");
 939                 return true;
 940         }
 941
 942         // Do NOT accept EOF as a valid terminator as it implies broken input.
 943         if(!cxxParserParseAndCondenseCurrentSubchain(
 944                         CXXTokenTypeOpeningParenthesis |
 945                                 CXXTokenTypeOpeningSquareParenthesis |
 946                                 CXXTokenTypeOpeningBracket,
 947                         false,
 948                         false
 949                 ))
 950         {
 951                 // Parsing and/or condensation of the subchain failed. This implies broken
 952                 // input (mismatched parenthesis/bracket, early EOF).
 953
 954                 CXX_DEBUG_LEAVE_TEXT("Failed to parse subchains. The input is broken...");
 955
 956                 cxxTokenDestroy(pAttrHead);
 957
 958                 // However our invariant (see comment at the beginning of the function)
 959                 // forbids us to return false if we didn't find an EOF. So we attempt
 960                 // to resume parsing anyway. If there is an EOF, cxxParserParseNextToken()
 961                 // will report it.
 962
 963                 // Kill the token chain
 964                 cxxTokenChainDestroyLast(g_cxx.pTokenChain);
 965
 966                 return cxxParserParseNextToken();
 967         }
 968
 969         CXX_DEBUG_ASSERT(
 970                         cxxTokenTypeIs(g_cxx.pToken,CXXTokenTypeParenthesisChain),
 971                         "Should have a parenthesis chain as last token!"
 972                 );
 973
 974         // Try to make sense of certain kinds of __attribute__.
 975         // the proper syntax is __attribute__(()), so look at the inner chain
 976
 977         CXXToken * pInner = cxxTokenChainFirst(g_cxx.pToken->pChain);
 978         if(pInner)
 979         {
 980                 if(pInner->pNext && cxxTokenTypeIs(pInner->pNext,CXXTokenTypeParenthesisChain))
 981                         cxxParserAnalyzeAttributeChain(pInner->pNext->pChain);
 982         }
 983
 984         CXXToken * pAttrArgs = cxxTokenChainTakeLast(g_cxx.pTokenChain);
 985         CXXToken * pAttrOwner = cxxTokenChainLast(g_cxx.pTokenChain);
 986
 987         // And finally extract yet another token.
 988         bool bRet = cxxParserParseNextToken();
 989
 990         if(pAttrOwner == NULL
 991            || cxxTokenTypeIs(pAttrOwner, CXXTokenTypeComma)) {
 992                 // If __attribute__ was at the beginning of the chain,
 993                 // we cannot attach the __attribute__ side chain to
 994                 // the previous token.
 995                 // In that case, we attach the side chain to the
 996                 // next token.
 997                 pAttrOwner = g_cxx.pToken;
 998         } else {
 999                 // Look up a previous identifier token.
1000                 CXXToken * p = cxxTokenChainPreviousTokenOfType(pAttrOwner,
1001                                                                                                                 CXXTokenTypeIdentifier);
1002                 if(p)
1003                         pAttrOwner = p;
1004         }
1005
1006         if(pAttrOwner)
1007         {
1008                 if(!pAttrOwner->pSideChain)
1009                         pAttrOwner->pSideChain = cxxTokenChainCreate();
1010                 cxxTokenChainAppend(pAttrOwner->pSideChain, pAttrHead);
1011                 cxxTokenChainAppend(pAttrOwner->pSideChain, pAttrArgs);
1012 #if 0
1013                 fprintf(stderr, "pAttrOwner(%s#%d): ",
1014                                 pAttrOwner == g_cxx.pToken? "next": "prev",
1015                                 pAttrOwner->iLineNumber);
1016                 CXX_DEBUG_TOKEN(pAttrOwner);
1017                 fprintf(stderr, "Side chain: ");
1018                 if(pAttrOwner->pSideChain)
1019                         CXX_DEBUG_CHAIN(pAttrOwner->pSideChain);
1020                 else
1021                         CXX_DEBUG_PRINT("NULL\n");
1022 #endif
1023         }
1024
1025         CXX_DEBUG_LEAVE();
1026         return bRet;
1027 }
1028
1029 //
1030 // We handle the attribute [[...]] sequence introduced in c++11 in the same way
1031 // as __attribute__((...)). We move it out of the parser's way as it complicates parsing.
1032 //
1033 // Returns false if it finds an EOF. This is an important invariant required by
1034 // cxxParserParseNextToken(), the only caller.
1035 //
1036 static bool cxxParserParseNextTokenCondenseCXX11Attribute(void)
1037 {
1038         CXX_DEBUG_ENTER();
1039
1040         CXX_DEBUG_ASSERT(
1041                         cxxTokenTypeIs(g_cxx.pToken, CXXTokenTypeOpeningSquareParenthesis),
1042                         "This function should be called only after we have parsed ["
1043                 );
1044
1045         // Input stream: [[...
1046         //   If the syntax is correct then this is an attribute sequence [[foo]]
1047         //
1048         // g_cxx.pToken points the first '['.
1049         // g_cxx.iChar points the second '['.
1050         //
1051         // A caller calls this function only when the second '[' is found.
1052
1053         if(!cxxParserParseAndCondenseCurrentSubchain(
1054                         CXXTokenTypeOpeningParenthesis |
1055                                 CXXTokenTypeOpeningSquareParenthesis |
1056                                 CXXTokenTypeOpeningBracket,
1057                         false,
1058                         false
1059                 ))
1060         {
1061                 // Parsing and/or condensation of the subchain failed. This implies broken
1062                 // input (mismatched parenthesis/bracket, early EOF).
1063
1064                 CXX_DEBUG_LEAVE_TEXT("Failed to parse subchains. The input is broken...");
1065
1066                 // However our invariant
1067                 // forbids us to return false if we didn't find an EOF. So we attempt
1068                 // to resume parsing anyway. If there is an EOF, cxxParserParseNextToken()
1069                 // will report it.
1070
1071                 // Kill the token chain
1072                 cxxTokenChainDestroyLast(g_cxx.pTokenChain);
1073
1074                 return cxxParserParseNextToken();
1075         }
1076
1077         // Now the current token should be replaced by a square parenthesis chain
1078         // that contains another square parenthesis chain.
1079         CXX_DEBUG_ASSERT(
1080                         cxxTokenTypeIs(g_cxx.pToken,CXXTokenTypeSquareParenthesisChain),
1081                         "Should have a parenthesis chain as last token!"
1082                 );
1083         CXX_DEBUG_ASSERT(
1084                         // at least [ + [*] + ]
1085                         (g_cxx.pToken->pChain->iCount >= 3) &&
1086                         cxxTokenTypeIs(
1087                                         cxxTokenChainAt(g_cxx.pToken->pChain,1),
1088                                         CXXTokenTypeSquareParenthesisChain
1089                                 ),
1090                         "Should have a nested parenthesis chain inside the last token!"
1091                 );
1092
1093         cxxParserAnalyzeAttributeChain(
1094                         cxxTokenChainAt(g_cxx.pToken->pChain,1)->pChain
1095                 );
1096
1097         // Now just kill it.
1098         cxxTokenChainDestroyLast(g_cxx.pTokenChain);
1099
1100         // And finally extract yet another token.
1101         bool bRet = cxxParserParseNextToken();
1102
1103         CXX_DEBUG_LEAVE();
1104         return bRet;
1105 }
1106
1107 // A macro token was encountered and it expects a parameter list.
1108 // The routine has to check if there is a following parenthesis
1109 // and eventually skip it but it MUST NOT parse the next token
1110 // if it is not a parenthesis. This is because the macro token
1111 // may have a replacement and is that one that has to be returned
1112 // back to the caller from cxxParserParseNextToken().
1113 static bool cxxParserParseNextTokenSkipMacroParenthesis(CXXToken ** ppChain)
1114 {
1115         CXX_DEBUG_ENTER();
1116
1117         CXX_DEBUG_ASSERT(ppChain,"ppChain should not be null here");
1118
1119         cxxParserSkipToNonWhiteSpace();
1120
1121         if(g_cxx.iChar != '(')
1122         {
1123                 *ppChain = NULL;
1124                 return true; // no parenthesis
1125         }
1126
1127         if(!cxxParserParseNextToken())
1128         {
1129                 CXX_DEBUG_LEAVE_TEXT("No next token after ignored identifier");
1130                 return false;
1131         }
1132
1133         if(!cxxTokenTypeIs(g_cxx.pToken,CXXTokenTypeOpeningParenthesis))
1134         {
1135                 CXX_DEBUG_ASSERT(false,"Should have found an open parenthesis token here!");
1136                 CXX_DEBUG_LEAVE_TEXT("Internal error");
1137                 return false;
1138         }
1139
1140         if(!cxxParserParseAndCondenseCurrentSubchain(
1141                         CXXTokenTypeOpeningParenthesis,
1142                         false,
1143                         false
1144                 ))
1145         {
1146                 CXX_DEBUG_LEAVE_TEXT("Failed to parse and condense subchains");
1147                 return false;
1148         }
1149
1150         CXX_DEBUG_ASSERT(
1151                         cxxTokenTypeIs(g_cxx.pToken,CXXTokenTypeParenthesisChain),
1152                         "Should have a parenthesis chain as last token!"
1153                 );
1154
1155         // Now just kill the chain.
1156         *ppChain = cxxTokenChainTakeLast(g_cxx.pTokenChain);
1157
1158         CXX_DEBUG_LEAVE();
1159         return true;
1160 }
1161
1162 static void cxxParserParseNextTokenApplyReplacement(
1163                 cppMacroInfo * pInfo,
1164                 CXXToken * pParameterChainToken
1165         )
1166 {
1167         CXX_DEBUG_ENTER();
1168
1169         CXX_DEBUG_ASSERT(pInfo,"Info must be not null");
1170         CXX_DEBUG_ASSERT(pInfo->replacements,"There should be a replacement");
1171
1172         if(!pInfo->hasParameterList)
1173         {
1174                 CXX_DEBUG_ASSERT(!pParameterChainToken,"This shouldn't have been extracted");
1175         }
1176
1177         CXXTokenChain * pParameters = NULL;
1178         const char ** aParameters = NULL;
1179         int iParameterCount = 0;
1180
1181         if(pInfo->hasParameterList && pParameterChainToken && (pParameterChainToken->pChain->iCount >= 3))
1182         {
1183                 // kill parenthesis
1184                 cxxTokenChainDestroyFirst(pParameterChainToken->pChain);
1185                 cxxTokenChainDestroyLast(pParameterChainToken->pChain);
1186
1187                 pParameters = cxxTokenChainSplitOnComma(
1188                                 pParameterChainToken->pChain
1189                         );
1190
1191                 aParameters = (const char **)eMalloc(sizeof(const char *) * pParameters->iCount);
1192                 CXXToken * pParam = cxxTokenChainFirst(pParameters);
1193                 while(pParam)
1194                 {
1195                         aParameters[iParameterCount] = vStringValue(pParam->pszWord);
1196                         iParameterCount++;
1197                         pParam = pParam->pNext;
1198                 }
1199
1200                 CXX_DEBUG_ASSERT(iParameterCount == pParameters->iCount,"Bad number of parameters found");
1201         }
1202
1203         vString * pReplacement = cppBuildMacroReplacement(pInfo,aParameters,iParameterCount);
1204
1205         if(pParameters)
1206         {
1207                 cxxTokenChainDestroy(pParameters);
1208                 eFree((char**)aParameters);
1209         }
1210
1211         CXX_DEBUG_PRINT("Applying complex replacement '%s'",vStringValue(pReplacement));
1212
1213         cppUngetStringBuiltByMacro(vStringValue(pReplacement),vStringLength(pReplacement), pInfo);
1214
1215         vStringDelete(pReplacement);
1216
1217         CXX_DEBUG_LEAVE();
1218 }
1219
1220 void cxxParserUngetCurrentToken(void)
1221 {
1222         CXX_DEBUG_ASSERT(
1223                         g_cxx.pToken &&
1224                         g_cxx.pTokenChain &&
1225                         (g_cxx.pTokenChain->iCount > 0),
1226                         "There should be at least one token to unget"
1227                 );
1228
1229         if(g_cxx.pUngetToken)
1230         {
1231                 if(g_cxx.pUngetToken->bFollowedBySpace)
1232                         cppUngetc(' ');
1233                 cppUngetString(vStringValue(g_cxx.pUngetToken->pszWord),vStringLength(g_cxx.pUngetToken->pszWord));
1234                 cxxTokenDestroy(g_cxx.pUngetToken);
1235         }
1236
1237         g_cxx.pUngetToken = cxxTokenChainTakeLast(g_cxx.pTokenChain);
1238
1239         CXX_DEBUG_ASSERT(g_cxx.pUngetToken == g_cxx.pToken,"Oops.. ungot a token that was not the chain tail");
1240
1241         g_cxx.pToken = cxxTokenChainLast(g_cxx.pTokenChain);
1242 }
1243
1244
1245 #define CXX_PARSER_MAXIMUM_TOKEN_CHAIN_SIZE 16384
1246
1247 // We stop applying macro replacements if the unget buffer gets too big
1248 // as it is a sign of recursive macro expansion
1249 #define CXX_PARSER_MAXIMUM_UNGET_BUFFER_SIZE_FOR_MACRO_REPLACEMENTS 65536
1250
1251 // We stop applying macro replacements if a macro is used so many
1252 // times in a recursive macro expansion.
1253 #define CXX_PARSER_MAXIMUM_MACRO_USE_COUNT 8
1254
1255 // Returns false if it finds an EOF. Returns true otherwise.
1256 //
1257 // In some special cases this function may parse more than one token,
1258 // however only a single token will always be returned.
1259 bool cxxParserParseNextToken(void)
1260 {
1261         // The token chain should not be allowed to grow arbitrarily large.
1262         // The token structures are quite big and it's easy to grow up to
1263         // 5-6GB or memory usage. However this limit should be large enough
1264         // to accommodate all the reasonable statements that could have some
1265         // information in them. This includes multiple function prototypes
1266         // in a single statement (ImageMagick has some examples) but probably
1267         // does NOT include large data tables.
1268         int iInitialTokenChainSize = g_cxx.pTokenChain->iCount;
1269         if(iInitialTokenChainSize >= CXX_PARSER_MAXIMUM_TOKEN_CHAIN_SIZE)
1270                 cxxTokenChainDestroyLast(g_cxx.pTokenChain);
1271
1272         if(g_cxx.pUngetToken)
1273         {
1274                 // got some tokens in the unget chain.
1275                 cxxTokenChainAppend(g_cxx.pTokenChain,g_cxx.pUngetToken);
1276
1277                 g_cxx.pToken = g_cxx.pUngetToken;
1278
1279                 g_cxx.pUngetToken = NULL;
1280
1281                 return !cxxTokenTypeIs(g_cxx.pToken,CXXTokenTypeEOF);
1282         }
1283
1284         CXXToken * t = cxxTokenCreate();
1285
1286         cxxTokenChainAppend(g_cxx.pTokenChain,t);
1287
1288         g_cxx.pToken = t;
1289
1290         cxxParserSkipToNonWhiteSpace();
1291
1292         // FIXME: this cpp handling is kind of broken:
1293         // it works only because the moon is in the correct phase.
1294         cppBeginStatement();
1295
1296         // This must be done after getting char from input
1297         t->iLineNumber = getInputLineNumber();
1298         t->oFilePosition = getInputFilePosition();
1299
1300         if(g_cxx.iChar == EOF)
1301         {
1302                 t->eType = CXXTokenTypeEOF;
1303                 t->bFollowedBySpace = false;
1304                 return false;
1305         }
1306
1307         unsigned int uInfo = UINFO(g_cxx.iChar);
1308
1309         //fprintf(stderr,"Char %c %02x info %u\n",g_cxx.iChar,g_cxx.iChar,uInfo);
1310
1311         if(uInfo & CXXCharTypeStartOfIdentifier)
1312         {
1313                 // word
1314                 t->eType = CXXTokenTypeIdentifier;
1315                 t->bFollowedBySpace = false;
1316
1317                 vStringPut(t->pszWord,g_cxx.iChar);
1318
1319                 // special case for tile, which may actually be an operator
1320                 if(g_cxx.iChar == '~')
1321                 {
1322                         // may be followed by space!
1323                         g_cxx.iChar = cppGetc();
1324                         if(cppIsspace(g_cxx.iChar))
1325                         {
1326                                 t->bFollowedBySpace = true;
1327                                 g_cxx.iChar = cppGetc();
1328                                 while(cppIsspace(g_cxx.iChar))
1329                                         g_cxx.iChar = cppGetc();
1330                         }
1331
1332                         // non space
1333                         uInfo = UINFO(g_cxx.iChar);
1334                         if(!(uInfo & CXXCharTypeStartOfIdentifier))
1335                         {
1336                                 // this is not an identifier after all
1337                                 t->eType = CXXTokenTypeOperator;
1338                                 if((!t->bFollowedBySpace) && g_cxx.iChar == '=')
1339                                 {
1340                                         // make ~= single token so it's not handled as
1341                                         // a separate assignment
1342                                         vStringPut(t->pszWord,g_cxx.iChar);
1343                                         g_cxx.iChar = cppGetc();
1344                                         t->bFollowedBySpace = cppIsspace(g_cxx.iChar);
1345                                 }
1346                                 return true;
1347                         }
1348                 } else {
1349                         g_cxx.iChar = cppGetc();
1350                 }
1351
1352                 for(;;)
1353                 {
1354                         uInfo = UINFO(g_cxx.iChar);
1355                         if(!(uInfo & CXXCharTypePartOfIdentifier))
1356                                 break;
1357                         vStringPut(t->pszWord,g_cxx.iChar);
1358                         g_cxx.iChar = cppGetc();
1359                 }
1360
1361                 int iCXXKeyword = lookupKeyword(t->pszWord->buffer,g_cxx.eLangType);
1362                 if(iCXXKeyword >= 0)
1363                 {
1364                         if(cxxKeywordIsDisabled((CXXKeyword)iCXXKeyword))
1365                         {
1366                                 t->eType = CXXTokenTypeIdentifier;
1367                         } else {
1368
1369                                 t->eType = CXXTokenTypeKeyword;
1370                                 t->eKeyword = (CXXKeyword)iCXXKeyword;
1371
1372
1373                                 if(cxxKeywordMayDropInTokenizer(iCXXKeyword))
1374                                 {
1375                                         // special handling for __attribute__ and __declspec
1376                                         return cxxParserParseNextTokenCondenseAttribute();
1377                                 }
1378                         }
1379                 } else {
1380
1381                         cppMacroInfo * pMacro = cppFindMacro(vStringValue(t->pszWord));
1382
1383 #ifdef DEBUG
1384                         if(pMacro && (pMacro->useCount >= CXX_PARSER_MAXIMUM_MACRO_USE_COUNT))
1385                         {
1386                                 /* If the macro is overly used, report it here. */
1387                                 CXX_DEBUG_PRINT("Overly uesd macro %s <%p> useCount: %d (> %d)",
1388                                                                 pMacro->name,
1389                                                                 pMacro, pMacro->useCount,
1390                                                                 CXX_PARSER_MAXIMUM_MACRO_USE_COUNT);
1391                         }
1392 #endif
1393
1394                         if(pMacro && (pMacro->useCount < CXX_PARSER_MAXIMUM_MACRO_USE_COUNT))
1395                         {
1396                                 CXX_DEBUG_PRINT("Macro %s <%p> useCount: %d", pMacro->name,
1397                                                                 pMacro, pMacro->useCount);
1398
1399                                 cxxTokenChainDestroyLast(g_cxx.pTokenChain);
1400
1401                                 CXXToken * pParameterChain = NULL;
1402
1403                                 if(pMacro->hasParameterList)
1404                                 {
1405                                         CXX_DEBUG_PRINT("Macro has parameter list");
1406                                         if(!cxxParserParseNextTokenSkipMacroParenthesis(&pParameterChain))
1407                                                 return false;
1408                                 }
1409
1410                                 // This is used to avoid infinite recursion in substitution
1411                                 // (things like -D foo=foo or similar)
1412
1413                                 if(pMacro->replacements)
1414                                 {
1415                                         CXX_DEBUG_PRINT("The token has replacements: applying");
1416
1417                                         if(
1418                                                 // Exclude possible cases of recursive macro expansion that
1419                                                 // causes level nesting
1420                                                 //    -D'x=y(x)'
1421                                                 (g_cxx.iNestingLevels < CXX_PARSER_MAXIMUM_NESTING_LEVELS) &&
1422                                                 // Exclude possible cases of recursive macro expansion that
1423                                                 // causes a single token chain to grow too big
1424                                                 //    -D'x=y.x'
1425                                                 (iInitialTokenChainSize < CXX_PARSER_MAXIMUM_TOKEN_CHAIN_SIZE) &&
1426                                                 // Detect other cases of nasty macro expansion that cause
1427                                                 // the unget buffer to grow fast (but the token chain to grow slowly)
1428                                                 //    -D'p=a' -D'a=p+p'
1429                                                 (cppUngetBufferSize() < CXX_PARSER_MAXIMUM_UNGET_BUFFER_SIZE_FOR_MACRO_REPLACEMENTS)
1430                                         )
1431                                         {
1432                                                 // unget last char
1433                                                 cppUngetc(g_cxx.iChar);
1434                                                 // unget the replacement
1435                                                 cxxParserParseNextTokenApplyReplacement(
1436                                                                 pMacro,
1437                                                                 pParameterChain
1438                                                         );
1439
1440                                                 g_cxx.iChar = cppGetc();
1441                                         } else {
1442                                                 // Possibly a recursive macro
1443                                                 CXX_DEBUG_PRINT(
1444                                                                 "Token has replacement but either nesting level is too "
1445                                                                 "big (%d), the token chain (%d) or the unget buffer (%d) "
1446                                                                 "have grown too large",
1447                                                                 g_cxx.iNestingLevels,
1448                                                                 g_cxx.pTokenChain->iCount,
1449                                                                 cppUngetBufferSize()
1450                                                         );
1451                                         }
1452                                 }
1453
1454                                 if(pParameterChain)
1455                                         cxxTokenDestroy(pParameterChain);
1456
1457                                 g_cxx.iNestingLevels++;
1458                                 // Have no token to return: parse it
1459                                 CXX_DEBUG_PRINT("Parse inner token");
1460                                 bool bRet = cxxParserParseNextToken();
1461                                 CXX_DEBUG_PRINT("Parsed inner token: %s type %d",g_cxx.pToken->pszWord->buffer,g_cxx.pToken->eType);
1462                                 g_cxx.iNestingLevels--;
1463                                 return bRet;
1464                         }
1465                 }
1466
1467                 t->bFollowedBySpace = cppIsspace(g_cxx.iChar);
1468
1469                 return true;
1470         }
1471
1472         if(g_cxx.iChar == '-')
1473         {
1474                 // special case for pointer
1475                 vStringPut(t->pszWord,g_cxx.iChar);
1476                 g_cxx.iChar = cppGetc();
1477                 if(g_cxx.iChar == '>')
1478                 {
1479                         t->eType = CXXTokenTypePointerOperator;
1480                         vStringPut(t->pszWord,g_cxx.iChar);
1481                         g_cxx.iChar = cppGetc();
1482                 } else {
1483                         t->eType = CXXTokenTypeOperator;
1484                         if(g_cxx.iChar == '-')
1485                         {
1486                                 vStringPut(t->pszWord,g_cxx.iChar);
1487                                 g_cxx.iChar = cppGetc();
1488                         }
1489                 }
1490                 t->bFollowedBySpace = cppIsspace(g_cxx.iChar);
1491                 return true;
1492         }
1493
1494 #if 0
1495         // As long as we use cppGetc() we don't need this
1496
1497         if(g_cxx.iChar == '"')
1498         {
1499                 // special case for strings
1500                 t->eType = CXXTokenTypeStringConstant;
1501                 vStringPut(t->pszWord,g_cxx.iChar);
1502                 // We don't even care of storing the other chars: we don't need
1503                 // them for parsing
1504                 // FIXME: We might need them in signature:() tag.. maybe add
1505                 // them up to a certain length only?
1506                 for(;;)
1507                 {
1508                         g_cxx.iChar = cppGetc();
1509                         if(g_cxx.iChar == EOF)
1510                         {
1511                                 t->bFollowedBySpace = false;
1512                                 return true;
1513                         }
1514                         if(g_cxx.iChar == '\\')
1515                         {
1516                                 // escape
1517                                 g_cxx.iChar = cppGetc();
1518                                 if(g_cxx.iChar == EOF)
1519                                 {
1520                                         t->bFollowedBySpace = false;
1521                                         return true;
1522                                 }
1523                         } else if(g_cxx.iChar == '"')
1524                         {
1525                                 g_cxx.iChar = cppGetc();
1526                                 break;
1527                         }
1528                 }
1529                 t->bFollowedBySpace = cppIsspace(g_cxx.iChar);
1530                 return true;
1531         }
1532 #else
1533         if(g_cxx.iChar == CPP_STRING_SYMBOL)
1534         {
1535                 t->eType = CXXTokenTypeStringConstant;
1536                 cppVStringPut(t->pszWord,g_cxx.iChar);
1537                 g_cxx.iChar = cppGetc();
1538                 t->bFollowedBySpace = cppIsspace(g_cxx.iChar);
1539                 return true;
1540         }
1541 #endif
1542
1543 #if 0
1544         // As long as we use cppGetc() we don't need this
1545         if(g_cxx.iChar == '\'')
1546         {
1547                 // special case for strings
1548                 t->eType = CXXTokenTypeCharacterConstant;
1549                 vStringPut(t->pszWord,g_cxx.iChar);
1550                 // We don't even care storing the other chars: we don't
1551                 // need them for parsing
1552                 for(;;)
1553                 {
1554                         g_cxx.iChar = cppGetc();
1555                         if(g_cxx.iChar == EOF)
1556                         {
1557                                 t->bFollowedBySpace = false;
1558                                 return true;
1559                         }
1560                         if(g_cxx.iChar == '\\')
1561                         {
1562                                 // escape
1563                                 g_cxx.iChar = cppGetc();
1564                                 if(g_cxx.iChar == EOF)
1565                                 {
1566                                         t->bFollowedBySpace = false;
1567                                         return true;
1568                                 }
1569                         } else if(g_cxx.iChar == '\'')
1570                         {
1571                                 g_cxx.iChar = cppGetc();
1572                                 break;
1573                         }
1574                 }
1575                 t->bFollowedBySpace = cppIsspace(g_cxx.iChar);
1576                 return true;
1577         }
1578 #else
1579         if(g_cxx.iChar == CPP_CHAR_SYMBOL)
1580         {
1581                 t->eType = CXXTokenTypeCharacterConstant;
1582                 cppVStringPut(t->pszWord,g_cxx.iChar);
1583                 g_cxx.iChar = cppGetc();
1584                 t->bFollowedBySpace = cppIsspace(g_cxx.iChar);
1585                 return true;
1586         }
1587 #endif
1588
1589         if(uInfo & CXXCharTypeDecimalDigit)
1590         {
1591                 // number
1592                 t->eType = CXXTokenTypeNumber;
1593                 vStringPut(t->pszWord,g_cxx.iChar);
1594
1595                 for(;;)
1596                 {
1597                         g_cxx.iChar = cppGetc();
1598                         uInfo = UINFO(g_cxx.iChar);
1599                         if(!(uInfo & CXXCharTypeValidInNumber))
1600                                 break;
1601                         vStringPut(t->pszWord,g_cxx.iChar);
1602                 }
1603
1604                 t->bFollowedBySpace = cppIsspace(g_cxx.iChar);
1605                 return true;
1606         }
1607
1608         if(uInfo & CXXCharTypeNamedSingleOrRepeatedCharToken)
1609         {
1610                 t->eType = g_aCharTable[g_cxx.iChar].uSingleTokenType;
1611                 vStringPut(t->pszWord,g_cxx.iChar);
1612                 int iChar = g_cxx.iChar;
1613                 g_cxx.iChar = cppGetc();
1614                 if(g_cxx.iChar == iChar)
1615                 {
1616                         t->eType = g_aCharTable[g_cxx.iChar].uMultiTokenType;
1617                         // We could signal a syntax error with more than two colons
1618                         // or equal signs...but we're tolerant
1619                         do {
1620                                 vStringPut(t->pszWord,g_cxx.iChar);
1621                                 g_cxx.iChar = cppGetc();
1622                         } while(g_cxx.iChar == iChar);
1623                 }
1624                 t->bFollowedBySpace = cppIsspace(g_cxx.iChar);
1625                 return true;
1626         }
1627
1628         if(uInfo & CXXCharTypeCustomHandling)
1629         {
1630                 t->eType = g_aCharTable[g_cxx.iChar].uSingleTokenType;
1631                 vStringPut(t->pszWord,g_cxx.iChar);
1632                 g_cxx.iChar = cppGetc();
1633                 switch(t->eType)
1634                 {
1635                         case CXXTokenTypeSmallerThanSign:
1636                                 // The < sign is used in templates and is problematic if parsed incorrectly.
1637                                 // We must exctract only the valid operator types: <, <<, <<=, <= <=>
1638                                 switch(g_cxx.iChar)
1639                                 {
1640                                         case '<':
1641                                                 // <<
1642                                                 t->eType = CXXTokenTypeOperator;
1643                                                 vStringPut(t->pszWord,g_cxx.iChar);
1644                                                 g_cxx.iChar = cppGetc();
1645                                                 if(g_cxx.iChar == '=')
1646                                                 {
1647                                                         // <<=
1648                                                         vStringPut(t->pszWord,g_cxx.iChar);
1649                                                         g_cxx.iChar = cppGetc();
1650                                                 }
1651                                         break;
1652                                         case '=':
1653                                                 // <=
1654                                                 t->eType = CXXTokenTypeOperator;
1655                                                 vStringPut(t->pszWord,g_cxx.iChar);
1656                                                 g_cxx.iChar = cppGetc();
1657                                                 if(g_cxx.iChar == '>')
1658                                                 {
1659                                                         // <=>
1660                                                         vStringPut(t->pszWord,g_cxx.iChar);
1661                                                         g_cxx.iChar = cppGetc();
1662                                                 }
1663                                         break;
1664                                         default:
1665                                                 // fall down
1666                                         break;
1667                                 }
1668
1669                                 t->bFollowedBySpace = cppIsspace(g_cxx.iChar);
1670                         break;
1671                         case CXXTokenTypeOpeningSquareParenthesis:
1672                                 // special handling for [[ attribute ]] which can appear almost anywhere
1673                                 // in the source code and is kind of annoying for the parser.
1674
1675                                 t->bFollowedBySpace = cppIsspace(g_cxx.iChar);
1676
1677                                 if(t->bFollowedBySpace)
1678                                 {
1679                                         // The tokens can be separated by a space, at least according to gcc.
1680                                         do {
1681                                                 g_cxx.iChar = cppGetc();
1682                                         } while(cppIsspace(g_cxx.iChar));
1683                                 }
1684
1685                                 if(g_cxx.iChar == '[')
1686                                         return cxxParserParseNextTokenCondenseCXX11Attribute();
1687                         break;
1688                         default:
1689                                 CXX_DEBUG_ASSERT(false,"There should be a custom handler for this token type");
1690                                 // treat as single token type in non debug builds
1691                                 t->bFollowedBySpace = cppIsspace(g_cxx.iChar);
1692                         break;
1693                 }
1694
1695                 return true;
1696         }
1697
1698         if(uInfo & CXXCharTypeNamedSingleCharToken)
1699         {
1700                 t->eType = g_aCharTable[g_cxx.iChar].uSingleTokenType;
1701                 vStringPut(t->pszWord,g_cxx.iChar);
1702                 g_cxx.iChar = cppGetc();
1703                 t->bFollowedBySpace = cppIsspace(g_cxx.iChar);
1704                 return true;
1705         }
1706
1707         if(uInfo & CXXCharTypeOperator)
1708         {
1709                 t->eType = CXXTokenTypeOperator;
1710                 vStringPut(t->pszWord,g_cxx.iChar);
1711                 g_cxx.iChar = cppGetc();
1712                 uInfo = UINFO(g_cxx.iChar);
1713                 while(uInfo & CXXCharTypeOperator)
1714                 {
1715                         vStringPut(t->pszWord,g_cxx.iChar);
1716                         g_cxx.iChar = cppGetc();
1717                         uInfo = UINFO(g_cxx.iChar);
1718                 }
1719                 t->bFollowedBySpace = cppIsspace(g_cxx.iChar);
1720                 return true;
1721         }
1722
1723         t->eType = CXXTokenTypeUnknown;
1724         cppVStringPut(t->pszWord,g_cxx.iChar);
1725         g_cxx.iChar = cppGetc();
1726         t->bFollowedBySpace = cppIsspace(g_cxx.iChar);
1727
1728         return true;
1729 }