scintilla/lexers/LexVerilog.cxx

   1 // Scintilla source code edit control
   2 /** @file LexVerilog.cxx
   3  ** Lexer for Verilog.
   4  ** Written by Avi Yegudin, based on C++ lexer by Neil Hodgson
   5  **/
   6 // Copyright 1998-2002 by Neil Hodgson <neilh@scintilla.org>
   7 // The License.txt file describes the conditions under which this software may be distributed.
   8
   9 #include <stdlib.h>
  10 #include <string.h>
  11 #include <stdio.h>
  12 #include <stdarg.h>
  13 #include <assert.h>
  14 #include <ctype.h>
  15
  16 #include <string>
  17 #include <vector>
  18 #include <map>
  19 #include <algorithm>
  20
  21 #include "ILexer.h"
  22 #include "Scintilla.h"
  23 #include "SciLexer.h"
  24
  25 #include "WordList.h"
  26 #include "LexAccessor.h"
  27 #include "Accessor.h"
  28 #include "StyleContext.h"
  29 #include "CharacterSet.h"
  30 #include "LexerModule.h"
  31
  32 #include "OptionSet.h"
  33 #include "SubStyles.h"
  34
  35 #ifdef SCI_NAMESPACE
  36 using namespace Scintilla;
  37 #endif
  38
  39 namespace {
  40         // Use an unnamed namespace to protect the functions and classes from name conflicts
  41
  42 struct PPDefinition {
  43         Sci_Position line;
  44         std::string key;
  45         std::string value;
  46         bool isUndef;
  47         std::string arguments;
  48         PPDefinition(Sci_Position line_, const std::string &key_, const std::string &value_, bool isUndef_ = false, std::string arguments_="") :
  49                 line(line_), key(key_), value(value_), isUndef(isUndef_), arguments(arguments_) {
  50         }
  51 };
  52
  53 class LinePPState {
  54         int state;
  55         int ifTaken;
  56         int level;
  57         bool ValidLevel() const {
  58                 return level >= 0 && level < 32;
  59         }
  60         int maskLevel() const {
  61                 return 1 << level;
  62         }
  63 public:
  64         LinePPState() : state(0), ifTaken(0), level(-1) {
  65         }
  66         bool IsInactive() const {
  67                 return state != 0;
  68         }
  69         bool CurrentIfTaken() const {
  70                 return (ifTaken & maskLevel()) != 0;
  71         }
  72         void StartSection(bool on) {
  73                 level++;
  74                 if (ValidLevel()) {
  75                         if (on) {
  76                                 state &= ~maskLevel();
  77                                 ifTaken |= maskLevel();
  78                         } else {
  79                                 state |= maskLevel();
  80                                 ifTaken &= ~maskLevel();
  81                         }
  82                 }
  83         }
  84         void EndSection() {
  85                 if (ValidLevel()) {
  86                         state &= ~maskLevel();
  87                         ifTaken &= ~maskLevel();
  88                 }
  89                 level--;
  90         }
  91         void InvertCurrentLevel() {
  92                 if (ValidLevel()) {
  93                         state ^= maskLevel();
  94                         ifTaken |= maskLevel();
  95                 }
  96         }
  97 };
  98
  99 // Hold the preprocessor state for each line seen.
 100 // Currently one entry per line but could become sparse with just one entry per preprocessor line.
 101 class PPStates {
 102         std::vector<LinePPState> vlls;
 103 public:
 104         LinePPState ForLine(Sci_Position line) const {
 105                 if ((line > 0) && (vlls.size() > static_cast<size_t>(line))) {
 106                         return vlls[line];
 107                 } else {
 108                         return LinePPState();
 109                 }
 110         }
 111         void Add(Sci_Position line, LinePPState lls) {
 112                 vlls.resize(line+1);
 113                 vlls[line] = lls;
 114         }
 115 };
 116
 117 // Options used for LexerVerilog
 118 struct OptionsVerilog {
 119         bool foldComment;
 120         bool foldPreprocessor;
 121         bool foldPreprocessorElse;
 122         bool foldCompact;
 123         bool foldAtElse;
 124         bool foldAtModule;
 125         bool trackPreprocessor;
 126         bool updatePreprocessor;
 127         bool portStyling;
 128         bool allUppercaseDocKeyword;
 129         OptionsVerilog() {
 130                 foldComment = false;
 131                 foldPreprocessor = false;
 132                 foldPreprocessorElse = false;
 133                 foldCompact = false;
 134                 foldAtElse = false;
 135                 foldAtModule = false;
 136                 // for backwards compatibility, preprocessor functionality is disabled by default
 137                 trackPreprocessor = false;
 138                 updatePreprocessor = false;
 139                 // for backwards compatibility, treat input/output/inout as regular keywords
 140                 portStyling = false;
 141                 // for backwards compatibility, don't treat all uppercase identifiers as documentation keywords
 142                 allUppercaseDocKeyword = false;
 143         }
 144 };
 145
 146 struct OptionSetVerilog : public OptionSet<OptionsVerilog> {
 147         OptionSetVerilog() {
 148                 DefineProperty("fold.comment", &OptionsVerilog::foldComment,
 149                         "This option enables folding multi-line comments when using the Verilog lexer.");
 150                 DefineProperty("fold.preprocessor", &OptionsVerilog::foldPreprocessor,
 151                         "This option enables folding preprocessor directives when using the Verilog lexer.");
 152                 DefineProperty("fold.compact", &OptionsVerilog::foldCompact);
 153                 DefineProperty("fold.at.else", &OptionsVerilog::foldAtElse,
 154                         "This option enables folding on the else line of an if statement.");
 155                 DefineProperty("fold.verilog.flags", &OptionsVerilog::foldAtModule,
 156                         "This option enables folding module definitions. Typically source files "
 157                         "contain only one module definition so this option is somewhat useless.");
 158                 DefineProperty("lexer.verilog.track.preprocessor", &OptionsVerilog::trackPreprocessor,
 159                         "Set to 1 to interpret `if/`else/`endif to grey out code that is not active.");
 160                 DefineProperty("lexer.verilog.update.preprocessor", &OptionsVerilog::updatePreprocessor,
 161                         "Set to 1 to update preprocessor definitions when `define, `undef, or `undefineall found.");
 162                 DefineProperty("lexer.verilog.portstyling", &OptionsVerilog::portStyling,
 163                         "Set to 1 to style input, output, and inout ports differently from regular keywords.");
 164                 DefineProperty("lexer.verilog.allupperkeywords", &OptionsVerilog::allUppercaseDocKeyword,
 165                         "Set to 1 to style identifiers that are all uppercase as documentation keyword.");
 166                 DefineProperty("lexer.verilog.fold.preprocessor.else", &OptionsVerilog::foldPreprocessorElse,
 167                         "This option enables folding on `else and `elsif preprocessor directives.");
 168         }
 169 };
 170
 171 const char styleSubable[] = {0};
 172
 173 }
 174
 175 class LexerVerilog : public ILexerWithSubStyles {
 176         CharacterSet setWord;
 177         WordList keywords;
 178         WordList keywords2;
 179         WordList keywords3;
 180         WordList keywords4;
 181         WordList keywords5;
 182         WordList ppDefinitions;
 183         PPStates vlls;
 184         std::vector<PPDefinition> ppDefineHistory;
 185         struct SymbolValue {
 186                 std::string value;
 187                 std::string arguments;
 188                 SymbolValue(const std::string &value_="", const std::string &arguments_="") : value(value_), arguments(arguments_) {
 189                 }
 190                 SymbolValue &operator = (const std::string &value_) {
 191                         value = value_;
 192                         arguments.clear();
 193                         return *this;
 194                 }
 195                 bool IsMacro() const {
 196                         return !arguments.empty();
 197                 }
 198         };
 199         typedef std::map<std::string, SymbolValue> SymbolTable;
 200         SymbolTable preprocessorDefinitionsStart;
 201         OptionsVerilog options;
 202         OptionSetVerilog osVerilog;
 203         enum { activeFlag = 0x40 };
 204         SubStyles subStyles;
 205
 206         // states at end of line (EOL) during fold operations:
 207         //              foldExternFlag: EOL while parsing an extern function/task declaration terminated by ';'
 208         //              foldWaitDisableFlag: EOL while parsing wait or disable statement, terminated by "fork" or '('
 209         //              typdefFlag: EOL while parsing typedef statement, terminated by ';'
 210         enum {foldExternFlag = 0x01, foldWaitDisableFlag = 0x02, typedefFlag = 0x04, protectedFlag = 0x08};
 211         // map using line number as key to store fold state information
 212         std::map<Sci_Position, int> foldState;
 213
 214 public:
 215         LexerVerilog() :
 216                 setWord(CharacterSet::setAlphaNum, "._", 0x80, true),
 217                 subStyles(styleSubable, 0x80, 0x40, activeFlag) {
 218                 }
 219         virtual ~LexerVerilog() {}
 220         int SCI_METHOD Version() const {
 221                 return lvSubStyles;
 222         }
 223         void SCI_METHOD Release() {
 224                 delete this;
 225         }
 226         const char* SCI_METHOD PropertyNames() {
 227                 return osVerilog.PropertyNames();
 228         }
 229         int SCI_METHOD PropertyType(const char* name) {
 230                 return osVerilog.PropertyType(name);
 231         }
 232         const char* SCI_METHOD DescribeProperty(const char* name) {
 233                 return osVerilog.DescribeProperty(name);
 234         }
 235         Sci_Position SCI_METHOD PropertySet(const char* key, const char* val) {
 236             return osVerilog.PropertySet(&options, key, val);
 237         }
 238         const char* SCI_METHOD DescribeWordListSets() {
 239                 return osVerilog.DescribeWordListSets();
 240         }
 241         Sci_Position SCI_METHOD WordListSet(int n, const char* wl);
 242         void SCI_METHOD Lex(Sci_PositionU startPos, Sci_Position length, int initStyle, IDocument *pAccess);
 243         void SCI_METHOD Fold(Sci_PositionU startPos, Sci_Position length, int initStyle, IDocument *pAccess);
 244         void* SCI_METHOD PrivateCall(int, void*) {
 245                 return 0;
 246         }
 247         int SCI_METHOD LineEndTypesSupported() {
 248                 return SC_LINE_END_TYPE_UNICODE;
 249         }
 250         int SCI_METHOD AllocateSubStyles(int styleBase, int numberStyles) {
 251                 return subStyles.Allocate(styleBase, numberStyles);
 252         }
 253         int SCI_METHOD SubStylesStart(int styleBase) {
 254                 return subStyles.Start(styleBase);
 255         }
 256         int SCI_METHOD SubStylesLength(int styleBase) {
 257                 return subStyles.Length(styleBase);
 258         }
 259         int SCI_METHOD StyleFromSubStyle(int subStyle) {
 260                 int styleBase = subStyles.BaseStyle(MaskActive(subStyle));
 261                 int active = subStyle & activeFlag;
 262                 return styleBase | active;
 263         }
 264         int SCI_METHOD PrimaryStyleFromStyle(int style) {
 265                 return MaskActive(style);
 266         }
 267         void SCI_METHOD FreeSubStyles() {
 268                 subStyles.Free();
 269         }
 270         void SCI_METHOD SetIdentifiers(int style, const char *identifiers) {
 271                 subStyles.SetIdentifiers(style, identifiers);
 272         }
 273         int SCI_METHOD DistanceToSecondaryStyles() {
 274                 return activeFlag;
 275         }
 276         const char * SCI_METHOD GetSubStyleBases() {
 277                 return styleSubable;
 278         }
 279         static ILexer* LexerFactoryVerilog() {
 280                 return new LexerVerilog();
 281         }
 282         static int MaskActive(int style) {
 283                 return style & ~activeFlag;
 284         }
 285         std::vector<std::string> Tokenize(const std::string &expr) const;
 286 };
 287
 288 Sci_Position SCI_METHOD LexerVerilog::WordListSet(int n, const char *wl) {
 289         WordList *wordListN = 0;
 290         switch (n) {
 291         case 0:
 292                 wordListN = &keywords;
 293                 break;
 294         case 1:
 295                 wordListN = &keywords2;
 296                 break;
 297         case 2:
 298                 wordListN = &keywords3;
 299                 break;
 300         case 3:
 301                 wordListN = &keywords4;
 302                 break;
 303         case 4:
 304                 wordListN = &keywords5;
 305                 break;
 306         case 5:
 307                 wordListN = &ppDefinitions;
 308                 break;
 309         }
 310         Sci_Position firstModification = -1;
 311         if (wordListN) {
 312                 WordList wlNew;
 313                 wlNew.Set(wl);
 314                 if (*wordListN != wlNew) {
 315                         wordListN->Set(wl);
 316                         firstModification = 0;
 317                         if (n == 5) {
 318                                 // Rebuild preprocessorDefinitions
 319                                 preprocessorDefinitionsStart.clear();
 320                                 for (int nDefinition = 0; nDefinition < ppDefinitions.Length(); nDefinition++) {
 321                                         const char *cpDefinition = ppDefinitions.WordAt(nDefinition);
 322                                         const char *cpEquals = strchr(cpDefinition, '=');
 323                                         if (cpEquals) {
 324                                                 std::string name(cpDefinition, cpEquals - cpDefinition);
 325                                                 std::string val(cpEquals+1);
 326                                                 size_t bracket = name.find('(');
 327                                                 size_t bracketEnd = name.find(')');
 328                                                 if ((bracket != std::string::npos) && (bracketEnd != std::string::npos)) {
 329                                                         // Macro
 330                                                         std::string args = name.substr(bracket + 1, bracketEnd - bracket - 1);
 331                                                         name = name.substr(0, bracket);
 332                                                         preprocessorDefinitionsStart[name] = SymbolValue(val, args);
 333                                                 } else {
 334                                                         preprocessorDefinitionsStart[name] = val;
 335                                                 }
 336                                         } else {
 337                                                 std::string name(cpDefinition);
 338                                                 std::string val("1");
 339                                                 preprocessorDefinitionsStart[name] = val;
 340                                         }
 341                                 }
 342                         }
 343                 }
 344         }
 345         return firstModification;
 346 }
 347
 348 static inline bool IsAWordChar(const int ch) {
 349         return (ch < 0x80) && (isalnum(ch) || ch == '_' || ch == '\''|| ch == '$');
 350 }
 351
 352 static inline bool IsAWordStart(const int ch) {
 353         return (ch < 0x80) && (isalnum(ch) || ch == '_' || ch == '$');
 354 }
 355
 356 static inline bool AllUpperCase(const char *a) {
 357         while (*a) {
 358                 if (*a >= 'a' && *a <= 'z') return false;
 359                 a++;
 360         }
 361         return true;
 362 }
 363
 364 // Functor used to truncate history
 365 struct After {
 366         Sci_Position line;
 367         explicit After(Sci_Position line_) : line(line_) {}
 368         bool operator()(PPDefinition &p) const {
 369                 return p.line > line;
 370         }
 371 };
 372
 373 static std::string GetRestOfLine(LexAccessor &styler, Sci_Position start, bool allowSpace) {
 374         std::string restOfLine;
 375         Sci_Position i =0;
 376         char ch = styler.SafeGetCharAt(start, '\n');
 377         Sci_Position endLine = styler.LineEnd(styler.GetLine(start));
 378         while (((start+i) < endLine) && (ch != '\r')) {
 379                 char chNext = styler.SafeGetCharAt(start + i + 1, '\n');
 380                 if (ch == '/' && (chNext == '/' || chNext == '*'))
 381                         break;
 382                 if (allowSpace || (ch != ' '))
 383                         restOfLine += ch;
 384                 i++;
 385                 ch = chNext;
 386         }
 387         return restOfLine;
 388 }
 389
 390 static bool IsSpaceOrTab(int ch) {
 391         return ch == ' ' || ch == '\t';
 392 }
 393
 394 void SCI_METHOD LexerVerilog::Lex(Sci_PositionU startPos, Sci_Position length, int initStyle, IDocument *pAccess)
 395 {
 396         LexAccessor styler(pAccess);
 397
 398         const int kwOther=0, kwDot=0x100, kwInput=0x200, kwOutput=0x300, kwInout=0x400, kwProtected=0x800;
 399         int lineState = kwOther;
 400         bool continuationLine = false;
 401
 402         Sci_Position curLine = styler.GetLine(startPos);
 403         if (curLine > 0) lineState = styler.GetLineState(curLine - 1);
 404
 405         // Do not leak onto next line
 406         if (initStyle == SCE_V_STRINGEOL)
 407                 initStyle = SCE_V_DEFAULT;
 408
 409         if ((MaskActive(initStyle) == SCE_V_PREPROCESSOR) ||
 410                         (MaskActive(initStyle) == SCE_V_COMMENTLINE) ||
 411                         (MaskActive(initStyle) == SCE_V_COMMENTLINEBANG)) {
 412                 // Set continuationLine if last character of previous line is '\'
 413                 if (curLine > 0) {
 414                         Sci_Position endLinePrevious = styler.LineEnd(curLine - 1);
 415                         if (endLinePrevious > 0) {
 416                                 continuationLine = styler.SafeGetCharAt(endLinePrevious-1) == '\\';
 417                         }
 418                 }
 419         }
 420
 421         StyleContext sc(startPos, length, initStyle, styler);
 422         LinePPState preproc = vlls.ForLine(curLine);
 423
 424         bool definitionsChanged = false;
 425
 426         // Truncate ppDefineHistory before current line
 427
 428         if (!options.updatePreprocessor)
 429                 ppDefineHistory.clear();
 430
 431         std::vector<PPDefinition>::iterator itInvalid = std::find_if(ppDefineHistory.begin(), ppDefineHistory.end(), After(curLine-1));
 432         if (itInvalid != ppDefineHistory.end()) {
 433                 ppDefineHistory.erase(itInvalid, ppDefineHistory.end());
 434                 definitionsChanged = true;
 435         }
 436
 437         SymbolTable preprocessorDefinitions = preprocessorDefinitionsStart;
 438         for (std::vector<PPDefinition>::iterator itDef = ppDefineHistory.begin(); itDef != ppDefineHistory.end(); ++itDef) {
 439                 if (itDef->isUndef)
 440                         preprocessorDefinitions.erase(itDef->key);
 441                 else
 442                         preprocessorDefinitions[itDef->key] = SymbolValue(itDef->value, itDef->arguments);
 443         }
 444
 445         int activitySet = preproc.IsInactive() ? activeFlag : 0;
 446         Sci_Position lineEndNext = styler.LineEnd(curLine);
 447         bool isEscapedId = false;    // true when parsing an escaped Identifier
 448         bool isProtected = (lineState&kwProtected) != 0;        // true when parsing a protected region
 449
 450         for (; sc.More(); sc.Forward()) {
 451                 if (sc.atLineStart) {
 452                         if (sc.state == SCE_V_STRING) {
 453                                 // Prevent SCE_V_STRINGEOL from leaking back to previous line
 454                                 sc.SetState(SCE_V_STRING);
 455                         }
 456                         if ((MaskActive(sc.state) == SCE_V_PREPROCESSOR) && (!continuationLine)) {
 457                                 sc.SetState(SCE_V_DEFAULT|activitySet);
 458                         }
 459                         if (preproc.IsInactive()) {
 460                                 activitySet = activeFlag;
 461                                 sc.SetState(sc.state | activitySet);
 462                         }
 463                 }
 464
 465                 if (sc.atLineEnd) {
 466                         curLine++;
 467                         lineEndNext = styler.LineEnd(curLine);
 468                         vlls.Add(curLine, preproc);
 469                         // Update the line state, so it can be seen by next line
 470                         styler.SetLineState(curLine, lineState);
 471                         isEscapedId = false;    // EOL terminates an escaped Identifier
 472                 }
 473
 474                 // Handle line continuation generically.
 475                 if (sc.ch == '\\') {
 476                         if (static_cast<Sci_Position>((sc.currentPos+1)) >= lineEndNext) {
 477                                 curLine++;
 478                                 lineEndNext = styler.LineEnd(curLine);
 479                                 vlls.Add(curLine, preproc);
 480                                 // Update the line state, so it can be seen by next line
 481                                 styler.SetLineState(curLine, lineState);
 482                                 sc.Forward();
 483                                 if (sc.ch == '\r' && sc.chNext == '\n') {
 484                                         // Even in UTF-8, \r and \n are separate
 485                                         sc.Forward();
 486                                 }
 487                                 continuationLine = true;
 488                                 sc.Forward();
 489                                 continue;
 490                         }
 491                 }
 492
 493                 // for comment keyword
 494                 if (MaskActive(sc.state) == SCE_V_COMMENT_WORD && !IsAWordChar(sc.ch)) {
 495                         char s[100];
 496                         int state = lineState & 0xff;
 497                         sc.GetCurrent(s, sizeof(s));
 498                         if (keywords5.InList(s)) {
 499                                 sc.ChangeState(SCE_V_COMMENT_WORD|activitySet);
 500                         } else {
 501                                 sc.ChangeState(state|activitySet);
 502                         }
 503                         sc.SetState(state|activitySet);
 504                 }
 505
 506                 const bool atLineEndBeforeSwitch = sc.atLineEnd;
 507
 508                 // Determine if the current state should terminate.
 509                 switch (MaskActive(sc.state)) {
 510                         case SCE_V_OPERATOR:
 511                                 sc.SetState(SCE_V_DEFAULT|activitySet);
 512                                 break;
 513                         case SCE_V_NUMBER:
 514                                 if (!(IsAWordChar(sc.ch) || (sc.ch == '?'))) {
 515                                         sc.SetState(SCE_V_DEFAULT|activitySet);
 516                                 }
 517                                 break;
 518                         case SCE_V_IDENTIFIER:
 519                                 if (!isEscapedId &&(!IsAWordChar(sc.ch) || (sc.ch == '.'))) {
 520                                         char s[100];
 521                                         lineState &= 0xff00;
 522                                         sc.GetCurrent(s, sizeof(s));
 523                                         if (options.portStyling && (strcmp(s, "input") == 0)) {
 524                                                 lineState = kwInput;
 525                                                 sc.ChangeState(SCE_V_INPUT|activitySet);
 526                                         } else if (options.portStyling && (strcmp(s, "output") == 0)) {
 527                                                 lineState = kwOutput;
 528                                                 sc.ChangeState(SCE_V_OUTPUT|activitySet);
 529                                         } else if (options.portStyling && (strcmp(s, "inout") == 0)) {
 530                                                 lineState = kwInout;
 531                                                 sc.ChangeState(SCE_V_INOUT|activitySet);
 532                                         } else if (lineState == kwInput) {
 533                                                 sc.ChangeState(SCE_V_INPUT|activitySet);
 534                                         } else if (lineState == kwOutput) {
 535                                                 sc.ChangeState(SCE_V_OUTPUT|activitySet);
 536                                         } else if (lineState == kwInout) {
 537                                                 sc.ChangeState(SCE_V_INOUT|activitySet);
 538                                         } else if (lineState == kwDot) {
 539                                                 lineState = kwOther;
 540                                                 if (options.portStyling)
 541                                                         sc.ChangeState(SCE_V_PORT_CONNECT|activitySet);
 542                                         } else if (keywords.InList(s)) {
 543                                                 sc.ChangeState(SCE_V_WORD|activitySet);
 544                                         } else if (keywords2.InList(s)) {
 545                                                 sc.ChangeState(SCE_V_WORD2|activitySet);
 546                                         } else if (keywords3.InList(s)) {
 547                                                 sc.ChangeState(SCE_V_WORD3|activitySet);
 548                                         } else if (keywords4.InList(s)) {
 549                                                 sc.ChangeState(SCE_V_USER|activitySet);
 550                                         } else if (options.allUppercaseDocKeyword && AllUpperCase(s)) {
 551                                                 sc.ChangeState(SCE_V_USER|activitySet);
 552                                         }
 553                                         sc.SetState(SCE_V_DEFAULT|activitySet);
 554                                 }
 555                                 break;
 556                         case SCE_V_PREPROCESSOR:
 557                                 if (!IsAWordChar(sc.ch) || sc.atLineEnd) {
 558                                         sc.SetState(SCE_V_DEFAULT|activitySet);
 559                                 }
 560                                 break;
 561                         case SCE_V_COMMENT:
 562                                 if (sc.Match('*', '/')) {
 563                                         sc.Forward();
 564                                         sc.ForwardSetState(SCE_V_DEFAULT|activitySet);
 565                                 } else if (IsAWordStart(sc.ch)) {
 566                                         lineState = sc.state | (lineState & 0xff00);
 567                                         sc.SetState(SCE_V_COMMENT_WORD|activitySet);
 568                                 }
 569                                 break;
 570                         case SCE_V_COMMENTLINE:
 571                         case SCE_V_COMMENTLINEBANG:
 572                                 if (sc.atLineStart) {
 573                                         sc.SetState(SCE_V_DEFAULT|activitySet);
 574                                 } else if (IsAWordStart(sc.ch)) {
 575                                         lineState = sc.state | (lineState & 0xff00);
 576                                         sc.SetState(SCE_V_COMMENT_WORD|activitySet);
 577                                 }
 578                                 break;
 579                         case SCE_V_STRING:
 580                                 if (sc.ch == '\\') {
 581                                         if (sc.chNext == '\"' || sc.chNext == '\'' || sc.chNext == '\\') {
 582                                                 sc.Forward();
 583                                         }
 584                                 } else if (sc.ch == '\"') {
 585                                         sc.ForwardSetState(SCE_V_DEFAULT|activitySet);
 586                                 } else if (sc.atLineEnd) {
 587                                         sc.ChangeState(SCE_V_STRINGEOL|activitySet);
 588                                         sc.ForwardSetState(SCE_V_DEFAULT|activitySet);
 589                                 }
 590                                 break;
 591                 }
 592
 593                 if (sc.atLineEnd && !atLineEndBeforeSwitch) {
 594                         // State exit processing consumed characters up to end of line.
 595                         curLine++;
 596                         lineEndNext = styler.LineEnd(curLine);
 597                         vlls.Add(curLine, preproc);
 598                         // Update the line state, so it can be seen by next line
 599                         styler.SetLineState(curLine, lineState);
 600                         isEscapedId = false;    // EOL terminates an escaped Identifier
 601                 }
 602
 603                 // Determine if a new state should be entered.
 604                 if (MaskActive(sc.state) == SCE_V_DEFAULT) {
 605                         if (sc.ch == '`') {
 606                                 sc.SetState(SCE_V_PREPROCESSOR|activitySet);
 607                                 // Skip whitespace between ` and preprocessor word
 608                                 do {
 609                                         sc.Forward();
 610                                 } while ((sc.ch == ' ' || sc.ch == '\t') && sc.More());
 611                                 if (sc.atLineEnd) {
 612                                         sc.SetState(SCE_V_DEFAULT|activitySet);
 613                                         styler.SetLineState(curLine, lineState);
 614                                 } else {
 615                                         if (sc.Match("protected")) {
 616                                                 isProtected = true;
 617                                                 lineState |= kwProtected;
 618                                                 styler.SetLineState(curLine, lineState);
 619                                         } else if (sc.Match("endprotected")) {
 620                                                 isProtected = false;
 621                                                 lineState &= ~kwProtected;
 622                                                 styler.SetLineState(curLine, lineState);
 623                                         } else if (!isProtected && options.trackPreprocessor) {
 624                                                 if (sc.Match("ifdef") || sc.Match("ifndef")) {
 625                                                         bool isIfDef = sc.Match("ifdef");
 626                                                         int i = isIfDef ? 5 : 6;
 627                                                         std::string restOfLine = GetRestOfLine(styler, sc.currentPos + i + 1, false);
 628                                                         bool foundDef = preprocessorDefinitions.find(restOfLine) != preprocessorDefinitions.end();
 629                                                         preproc.StartSection(isIfDef == foundDef);
 630                                                 } else if (sc.Match("else")) {
 631                                                         if (!preproc.CurrentIfTaken()) {
 632                                                                 preproc.InvertCurrentLevel();
 633                                                                 activitySet = preproc.IsInactive() ? activeFlag : 0;
 634                                                                 if (!activitySet) {
 635                                                                         sc.ChangeState(SCE_V_PREPROCESSOR|activitySet);
 636                                                                 }
 637                                                         } else if (!preproc.IsInactive()) {
 638                                                                 preproc.InvertCurrentLevel();
 639                                                                 activitySet = preproc.IsInactive() ? activeFlag : 0;
 640                                                                 if (!activitySet) {
 641                                                                         sc.ChangeState(SCE_V_PREPROCESSOR|activitySet);
 642                                                                 }
 643                                                         }
 644                                                 } else if (sc.Match("elsif")) {
 645                                                         // Ensure only one chosen out of `if .. `elsif .. `elsif .. `else .. `endif
 646                                                         if (!preproc.CurrentIfTaken()) {
 647                                                                 // Similar to `ifdef
 648                                                                 std::string restOfLine = GetRestOfLine(styler, sc.currentPos + 6, true);
 649                                                                 bool ifGood = preprocessorDefinitions.find(restOfLine) != preprocessorDefinitions.end();
 650                                                                 if (ifGood) {
 651                                                                         preproc.InvertCurrentLevel();
 652                                                                         activitySet = preproc.IsInactive() ? activeFlag : 0;
 653                                                                         if (!activitySet)
 654                                                                                 sc.ChangeState(SCE_V_PREPROCESSOR|activitySet);
 655                                                                 }
 656                                                         } else if (!preproc.IsInactive()) {
 657                                                                 preproc.InvertCurrentLevel();
 658                                                                 activitySet = preproc.IsInactive() ? activeFlag : 0;
 659                                                                 if (!activitySet)
 660                                                                         sc.ChangeState(SCE_V_PREPROCESSOR|activitySet);
 661                                                         }
 662                                                 } else if (sc.Match("endif")) {
 663                                                         preproc.EndSection();
 664                                                         activitySet = preproc.IsInactive() ? activeFlag : 0;
 665                                                         sc.ChangeState(SCE_V_PREPROCESSOR|activitySet);
 666                                                 } else if (sc.Match("define")) {
 667                                                         if (options.updatePreprocessor && !preproc.IsInactive()) {
 668                                                                 std::string restOfLine = GetRestOfLine(styler, sc.currentPos + 6, true);
 669                                                                 size_t startName = 0;
 670                                                                 while ((startName < restOfLine.length()) && IsSpaceOrTab(restOfLine[startName]))
 671                                                                         startName++;
 672                                                                 size_t endName = startName;
 673                                                                 while ((endName < restOfLine.length()) && setWord.Contains(static_cast<unsigned char>(restOfLine[endName])))
 674                                                                         endName++;
 675                                                                 std::string key = restOfLine.substr(startName, endName-startName);
 676                                                                 if ((endName < restOfLine.length()) && (restOfLine.at(endName) == '(')) {
 677                                                                         // Macro
 678                                                                         size_t endArgs = endName;
 679                                                                         while ((endArgs < restOfLine.length()) && (restOfLine[endArgs] != ')'))
 680                                                                                 endArgs++;
 681                                                                         std::string args = restOfLine.substr(endName + 1, endArgs - endName - 1);
 682                                                                         size_t startValue = endArgs+1;
 683                                                                         while ((startValue < restOfLine.length()) && IsSpaceOrTab(restOfLine[startValue]))
 684                                                                                 startValue++;
 685                                                                         std::string value;
 686                                                                         if (startValue < restOfLine.length())
 687                                                                                 value = restOfLine.substr(startValue);
 688                                                                         preprocessorDefinitions[key] = SymbolValue(value, args);
 689                                                                         ppDefineHistory.push_back(PPDefinition(curLine, key, value, false, args));
 690                                                                         definitionsChanged = true;
 691                                                                 } else {
 692                                                                         // Value
 693                                                                         size_t startValue = endName;
 694                                                                         while ((startValue < restOfLine.length()) && IsSpaceOrTab(restOfLine[startValue]))
 695                                                                                 startValue++;
 696                                                                         std::string value = restOfLine.substr(startValue);
 697                                                                         preprocessorDefinitions[key] = value;
 698                                                                         ppDefineHistory.push_back(PPDefinition(curLine, key, value));
 699                                                                         definitionsChanged = true;
 700                                                                 }
 701                                                         }
 702                                                 } else if (sc.Match("undefineall")) {
 703                                                         if (options.updatePreprocessor && !preproc.IsInactive()) {
 704                                                                 // remove all preprocessor definitions
 705                                                                 std::map<std::string, SymbolValue>::iterator itDef;
 706                                                                 for(itDef = preprocessorDefinitions.begin(); itDef != preprocessorDefinitions.end(); ++itDef) {
 707                                                                         ppDefineHistory.push_back(PPDefinition(curLine, itDef->first, "", true));
 708                                                                 }
 709                                                                 preprocessorDefinitions.clear();
 710                                                                 definitionsChanged = true;
 711                                                         }
 712                                                 } else if (sc.Match("undef")) {
 713                                                         if (options.updatePreprocessor && !preproc.IsInactive()) {
 714                                                                 std::string restOfLine = GetRestOfLine(styler, sc.currentPos + 5, true);
 715                                                                 std::vector<std::string> tokens = Tokenize(restOfLine);
 716                                                                 std::string key;
 717                                                                 if (tokens.size() >= 1) {
 718                                                                         key = tokens[0];
 719                                                                         preprocessorDefinitions.erase(key);
 720                                                                         ppDefineHistory.push_back(PPDefinition(curLine, key, "", true));
 721                                                                         definitionsChanged = true;
 722                                                                 }
 723                                                         }
 724                                                 }
 725                                         }
 726                                 }
 727                         } else if (!isProtected) {
 728                                 if (IsADigit(sc.ch) || (sc.ch == '\'') || (sc.ch == '.' && IsADigit(sc.chNext))) {
 729                                         sc.SetState(SCE_V_NUMBER|activitySet);
 730                                 } else if (IsAWordStart(sc.ch)) {
 731                                         sc.SetState(SCE_V_IDENTIFIER|activitySet);
 732                                 } else if (sc.Match('/', '*')) {
 733                                         sc.SetState(SCE_V_COMMENT|activitySet);
 734                                         sc.Forward();   // Eat the * so it isn't used for the end of the comment
 735                                 } else if (sc.Match('/', '/')) {
 736                                         if (sc.Match("//!"))    // Nice to have a different comment style
 737                                                 sc.SetState(SCE_V_COMMENTLINEBANG|activitySet);
 738                                         else
 739                                                 sc.SetState(SCE_V_COMMENTLINE|activitySet);
 740                                 } else if (sc.ch == '\"') {
 741                                         sc.SetState(SCE_V_STRING|activitySet);
 742                                 } else if (sc.ch == '\\') {
 743                                         // escaped identifier, everything is ok up to whitespace
 744                                         isEscapedId = true;
 745                                         sc.SetState(SCE_V_IDENTIFIER|activitySet);
 746                                 } else if (isoperator(static_cast<char>(sc.ch)) || sc.ch == '@' || sc.ch == '#') {
 747                                         sc.SetState(SCE_V_OPERATOR|activitySet);
 748                                         if (sc.ch == '.') lineState = kwDot;
 749                                         if (sc.ch == ';') lineState = kwOther;
 750                                 }
 751                         }
 752                 }
 753                 if (isEscapedId && isspacechar(sc.ch)) {
 754                         isEscapedId = false;
 755                 }
 756         }
 757         if (definitionsChanged) {
 758                 styler.ChangeLexerState(startPos, startPos + length);
 759         }
 760         sc.Complete();
 761 }
 762
 763 static bool IsStreamCommentStyle(int style) {
 764         return style == SCE_V_COMMENT;
 765 }
 766
 767 static bool IsCommentLine(Sci_Position line, LexAccessor &styler) {
 768         Sci_Position pos = styler.LineStart(line);
 769         Sci_Position eolPos = styler.LineStart(line + 1) - 1;
 770         for (Sci_Position i = pos; i < eolPos; i++) {
 771                 char ch = styler[i];
 772                 char chNext = styler.SafeGetCharAt(i + 1);
 773                 int style = styler.StyleAt(i);
 774                 if (ch == '/' && chNext == '/' &&
 775                    (style == SCE_V_COMMENTLINE || style == SCE_V_COMMENTLINEBANG)) {
 776                         return true;
 777                 } else if (!IsASpaceOrTab(ch)) {
 778                         return false;
 779                 }
 780         }
 781         return false;
 782 }
 783
 784 // Store both the current line's fold level and the next lines in the
 785 // level store to make it easy to pick up with each increment
 786 // and to make it possible to fiddle the current level for "} else {".
 787 void SCI_METHOD LexerVerilog::Fold(Sci_PositionU startPos, Sci_Position length, int initStyle, IDocument *pAccess)
 788 {
 789         LexAccessor styler(pAccess);
 790         bool foldAtBrace  = 1;
 791         bool foldAtParenthese  = 1;
 792
 793         Sci_Position lineCurrent = styler.GetLine(startPos);
 794         // Move back one line to be compatible with LexerModule::Fold behavior, fixes problem with foldComment behavior
 795         if (lineCurrent > 0) {
 796                 lineCurrent--;
 797                 Sci_Position newStartPos = styler.LineStart(lineCurrent);
 798                 length += startPos - newStartPos;
 799                 startPos = newStartPos;
 800                 initStyle = 0;
 801                 if (startPos > 0) {
 802                         initStyle = styler.StyleAt(startPos - 1);
 803                 }
 804         }
 805         Sci_PositionU endPos = startPos + length;
 806         int visibleChars = 0;
 807         int levelCurrent = SC_FOLDLEVELBASE;
 808         if (lineCurrent > 0)
 809                 levelCurrent = styler.LevelAt(lineCurrent-1) >> 16;
 810         int levelMinCurrent = levelCurrent;
 811         int levelNext = levelCurrent;
 812         char chNext = styler[startPos];
 813         int styleNext = MaskActive(styler.StyleAt(startPos));
 814         int style = MaskActive(initStyle);
 815
 816         // restore fold state (if it exists) for prior line
 817         int stateCurrent = 0;
 818         std::map<Sci_Position,int>::iterator foldStateIterator = foldState.find(lineCurrent-1);
 819         if (foldStateIterator != foldState.end()) {
 820                 stateCurrent = foldStateIterator->second;
 821         }
 822
 823         // remove all foldState entries after lineCurrent-1
 824         foldStateIterator = foldState.upper_bound(lineCurrent-1);
 825         if (foldStateIterator != foldState.end()) {
 826                 foldState.erase(foldStateIterator, foldState.end());
 827         }
 828
 829         for (Sci_PositionU i = startPos; i < endPos; i++) {
 830                 char ch = chNext;
 831                 chNext = styler.SafeGetCharAt(i + 1);
 832                 int stylePrev = style;
 833                 style = styleNext;
 834                 styleNext = MaskActive(styler.StyleAt(i + 1));
 835                 bool atEOL = (ch == '\r' && chNext != '\n') || (ch == '\n');
 836                 if (!(stateCurrent & protectedFlag)) {
 837                         if (options.foldComment && IsStreamCommentStyle(style)) {
 838                                 if (!IsStreamCommentStyle(stylePrev)) {
 839                                         levelNext++;
 840                                 } else if (!IsStreamCommentStyle(styleNext) && !atEOL) {
 841                                         // Comments don't end at end of line and the next character may be unstyled.
 842                                         levelNext--;
 843                                 }
 844                         }
 845                         if (options.foldComment && atEOL && IsCommentLine(lineCurrent, styler))
 846                         {
 847                                 if (!IsCommentLine(lineCurrent - 1, styler)
 848                                         && IsCommentLine(lineCurrent + 1, styler))
 849                                         levelNext++;
 850                                 else if (IsCommentLine(lineCurrent - 1, styler)
 851                                                  && !IsCommentLine(lineCurrent+1, styler))
 852                                         levelNext--;
 853                         }
 854                         if (options.foldComment && (style == SCE_V_COMMENTLINE)) {
 855                                 if ((ch == '/') && (chNext == '/')) {
 856                                         char chNext2 = styler.SafeGetCharAt(i + 2);
 857                                         if (chNext2 == '{') {
 858                                                 levelNext++;
 859                                         } else if (chNext2 == '}') {
 860                                                 levelNext--;
 861                                         }
 862                                 }
 863                         }
 864                 }
 865                 if (ch == '`') {
 866                         Sci_PositionU j = i + 1;
 867                         while ((j < endPos) && IsASpaceOrTab(styler.SafeGetCharAt(j))) {
 868                                 j++;
 869                         }
 870                         if (styler.Match(j, "protected")) {
 871                                 stateCurrent |= protectedFlag;
 872                                 levelNext++;
 873                         } else if (styler.Match(j, "endprotected")) {
 874                                 stateCurrent &= ~protectedFlag;
 875                                 levelNext--;
 876                         } else if (!(stateCurrent & protectedFlag) && options.foldPreprocessor && (style == SCE_V_PREPROCESSOR)) {
 877                                 if (styler.Match(j, "if")) {
 878                                         if (options.foldPreprocessorElse) {
 879                                                 // Measure the minimum before a begin to allow
 880                                                 // folding on "end else begin"
 881                                                 if (levelMinCurrent > levelNext) {
 882                                                         levelMinCurrent = levelNext;
 883                                                 }
 884                                         }
 885                                         levelNext++;
 886                                 } else if (options.foldPreprocessorElse && styler.Match(j, "else")) {
 887                                         levelNext--;
 888                                         if (levelMinCurrent > levelNext) {
 889                                                 levelMinCurrent = levelNext;
 890                                         }
 891                                         levelNext++;
 892                                 } else if (options.foldPreprocessorElse && styler.Match(j, "elsif")) {
 893                                         levelNext--;
 894                                         // Measure the minimum before a begin to allow
 895                                         // folding on "end else begin"
 896                                         if (levelMinCurrent > levelNext) {
 897                                                 levelMinCurrent = levelNext;
 898                                         }
 899                                         levelNext++;
 900                                 } else if (styler.Match(j, "endif")) {
 901                                         levelNext--;
 902                                 }
 903                         }
 904                 }
 905                 if (style == SCE_V_OPERATOR) {
 906                         if (foldAtParenthese) {
 907                                 if (ch == '(') {
 908                                         levelNext++;
 909                                 } else if (ch == ')') {
 910                                         levelNext--;
 911                                 }
 912                         }
 913                         // semicolons terminate external declarations
 914                         if (ch == ';') {
 915                                 // extern and pure virtual declarations terminated by semicolon
 916                                 if (stateCurrent & foldExternFlag) {
 917                                         levelNext--;
 918                                         stateCurrent &= ~foldExternFlag;
 919                                 }
 920                                 // wait and disable statements terminated by semicolon
 921                                 if (stateCurrent & foldWaitDisableFlag) {
 922                                         stateCurrent &= ~foldWaitDisableFlag;
 923                                 }
 924                                 // typedef statements terminated by semicolon
 925                                 if (stateCurrent & typedefFlag) {
 926                                         stateCurrent &= ~typedefFlag;
 927                                 }
 928                         }
 929                         // wait and disable statements containing '(' will not contain "fork" keyword, special processing is not needed
 930                         if (ch == '(') {
 931                                 if (stateCurrent & foldWaitDisableFlag) {
 932                                         stateCurrent &= ~foldWaitDisableFlag;
 933                                 }
 934                         }
 935                 }
 936                 if (style == SCE_V_OPERATOR) {
 937                         if (foldAtBrace) {
 938                                 if (ch == '{') {
 939                                         levelNext++;
 940                                 } else if (ch == '}') {
 941                                         levelNext--;
 942                                 }
 943                         }
 944                 }
 945                 if (style == SCE_V_WORD && stylePrev != SCE_V_WORD) {
 946                         Sci_PositionU j = i;
 947                         if (styler.Match(j, "case") ||
 948                                 styler.Match(j, "casex") ||
 949                                 styler.Match(j, "casez") ||
 950                                 styler.Match(j, "covergroup") ||
 951                                 styler.Match(j, "function") ||
 952                                 styler.Match(j, "generate") ||
 953                                 styler.Match(j, "interface") ||
 954                                 styler.Match(j, "package") ||
 955                                 styler.Match(j, "primitive") ||
 956                                 styler.Match(j, "program") ||
 957                                 styler.Match(j, "sequence") ||
 958                                 styler.Match(j, "specify") ||
 959                                 styler.Match(j, "table") ||
 960                                 styler.Match(j, "task") ||
 961                                 (styler.Match(j, "module") && options.foldAtModule)) {
 962                                 levelNext++;
 963                         } else if (styler.Match(j, "begin")) {
 964                                 // Measure the minimum before a begin to allow
 965                                 // folding on "end else begin"
 966                                 if (levelMinCurrent > levelNext) {
 967                                         levelMinCurrent = levelNext;
 968                                 }
 969                                 levelNext++;
 970                         } else if (styler.Match(j, "class")) {
 971                                 // class does not introduce a block when used in a typedef statement
 972                                 if (!(stateCurrent & typedefFlag))
 973                                         levelNext++;
 974                         } else if (styler.Match(j, "fork")) {
 975                                 // fork does not introduce a block when used in a wait or disable statement
 976                                 if (stateCurrent & foldWaitDisableFlag) {
 977                                         stateCurrent &= ~foldWaitDisableFlag;
 978                                 } else
 979                                         levelNext++;
 980                         } else if (styler.Match(j, "endcase") ||
 981                                 styler.Match(j, "endclass") ||
 982                                 styler.Match(j, "endfunction") ||
 983                                 styler.Match(j, "endgenerate") ||
 984                                 styler.Match(j, "endgroup") ||
 985                                 styler.Match(j, "endinterface") ||
 986                                 styler.Match(j, "endpackage") ||
 987                                 styler.Match(j, "endprimitive") ||
 988                                 styler.Match(j, "endprogram") ||
 989                                 styler.Match(j, "endsequence") ||
 990                                 styler.Match(j, "endspecify") ||
 991                                 styler.Match(j, "endtable") ||
 992                                 styler.Match(j, "endtask") ||
 993                                 styler.Match(j, "join") ||
 994                                 styler.Match(j, "join_any") ||
 995                                 styler.Match(j, "join_none") ||
 996                                 (styler.Match(j, "endmodule") && options.foldAtModule) ||
 997                                 (styler.Match(j, "end") && !IsAWordChar(styler.SafeGetCharAt(j + 3)))) {
 998                                 levelNext--;
 999                         } else if (styler.Match(j, "extern") ||
1000                                 styler.Match(j, "pure")) {
1001                                 // extern and pure virtual functions/tasks are terminated by ';' not endfunction/endtask
1002                                 stateCurrent |= foldExternFlag;
1003                         } else if (styler.Match(j, "disable") ||
1004                                 styler.Match(j, "wait")) {
1005                                 // fork does not introduce a block when used in a wait or disable statement
1006                                 stateCurrent |= foldWaitDisableFlag;
1007                         } else if (styler.Match(j, "typedef")) {
1008                                 stateCurrent |= typedefFlag;
1009                         }
1010                 }
1011                 if (atEOL) {
1012                         int levelUse = levelCurrent;
1013                         if (options.foldAtElse||options.foldPreprocessorElse) {
1014                                 levelUse = levelMinCurrent;
1015                         }
1016                         int lev = levelUse | levelNext << 16;
1017                         if (visibleChars == 0 && options.foldCompact)
1018                                 lev |= SC_FOLDLEVELWHITEFLAG;
1019                         if (levelUse < levelNext)
1020                                 lev |= SC_FOLDLEVELHEADERFLAG;
1021                         if (stateCurrent) {
1022                                 foldState[lineCurrent] = stateCurrent;
1023                         }
1024                         if (lev != styler.LevelAt(lineCurrent)) {
1025                                 styler.SetLevel(lineCurrent, lev);
1026                         }
1027                         lineCurrent++;
1028                         levelCurrent = levelNext;
1029                         levelMinCurrent = levelCurrent;
1030                         visibleChars = 0;
1031                 }
1032                 if (!isspacechar(ch))
1033                         visibleChars++;
1034         }
1035 }
1036
1037 std::vector<std::string> LexerVerilog::Tokenize(const std::string &expr) const {
1038         // Break into tokens
1039         std::vector<std::string> tokens;
1040         const char *cp = expr.c_str();
1041         while (*cp) {
1042                 std::string word;
1043                 if (setWord.Contains(static_cast<unsigned char>(*cp))) {
1044                         // Identifiers and numbers
1045                         while (setWord.Contains(static_cast<unsigned char>(*cp))) {
1046                                 word += *cp;
1047                                 cp++;
1048                         }
1049                 } else if (IsSpaceOrTab(*cp)) {
1050                         while (IsSpaceOrTab(*cp)) {
1051                                 cp++;
1052                         }
1053                         continue;
1054                 } else {
1055                         // Should handle strings, characters, and comments here
1056                         word += *cp;
1057                         cp++;
1058                 }
1059                 tokens.push_back(word);
1060         }
1061         return tokens;
1062 }
1063
1064 static const char * const verilogWordLists[] = {
1065             "Primary keywords and identifiers",
1066             "Secondary keywords and identifiers",
1067             "System Tasks",
1068             "User defined tasks and identifiers",
1069             "Documentation comment keywords",
1070             "Preprocessor definitions",
1071             0,
1072         };
1073
1074 LexerModule lmVerilog(SCLEX_VERILOG, LexerVerilog::LexerFactoryVerilog, "verilog", verilogWordLists);