ext/scintilla/lexers/LexTADS3.cxx

   1 // Scintilla source code edit control
   2 /** @file LexTADS3.cxx
   3  ** Lexer for TADS3.
   4  **/
   5 // Copyright 1998-2006 by Neil Hodgson <neilh@scintilla.org>
   6 // The License.txt file describes the conditions under which this software may be distributed.
   7
   8 /*
   9  * TADS3 is a language designed by Michael J. Roberts for the writing of text
  10  * based games.  TADS comes from Text Adventure Development System.  It has good
  11  * support for the processing and outputting of formatted text and much of a
  12  * TADS program listing consists of strings.
  13  *
  14  * TADS has two types of strings, those enclosed in single quotes (') and those
  15  * enclosed in double quotes (").  These strings have different symantics and
  16  * can be given different highlighting if desired.
  17  *
  18  * There can be embedded within both types of strings html tags
  19  * ( <tag key=value> ), library directives ( <.directive> ), and message
  20  * parameters ( {The doctor's/his} ).
  21  *
  22  * Double quoted strings can also contain interpolated expressions
  23  * ( << rug.moved ? ' and a hole in the floor. ' : nil >> ).  These expressions
  24  * may themselves contain single or double quoted strings, although the double
  25  * quoted strings may not contain interpolated expressions.
  26  *
  27  * These embedded constructs influence the output and formatting and are an
  28  * important part of a program and require highlighting.
  29  *
  30  * LINKS
  31  * http://www.tads.org/
  32  */
  33
  34 #include <stdlib.h>
  35 #include <string.h>
  36 #include <stdio.h>
  37 #include <stdarg.h>
  38 #include <assert.h>
  39 #include <ctype.h>
  40
  41 #include "ILexer.h"
  42 #include "Scintilla.h"
  43 #include "SciLexer.h"
  44
  45 #include "WordList.h"
  46 #include "LexAccessor.h"
  47 #include "Accessor.h"
  48 #include "StyleContext.h"
  49 #include "CharacterSet.h"
  50 #include "LexerModule.h"
  51
  52 #ifdef SCI_NAMESPACE
  53 using namespace Scintilla;
  54 #endif
  55
  56 static const int T3_SINGLE_QUOTE = 1;
  57 static const int T3_INT_EXPRESSION = 2;
  58 static const int T3_INT_EXPRESSION_IN_TAG = 4;
  59 static const int T3_HTML_SQUOTE = 8;
  60
  61 static inline bool IsEOL(const int ch, const int chNext) {
  62         return (ch == '\r' && chNext != '\n') || (ch == '\n');
  63 }
  64
  65 /*
  66  *   Test the current character to see if it's the START of an EOL sequence;
  67  *   if so, skip ahead to the last character of the sequence and return true,
  68  *   and if not just return false.  There are a few places where we want to
  69  *   check to see if a newline sequence occurs at a particular point, but
  70  *   where a caller expects a subroutine to stop only upon reaching the END
  71  *   of a newline sequence (in particular, CR-LF on Windows).  That's why
  72  *   IsEOL() above only returns true on CR if the CR isn't followed by an LF
  73  *   - it doesn't want to admit that there's a newline until reaching the END
  74  *   of the sequence.  We meet both needs by saying that there's a newline
  75  *   when we see the CR in a CR-LF, but skipping the CR before returning so
  76  *   that the caller's caller will see that we've stopped at the LF.
  77  */
  78 static inline bool IsEOLSkip(StyleContext &sc)
  79 {
  80     /* test for CR-LF */
  81     if (sc.ch == '\r' && sc.chNext == '\n')
  82     {
  83         /* got CR-LF - skip the CR and indicate that we're at a newline */
  84         sc.Forward();
  85         return true;
  86     }
  87
  88     /*
  89      *   in other cases, we have at most a 1-character newline, so do the
  90      *   normal IsEOL test
  91      */
  92     return IsEOL(sc.ch, sc.chNext);
  93 }
  94
  95 static inline bool IsATADS3Operator(const int ch) {
  96         return ch == '=' || ch == '{' || ch == '}' || ch == '(' || ch == ')'
  97                 || ch == '[' || ch == ']' || ch == ',' || ch == ':' || ch == ';'
  98                 || ch == '+' || ch == '-' || ch == '*' || ch == '/' || ch == '%'
  99                 || ch == '?' || ch == '!' || ch == '<' || ch == '>' || ch == '|'
 100                 || ch == '@' || ch == '&' || ch == '~';
 101 }
 102
 103 static inline bool IsAWordChar(const int ch) {
 104         return isalnum(ch) || ch == '_';
 105 }
 106
 107 static inline bool IsAWordStart(const int ch) {
 108         return isalpha(ch) || ch == '_';
 109 }
 110
 111 static inline bool IsAHexDigit(const int ch) {
 112         int lch = tolower(ch);
 113         return isdigit(lch) || lch == 'a' || lch == 'b' || lch == 'c'
 114                 || lch == 'd' || lch == 'e' || lch == 'f';
 115 }
 116
 117 static inline bool IsAnHTMLChar(int ch) {
 118         return isalnum(ch) || ch == '-' || ch == '_' || ch == '.';
 119 }
 120
 121 static inline bool IsADirectiveChar(int ch) {
 122         return isalnum(ch) || isspace(ch) || ch == '-' || ch == '/';
 123 }
 124
 125 static inline bool IsANumberStart(StyleContext &sc) {
 126         return isdigit(sc.ch)
 127                 || (!isdigit(sc.chPrev) && sc.ch == '.' && isdigit(sc.chNext));
 128 }
 129
 130 inline static void ColouriseTADS3Operator(StyleContext &sc) {
 131         int initState = sc.state;
 132         int c = sc.ch;
 133         sc.SetState(c == '{' || c == '}' ? SCE_T3_BRACE : SCE_T3_OPERATOR);
 134         sc.ForwardSetState(initState);
 135 }
 136
 137 static void ColouriseTADSHTMLString(StyleContext &sc, int &lineState) {
 138         int endState = sc.state;
 139         int chQuote = sc.ch;
 140         int chString = (lineState & T3_SINGLE_QUOTE) ? '\'' : '"';
 141         if (endState == SCE_T3_HTML_STRING) {
 142                 if (lineState&T3_SINGLE_QUOTE) {
 143                         endState = SCE_T3_S_STRING;
 144                         chString = '\'';
 145                 } else if (lineState&T3_INT_EXPRESSION) {
 146                         endState = SCE_T3_X_STRING;
 147                         chString = '"';
 148                 } else {
 149                         endState = SCE_T3_HTML_DEFAULT;
 150                         chString = '"';
 151                 }
 152                 chQuote = (lineState & T3_HTML_SQUOTE) ? '\'' : '"';
 153         } else {
 154                 sc.SetState(SCE_T3_HTML_STRING);
 155                 sc.Forward();
 156         }
 157         if (chQuote == '"')
 158                 lineState &= ~T3_HTML_SQUOTE;
 159         else
 160                 lineState |= T3_HTML_SQUOTE;
 161
 162         while (sc.More()) {
 163                 if (IsEOL(sc.ch, sc.chNext)) {
 164                         return;
 165                 }
 166                 if (sc.ch == chQuote) {
 167                         sc.ForwardSetState(endState);
 168                         return;
 169                 }
 170                 if (sc.Match('\\', static_cast<char>(chQuote))) {
 171                         sc.Forward(2);
 172                         sc.SetState(endState);
 173                         return;
 174                 }
 175                 if (sc.ch == chString) {
 176                         sc.SetState(SCE_T3_DEFAULT);
 177                         return;
 178                 }
 179
 180                 if (sc.Match('<', '<')) {
 181                         lineState |= T3_INT_EXPRESSION | T3_INT_EXPRESSION_IN_TAG;
 182                         sc.SetState(SCE_T3_X_DEFAULT);
 183                         sc.Forward(2);
 184                         return;
 185                 }
 186
 187                 if (sc.Match('\\', static_cast<char>(chQuote))
 188                         || sc.Match('\\', static_cast<char>(chString))
 189                         || sc.Match('\\', '\\')) {
 190                         sc.Forward(2);
 191                 } else {
 192                         sc.Forward();
 193                 }
 194         }
 195 }
 196
 197 static void ColouriseTADS3HTMLTagStart(StyleContext &sc) {
 198         sc.SetState(SCE_T3_HTML_TAG);
 199         sc.Forward();
 200         if (sc.ch == '/') {
 201                 sc.Forward();
 202         }
 203         while (IsAnHTMLChar(sc.ch)) {
 204                 sc.Forward();
 205         }
 206 }
 207
 208 static void ColouriseTADS3HTMLTag(StyleContext &sc, int &lineState) {
 209         int endState = sc.state;
 210         int chQuote = '"';
 211         int chString = '\'';
 212         switch (endState) {
 213                 case SCE_T3_S_STRING:
 214                         ColouriseTADS3HTMLTagStart(sc);
 215                         sc.SetState(SCE_T3_HTML_DEFAULT);
 216                         chQuote = '\'';
 217                         chString = '"';
 218                         break;
 219                 case SCE_T3_D_STRING:
 220                 case SCE_T3_X_STRING:
 221                         ColouriseTADS3HTMLTagStart(sc);
 222                         sc.SetState(SCE_T3_HTML_DEFAULT);
 223                         break;
 224                 case SCE_T3_HTML_DEFAULT:
 225                         if (lineState&T3_SINGLE_QUOTE) {
 226                                 endState = SCE_T3_S_STRING;
 227                                 chQuote = '\'';
 228                                 chString = '"';
 229                         } else if (lineState&T3_INT_EXPRESSION) {
 230                                 endState = SCE_T3_X_STRING;
 231                         } else {
 232                                 endState = SCE_T3_D_STRING;
 233                         }
 234                         break;
 235         }
 236
 237         while (sc.More()) {
 238                 if (IsEOL(sc.ch, sc.chNext)) {
 239                         return;
 240                 }
 241                 if (sc.Match('/', '>')) {
 242                         sc.SetState(SCE_T3_HTML_TAG);
 243                         sc.Forward(2);
 244                         sc.SetState(endState);
 245                         return;
 246                 }
 247                 if (sc.ch == '>') {
 248                         sc.SetState(SCE_T3_HTML_TAG);
 249                         sc.ForwardSetState(endState);
 250                         return;
 251                 }
 252                 if (sc.ch == chQuote) {
 253                         sc.SetState(endState);
 254                         return;
 255                 }
 256                 if (sc.Match('\\', static_cast<char>(chQuote))) {
 257                         sc.Forward();
 258                         ColouriseTADSHTMLString(sc, lineState);
 259                         if (sc.state == SCE_T3_X_DEFAULT)
 260                             break;
 261                 } else if (sc.ch == chString) {
 262                         ColouriseTADSHTMLString(sc, lineState);
 263                 } else if (sc.ch == '=') {
 264                         ColouriseTADS3Operator(sc);
 265                 } else {
 266                         sc.Forward();
 267                 }
 268         }
 269 }
 270
 271 static void ColouriseTADS3Keyword(StyleContext &sc,
 272                                                         WordList *keywordlists[],       unsigned int endPos) {
 273         char s[250];
 274         WordList &keywords = *keywordlists[0];
 275         WordList &userwords1 = *keywordlists[1];
 276         WordList &userwords2 = *keywordlists[2];
 277         WordList &userwords3 = *keywordlists[3];
 278         int initState = sc.state;
 279         sc.SetState(SCE_T3_IDENTIFIER);
 280         while (sc.More() && (IsAWordChar(sc.ch))) {
 281                 sc.Forward();
 282         }
 283         sc.GetCurrent(s, sizeof(s));
 284         if ( strcmp(s, "is") == 0 || strcmp(s, "not") == 0) {
 285                 // have to find if "in" is next
 286                 int n = 1;
 287                 while (n + sc.currentPos < endPos && IsASpaceOrTab(sc.GetRelative(n)))
 288                         n++;
 289                 if (sc.GetRelative(n) == 'i' && sc.GetRelative(n+1) == 'n') {
 290                         sc.Forward(n+2);
 291                         sc.ChangeState(SCE_T3_KEYWORD);
 292                 }
 293         } else if (keywords.InList(s)) {
 294                 sc.ChangeState(SCE_T3_KEYWORD);
 295         } else if (userwords3.InList(s)) {
 296                 sc.ChangeState(SCE_T3_USER3);
 297         } else if (userwords2.InList(s)) {
 298                 sc.ChangeState(SCE_T3_USER2);
 299         } else if (userwords1.InList(s)) {
 300                 sc.ChangeState(SCE_T3_USER1);
 301         }
 302         sc.SetState(initState);
 303 }
 304
 305 static void ColouriseTADS3MsgParam(StyleContext &sc, int &lineState) {
 306         int endState = sc.state;
 307         int chQuote = '"';
 308         switch (endState) {
 309                 case SCE_T3_S_STRING:
 310                         sc.SetState(SCE_T3_MSG_PARAM);
 311                         sc.Forward();
 312                         chQuote = '\'';
 313                         break;
 314                 case SCE_T3_D_STRING:
 315                 case SCE_T3_X_STRING:
 316                         sc.SetState(SCE_T3_MSG_PARAM);
 317                         sc.Forward();
 318                         break;
 319                 case SCE_T3_MSG_PARAM:
 320                         if (lineState&T3_SINGLE_QUOTE) {
 321                                 endState = SCE_T3_S_STRING;
 322                                 chQuote = '\'';
 323                         } else if (lineState&T3_INT_EXPRESSION) {
 324                                 endState = SCE_T3_X_STRING;
 325                         } else {
 326                                 endState = SCE_T3_D_STRING;
 327                         }
 328                         break;
 329         }
 330         while (sc.More() && sc.ch != '}' && sc.ch != chQuote) {
 331                 if (IsEOL(sc.ch, sc.chNext)) {
 332                         return;
 333                 }
 334                 if (sc.ch == '\\') {
 335                         sc.Forward();
 336                 }
 337                 sc.Forward();
 338         }
 339         if (sc.ch == chQuote) {
 340                 sc.SetState(endState);
 341         } else {
 342                 sc.ForwardSetState(endState);
 343         }
 344 }
 345
 346 static void ColouriseTADS3LibDirective(StyleContext &sc, int &lineState) {
 347         int initState = sc.state;
 348         int chQuote = '"';
 349         switch (initState) {
 350                 case SCE_T3_S_STRING:
 351                         sc.SetState(SCE_T3_LIB_DIRECTIVE);
 352                         sc.Forward(2);
 353                         chQuote = '\'';
 354                         break;
 355                 case SCE_T3_D_STRING:
 356                         sc.SetState(SCE_T3_LIB_DIRECTIVE);
 357                         sc.Forward(2);
 358                         break;
 359                 case SCE_T3_LIB_DIRECTIVE:
 360                         if (lineState&T3_SINGLE_QUOTE) {
 361                                 initState = SCE_T3_S_STRING;
 362                                 chQuote = '\'';
 363                         } else {
 364                                 initState = SCE_T3_D_STRING;
 365                         }
 366                         break;
 367         }
 368         while (sc.More() && IsADirectiveChar(sc.ch)) {
 369                 if (IsEOL(sc.ch, sc.chNext)) {
 370                         return;
 371                 }
 372                 sc.Forward();
 373         };
 374         if (sc.ch == '>' || !sc.More()) {
 375                 sc.ForwardSetState(initState);
 376         } else if (sc.ch == chQuote) {
 377                 sc.SetState(initState);
 378         } else {
 379                 sc.ChangeState(initState);
 380                 sc.Forward();
 381         }
 382 }
 383
 384 static void ColouriseTADS3String(StyleContext &sc, int &lineState) {
 385         int chQuote = sc.ch;
 386         int endState = sc.state;
 387         switch (sc.state) {
 388                 case SCE_T3_DEFAULT:
 389                 case SCE_T3_X_DEFAULT:
 390                         if (chQuote == '"') {
 391                                 if (sc.state == SCE_T3_DEFAULT) {
 392                                         sc.SetState(SCE_T3_D_STRING);
 393                                 } else {
 394                                         sc.SetState(SCE_T3_X_STRING);
 395                                 }
 396                                 lineState &= ~T3_SINGLE_QUOTE;
 397                         } else {
 398                                 sc.SetState(SCE_T3_S_STRING);
 399                                 lineState |= T3_SINGLE_QUOTE;
 400                         }
 401                         sc.Forward();
 402                         break;
 403                 case SCE_T3_S_STRING:
 404                         chQuote = '\'';
 405                         endState = lineState&T3_INT_EXPRESSION ?
 406                                 SCE_T3_X_DEFAULT : SCE_T3_DEFAULT;
 407                         break;
 408                 case SCE_T3_D_STRING:
 409                         chQuote = '"';
 410                         endState = SCE_T3_DEFAULT;
 411                         break;
 412                 case SCE_T3_X_STRING:
 413                         chQuote = '"';
 414                         endState = SCE_T3_X_DEFAULT;
 415                         break;
 416         }
 417         while (sc.More()) {
 418                 if (IsEOL(sc.ch, sc.chNext)) {
 419                         return;
 420                 }
 421                 if (sc.ch == chQuote) {
 422                         sc.ForwardSetState(endState);
 423                         return;
 424                 }
 425                 if (sc.state == SCE_T3_D_STRING && sc.Match('<', '<')) {
 426                         lineState |= T3_INT_EXPRESSION;
 427                         sc.SetState(SCE_T3_X_DEFAULT);
 428                         sc.Forward(2);
 429                         return;
 430                 }
 431                 if (sc.Match('\\', static_cast<char>(chQuote))
 432                     || sc.Match('\\', '\\')) {
 433                         sc.Forward(2);
 434                 } else if (sc.ch == '{') {
 435                         ColouriseTADS3MsgParam(sc, lineState);
 436                 } else if (sc.Match('<', '.')) {
 437                         ColouriseTADS3LibDirective(sc, lineState);
 438                 } else if (sc.ch == '<') {
 439                         ColouriseTADS3HTMLTag(sc, lineState);
 440                         if (sc.state == SCE_T3_X_DEFAULT)
 441                                 return;
 442                 } else {
 443                         sc.Forward();
 444                 }
 445         }
 446 }
 447
 448 static void ColouriseTADS3Comment(StyleContext &sc, int endState) {
 449         sc.SetState(SCE_T3_BLOCK_COMMENT);
 450         while (sc.More()) {
 451                 if (IsEOL(sc.ch, sc.chNext)) {
 452                         return;
 453                 }
 454                 if (sc.Match('*', '/')) {
 455                         sc.Forward(2);
 456                         sc.SetState(endState);
 457                         return;
 458                 }
 459                 sc.Forward();
 460         }
 461 }
 462
 463 static void ColouriseToEndOfLine(StyleContext &sc, int initState, int endState) {
 464         sc.SetState(initState);
 465         while (sc.More()) {
 466                 if (sc.ch == '\\') {
 467                         sc.Forward();
 468                         if (IsEOLSkip(sc)) {
 469                                         return;
 470                         }
 471                 }
 472                 if (IsEOL(sc.ch, sc.chNext)) {
 473                         sc.SetState(endState);
 474                         return;
 475                 }
 476                 sc.Forward();
 477         }
 478 }
 479
 480 static void ColouriseTADS3Number(StyleContext &sc) {
 481         int endState = sc.state;
 482         bool inHexNumber = false;
 483         bool seenE = false;
 484         bool seenDot = sc.ch == '.';
 485         sc.SetState(SCE_T3_NUMBER);
 486         if (sc.More()) {
 487                 sc.Forward();
 488         }
 489         if (sc.chPrev == '0' && tolower(sc.ch) == 'x') {
 490                 inHexNumber = true;
 491                 sc.Forward();
 492         }
 493         while (sc.More()) {
 494                 if (inHexNumber) {
 495                         if (!IsAHexDigit(sc.ch)) {
 496                                 break;
 497                         }
 498                 } else if (!isdigit(sc.ch)) {
 499                         if (!seenE && tolower(sc.ch) == 'e') {
 500                                 seenE = true;
 501                                 seenDot = true;
 502                                 if (sc.chNext == '+' || sc.chNext == '-') {
 503                                         sc.Forward();
 504                                 }
 505                         } else if (!seenDot && sc.ch == '.') {
 506                                 seenDot = true;
 507                         } else {
 508                                 break;
 509                         }
 510                 }
 511                 sc.Forward();
 512         }
 513         sc.SetState(endState);
 514 }
 515
 516 static void ColouriseTADS3Doc(unsigned int startPos, int length, int initStyle,
 517                                                            WordList *keywordlists[], Accessor &styler) {
 518         int visibleChars = 0;
 519         int bracketLevel = 0;
 520         int lineState = 0;
 521         unsigned int endPos = startPos + length;
 522         int lineCurrent = styler.GetLine(startPos);
 523         if (lineCurrent > 0) {
 524                 lineState = styler.GetLineState(lineCurrent-1);
 525         }
 526         StyleContext sc(startPos, length, initStyle, styler);
 527
 528         while (sc.More()) {
 529
 530                 if (IsEOL(sc.ch, sc.chNext)) {
 531                         styler.SetLineState(lineCurrent, lineState);
 532                         lineCurrent++;
 533                         visibleChars = 0;
 534                         sc.Forward();
 535                         if (sc.ch == '\n') {
 536                                 sc.Forward();
 537                         }
 538                 }
 539
 540                 switch(sc.state) {
 541                         case SCE_T3_PREPROCESSOR:
 542                         case SCE_T3_LINE_COMMENT:
 543                                 ColouriseToEndOfLine(sc, sc.state, lineState&T3_INT_EXPRESSION ?
 544                                         SCE_T3_X_DEFAULT : SCE_T3_DEFAULT);
 545                                 break;
 546                         case SCE_T3_S_STRING:
 547                         case SCE_T3_D_STRING:
 548                         case SCE_T3_X_STRING:
 549                                 ColouriseTADS3String(sc, lineState);
 550                                 visibleChars++;
 551                                 break;
 552                         case SCE_T3_MSG_PARAM:
 553                                 ColouriseTADS3MsgParam(sc, lineState);
 554                                 break;
 555                         case SCE_T3_LIB_DIRECTIVE:
 556                                 ColouriseTADS3LibDirective(sc, lineState);
 557                                 break;
 558                         case SCE_T3_HTML_DEFAULT:
 559                                 ColouriseTADS3HTMLTag(sc, lineState);
 560                                 break;
 561                         case SCE_T3_HTML_STRING:
 562                                 ColouriseTADSHTMLString(sc, lineState);
 563                                 break;
 564                         case SCE_T3_BLOCK_COMMENT:
 565                                 ColouriseTADS3Comment(sc, lineState&T3_INT_EXPRESSION ?
 566                                         SCE_T3_X_DEFAULT : SCE_T3_DEFAULT);
 567                                 break;
 568                         case SCE_T3_DEFAULT:
 569                         case SCE_T3_X_DEFAULT:
 570                                 if (IsASpaceOrTab(sc.ch)) {
 571                                         sc.Forward();
 572                                 } else if (sc.ch == '#' && visibleChars == 0) {
 573                                         ColouriseToEndOfLine(sc, SCE_T3_PREPROCESSOR, sc.state);
 574                                 } else if (sc.Match('/', '*')) {
 575                                         ColouriseTADS3Comment(sc, sc.state);
 576                                         visibleChars++;
 577                                 } else if (sc.Match('/', '/')) {
 578                                         ColouriseToEndOfLine(sc, SCE_T3_LINE_COMMENT, sc.state);
 579                                 } else if (sc.ch == '"') {
 580                                         bracketLevel = 0;
 581                                         ColouriseTADS3String(sc, lineState);
 582                                         visibleChars++;
 583                                 } else if (sc.ch == '\'') {
 584                                         ColouriseTADS3String(sc, lineState);
 585                                         visibleChars++;
 586                                 } else if (sc.state == SCE_T3_X_DEFAULT && bracketLevel == 0
 587                                                    && sc.Match('>', '>')) {
 588                                         sc.Forward(2);
 589                                         sc.SetState(SCE_T3_D_STRING);
 590                                         if (lineState & T3_INT_EXPRESSION_IN_TAG)
 591                                                 sc.SetState(SCE_T3_HTML_STRING);
 592                                         lineState &= ~(T3_SINGLE_QUOTE|T3_INT_EXPRESSION
 593                                                        |T3_INT_EXPRESSION_IN_TAG);
 594                                 } else if (IsATADS3Operator(sc.ch)) {
 595                                         if (sc.state == SCE_T3_X_DEFAULT) {
 596                                                 if (sc.ch == '(') {
 597                                                         bracketLevel++;
 598                                                 } else if (sc.ch == ')' && bracketLevel > 0) {
 599                                                         bracketLevel--;
 600                                                 }
 601                                         }
 602                                         ColouriseTADS3Operator(sc);
 603                                         visibleChars++;
 604                                 } else if (IsANumberStart(sc)) {
 605                                         ColouriseTADS3Number(sc);
 606                                         visibleChars++;
 607                                 } else if (IsAWordStart(sc.ch)) {
 608                                         ColouriseTADS3Keyword(sc, keywordlists, endPos);
 609                                         visibleChars++;
 610                                 } else if (sc.Match("...")) {
 611                                         sc.SetState(SCE_T3_IDENTIFIER);
 612                                         sc.Forward(3);
 613                                         sc.SetState(SCE_T3_DEFAULT);
 614                                 } else {
 615                                         sc.Forward();
 616                                         visibleChars++;
 617                                 }
 618                                 break;
 619                         default:
 620                                 sc.SetState(SCE_T3_DEFAULT);
 621                                 sc.Forward();
 622                 }
 623         }
 624         sc.Complete();
 625 }
 626
 627 /*
 628  TADS3 has two styles of top level block (TLB). Eg
 629
 630  // default style
 631  silverKey : Key 'small silver key' 'small silver key'
 632         "A small key glints in the sunlight. "
 633  ;
 634
 635  and
 636
 637  silverKey : Key {
 638         'small silver key'
 639         'small silver key'
 640         "A small key glints in the sunlight. "
 641  }
 642
 643  Some constructs mandate one or the other, but usually the author has may choose
 644  either.
 645
 646  T3_SEENSTART is used to indicate that a braceless TLB has been (potentially)
 647  seen and is also used to match the closing ';' of the default style.
 648
 649  T3_EXPECTINGIDENTIFIER and T3_EXPECTINGPUNCTUATION are used to keep track of
 650  what characters may be seen without incrementing the block level.  The general
 651  pattern is identifier <punc> identifier, acceptable punctuation characters
 652  are ':', ',', '(' and ')'.  No attempt is made to ensure that punctuation
 653  characters are syntactically correct, eg parentheses match. A ')' always
 654  signifies the start of a block.  We just need to check if it is followed by a
 655  '{', in which case we let the brace handling code handle the folding level.
 656
 657  expectingIdentifier == false && expectingIdentifier == false
 658  Before the start of a TLB.
 659
 660  expectingIdentifier == true && expectingIdentifier == true
 661  Currently in an identifier.  Will accept identifier or punctuation.
 662
 663  expectingIdentifier == true && expectingIdentifier == false
 664  Just seen a punctuation character & now waiting for an identifier to start.
 665
 666  expectingIdentifier == false && expectingIdentifier == truee
 667  We were in an identifier and have seen space.  Now waiting to see a punctuation
 668  character
 669
 670  Space, comments & preprocessor directives are always acceptable and are
 671  equivalent.
 672 */
 673
 674 static const int T3_SEENSTART = 1 << 12;
 675 static const int T3_EXPECTINGIDENTIFIER = 1 << 13;
 676 static const int T3_EXPECTINGPUNCTUATION = 1 << 14;
 677
 678 static inline bool IsStringTransition(int s1, int s2) {
 679         return s1 != s2
 680                 && (s1 == SCE_T3_S_STRING || s1 == SCE_T3_X_STRING
 681                         || (s1 == SCE_T3_D_STRING && s2 != SCE_T3_X_DEFAULT))
 682                 && s2 != SCE_T3_LIB_DIRECTIVE
 683                 && s2 != SCE_T3_MSG_PARAM
 684                 && s2 != SCE_T3_HTML_TAG
 685                 && s2 != SCE_T3_HTML_STRING;
 686 }
 687
 688 static inline bool IsATADS3Punctuation(const int ch) {
 689         return ch == ':' || ch == ',' || ch == '(' || ch == ')';
 690 }
 691
 692 static inline bool IsAnIdentifier(const int style) {
 693         return style == SCE_T3_IDENTIFIER
 694                 || style == SCE_T3_USER1
 695                 || style == SCE_T3_USER2
 696                 || style == SCE_T3_USER3;
 697 }
 698
 699 static inline bool IsAnOperator(const int style) {
 700     return style == SCE_T3_OPERATOR || SCE_T3_BRACE;
 701 }
 702
 703 static inline bool IsSpaceEquivalent(const int ch, const int style) {
 704         return isspace(ch)
 705                 || style == SCE_T3_BLOCK_COMMENT
 706                 || style == SCE_T3_LINE_COMMENT
 707                 || style == SCE_T3_PREPROCESSOR;
 708 }
 709
 710 static char peekAhead(unsigned int startPos, unsigned int endPos,
 711                                           Accessor &styler) {
 712         for (unsigned int i = startPos; i < endPos; i++) {
 713                 int style = styler.StyleAt(i);
 714                 char ch = styler[i];
 715                 if (!IsSpaceEquivalent(ch, style)) {
 716                         if (IsAnIdentifier(style)) {
 717                                 return 'a';
 718                         }
 719                         if (IsATADS3Punctuation(ch)) {
 720                                 return ':';
 721                         }
 722                         if (ch == '{') {
 723                                 return '{';
 724                         }
 725                         return '*';
 726                 }
 727         }
 728         return ' ';
 729 }
 730
 731 static void FoldTADS3Doc(unsigned int startPos, int length, int initStyle,
 732                             WordList *[], Accessor &styler) {
 733         unsigned int endPos = startPos + length;
 734         int lineCurrent = styler.GetLine(startPos);
 735         int levelCurrent = SC_FOLDLEVELBASE;
 736         if (lineCurrent > 0)
 737                 levelCurrent = styler.LevelAt(lineCurrent-1) >> 16;
 738         int seenStart = levelCurrent & T3_SEENSTART;
 739         int expectingIdentifier = levelCurrent & T3_EXPECTINGIDENTIFIER;
 740         int expectingPunctuation = levelCurrent & T3_EXPECTINGPUNCTUATION;
 741         levelCurrent &= SC_FOLDLEVELNUMBERMASK;
 742         int levelMinCurrent = levelCurrent;
 743         int levelNext = levelCurrent;
 744         char chNext = styler[startPos];
 745         int styleNext = styler.StyleAt(startPos);
 746         int style = initStyle;
 747         char ch = chNext;
 748         int stylePrev = style;
 749         bool redo = false;
 750         for (unsigned int i = startPos; i < endPos; i++) {
 751                 if (redo) {
 752                         redo = false;
 753                         i--;
 754                 } else {
 755                         ch = chNext;
 756                         chNext = styler.SafeGetCharAt(i + 1);
 757                         stylePrev = style;
 758                         style = styleNext;
 759                         styleNext = styler.StyleAt(i + 1);
 760                 }
 761                 bool atEOL = IsEOL(ch, chNext);
 762
 763                 if (levelNext == SC_FOLDLEVELBASE) {
 764                         if (IsSpaceEquivalent(ch, style)) {
 765                                 if (expectingPunctuation) {
 766                                         expectingIdentifier = 0;
 767                                 }
 768                                 if (style == SCE_T3_BLOCK_COMMENT) {
 769                                         levelNext++;
 770                                 }
 771                         } else if (ch == '{') {
 772                                 levelNext++;
 773                                 seenStart = 0;
 774                         } else if (ch == '\'' || ch == '"' || ch == '[') {
 775                                 levelNext++;
 776                                 if (seenStart) {
 777                                         redo = true;
 778                                 }
 779                         } else if (ch == ';') {
 780                                 seenStart = 0;
 781                                 expectingIdentifier = 0;
 782                                 expectingPunctuation = 0;
 783                         } else if (expectingIdentifier && expectingPunctuation) {
 784                                 if (IsATADS3Punctuation(ch)) {
 785                                         if (ch == ')' && peekAhead(i+1, endPos, styler) != '{') {
 786                                                 levelNext++;
 787                                         } else {
 788                                                 expectingPunctuation = 0;
 789                                         }
 790                                 } else if (!IsAnIdentifier(style)) {
 791                                         levelNext++;
 792                                 }
 793                         } else if (expectingIdentifier && !expectingPunctuation) {
 794                                 if (!IsAnIdentifier(style)) {
 795                                         levelNext++;
 796                                 } else {
 797                                         expectingPunctuation = T3_EXPECTINGPUNCTUATION;
 798                                 }
 799                         } else if (!expectingIdentifier && expectingPunctuation) {
 800                                 if (!IsATADS3Punctuation(ch)) {
 801                                         levelNext++;
 802                                 } else {
 803                                         if (ch == ')' && peekAhead(i+1, endPos, styler) != '{') {
 804                                                 levelNext++;
 805                                         } else {
 806                                                 expectingIdentifier = T3_EXPECTINGIDENTIFIER;
 807                                                 expectingPunctuation = 0;
 808                                         }
 809                                 }
 810                         } else if (!expectingIdentifier && !expectingPunctuation) {
 811                                 if (IsAnIdentifier(style)) {
 812                                         seenStart = T3_SEENSTART;
 813                                         expectingIdentifier = T3_EXPECTINGIDENTIFIER;
 814                                         expectingPunctuation = T3_EXPECTINGPUNCTUATION;
 815                                 }
 816                         }
 817
 818                         if (levelNext != SC_FOLDLEVELBASE && style != SCE_T3_BLOCK_COMMENT) {
 819                                 expectingIdentifier = 0;
 820                                 expectingPunctuation = 0;
 821                         }
 822
 823                 } else if (levelNext == SC_FOLDLEVELBASE+1 && seenStart
 824                                    && ch == ';' && IsAnOperator(style)) {
 825                         levelNext--;
 826                         seenStart = 0;
 827                 } else if (style == SCE_T3_BLOCK_COMMENT) {
 828                         if (stylePrev != SCE_T3_BLOCK_COMMENT) {
 829                                 levelNext++;
 830                         } else if (styleNext != SCE_T3_BLOCK_COMMENT && !atEOL) {
 831                                 // Comments don't end at end of line and the next character may be unstyled.
 832                                 levelNext--;
 833                         }
 834                 } else if (ch == '\'' || ch == '"') {
 835                         if (IsStringTransition(style, stylePrev)) {
 836                                 if (levelMinCurrent > levelNext) {
 837                                         levelMinCurrent = levelNext;
 838                                 }
 839                                 levelNext++;
 840                         } else if (IsStringTransition(style, styleNext)) {
 841                                 levelNext--;
 842                         }
 843                 } else if (IsAnOperator(style)) {
 844                         if (ch == '{' || ch == '[') {
 845                                 // Measure the minimum before a '{' to allow
 846                                 // folding on "} else {"
 847                                 if (levelMinCurrent > levelNext) {
 848                                         levelMinCurrent = levelNext;
 849                                 }
 850                                 levelNext++;
 851                         } else if (ch == '}' || ch == ']') {
 852                                 levelNext--;
 853                         }
 854                 }
 855
 856                 if (atEOL) {
 857                         if (seenStart && levelNext == SC_FOLDLEVELBASE) {
 858                                 switch (peekAhead(i+1, endPos, styler)) {
 859                                         case ' ':
 860                                         case '{':
 861                                                 break;
 862                                         case '*':
 863                                                 levelNext++;
 864                                                 break;
 865                                         case 'a':
 866                                                 if (expectingPunctuation) {
 867                                                         levelNext++;
 868                                                 }
 869                                                 break;
 870                                         case ':':
 871                                                 if (expectingIdentifier) {
 872                                                         levelNext++;
 873                                                 }
 874                                                 break;
 875                                 }
 876                                 if (levelNext != SC_FOLDLEVELBASE) {
 877                                         expectingIdentifier = 0;
 878                                         expectingPunctuation = 0;
 879                                 }
 880                         }
 881                         int lev = levelMinCurrent | (levelNext | expectingIdentifier
 882                                 | expectingPunctuation | seenStart) << 16;
 883                         if (levelMinCurrent < levelNext)
 884                                 lev |= SC_FOLDLEVELHEADERFLAG;
 885                         if (lev != styler.LevelAt(lineCurrent)) {
 886                                 styler.SetLevel(lineCurrent, lev);
 887                         }
 888                         lineCurrent++;
 889                         levelCurrent = levelNext;
 890                         levelMinCurrent = levelCurrent;
 891                 }
 892         }
 893 }
 894
 895 static const char * const tads3WordList[] = {
 896         "TADS3 Keywords",
 897         "User defined 1",
 898         "User defined 2",
 899         "User defined 3",
 900         0
 901 };
 902
 903 LexerModule lmTADS3(SCLEX_TADS3, ColouriseTADS3Doc, "tads3", FoldTADS3Doc, tads3WordList);