ext/scintilla/lexers/LexRust.cxx

   1 /** @file LexRust.cxx
   2  ** Lexer for Rust.
   3  **
   4  ** Copyright (c) 2013 by SiegeLord <slabode@aim.com>
   5  ** Converted to lexer object and added further folding features/properties by "Udo Lechner" <dlchnr(at)gmx(dot)net>
   6  **/
   7 // Copyright 1998-2005 by Neil Hodgson <neilh@scintilla.org>
   8 // The License.txt file describes the conditions under which this software may be distributed.
   9
  10 #include <stdlib.h>
  11 #include <string.h>
  12 #include <stdio.h>
  13 #include <stdarg.h>
  14 #include <assert.h>
  15 #include <ctype.h>
  16
  17 #include <string>
  18 #include <map>
  19
  20 #include "ILexer.h"
  21 #include "Scintilla.h"
  22 #include "SciLexer.h"
  23
  24 #include "WordList.h"
  25 #include "LexAccessor.h"
  26 #include "Accessor.h"
  27 #include "StyleContext.h"
  28 #include "CharacterSet.h"
  29 #include "LexerModule.h"
  30 #include "OptionSet.h"
  31 #include "PropSetSimple.h"
  32
  33 #ifdef SCI_NAMESPACE
  34 using namespace Scintilla;
  35 #endif
  36
  37 static const int NUM_RUST_KEYWORD_LISTS = 7;
  38 static const int MAX_RUST_IDENT_CHARS = 1023;
  39
  40 static bool IsStreamCommentStyle(int style) {
  41         return style == SCE_RUST_COMMENTBLOCK ||
  42                    style == SCE_RUST_COMMENTBLOCKDOC;
  43 }
  44
  45 // Options used for LexerRust
  46 struct OptionsRust {
  47         bool fold;
  48         bool foldSyntaxBased;
  49         bool foldComment;
  50         bool foldCommentMultiline;
  51         bool foldCommentExplicit;
  52         std::string foldExplicitStart;
  53         std::string foldExplicitEnd;
  54         bool foldExplicitAnywhere;
  55         bool foldCompact;
  56         int  foldAtElseInt;
  57         bool foldAtElse;
  58         OptionsRust() {
  59                 fold = false;
  60                 foldSyntaxBased = true;
  61                 foldComment = false;
  62                 foldCommentMultiline = true;
  63                 foldCommentExplicit = true;
  64                 foldExplicitStart = "";
  65                 foldExplicitEnd   = "";
  66                 foldExplicitAnywhere = false;
  67                 foldCompact = true;
  68                 foldAtElseInt = -1;
  69                 foldAtElse = false;
  70         }
  71 };
  72
  73 static const char * const rustWordLists[NUM_RUST_KEYWORD_LISTS + 1] = {
  74                         "Primary keywords and identifiers",
  75                         "Built in types",
  76                         "Other keywords",
  77                         "Keywords 4",
  78                         "Keywords 5",
  79                         "Keywords 6",
  80                         "Keywords 7",
  81                         0,
  82                 };
  83
  84 struct OptionSetRust : public OptionSet<OptionsRust> {
  85         OptionSetRust() {
  86                 DefineProperty("fold", &OptionsRust::fold);
  87
  88                 DefineProperty("fold.comment", &OptionsRust::foldComment);
  89
  90                 DefineProperty("fold.compact", &OptionsRust::foldCompact);
  91
  92                 DefineProperty("fold.at.else", &OptionsRust::foldAtElse);
  93
  94                 DefineProperty("fold.rust.syntax.based", &OptionsRust::foldSyntaxBased,
  95                         "Set this property to 0 to disable syntax based folding.");
  96
  97                 DefineProperty("fold.rust.comment.multiline", &OptionsRust::foldCommentMultiline,
  98                         "Set this property to 0 to disable folding multi-line comments when fold.comment=1.");
  99
 100                 DefineProperty("fold.rust.comment.explicit", &OptionsRust::foldCommentExplicit,
 101                         "Set this property to 0 to disable folding explicit fold points when fold.comment=1.");
 102
 103                 DefineProperty("fold.rust.explicit.start", &OptionsRust::foldExplicitStart,
 104                         "The string to use for explicit fold start points, replacing the standard //{.");
 105
 106                 DefineProperty("fold.rust.explicit.end", &OptionsRust::foldExplicitEnd,
 107                         "The string to use for explicit fold end points, replacing the standard //}.");
 108
 109                 DefineProperty("fold.rust.explicit.anywhere", &OptionsRust::foldExplicitAnywhere,
 110                         "Set this property to 1 to enable explicit fold points anywhere, not just in line comments.");
 111
 112                 DefineProperty("lexer.rust.fold.at.else", &OptionsRust::foldAtElseInt,
 113                         "This option enables Rust folding on a \"} else {\" line of an if statement.");
 114
 115                 DefineWordListSets(rustWordLists);
 116         }
 117 };
 118
 119 class LexerRust : public ILexer {
 120         WordList keywords[NUM_RUST_KEYWORD_LISTS];
 121         OptionsRust options;
 122         OptionSetRust osRust;
 123 public:
 124         virtual ~LexerRust() {
 125         }
 126         void SCI_METHOD Release() {
 127                 delete this;
 128         }
 129         int SCI_METHOD Version() const {
 130                 return lvOriginal;
 131         }
 132         const char * SCI_METHOD PropertyNames() {
 133                 return osRust.PropertyNames();
 134         }
 135         int SCI_METHOD PropertyType(const char *name) {
 136                 return osRust.PropertyType(name);
 137         }
 138         const char * SCI_METHOD DescribeProperty(const char *name) {
 139                 return osRust.DescribeProperty(name);
 140         }
 141         int SCI_METHOD PropertySet(const char *key, const char *val);
 142         const char * SCI_METHOD DescribeWordListSets() {
 143                 return osRust.DescribeWordListSets();
 144         }
 145         int SCI_METHOD WordListSet(int n, const char *wl);
 146         void SCI_METHOD Lex(unsigned int startPos, int length, int initStyle, IDocument *pAccess);
 147         void SCI_METHOD Fold(unsigned int startPos, int length, int initStyle, IDocument *pAccess);
 148         void * SCI_METHOD PrivateCall(int, void *) {
 149                 return 0;
 150         }
 151         static ILexer *LexerFactoryRust() {
 152                 return new LexerRust();
 153         }
 154 };
 155
 156 int SCI_METHOD LexerRust::PropertySet(const char *key, const char *val) {
 157         if (osRust.PropertySet(&options, key, val)) {
 158                 return 0;
 159         }
 160         return -1;
 161 }
 162
 163 int SCI_METHOD LexerRust::WordListSet(int n, const char *wl) {
 164         int firstModification = -1;
 165         if (n < NUM_RUST_KEYWORD_LISTS) {
 166                 WordList *wordListN = &keywords[n];
 167                 WordList wlNew;
 168                 wlNew.Set(wl);
 169                 if (*wordListN != wlNew) {
 170                         wordListN->Set(wl);
 171                         firstModification = 0;
 172                 }
 173         }
 174         return firstModification;
 175 }
 176
 177 static bool IsWhitespace(int c) {
 178     return c == ' ' || c == '\t' || c == '\r' || c == '\n';
 179 }
 180
 181 /* This isn't quite right for Unicode identifiers */
 182 static bool IsIdentifierStart(int ch) {
 183         return (IsASCII(ch) && (isalpha(ch) || ch == '_')) || !IsASCII(ch);
 184 }
 185
 186 /* This isn't quite right for Unicode identifiers */
 187 static bool IsIdentifierContinue(int ch) {
 188         return (IsASCII(ch) && (isalnum(ch) || ch == '_')) || !IsASCII(ch);
 189 }
 190
 191 static void ScanWhitespace(Accessor& styler, int& pos, int max) {
 192         while (IsWhitespace(styler.SafeGetCharAt(pos, '\0')) && pos < max) {
 193                 if (pos == styler.LineEnd(styler.GetLine(pos)))
 194                         styler.SetLineState(styler.GetLine(pos), 0);
 195                 pos++;
 196         }
 197         styler.ColourTo(pos-1, SCE_RUST_DEFAULT);
 198 }
 199
 200 static void GrabString(char* s, Accessor& styler, int start, int len) {
 201         for (int ii = 0; ii < len; ii++)
 202                 s[ii] = styler[ii + start];
 203         s[len] = '\0';
 204 }
 205
 206 static void ScanIdentifier(Accessor& styler, int& pos, WordList *keywords) {
 207         int start = pos;
 208         while (IsIdentifierContinue(styler.SafeGetCharAt(pos, '\0')))
 209                 pos++;
 210
 211         if (styler.SafeGetCharAt(pos, '\0') == '!') {
 212                 pos++;
 213                 styler.ColourTo(pos - 1, SCE_RUST_MACRO);
 214         } else {
 215                 char s[MAX_RUST_IDENT_CHARS + 1];
 216                 int len = pos - start;
 217                 len = len > MAX_RUST_IDENT_CHARS ? MAX_RUST_IDENT_CHARS : len;
 218                 GrabString(s, styler, start, len);
 219                 bool keyword = false;
 220                 for (int ii = 0; ii < NUM_RUST_KEYWORD_LISTS; ii++) {
 221                         if (keywords[ii].InList(s)) {
 222                                 styler.ColourTo(pos - 1, SCE_RUST_WORD + ii);
 223                                 keyword = true;
 224                                 break;
 225                         }
 226                 }
 227                 if (!keyword) {
 228                         styler.ColourTo(pos - 1, SCE_RUST_IDENTIFIER);
 229                 }
 230         }
 231 }
 232
 233 static void ScanDigits(Accessor& styler, int& pos, int base) {
 234         for (;;) {
 235                 int c = styler.SafeGetCharAt(pos, '\0');
 236                 if (IsADigit(c, base) || c == '_')
 237                         pos++;
 238                 else
 239                         break;
 240         }
 241 }
 242
 243 static void ScanNumber(Accessor& styler, int& pos) {
 244         int base = 10;
 245         int c = styler.SafeGetCharAt(pos, '\0');
 246         int n = styler.SafeGetCharAt(pos + 1, '\0');
 247         bool error = false;
 248         if (c == '0' && n == 'x') {
 249                 pos += 2;
 250                 base = 16;
 251         } else if (c == '0' && n == 'b') {
 252                 pos += 2;
 253                 base = 2;
 254         } else if (c == '0' && n == 'o') {
 255                 pos += 2;
 256                 base = 8;
 257         }
 258         int old_pos = pos;
 259         ScanDigits(styler, pos, base);
 260         c = styler.SafeGetCharAt(pos, '\0');
 261         if (c == 'u' || c == 'i') {
 262                 pos++;
 263                 c = styler.SafeGetCharAt(pos, '\0');
 264                 n = styler.SafeGetCharAt(pos + 1, '\0');
 265                 if (c == '8') {
 266                         pos++;
 267                 } else if (c == '1' && n == '6') {
 268                         pos += 2;
 269                 } else if (c == '3' && n == '2') {
 270                         pos += 2;
 271                 } else if (c == '6' && n == '4') {
 272                         pos += 2;
 273                 }
 274         } else {
 275                 n = styler.SafeGetCharAt(pos + 1, '\0');
 276                 if (c == '.' && !(IsIdentifierStart(n) || n == '.')) {
 277                         error |= base != 10;
 278                         pos++;
 279                         ScanDigits(styler, pos, 10);
 280                 }
 281
 282                 c = styler.SafeGetCharAt(pos, '\0');
 283                 if (c == 'e' || c == 'E') {
 284                         error |= base != 10;
 285                         pos++;
 286                         c = styler.SafeGetCharAt(pos, '\0');
 287                         if (c == '-' || c == '+')
 288                                 pos++;
 289                         int old_pos = pos;
 290                         ScanDigits(styler, pos, 10);
 291                         if (old_pos == pos) {
 292                                 error = true;
 293                         }
 294                 }
 295
 296                 c = styler.SafeGetCharAt(pos, '\0');
 297                 if (c == 'f') {
 298                         error |= base != 10;
 299                         pos++;
 300                         c = styler.SafeGetCharAt(pos, '\0');
 301                         n = styler.SafeGetCharAt(pos + 1, '\0');
 302                         if (c == '3' && n == '2') {
 303                                 pos += 2;
 304                         } else if (c == '6' && n == '4') {
 305                                 pos += 2;
 306                         } else {
 307                                 error = true;
 308                         }
 309                 }
 310         }
 311         if (old_pos == pos) {
 312                 error = true;
 313         }
 314         if (error)
 315                 styler.ColourTo(pos - 1, SCE_RUST_LEXERROR);
 316         else
 317                 styler.ColourTo(pos - 1, SCE_RUST_NUMBER);
 318 }
 319
 320 static bool IsOneCharOperator(int c) {
 321         return c == ';' || c == ',' || c == '(' || c == ')'
 322             || c == '{' || c == '}' || c == '[' || c == ']'
 323             || c == '@' || c == '#' || c == '~' || c == '+'
 324             || c == '*' || c == '/' || c == '^' || c == '%'
 325             || c == '.' || c == ':' || c == '!' || c == '<'
 326             || c == '>' || c == '=' || c == '-' || c == '&'
 327             || c == '|' || c == '$';
 328 }
 329
 330 static bool IsTwoCharOperator(int c, int n) {
 331         return (c == '.' && n == '.') || (c == ':' && n == ':')
 332             || (c == '!' && n == '=') || (c == '<' && n == '<')
 333             || (c == '<' && n == '=') || (c == '>' && n == '>')
 334             || (c == '>' && n == '=') || (c == '=' && n == '=')
 335             || (c == '=' && n == '>') || (c == '-' && n == '>')
 336             || (c == '&' && n == '&') || (c == '|' && n == '|')
 337             || (c == '-' && n == '=') || (c == '&' && n == '=')
 338             || (c == '|' && n == '=') || (c == '+' && n == '=')
 339             || (c == '*' && n == '=') || (c == '/' && n == '=')
 340             || (c == '^' && n == '=') || (c == '%' && n == '=');
 341 }
 342
 343 static bool IsThreeCharOperator(int c, int n, int n2) {
 344         return (c == '<' && n == '<' && n2 == '=')
 345             || (c == '>' && n == '>' && n2 == '=');
 346 }
 347
 348 static bool IsValidCharacterEscape(int c) {
 349         return c == 'n'  || c == 'r' || c == 't' || c == '\\'
 350             || c == '\'' || c == '"' || c == '0';
 351 }
 352
 353 static bool IsValidStringEscape(int c) {
 354         return IsValidCharacterEscape(c) || c == '\n';
 355 }
 356
 357 static bool ScanNumericEscape(Accessor &styler, int& pos, int num_digits, bool stop_asap) {
 358         for (;;) {
 359                 int c = styler.SafeGetCharAt(pos, '\0');
 360                 if (!IsADigit(c, 16))
 361                         break;
 362                 num_digits--;
 363                 pos++;
 364                 if (num_digits == 0 && stop_asap)
 365                         return true;
 366         }
 367         if (num_digits == 0) {
 368                 return true;
 369         } else {
 370                 return false;
 371         }
 372 }
 373
 374 /* This is overly permissive for character literals in order to accept UTF-8 encoded
 375  * character literals. */
 376 static void ScanCharacterLiteralOrLifetime(Accessor &styler, int& pos) {
 377         pos++;
 378         int c = styler.SafeGetCharAt(pos, '\0');
 379         int n = styler.SafeGetCharAt(pos + 1, '\0');
 380         bool done = false;
 381         bool valid_lifetime = IsIdentifierStart(c);
 382         bool valid_char = true;
 383         bool first = true;
 384         while (!done) {
 385                 switch (c) {
 386                         case '\\':
 387                                 done = true;
 388                                 if (IsValidCharacterEscape(n)) {
 389                                         pos += 2;
 390                                 } else if (n == 'x') {
 391                                         pos += 2;
 392                                         valid_char = ScanNumericEscape(styler, pos, 2, false);
 393                                 } else if (n == 'u') {
 394                                         pos += 2;
 395                                         valid_char = ScanNumericEscape(styler, pos, 4, false);
 396                                 } else if (n == 'U') {
 397                                         pos += 2;
 398                                         valid_char = ScanNumericEscape(styler, pos, 8, false);
 399                                 } else {
 400                                         valid_char = false;
 401                                 }
 402                                 break;
 403                         case '\'':
 404                                 valid_char = !first;
 405                                 done = true;
 406                                 break;
 407                         case '\t':
 408                         case '\n':
 409                         case '\r':
 410                         case '\0':
 411                                 valid_char = false;
 412                                 done = true;
 413                                 break;
 414                         default:
 415                                 if (!IsIdentifierContinue(c) && !first) {
 416                                         done = true;
 417                                 } else {
 418                                         pos++;
 419                                 }
 420                                 break;
 421                 }
 422                 c = styler.SafeGetCharAt(pos, '\0');
 423                 n = styler.SafeGetCharAt(pos + 1, '\0');
 424
 425                 first = false;
 426         }
 427         if (styler.SafeGetCharAt(pos, '\0') == '\'') {
 428                 valid_lifetime = false;
 429         } else {
 430                 valid_char = false;
 431         }
 432         if (valid_lifetime) {
 433                 styler.ColourTo(pos - 1, SCE_RUST_LIFETIME);
 434         } else if (valid_char) {
 435                 pos++;
 436                 styler.ColourTo(pos - 1, SCE_RUST_CHARACTER);
 437         } else {
 438                 styler.ColourTo(pos - 1, SCE_RUST_LEXERROR);
 439         }
 440 }
 441
 442 enum CommentState {
 443         UnknownComment,
 444         DocComment,
 445         NotDocComment
 446 };
 447
 448 /*
 449  * The rule for block-doc comments is as follows: /xxN and /x! (where x is an asterisk, N is a non-asterisk) start doc comments.
 450  * Otherwise it's a regular comment.
 451  */
 452 static void ResumeBlockComment(Accessor &styler, int& pos, int max, CommentState state, int level) {
 453         int c = styler.SafeGetCharAt(pos, '\0');
 454         bool maybe_doc_comment = false;
 455         if (c == '*') {
 456                 int n = styler.SafeGetCharAt(pos + 1, '\0');
 457                 if (n != '*' && n != '/') {
 458                         maybe_doc_comment = true;
 459                 }
 460         } else if (c == '!') {
 461                 maybe_doc_comment = true;
 462         }
 463
 464         for (;;) {
 465                 int n = styler.SafeGetCharAt(pos + 1, '\0');
 466                 if (pos == styler.LineEnd(styler.GetLine(pos)))
 467                         styler.SetLineState(styler.GetLine(pos), level);
 468                 if (c == '*') {
 469                         pos++;
 470                         if (n == '/') {
 471                                 pos++;
 472                                 level--;
 473                                 if (level == 0) {
 474                                         styler.SetLineState(styler.GetLine(pos), 0);
 475                                         if (state == DocComment || (state == UnknownComment && maybe_doc_comment))
 476                                                 styler.ColourTo(pos - 1, SCE_RUST_COMMENTBLOCKDOC);
 477                                         else
 478                                                 styler.ColourTo(pos - 1, SCE_RUST_COMMENTBLOCK);
 479                                         break;
 480                                 }
 481                         }
 482                 } else if (c == '/') {
 483                         pos++;
 484                         if (n == '*') {
 485                                 pos++;
 486                                 level++;
 487                         }
 488                 }
 489                 else {
 490                         pos++;
 491                 }
 492                 if (pos >= max) {
 493                         if (state == DocComment || (state == UnknownComment && maybe_doc_comment))
 494                                 styler.ColourTo(pos - 1, SCE_RUST_COMMENTBLOCKDOC);
 495                         else
 496                                 styler.ColourTo(pos - 1, SCE_RUST_COMMENTBLOCK);
 497                         break;
 498                 }
 499                 c = styler.SafeGetCharAt(pos, '\0');
 500         }
 501 }
 502
 503 /*
 504  * The rule for line-doc comments is as follows... ///N and //! (where N is a non slash) start doc comments.
 505  * Otherwise it's a normal line comment.
 506  */
 507 static void ResumeLineComment(Accessor &styler, int& pos, int max, CommentState state) {
 508         bool maybe_doc_comment = false;
 509         int c = styler.SafeGetCharAt(pos, '\0');
 510         if (c == '/') {
 511                 if (pos < max) {
 512                         pos++;
 513                         c = styler.SafeGetCharAt(pos, '\0');
 514                         if (c != '/') {
 515                                 maybe_doc_comment = true;
 516                         }
 517                 }
 518         } else if (c == '!') {
 519                 maybe_doc_comment = true;
 520         }
 521
 522         while (pos < max && c != '\n') {
 523                 if (pos == styler.LineEnd(styler.GetLine(pos)))
 524                         styler.SetLineState(styler.GetLine(pos), 0);
 525                 pos++;
 526                 c = styler.SafeGetCharAt(pos, '\0');
 527         }
 528
 529         if (state == DocComment || (state == UnknownComment && maybe_doc_comment))
 530                 styler.ColourTo(pos - 1, SCE_RUST_COMMENTLINEDOC);
 531         else
 532                 styler.ColourTo(pos - 1, SCE_RUST_COMMENTLINE);
 533 }
 534
 535 static void ScanComments(Accessor &styler, int& pos, int max) {
 536         pos++;
 537         int c = styler.SafeGetCharAt(pos, '\0');
 538         pos++;
 539         if (c == '/')
 540                 ResumeLineComment(styler, pos, max, UnknownComment);
 541         else if (c == '*')
 542                 ResumeBlockComment(styler, pos, max, UnknownComment, 1);
 543 }
 544
 545 static void ResumeString(Accessor &styler, int& pos, int max) {
 546         int c = styler.SafeGetCharAt(pos, '\0');
 547         bool error = false;
 548         while (c != '"' && !error) {
 549                 if (pos >= max) {
 550                         error = true;
 551                         break;
 552                 }
 553                 if (pos == styler.LineEnd(styler.GetLine(pos)))
 554                         styler.SetLineState(styler.GetLine(pos), 0);
 555                 if (c == '\\') {
 556                         int n = styler.SafeGetCharAt(pos + 1, '\0');
 557                         if (IsValidStringEscape(n)) {
 558                                 pos += 2;
 559                         } else if (n == 'x') {
 560                                 pos += 2;
 561                                 error = !ScanNumericEscape(styler, pos, 2, true);
 562                         } else if (n == 'u') {
 563                                 pos += 2;
 564                                 error = !ScanNumericEscape(styler, pos, 4, true);
 565                         } else if (n == 'U') {
 566                                 pos += 2;
 567                                 error = !ScanNumericEscape(styler, pos, 8, true);
 568                         } else {
 569                                 pos += 1;
 570                                 error = true;
 571                         }
 572                 } else {
 573                         pos++;
 574                 }
 575                 c = styler.SafeGetCharAt(pos, '\0');
 576         }
 577         if (!error)
 578                 pos++;
 579         styler.ColourTo(pos - 1, SCE_RUST_STRING);
 580 }
 581
 582 static void ResumeRawString(Accessor &styler, int& pos, int max, int num_hashes) {
 583         for (;;) {
 584                 if (pos == styler.LineEnd(styler.GetLine(pos)))
 585                         styler.SetLineState(styler.GetLine(pos), num_hashes);
 586
 587                 int c = styler.SafeGetCharAt(pos, '\0');
 588                 if (c == '"') {
 589                         pos++;
 590                         int trailing_num_hashes = 0;
 591                         while (styler.SafeGetCharAt(pos, '\0') == '#' && trailing_num_hashes < num_hashes) {
 592                                 trailing_num_hashes++;
 593                                 pos++;
 594                         }
 595                         if (trailing_num_hashes == num_hashes) {
 596                                 styler.SetLineState(styler.GetLine(pos), 0);
 597                                 styler.ColourTo(pos - 1, SCE_RUST_STRINGR);
 598                                 break;
 599                         }
 600                 } else if (pos >= max) {
 601                         styler.ColourTo(pos - 1, SCE_RUST_STRINGR);
 602                         break;
 603                 } else {
 604                         pos++;
 605                 }
 606         }
 607 }
 608
 609 static void ScanRawString(Accessor &styler, int& pos, int max) {
 610         pos++;
 611         int num_hashes = 0;
 612         while (styler.SafeGetCharAt(pos, '\0') == '#') {
 613                 num_hashes++;
 614                 pos++;
 615         }
 616         if (styler.SafeGetCharAt(pos, '\0') != '"') {
 617                 styler.ColourTo(pos - 1, SCE_RUST_LEXERROR);
 618         } else {
 619                 pos++;
 620                 ResumeRawString(styler, pos, max, num_hashes);
 621         }
 622 }
 623
 624 void SCI_METHOD LexerRust::Lex(unsigned int startPos, int length, int initStyle, IDocument *pAccess) {
 625         PropSetSimple props;
 626         Accessor styler(pAccess, &props);
 627         int pos = startPos;
 628         int max = pos + length;
 629
 630         styler.StartAt(pos);
 631         styler.StartSegment(pos);
 632
 633         if (initStyle == SCE_RUST_COMMENTBLOCK || initStyle == SCE_RUST_COMMENTBLOCKDOC) {
 634                 ResumeBlockComment(styler, pos, max, initStyle == SCE_RUST_COMMENTBLOCKDOC ? DocComment : NotDocComment, styler.GetLineState(styler.GetLine(pos) - 1));
 635         } else if (initStyle == SCE_RUST_COMMENTLINE || initStyle == SCE_RUST_COMMENTLINEDOC) {
 636                 ResumeLineComment(styler, pos, max, initStyle == SCE_RUST_COMMENTLINEDOC ? DocComment : NotDocComment);
 637         } else if (initStyle == SCE_RUST_STRING) {
 638                 ResumeString(styler, pos, max);
 639         } else if (initStyle == SCE_RUST_STRINGR) {
 640                 ResumeRawString(styler, pos, max, styler.GetLineState(styler.GetLine(pos) - 1));
 641         }
 642
 643         while (pos < max) {
 644                 int c = styler.SafeGetCharAt(pos, '\0');
 645                 int n = styler.SafeGetCharAt(pos + 1, '\0');
 646                 int n2 = styler.SafeGetCharAt(pos + 2, '\0');
 647
 648                 if (pos == 0 && c == '#' && n == '!') {
 649                         pos += 2;
 650                         ResumeLineComment(styler, pos, max, NotDocComment);
 651                 } else if (IsWhitespace(c)) {
 652                         ScanWhitespace(styler, pos, max);
 653                 } else if (c == '/' && (n == '/' || n == '*')) {
 654                         ScanComments(styler, pos, max);
 655                 } else if (c == 'r' && (n == '#' || n == '"')) {
 656                         ScanRawString(styler, pos, max);
 657                 } else if (IsIdentifierStart(c)) {
 658                         ScanIdentifier(styler, pos, keywords);
 659                 } else if (IsADigit(c)) {
 660                         ScanNumber(styler, pos);
 661                 } else if (IsThreeCharOperator(c, n, n2)) {
 662                         pos += 3;
 663                         styler.ColourTo(pos - 1, SCE_RUST_OPERATOR);
 664                 } else if (IsTwoCharOperator(c, n)) {
 665                         pos += 2;
 666                         styler.ColourTo(pos - 1, SCE_RUST_OPERATOR);
 667                 } else if (IsOneCharOperator(c)) {
 668                         pos++;
 669                         styler.ColourTo(pos - 1, SCE_RUST_OPERATOR);
 670                 } else if (c == '\'') {
 671                         ScanCharacterLiteralOrLifetime(styler, pos);
 672                 } else if (c == '"') {
 673                         pos++;
 674                         ResumeString(styler, pos, max);
 675                 } else {
 676                         pos++;
 677                         styler.ColourTo(pos - 1, SCE_RUST_LEXERROR);
 678                 }
 679         }
 680         styler.ColourTo(pos - 1, SCE_RUST_DEFAULT);
 681         styler.Flush();
 682 }
 683
 684 void SCI_METHOD LexerRust::Fold(unsigned int startPos, int length, int initStyle, IDocument *pAccess) {
 685
 686         if (!options.fold)
 687                 return;
 688
 689         LexAccessor styler(pAccess);
 690
 691         unsigned int endPos = startPos + length;
 692         int visibleChars = 0;
 693         bool inLineComment = false;
 694         int lineCurrent = styler.GetLine(startPos);
 695         int levelCurrent = SC_FOLDLEVELBASE;
 696         if (lineCurrent > 0)
 697                 levelCurrent = styler.LevelAt(lineCurrent-1) >> 16;
 698         unsigned int lineStartNext = styler.LineStart(lineCurrent+1);
 699         int levelMinCurrent = levelCurrent;
 700         int levelNext = levelCurrent;
 701         char chNext = styler[startPos];
 702         int styleNext = styler.StyleAt(startPos);
 703         int style = initStyle;
 704         const bool userDefinedFoldMarkers = !options.foldExplicitStart.empty() && !options.foldExplicitEnd.empty();
 705         for (unsigned int i = startPos; i < endPos; i++) {
 706                 char ch = chNext;
 707                 chNext = styler.SafeGetCharAt(i + 1);
 708                 int stylePrev = style;
 709                 style = styleNext;
 710                 styleNext = styler.StyleAt(i + 1);
 711                 bool atEOL = i == (lineStartNext-1);
 712                 if ((style == SCE_RUST_COMMENTLINE) || (style == SCE_RUST_COMMENTLINEDOC))
 713                         inLineComment = true;
 714                 if (options.foldComment && options.foldCommentMultiline && IsStreamCommentStyle(style) && !inLineComment) {
 715                         if (!IsStreamCommentStyle(stylePrev)) {
 716                                 levelNext++;
 717                         } else if (!IsStreamCommentStyle(styleNext) && !atEOL) {
 718                                 // Comments don't end at end of line and the next character may be unstyled.
 719                                 levelNext--;
 720                         }
 721                 }
 722                 if (options.foldComment && options.foldCommentExplicit && ((style == SCE_RUST_COMMENTLINE) || options.foldExplicitAnywhere)) {
 723                         if (userDefinedFoldMarkers) {
 724                                 if (styler.Match(i, options.foldExplicitStart.c_str())) {
 725                                         levelNext++;
 726                                 } else if (styler.Match(i, options.foldExplicitEnd.c_str())) {
 727                                         levelNext--;
 728                                 }
 729                         } else {
 730                                 if ((ch == '/') && (chNext == '/')) {
 731                                         char chNext2 = styler.SafeGetCharAt(i + 2);
 732                                         if (chNext2 == '{') {
 733                                                 levelNext++;
 734                                         } else if (chNext2 == '}') {
 735                                                 levelNext--;
 736                                         }
 737                                 }
 738                         }
 739                 }
 740                 if (options.foldSyntaxBased && (style == SCE_RUST_OPERATOR)) {
 741                         if (ch == '{') {
 742                                 // Measure the minimum before a '{' to allow
 743                                 // folding on "} else {"
 744                                 if (levelMinCurrent > levelNext) {
 745                                         levelMinCurrent = levelNext;
 746                                 }
 747                                 levelNext++;
 748                         } else if (ch == '}') {
 749                                 levelNext--;
 750                         }
 751                 }
 752                 if (!IsASpace(ch))
 753                         visibleChars++;
 754                 if (atEOL || (i == endPos-1)) {
 755                         int levelUse = levelCurrent;
 756                         if (options.foldSyntaxBased && options.foldAtElse) {
 757                                 levelUse = levelMinCurrent;
 758                         }
 759                         int lev = levelUse | levelNext << 16;
 760                         if (visibleChars == 0 && options.foldCompact)
 761                                 lev |= SC_FOLDLEVELWHITEFLAG;
 762                         if (levelUse < levelNext)
 763                                 lev |= SC_FOLDLEVELHEADERFLAG;
 764                         if (lev != styler.LevelAt(lineCurrent)) {
 765                                 styler.SetLevel(lineCurrent, lev);
 766                         }
 767                         lineCurrent++;
 768                         lineStartNext = styler.LineStart(lineCurrent+1);
 769                         levelCurrent = levelNext;
 770                         levelMinCurrent = levelCurrent;
 771                         if (atEOL && (i == static_cast<unsigned int>(styler.Length()-1))) {
 772                                 // There is an empty line at end of file so give it same level and empty
 773                                 styler.SetLevel(lineCurrent, (levelCurrent | levelCurrent << 16) | SC_FOLDLEVELWHITEFLAG);
 774                         }
 775                         visibleChars = 0;
 776                         inLineComment = false;
 777                 }
 778         }
 779 }
 780
 781 LexerModule lmRust(SCLEX_RUST, LexerRust::LexerFactoryRust, "rust", rustWordLists);