scintilla/lexers/LexRust.cxx

   1 /** @file LexRust.cxx
   2  ** Lexer for Rust.
   3  **
   4  ** Copyright (c) 2013 by SiegeLord <slabode@aim.com>
   5  ** Converted to lexer object and added further folding features/properties by "Udo Lechner" <dlchnr(at)gmx(dot)net>
   6  **/
   7 // Copyright 1998-2005 by Neil Hodgson <neilh@scintilla.org>
   8 // The License.txt file describes the conditions under which this software may be distributed.
   9
  10 #include <stdlib.h>
  11 #include <string.h>
  12 #include <stdio.h>
  13 #include <stdarg.h>
  14 #include <assert.h>
  15 #include <ctype.h>
  16
  17 #include <string>
  18 #include <map>
  19
  20 #include "ILexer.h"
  21 #include "Scintilla.h"
  22 #include "SciLexer.h"
  23
  24 #include "PropSetSimple.h"
  25 #include "WordList.h"
  26 #include "LexAccessor.h"
  27 #include "Accessor.h"
  28 #include "StyleContext.h"
  29 #include "CharacterSet.h"
  30 #include "LexerModule.h"
  31 #include "OptionSet.h"
  32
  33 #ifdef SCI_NAMESPACE
  34 using namespace Scintilla;
  35 #endif
  36
  37 static const int NUM_RUST_KEYWORD_LISTS = 7;
  38 static const int MAX_RUST_IDENT_CHARS = 1023;
  39
  40 static bool IsStreamCommentStyle(int style) {
  41         return style == SCE_RUST_COMMENTBLOCK ||
  42                    style == SCE_RUST_COMMENTBLOCKDOC;
  43 }
  44
  45 // Options used for LexerRust
  46 struct OptionsRust {
  47         bool fold;
  48         bool foldSyntaxBased;
  49         bool foldComment;
  50         bool foldCommentMultiline;
  51         bool foldCommentExplicit;
  52         std::string foldExplicitStart;
  53         std::string foldExplicitEnd;
  54         bool foldExplicitAnywhere;
  55         bool foldCompact;
  56         int  foldAtElseInt;
  57         bool foldAtElse;
  58         OptionsRust() {
  59                 fold = false;
  60                 foldSyntaxBased = true;
  61                 foldComment = false;
  62                 foldCommentMultiline = true;
  63                 foldCommentExplicit = true;
  64                 foldExplicitStart = "";
  65                 foldExplicitEnd   = "";
  66                 foldExplicitAnywhere = false;
  67                 foldCompact = true;
  68                 foldAtElseInt = -1;
  69                 foldAtElse = false;
  70         }
  71 };
  72
  73 static const char * const rustWordLists[NUM_RUST_KEYWORD_LISTS + 1] = {
  74                         "Primary keywords and identifiers",
  75                         "Built in types",
  76                         "Other keywords",
  77                         "Keywords 4",
  78                         "Keywords 5",
  79                         "Keywords 6",
  80                         "Keywords 7",
  81                         0,
  82                 };
  83
  84 struct OptionSetRust : public OptionSet<OptionsRust> {
  85         OptionSetRust() {
  86                 DefineProperty("fold", &OptionsRust::fold);
  87
  88                 DefineProperty("fold.comment", &OptionsRust::foldComment);
  89
  90                 DefineProperty("fold.compact", &OptionsRust::foldCompact);
  91
  92                 DefineProperty("fold.at.else", &OptionsRust::foldAtElse);
  93
  94                 DefineProperty("fold.rust.syntax.based", &OptionsRust::foldSyntaxBased,
  95                         "Set this property to 0 to disable syntax based folding.");
  96
  97                 DefineProperty("fold.rust.comment.multiline", &OptionsRust::foldCommentMultiline,
  98                         "Set this property to 0 to disable folding multi-line comments when fold.comment=1.");
  99
 100                 DefineProperty("fold.rust.comment.explicit", &OptionsRust::foldCommentExplicit,
 101                         "Set this property to 0 to disable folding explicit fold points when fold.comment=1.");
 102
 103                 DefineProperty("fold.rust.explicit.start", &OptionsRust::foldExplicitStart,
 104                         "The string to use for explicit fold start points, replacing the standard //{.");
 105
 106                 DefineProperty("fold.rust.explicit.end", &OptionsRust::foldExplicitEnd,
 107                         "The string to use for explicit fold end points, replacing the standard //}.");
 108
 109                 DefineProperty("fold.rust.explicit.anywhere", &OptionsRust::foldExplicitAnywhere,
 110                         "Set this property to 1 to enable explicit fold points anywhere, not just in line comments.");
 111
 112                 DefineProperty("lexer.rust.fold.at.else", &OptionsRust::foldAtElseInt,
 113                         "This option enables Rust folding on a \"} else {\" line of an if statement.");
 114
 115                 DefineWordListSets(rustWordLists);
 116         }
 117 };
 118
 119 class LexerRust : public ILexer {
 120         WordList keywords[NUM_RUST_KEYWORD_LISTS];
 121         OptionsRust options;
 122         OptionSetRust osRust;
 123 public:
 124         virtual ~LexerRust() {
 125         }
 126         void SCI_METHOD Release() {
 127                 delete this;
 128         }
 129         int SCI_METHOD Version() const {
 130                 return lvOriginal;
 131         }
 132         const char * SCI_METHOD PropertyNames() {
 133                 return osRust.PropertyNames();
 134         }
 135         int SCI_METHOD PropertyType(const char *name) {
 136                 return osRust.PropertyType(name);
 137         }
 138         const char * SCI_METHOD DescribeProperty(const char *name) {
 139                 return osRust.DescribeProperty(name);
 140         }
 141         Sci_Position SCI_METHOD PropertySet(const char *key, const char *val);
 142         const char * SCI_METHOD DescribeWordListSets() {
 143                 return osRust.DescribeWordListSets();
 144         }
 145         Sci_Position SCI_METHOD WordListSet(int n, const char *wl);
 146         void SCI_METHOD Lex(Sci_PositionU startPos, Sci_Position length, int initStyle, IDocument *pAccess);
 147         void SCI_METHOD Fold(Sci_PositionU startPos, Sci_Position length, int initStyle, IDocument *pAccess);
 148         void * SCI_METHOD PrivateCall(int, void *) {
 149                 return 0;
 150         }
 151         static ILexer *LexerFactoryRust() {
 152                 return new LexerRust();
 153         }
 154 };
 155
 156 Sci_Position SCI_METHOD LexerRust::PropertySet(const char *key, const char *val) {
 157         if (osRust.PropertySet(&options, key, val)) {
 158                 return 0;
 159         }
 160         return -1;
 161 }
 162
 163 Sci_Position SCI_METHOD LexerRust::WordListSet(int n, const char *wl) {
 164         Sci_Position firstModification = -1;
 165         if (n < NUM_RUST_KEYWORD_LISTS) {
 166                 WordList *wordListN = &keywords[n];
 167                 WordList wlNew;
 168                 wlNew.Set(wl);
 169                 if (*wordListN != wlNew) {
 170                         wordListN->Set(wl);
 171                         firstModification = 0;
 172                 }
 173         }
 174         return firstModification;
 175 }
 176
 177 static bool IsWhitespace(int c) {
 178     return c == ' ' || c == '\t' || c == '\r' || c == '\n';
 179 }
 180
 181 /* This isn't quite right for Unicode identifiers */
 182 static bool IsIdentifierStart(int ch) {
 183         return (IsASCII(ch) && (isalpha(ch) || ch == '_')) || !IsASCII(ch);
 184 }
 185
 186 /* This isn't quite right for Unicode identifiers */
 187 static bool IsIdentifierContinue(int ch) {
 188         return (IsASCII(ch) && (isalnum(ch) || ch == '_')) || !IsASCII(ch);
 189 }
 190
 191 static void ScanWhitespace(Accessor& styler, Sci_Position& pos, Sci_Position max) {
 192         while (IsWhitespace(styler.SafeGetCharAt(pos, '\0')) && pos < max) {
 193                 if (pos == styler.LineEnd(styler.GetLine(pos)))
 194                         styler.SetLineState(styler.GetLine(pos), 0);
 195                 pos++;
 196         }
 197         styler.ColourTo(pos-1, SCE_RUST_DEFAULT);
 198 }
 199
 200 static void GrabString(char* s, Accessor& styler, Sci_Position start, Sci_Position len) {
 201         for (Sci_Position ii = 0; ii < len; ii++)
 202                 s[ii] = styler[ii + start];
 203         s[len] = '\0';
 204 }
 205
 206 static void ScanIdentifier(Accessor& styler, Sci_Position& pos, WordList *keywords) {
 207         Sci_Position start = pos;
 208         while (IsIdentifierContinue(styler.SafeGetCharAt(pos, '\0')))
 209                 pos++;
 210
 211         if (styler.SafeGetCharAt(pos, '\0') == '!') {
 212                 pos++;
 213                 styler.ColourTo(pos - 1, SCE_RUST_MACRO);
 214         } else {
 215                 char s[MAX_RUST_IDENT_CHARS + 1];
 216                 int len = pos - start;
 217                 len = len > MAX_RUST_IDENT_CHARS ? MAX_RUST_IDENT_CHARS : len;
 218                 GrabString(s, styler, start, len);
 219                 bool keyword = false;
 220                 for (int ii = 0; ii < NUM_RUST_KEYWORD_LISTS; ii++) {
 221                         if (keywords[ii].InList(s)) {
 222                                 styler.ColourTo(pos - 1, SCE_RUST_WORD + ii);
 223                                 keyword = true;
 224                                 break;
 225                         }
 226                 }
 227                 if (!keyword) {
 228                         styler.ColourTo(pos - 1, SCE_RUST_IDENTIFIER);
 229                 }
 230         }
 231 }
 232
 233 /* Scans a sequence of digits, returning true if it found any. */
 234 static bool ScanDigits(Accessor& styler, Sci_Position& pos, int base) {
 235         Sci_Position old_pos = pos;
 236         for (;;) {
 237                 int c = styler.SafeGetCharAt(pos, '\0');
 238                 if (IsADigit(c, base) || c == '_')
 239                         pos++;
 240                 else
 241                         break;
 242         }
 243         return old_pos != pos;
 244 }
 245
 246 /* Scans an integer and floating point literals. */
 247 static void ScanNumber(Accessor& styler, Sci_Position& pos) {
 248         int base = 10;
 249         int c = styler.SafeGetCharAt(pos, '\0');
 250         int n = styler.SafeGetCharAt(pos + 1, '\0');
 251         bool error = false;
 252         /* Scan the prefix, thus determining the base.
 253          * 10 is default if there's no prefix. */
 254         if (c == '0' && n == 'x') {
 255                 pos += 2;
 256                 base = 16;
 257         } else if (c == '0' && n == 'b') {
 258                 pos += 2;
 259                 base = 2;
 260         } else if (c == '0' && n == 'o') {
 261                 pos += 2;
 262                 base = 8;
 263         }
 264
 265         /* Scan initial digits. The literal is malformed if there are none. */
 266         error |= !ScanDigits(styler, pos, base);
 267         /* See if there's an integer suffix. We mimic the Rust's lexer
 268          * and munch it even if there was an error above. */
 269         c = styler.SafeGetCharAt(pos, '\0');
 270         if (c == 'u' || c == 'i') {
 271                 pos++;
 272                 c = styler.SafeGetCharAt(pos, '\0');
 273                 n = styler.SafeGetCharAt(pos + 1, '\0');
 274                 if (c == '8' || c == 's') {
 275                         pos++;
 276                 } else if (c == '1' && n == '6') {
 277                         pos += 2;
 278                 } else if (c == '3' && n == '2') {
 279                         pos += 2;
 280                 } else if (c == '6' && n == '4') {
 281                         pos += 2;
 282                 } else {
 283                         error = true;
 284                 }
 285         /* See if it's a floating point literal. These literals have to be base 10.
 286          */
 287         } else if (!error) {
 288                 /* If there's a period, it's a floating point literal unless it's
 289                  * followed by an identifier (meaning this is a method call, e.g.
 290                  * `1.foo()`) or another period, in which case it's a range (e.g. 1..2)
 291                  */
 292                 n = styler.SafeGetCharAt(pos + 1, '\0');
 293                 if (c == '.' && !(IsIdentifierStart(n) || n == '.')) {
 294                         error |= base != 10;
 295                         pos++;
 296                         /* It's ok to have no digits after the period. */
 297                         ScanDigits(styler, pos, 10);
 298                 }
 299
 300                 /* Look for the exponentiation. */
 301                 c = styler.SafeGetCharAt(pos, '\0');
 302                 if (c == 'e' || c == 'E') {
 303                         error |= base != 10;
 304                         pos++;
 305                         c = styler.SafeGetCharAt(pos, '\0');
 306                         if (c == '-' || c == '+')
 307                                 pos++;
 308                         /* It is invalid to have no digits in the exponent. */
 309                         error |= !ScanDigits(styler, pos, 10);
 310                 }
 311
 312                 /* Scan the floating point suffix. */
 313                 c = styler.SafeGetCharAt(pos, '\0');
 314                 if (c == 'f') {
 315                         error |= base != 10;
 316                         pos++;
 317                         c = styler.SafeGetCharAt(pos, '\0');
 318                         n = styler.SafeGetCharAt(pos + 1, '\0');
 319                         if (c == '3' && n == '2') {
 320                                 pos += 2;
 321                         } else if (c == '6' && n == '4') {
 322                                 pos += 2;
 323                         } else {
 324                                 error = true;
 325                         }
 326                 }
 327         }
 328
 329         if (error)
 330                 styler.ColourTo(pos - 1, SCE_RUST_LEXERROR);
 331         else
 332                 styler.ColourTo(pos - 1, SCE_RUST_NUMBER);
 333 }
 334
 335 static bool IsOneCharOperator(int c) {
 336         return c == ';' || c == ',' || c == '(' || c == ')'
 337             || c == '{' || c == '}' || c == '[' || c == ']'
 338             || c == '@' || c == '#' || c == '~' || c == '+'
 339             || c == '*' || c == '/' || c == '^' || c == '%'
 340             || c == '.' || c == ':' || c == '!' || c == '<'
 341             || c == '>' || c == '=' || c == '-' || c == '&'
 342             || c == '|' || c == '$';
 343 }
 344
 345 static bool IsTwoCharOperator(int c, int n) {
 346         return (c == '.' && n == '.') || (c == ':' && n == ':')
 347             || (c == '!' && n == '=') || (c == '<' && n == '<')
 348             || (c == '<' && n == '=') || (c == '>' && n == '>')
 349             || (c == '>' && n == '=') || (c == '=' && n == '=')
 350             || (c == '=' && n == '>') || (c == '-' && n == '>')
 351             || (c == '&' && n == '&') || (c == '|' && n == '|')
 352             || (c == '-' && n == '=') || (c == '&' && n == '=')
 353             || (c == '|' && n == '=') || (c == '+' && n == '=')
 354             || (c == '*' && n == '=') || (c == '/' && n == '=')
 355             || (c == '^' && n == '=') || (c == '%' && n == '=');
 356 }
 357
 358 static bool IsThreeCharOperator(int c, int n, int n2) {
 359         return (c == '<' && n == '<' && n2 == '=')
 360             || (c == '>' && n == '>' && n2 == '=');
 361 }
 362
 363 static bool IsValidCharacterEscape(int c) {
 364         return c == 'n'  || c == 'r' || c == 't' || c == '\\'
 365             || c == '\'' || c == '"' || c == '0';
 366 }
 367
 368 static bool IsValidStringEscape(int c) {
 369         return IsValidCharacterEscape(c) || c == '\n' || c == '\r';
 370 }
 371
 372 static bool ScanNumericEscape(Accessor &styler, Sci_Position& pos, Sci_Position num_digits, bool stop_asap) {
 373         for (;;) {
 374                 int c = styler.SafeGetCharAt(pos, '\0');
 375                 if (!IsADigit(c, 16))
 376                         break;
 377                 num_digits--;
 378                 pos++;
 379                 if (num_digits == 0 && stop_asap)
 380                         return true;
 381         }
 382         if (num_digits == 0) {
 383                 return true;
 384         } else {
 385                 return false;
 386         }
 387 }
 388
 389 /* This is overly permissive for character literals in order to accept UTF-8 encoded
 390  * character literals. */
 391 static void ScanCharacterLiteralOrLifetime(Accessor &styler, Sci_Position& pos, bool ascii_only) {
 392         pos++;
 393         int c = styler.SafeGetCharAt(pos, '\0');
 394         int n = styler.SafeGetCharAt(pos + 1, '\0');
 395         bool done = false;
 396         bool valid_lifetime = !ascii_only && IsIdentifierStart(c);
 397         bool valid_char = true;
 398         bool first = true;
 399         while (!done) {
 400                 switch (c) {
 401                         case '\\':
 402                                 done = true;
 403                                 if (IsValidCharacterEscape(n)) {
 404                                         pos += 2;
 405                                 } else if (n == 'x') {
 406                                         pos += 2;
 407                                         valid_char = ScanNumericEscape(styler, pos, 2, false);
 408                                 } else if (n == 'u' && !ascii_only) {
 409                                         pos += 2;
 410                                         valid_char = ScanNumericEscape(styler, pos, 4, false);
 411                                 } else if (n == 'U' && !ascii_only) {
 412                                         pos += 2;
 413                                         valid_char = ScanNumericEscape(styler, pos, 8, false);
 414                                 } else {
 415                                         valid_char = false;
 416                                 }
 417                                 break;
 418                         case '\'':
 419                                 valid_char = !first;
 420                                 done = true;
 421                                 break;
 422                         case '\t':
 423                         case '\n':
 424                         case '\r':
 425                         case '\0':
 426                                 valid_char = false;
 427                                 done = true;
 428                                 break;
 429                         default:
 430                                 if (ascii_only && !IsASCII((char)c)) {
 431                                         done = true;
 432                                         valid_char = false;
 433                                 } else if (!IsIdentifierContinue(c) && !first) {
 434                                         done = true;
 435                                 } else {
 436                                         pos++;
 437                                 }
 438                                 break;
 439                 }
 440                 c = styler.SafeGetCharAt(pos, '\0');
 441                 n = styler.SafeGetCharAt(pos + 1, '\0');
 442
 443                 first = false;
 444         }
 445         if (styler.SafeGetCharAt(pos, '\0') == '\'') {
 446                 valid_lifetime = false;
 447         } else {
 448                 valid_char = false;
 449         }
 450         if (valid_lifetime) {
 451                 styler.ColourTo(pos - 1, SCE_RUST_LIFETIME);
 452         } else if (valid_char) {
 453                 pos++;
 454                 styler.ColourTo(pos - 1, ascii_only ? SCE_RUST_BYTECHARACTER : SCE_RUST_CHARACTER);
 455         } else {
 456                 styler.ColourTo(pos - 1, SCE_RUST_LEXERROR);
 457         }
 458 }
 459
 460 enum CommentState {
 461         UnknownComment,
 462         DocComment,
 463         NotDocComment
 464 };
 465
 466 /*
 467  * The rule for block-doc comments is as follows: /xxN and /x! (where x is an asterisk, N is a non-asterisk) start doc comments.
 468  * Otherwise it's a regular comment.
 469  */
 470 static void ResumeBlockComment(Accessor &styler, Sci_Position& pos, Sci_Position max, CommentState state, int level) {
 471         int c = styler.SafeGetCharAt(pos, '\0');
 472         bool maybe_doc_comment = false;
 473         if (c == '*') {
 474                 int n = styler.SafeGetCharAt(pos + 1, '\0');
 475                 if (n != '*' && n != '/') {
 476                         maybe_doc_comment = true;
 477                 }
 478         } else if (c == '!') {
 479                 maybe_doc_comment = true;
 480         }
 481
 482         for (;;) {
 483                 int n = styler.SafeGetCharAt(pos + 1, '\0');
 484                 if (pos == styler.LineEnd(styler.GetLine(pos)))
 485                         styler.SetLineState(styler.GetLine(pos), level);
 486                 if (c == '*') {
 487                         pos++;
 488                         if (n == '/') {
 489                                 pos++;
 490                                 level--;
 491                                 if (level == 0) {
 492                                         styler.SetLineState(styler.GetLine(pos), 0);
 493                                         if (state == DocComment || (state == UnknownComment && maybe_doc_comment))
 494                                                 styler.ColourTo(pos - 1, SCE_RUST_COMMENTBLOCKDOC);
 495                                         else
 496                                                 styler.ColourTo(pos - 1, SCE_RUST_COMMENTBLOCK);
 497                                         break;
 498                                 }
 499                         }
 500                 } else if (c == '/') {
 501                         pos++;
 502                         if (n == '*') {
 503                                 pos++;
 504                                 level++;
 505                         }
 506                 }
 507                 else {
 508                         pos++;
 509                 }
 510                 if (pos >= max) {
 511                         if (state == DocComment || (state == UnknownComment && maybe_doc_comment))
 512                                 styler.ColourTo(pos - 1, SCE_RUST_COMMENTBLOCKDOC);
 513                         else
 514                                 styler.ColourTo(pos - 1, SCE_RUST_COMMENTBLOCK);
 515                         break;
 516                 }
 517                 c = styler.SafeGetCharAt(pos, '\0');
 518         }
 519 }
 520
 521 /*
 522  * The rule for line-doc comments is as follows... ///N and //! (where N is a non slash) start doc comments.
 523  * Otherwise it's a normal line comment.
 524  */
 525 static void ResumeLineComment(Accessor &styler, Sci_Position& pos, Sci_Position max, CommentState state) {
 526         bool maybe_doc_comment = false;
 527         int c = styler.SafeGetCharAt(pos, '\0');
 528         if (c == '/') {
 529                 if (pos < max) {
 530                         pos++;
 531                         c = styler.SafeGetCharAt(pos, '\0');
 532                         if (c != '/') {
 533                                 maybe_doc_comment = true;
 534                         }
 535                 }
 536         } else if (c == '!') {
 537                 maybe_doc_comment = true;
 538         }
 539
 540         while (pos < max && c != '\n') {
 541                 if (pos == styler.LineEnd(styler.GetLine(pos)))
 542                         styler.SetLineState(styler.GetLine(pos), 0);
 543                 pos++;
 544                 c = styler.SafeGetCharAt(pos, '\0');
 545         }
 546
 547         if (state == DocComment || (state == UnknownComment && maybe_doc_comment))
 548                 styler.ColourTo(pos - 1, SCE_RUST_COMMENTLINEDOC);
 549         else
 550                 styler.ColourTo(pos - 1, SCE_RUST_COMMENTLINE);
 551 }
 552
 553 static void ScanComments(Accessor &styler, Sci_Position& pos, Sci_Position max) {
 554         pos++;
 555         int c = styler.SafeGetCharAt(pos, '\0');
 556         pos++;
 557         if (c == '/')
 558                 ResumeLineComment(styler, pos, max, UnknownComment);
 559         else if (c == '*')
 560                 ResumeBlockComment(styler, pos, max, UnknownComment, 1);
 561 }
 562
 563 static void ResumeString(Accessor &styler, Sci_Position& pos, Sci_Position max, bool ascii_only) {
 564         int c = styler.SafeGetCharAt(pos, '\0');
 565         bool error = false;
 566         while (c != '"' && !error) {
 567                 if (pos >= max) {
 568                         error = true;
 569                         break;
 570                 }
 571                 if (pos == styler.LineEnd(styler.GetLine(pos)))
 572                         styler.SetLineState(styler.GetLine(pos), 0);
 573                 if (c == '\\') {
 574                         int n = styler.SafeGetCharAt(pos + 1, '\0');
 575                         if (IsValidStringEscape(n)) {
 576                                 pos += 2;
 577                         } else if (n == 'x') {
 578                                 pos += 2;
 579                                 error = !ScanNumericEscape(styler, pos, 2, true);
 580                         } else if (n == 'u' && !ascii_only) {
 581                                 pos += 2;
 582                                 error = !ScanNumericEscape(styler, pos, 4, true);
 583                         } else if (n == 'U' && !ascii_only) {
 584                                 pos += 2;
 585                                 error = !ScanNumericEscape(styler, pos, 8, true);
 586                         } else {
 587                                 pos += 1;
 588                                 error = true;
 589                         }
 590                 } else {
 591                         if (ascii_only && !IsASCII((char)c))
 592                                 error = true;
 593                         else
 594                                 pos++;
 595                 }
 596                 c = styler.SafeGetCharAt(pos, '\0');
 597         }
 598         if (!error)
 599                 pos++;
 600         styler.ColourTo(pos - 1, ascii_only ? SCE_RUST_BYTESTRING : SCE_RUST_STRING);
 601 }
 602
 603 static void ResumeRawString(Accessor &styler, Sci_Position& pos, Sci_Position max, int num_hashes, bool ascii_only) {
 604         for (;;) {
 605                 if (pos == styler.LineEnd(styler.GetLine(pos)))
 606                         styler.SetLineState(styler.GetLine(pos), num_hashes);
 607
 608                 int c = styler.SafeGetCharAt(pos, '\0');
 609                 if (c == '"') {
 610                         pos++;
 611                         int trailing_num_hashes = 0;
 612                         while (styler.SafeGetCharAt(pos, '\0') == '#' && trailing_num_hashes < num_hashes) {
 613                                 trailing_num_hashes++;
 614                                 pos++;
 615                         }
 616                         if (trailing_num_hashes == num_hashes) {
 617                                 styler.SetLineState(styler.GetLine(pos), 0);
 618                                 break;
 619                         }
 620                 } else if (pos >= max) {
 621                         break;
 622                 } else {
 623                         if (ascii_only && !IsASCII((char)c))
 624                                 break;
 625                         pos++;
 626                 }
 627         }
 628         styler.ColourTo(pos - 1, ascii_only ? SCE_RUST_BYTESTRINGR : SCE_RUST_STRINGR);
 629 }
 630
 631 static void ScanRawString(Accessor &styler, Sci_Position& pos, Sci_Position max, bool ascii_only) {
 632         pos++;
 633         int num_hashes = 0;
 634         while (styler.SafeGetCharAt(pos, '\0') == '#') {
 635                 num_hashes++;
 636                 pos++;
 637         }
 638         if (styler.SafeGetCharAt(pos, '\0') != '"') {
 639                 styler.ColourTo(pos - 1, SCE_RUST_LEXERROR);
 640         } else {
 641                 pos++;
 642                 ResumeRawString(styler, pos, max, num_hashes, ascii_only);
 643         }
 644 }
 645
 646 void SCI_METHOD LexerRust::Lex(Sci_PositionU startPos, Sci_Position length, int initStyle, IDocument *pAccess) {
 647         PropSetSimple props;
 648         Accessor styler(pAccess, &props);
 649         Sci_Position pos = startPos;
 650         Sci_Position max = pos + length;
 651
 652         styler.StartAt(pos);
 653         styler.StartSegment(pos);
 654
 655         if (initStyle == SCE_RUST_COMMENTBLOCK || initStyle == SCE_RUST_COMMENTBLOCKDOC) {
 656                 ResumeBlockComment(styler, pos, max, initStyle == SCE_RUST_COMMENTBLOCKDOC ? DocComment : NotDocComment, styler.GetLineState(styler.GetLine(pos) - 1));
 657         } else if (initStyle == SCE_RUST_COMMENTLINE || initStyle == SCE_RUST_COMMENTLINEDOC) {
 658                 ResumeLineComment(styler, pos, max, initStyle == SCE_RUST_COMMENTLINEDOC ? DocComment : NotDocComment);
 659         } else if (initStyle == SCE_RUST_STRING) {
 660                 ResumeString(styler, pos, max, false);
 661         } else if (initStyle == SCE_RUST_BYTESTRING) {
 662                 ResumeString(styler, pos, max, true);
 663         } else if (initStyle == SCE_RUST_STRINGR) {
 664                 ResumeRawString(styler, pos, max, styler.GetLineState(styler.GetLine(pos) - 1), false);
 665         } else if (initStyle == SCE_RUST_BYTESTRINGR) {
 666                 ResumeRawString(styler, pos, max, styler.GetLineState(styler.GetLine(pos) - 1), true);
 667         }
 668
 669         while (pos < max) {
 670                 int c = styler.SafeGetCharAt(pos, '\0');
 671                 int n = styler.SafeGetCharAt(pos + 1, '\0');
 672                 int n2 = styler.SafeGetCharAt(pos + 2, '\0');
 673
 674                 if (pos == 0 && c == '#' && n == '!' && n2 != '[') {
 675                         pos += 2;
 676                         ResumeLineComment(styler, pos, max, NotDocComment);
 677                 } else if (IsWhitespace(c)) {
 678                         ScanWhitespace(styler, pos, max);
 679                 } else if (c == '/' && (n == '/' || n == '*')) {
 680                         ScanComments(styler, pos, max);
 681                 } else if (c == 'r' && (n == '#' || n == '"')) {
 682                         ScanRawString(styler, pos, max, false);
 683                 } else if (c == 'b' && n == 'r' && (n2 == '#' || n2 == '"')) {
 684                         pos++;
 685                         ScanRawString(styler, pos, max, true);
 686                 } else if (c == 'b' && n == '"') {
 687                         pos += 2;
 688                         ResumeString(styler, pos, max, true);
 689                 } else if (c == 'b' && n == '\'') {
 690                         pos++;
 691                         ScanCharacterLiteralOrLifetime(styler, pos, true);
 692                 } else if (IsIdentifierStart(c)) {
 693                         ScanIdentifier(styler, pos, keywords);
 694                 } else if (IsADigit(c)) {
 695                         ScanNumber(styler, pos);
 696                 } else if (IsThreeCharOperator(c, n, n2)) {
 697                         pos += 3;
 698                         styler.ColourTo(pos - 1, SCE_RUST_OPERATOR);
 699                 } else if (IsTwoCharOperator(c, n)) {
 700                         pos += 2;
 701                         styler.ColourTo(pos - 1, SCE_RUST_OPERATOR);
 702                 } else if (IsOneCharOperator(c)) {
 703                         pos++;
 704                         styler.ColourTo(pos - 1, SCE_RUST_OPERATOR);
 705                 } else if (c == '\'') {
 706                         ScanCharacterLiteralOrLifetime(styler, pos, false);
 707                 } else if (c == '"') {
 708                         pos++;
 709                         ResumeString(styler, pos, max, false);
 710                 } else {
 711                         pos++;
 712                         styler.ColourTo(pos - 1, SCE_RUST_LEXERROR);
 713                 }
 714         }
 715         styler.ColourTo(pos - 1, SCE_RUST_DEFAULT);
 716         styler.Flush();
 717 }
 718
 719 void SCI_METHOD LexerRust::Fold(Sci_PositionU startPos, Sci_Position length, int initStyle, IDocument *pAccess) {
 720
 721         if (!options.fold)
 722                 return;
 723
 724         LexAccessor styler(pAccess);
 725
 726         Sci_PositionU endPos = startPos + length;
 727         int visibleChars = 0;
 728         bool inLineComment = false;
 729         Sci_Position lineCurrent = styler.GetLine(startPos);
 730         int levelCurrent = SC_FOLDLEVELBASE;
 731         if (lineCurrent > 0)
 732                 levelCurrent = styler.LevelAt(lineCurrent-1) >> 16;
 733         Sci_PositionU lineStartNext = styler.LineStart(lineCurrent+1);
 734         int levelMinCurrent = levelCurrent;
 735         int levelNext = levelCurrent;
 736         char chNext = styler[startPos];
 737         int styleNext = styler.StyleAt(startPos);
 738         int style = initStyle;
 739         const bool userDefinedFoldMarkers = !options.foldExplicitStart.empty() && !options.foldExplicitEnd.empty();
 740         for (Sci_PositionU i = startPos; i < endPos; i++) {
 741                 char ch = chNext;
 742                 chNext = styler.SafeGetCharAt(i + 1);
 743                 int stylePrev = style;
 744                 style = styleNext;
 745                 styleNext = styler.StyleAt(i + 1);
 746                 bool atEOL = i == (lineStartNext-1);
 747                 if ((style == SCE_RUST_COMMENTLINE) || (style == SCE_RUST_COMMENTLINEDOC))
 748                         inLineComment = true;
 749                 if (options.foldComment && options.foldCommentMultiline && IsStreamCommentStyle(style) && !inLineComment) {
 750                         if (!IsStreamCommentStyle(stylePrev)) {
 751                                 levelNext++;
 752                         } else if (!IsStreamCommentStyle(styleNext) && !atEOL) {
 753                                 // Comments don't end at end of line and the next character may be unstyled.
 754                                 levelNext--;
 755                         }
 756                 }
 757                 if (options.foldComment && options.foldCommentExplicit && ((style == SCE_RUST_COMMENTLINE) || options.foldExplicitAnywhere)) {
 758                         if (userDefinedFoldMarkers) {
 759                                 if (styler.Match(i, options.foldExplicitStart.c_str())) {
 760                                         levelNext++;
 761                                 } else if (styler.Match(i, options.foldExplicitEnd.c_str())) {
 762                                         levelNext--;
 763                                 }
 764                         } else {
 765                                 if ((ch == '/') && (chNext == '/')) {
 766                                         char chNext2 = styler.SafeGetCharAt(i + 2);
 767                                         if (chNext2 == '{') {
 768                                                 levelNext++;
 769                                         } else if (chNext2 == '}') {
 770                                                 levelNext--;
 771                                         }
 772                                 }
 773                         }
 774                 }
 775                 if (options.foldSyntaxBased && (style == SCE_RUST_OPERATOR)) {
 776                         if (ch == '{') {
 777                                 // Measure the minimum before a '{' to allow
 778                                 // folding on "} else {"
 779                                 if (levelMinCurrent > levelNext) {
 780                                         levelMinCurrent = levelNext;
 781                                 }
 782                                 levelNext++;
 783                         } else if (ch == '}') {
 784                                 levelNext--;
 785                         }
 786                 }
 787                 if (!IsASpace(ch))
 788                         visibleChars++;
 789                 if (atEOL || (i == endPos-1)) {
 790                         int levelUse = levelCurrent;
 791                         if (options.foldSyntaxBased && options.foldAtElse) {
 792                                 levelUse = levelMinCurrent;
 793                         }
 794                         int lev = levelUse | levelNext << 16;
 795                         if (visibleChars == 0 && options.foldCompact)
 796                                 lev |= SC_FOLDLEVELWHITEFLAG;
 797                         if (levelUse < levelNext)
 798                                 lev |= SC_FOLDLEVELHEADERFLAG;
 799                         if (lev != styler.LevelAt(lineCurrent)) {
 800                                 styler.SetLevel(lineCurrent, lev);
 801                         }
 802                         lineCurrent++;
 803                         lineStartNext = styler.LineStart(lineCurrent+1);
 804                         levelCurrent = levelNext;
 805                         levelMinCurrent = levelCurrent;
 806                         if (atEOL && (i == static_cast<Sci_PositionU>(styler.Length()-1))) {
 807                                 // There is an empty line at end of file so give it same level and empty
 808                                 styler.SetLevel(lineCurrent, (levelCurrent | levelCurrent << 16) | SC_FOLDLEVELWHITEFLAG);
 809                         }
 810                         visibleChars = 0;
 811                         inLineComment = false;
 812                 }
 813         }
 814 }
 815
 816 LexerModule lmRust(SCLEX_RUST, LexerRust::LexerFactoryRust, "rust", rustWordLists);