scintilla/lexers/LexPython.cxx

   1 // Scintilla source code edit control
   2 /** @file LexPython.cxx
   3  ** Lexer for Python.
   4  **/
   5 // Copyright 1998-2002 by Neil Hodgson <neilh@scintilla.org>
   6 // The License.txt file describes the conditions under which this software may be distributed.
   7
   8 #include <stdlib.h>
   9 #include <string.h>
  10 #include <stdio.h>
  11 #include <stdarg.h>
  12 #include <assert.h>
  13 #include <ctype.h>
  14
  15 #include <string>
  16 #include <vector>
  17 #include <map>
  18
  19 #include "ILexer.h"
  20 #include "Scintilla.h"
  21 #include "SciLexer.h"
  22
  23 #include "WordList.h"
  24 #include "LexAccessor.h"
  25 #include "Accessor.h"
  26 #include "StyleContext.h"
  27 #include "CharacterSet.h"
  28 #include "LexerModule.h"
  29 #include "OptionSet.h"
  30 #include "SubStyles.h"
  31
  32 #ifdef SCI_NAMESPACE
  33 using namespace Scintilla;
  34 #endif
  35
  36 namespace {
  37         // Use an unnamed namespace to protect the functions and classes from name conflicts
  38
  39 /* kwCDef, kwCTypeName only used for Cython */
  40 enum kwType { kwOther, kwClass, kwDef, kwImport, kwCDef, kwCTypeName, kwCPDef };
  41
  42 enum literalsAllowed { litNone = 0, litU = 1, litB = 2 };
  43
  44 const int indicatorWhitespace = 1;
  45
  46 bool IsPyComment(Accessor &styler, Sci_Position pos, Sci_Position len) {
  47         return len > 0 && styler[pos] == '#';
  48 }
  49
  50 bool IsPyStringTypeChar(int ch, literalsAllowed allowed) {
  51         return
  52                 ((allowed & litB) && (ch == 'b' || ch == 'B')) ||
  53                 ((allowed & litU) && (ch == 'u' || ch == 'U'));
  54 }
  55
  56 bool IsPyStringStart(int ch, int chNext, int chNext2, literalsAllowed allowed) {
  57         if (ch == '\'' || ch == '"')
  58                 return true;
  59         if (IsPyStringTypeChar(ch, allowed)) {
  60                 if (chNext == '"' || chNext == '\'')
  61                         return true;
  62                 if ((chNext == 'r' || chNext == 'R') && (chNext2 == '"' || chNext2 == '\''))
  63                         return true;
  64         }
  65         if ((ch == 'r' || ch == 'R') && (chNext == '"' || chNext == '\''))
  66                 return true;
  67
  68         return false;
  69 }
  70
  71 /* Return the state to use for the string starting at i; *nextIndex will be set to the first index following the quote(s) */
  72 int GetPyStringState(Accessor &styler, Sci_Position i, Sci_PositionU *nextIndex, literalsAllowed allowed) {
  73         char ch = styler.SafeGetCharAt(i);
  74         char chNext = styler.SafeGetCharAt(i + 1);
  75
  76         // Advance beyond r, u, or ur prefix (or r, b, or br in Python 3.0), but bail if there are any unexpected chars
  77         if (ch == 'r' || ch == 'R') {
  78                 i++;
  79                 ch = styler.SafeGetCharAt(i);
  80                 chNext = styler.SafeGetCharAt(i + 1);
  81         } else if (IsPyStringTypeChar(ch, allowed)) {
  82                 if (chNext == 'r' || chNext == 'R')
  83                         i += 2;
  84                 else
  85                         i += 1;
  86                 ch = styler.SafeGetCharAt(i);
  87                 chNext = styler.SafeGetCharAt(i + 1);
  88         }
  89
  90         if (ch != '"' && ch != '\'') {
  91                 *nextIndex = i + 1;
  92                 return SCE_P_DEFAULT;
  93         }
  94
  95         if (ch == chNext && ch == styler.SafeGetCharAt(i + 2)) {
  96                 *nextIndex = i + 3;
  97
  98                 if (ch == '"')
  99                         return SCE_P_TRIPLEDOUBLE;
 100                 else
 101                         return SCE_P_TRIPLE;
 102         } else {
 103                 *nextIndex = i + 1;
 104
 105                 if (ch == '"')
 106                         return SCE_P_STRING;
 107                 else
 108                         return SCE_P_CHARACTER;
 109         }
 110 }
 111
 112 inline bool IsAWordChar(int ch) {
 113         return (ch < 0x80) && (isalnum(ch) || ch == '.' || ch == '_');
 114 }
 115
 116 inline bool IsAWordStart(int ch) {
 117         return (ch < 0x80) && (isalnum(ch) || ch == '_');
 118 }
 119
 120 static bool IsFirstNonWhitespace(Sci_Position pos, Accessor &styler) {
 121         Sci_Position line = styler.GetLine(pos);
 122         Sci_Position start_pos = styler.LineStart(line);
 123         for (Sci_Position i = start_pos; i < pos; i++) {
 124                 char ch = styler[i];
 125                 if (!(ch == ' ' || ch == '\t'))
 126                         return false;
 127         }
 128         return true;
 129 }
 130
 131 // Options used for LexerPython
 132 struct OptionsPython {
 133         int whingeLevel;
 134         bool base2or8Literals;
 135         bool stringsU;
 136         bool stringsB;
 137         bool stringsOverNewline;
 138         bool keywords2NoSubIdentifiers;
 139         bool fold;
 140         bool foldQuotes;
 141         bool foldCompact;
 142
 143         OptionsPython() {
 144                 whingeLevel = 0;
 145                 base2or8Literals = true;
 146                 stringsU = true;
 147                 stringsB = true;
 148                 stringsOverNewline = false;
 149                 keywords2NoSubIdentifiers = false;
 150                 fold = false;
 151                 foldQuotes = false;
 152                 foldCompact = false;
 153         }
 154
 155         literalsAllowed AllowedLiterals() const {
 156                 literalsAllowed allowedLiterals = stringsU ? litU : litNone;
 157                 if (stringsB)
 158                         allowedLiterals = static_cast<literalsAllowed>(allowedLiterals | litB);
 159                 return allowedLiterals;
 160         }
 161 };
 162
 163 static const char *const pythonWordListDesc[] = {
 164         "Keywords",
 165         "Highlighted identifiers",
 166         0
 167 };
 168
 169 struct OptionSetPython : public OptionSet<OptionsPython> {
 170         OptionSetPython() {
 171                 DefineProperty("tab.timmy.whinge.level", &OptionsPython::whingeLevel,
 172                         "For Python code, checks whether indenting is consistent. "
 173                         "The default, 0 turns off indentation checking, "
 174                         "1 checks whether each line is potentially inconsistent with the previous line, "
 175                         "2 checks whether any space characters occur before a tab character in the indentation, "
 176                         "3 checks whether any spaces are in the indentation, and "
 177                         "4 checks for any tab characters in the indentation. "
 178                         "1 is a good level to use.");
 179
 180                 DefineProperty("lexer.python.literals.binary", &OptionsPython::base2or8Literals,
 181                         "Set to 0 to not recognise Python 3 binary and octal literals: 0b1011 0o712.");
 182
 183                 DefineProperty("lexer.python.strings.u", &OptionsPython::stringsU,
 184                         "Set to 0 to not recognise Python Unicode literals u\"x\" as used before Python 3.");
 185
 186                 DefineProperty("lexer.python.strings.b", &OptionsPython::stringsB,
 187                         "Set to 0 to not recognise Python 3 bytes literals b\"x\".");
 188
 189                 DefineProperty("lexer.python.strings.over.newline", &OptionsPython::stringsOverNewline,
 190                         "Set to 1 to allow strings to span newline characters.");
 191
 192                 DefineProperty("lexer.python.keywords2.no.sub.identifiers", &OptionsPython::keywords2NoSubIdentifiers,
 193                         "When enabled, it will not style keywords2 items that are used as a sub-identifier. "
 194                         "Example: when set, will not highlight \"foo.open\" when \"open\" is a keywords2 item.");
 195
 196                 DefineProperty("fold", &OptionsPython::fold);
 197
 198                 DefineProperty("fold.quotes.python", &OptionsPython::foldQuotes,
 199                         "This option enables folding multi-line quoted strings when using the Python lexer.");
 200
 201                 DefineProperty("fold.compact", &OptionsPython::foldCompact);
 202
 203                 DefineWordListSets(pythonWordListDesc);
 204         }
 205 };
 206
 207 const char styleSubable[] = { SCE_P_IDENTIFIER, 0 };
 208
 209 }
 210
 211 class LexerPython : public ILexerWithSubStyles {
 212         WordList keywords;
 213         WordList keywords2;
 214         OptionsPython options;
 215         OptionSetPython osPython;
 216         enum { ssIdentifier };
 217         SubStyles subStyles;
 218 public:
 219         explicit LexerPython() :
 220                 subStyles(styleSubable, 0x80, 0x40, 0) {
 221         }
 222         virtual ~LexerPython() {
 223         }
 224         void SCI_METHOD Release() {
 225                 delete this;
 226         }
 227         int SCI_METHOD Version() const {
 228                 return lvSubStyles;
 229         }
 230         const char * SCI_METHOD PropertyNames() {
 231                 return osPython.PropertyNames();
 232         }
 233         int SCI_METHOD PropertyType(const char *name) {
 234                 return osPython.PropertyType(name);
 235         }
 236         const char * SCI_METHOD DescribeProperty(const char *name) {
 237                 return osPython.DescribeProperty(name);
 238         }
 239         Sci_Position SCI_METHOD PropertySet(const char *key, const char *val);
 240         const char * SCI_METHOD DescribeWordListSets() {
 241                 return osPython.DescribeWordListSets();
 242         }
 243         Sci_Position SCI_METHOD WordListSet(int n, const char *wl);
 244         void SCI_METHOD Lex(Sci_PositionU startPos, Sci_Position length, int initStyle, IDocument *pAccess);
 245         void SCI_METHOD Fold(Sci_PositionU startPos, Sci_Position length, int initStyle, IDocument *pAccess);
 246
 247         void * SCI_METHOD PrivateCall(int, void *) {
 248                 return 0;
 249         }
 250
 251         int SCI_METHOD LineEndTypesSupported() {
 252                 return SC_LINE_END_TYPE_UNICODE;
 253         }
 254
 255         int SCI_METHOD AllocateSubStyles(int styleBase, int numberStyles) {
 256                 return subStyles.Allocate(styleBase, numberStyles);
 257         }
 258         int SCI_METHOD SubStylesStart(int styleBase) {
 259                 return subStyles.Start(styleBase);
 260         }
 261         int SCI_METHOD SubStylesLength(int styleBase) {
 262                 return subStyles.Length(styleBase);
 263         }
 264         int SCI_METHOD StyleFromSubStyle(int subStyle) {
 265                 int styleBase = subStyles.BaseStyle(subStyle);
 266                 return styleBase;
 267         }
 268         int SCI_METHOD PrimaryStyleFromStyle(int style) {
 269                 return style;
 270         }
 271         void SCI_METHOD FreeSubStyles() {
 272                 subStyles.Free();
 273         }
 274         void SCI_METHOD SetIdentifiers(int style, const char *identifiers) {
 275                 subStyles.SetIdentifiers(style, identifiers);
 276         }
 277         int SCI_METHOD DistanceToSecondaryStyles() {
 278                 return 0;
 279         }
 280         const char * SCI_METHOD GetSubStyleBases() {
 281                 return styleSubable;
 282         }
 283
 284         static ILexer *LexerFactoryPython() {
 285                 return new LexerPython();
 286         }
 287 };
 288
 289 Sci_Position SCI_METHOD LexerPython::PropertySet(const char *key, const char *val) {
 290         if (osPython.PropertySet(&options, key, val)) {
 291                 return 0;
 292         }
 293         return -1;
 294 }
 295
 296 Sci_Position SCI_METHOD LexerPython::WordListSet(int n, const char *wl) {
 297         WordList *wordListN = 0;
 298         switch (n) {
 299         case 0:
 300                 wordListN = &keywords;
 301                 break;
 302         case 1:
 303                 wordListN = &keywords2;
 304                 break;
 305         }
 306         Sci_Position firstModification = -1;
 307         if (wordListN) {
 308                 WordList wlNew;
 309                 wlNew.Set(wl);
 310                 if (*wordListN != wlNew) {
 311                         wordListN->Set(wl);
 312                         firstModification = 0;
 313                 }
 314         }
 315         return firstModification;
 316 }
 317
 318 void SCI_METHOD LexerPython::Lex(Sci_PositionU startPos, Sci_Position length, int initStyle, IDocument *pAccess) {
 319         Accessor styler(pAccess, NULL);
 320
 321         const Sci_Position endPos = startPos + length;
 322
 323         // Backtrack to previous line in case need to fix its tab whinging
 324         Sci_Position lineCurrent = styler.GetLine(startPos);
 325         if (startPos > 0) {
 326                 if (lineCurrent > 0) {
 327                         lineCurrent--;
 328                         // Look for backslash-continued lines
 329                         while (lineCurrent > 0) {
 330                                 Sci_Position eolPos = styler.LineStart(lineCurrent) - 1;
 331                                 int eolStyle = styler.StyleAt(eolPos);
 332                                 if (eolStyle == SCE_P_STRING
 333                                     || eolStyle == SCE_P_CHARACTER
 334                                     || eolStyle == SCE_P_STRINGEOL) {
 335                                         lineCurrent -= 1;
 336                                 } else {
 337                                         break;
 338                                 }
 339                         }
 340                         startPos = styler.LineStart(lineCurrent);
 341                 }
 342                 initStyle = startPos == 0 ? SCE_P_DEFAULT : styler.StyleAt(startPos - 1);
 343         }
 344
 345         const literalsAllowed allowedLiterals = options.AllowedLiterals();
 346
 347         initStyle = initStyle & 31;
 348         if (initStyle == SCE_P_STRINGEOL) {
 349                 initStyle = SCE_P_DEFAULT;
 350         }
 351
 352         kwType kwLast = kwOther;
 353         int spaceFlags = 0;
 354         styler.IndentAmount(lineCurrent, &spaceFlags, IsPyComment);
 355         bool base_n_number = false;
 356
 357         const WordClassifier &classifierIdentifiers = subStyles.Classifier(SCE_P_IDENTIFIER);
 358
 359         StyleContext sc(startPos, endPos - startPos, initStyle, styler);
 360
 361         bool indentGood = true;
 362         Sci_Position startIndicator = sc.currentPos;
 363         bool inContinuedString = false;
 364
 365         for (; sc.More(); sc.Forward()) {
 366
 367                 if (sc.atLineStart) {
 368                         styler.IndentAmount(lineCurrent, &spaceFlags, IsPyComment);
 369                         indentGood = true;
 370                         if (options.whingeLevel == 1) {
 371                                 indentGood = (spaceFlags & wsInconsistent) == 0;
 372                         } else if (options.whingeLevel == 2) {
 373                                 indentGood = (spaceFlags & wsSpaceTab) == 0;
 374                         } else if (options.whingeLevel == 3) {
 375                                 indentGood = (spaceFlags & wsSpace) == 0;
 376                         } else if (options.whingeLevel == 4) {
 377                                 indentGood = (spaceFlags & wsTab) == 0;
 378                         }
 379                         if (!indentGood) {
 380                                 styler.IndicatorFill(startIndicator, sc.currentPos, indicatorWhitespace, 0);
 381                                 startIndicator = sc.currentPos;
 382                         }
 383                 }
 384
 385                 if (sc.atLineEnd) {
 386                         if ((sc.state == SCE_P_DEFAULT) ||
 387                                 (sc.state == SCE_P_TRIPLE) ||
 388                                 (sc.state == SCE_P_TRIPLEDOUBLE)) {
 389                                 // Perform colourisation of white space and triple quoted strings at end of each line to allow
 390                                 // tab marking to work inside white space and triple quoted strings
 391                                 sc.SetState(sc.state);
 392                         }
 393                         lineCurrent++;
 394                         if ((sc.state == SCE_P_STRING) || (sc.state == SCE_P_CHARACTER)) {
 395                                 if (inContinuedString || options.stringsOverNewline) {
 396                                         inContinuedString = false;
 397                                 } else {
 398                                         sc.ChangeState(SCE_P_STRINGEOL);
 399                                         sc.ForwardSetState(SCE_P_DEFAULT);
 400                                 }
 401                         }
 402                         if (!sc.More())
 403                                 break;
 404                 }
 405
 406                 bool needEOLCheck = false;
 407
 408                 // Check for a state end
 409                 if (sc.state == SCE_P_OPERATOR) {
 410                         kwLast = kwOther;
 411                         sc.SetState(SCE_P_DEFAULT);
 412                 } else if (sc.state == SCE_P_NUMBER) {
 413                         if (!IsAWordChar(sc.ch) &&
 414                                 !(!base_n_number && ((sc.ch == '+' || sc.ch == '-') && (sc.chPrev == 'e' || sc.chPrev == 'E')))) {
 415                                 sc.SetState(SCE_P_DEFAULT);
 416                         }
 417                 } else if (sc.state == SCE_P_IDENTIFIER) {
 418                         if ((sc.ch == '.') || (!IsAWordChar(sc.ch))) {
 419                                 char s[100];
 420                                 sc.GetCurrent(s, sizeof(s));
 421                                 int style = SCE_P_IDENTIFIER;
 422                                 if ((kwLast == kwImport) && (strcmp(s, "as") == 0)) {
 423                                         style = SCE_P_WORD;
 424                                 } else if (keywords.InList(s)) {
 425                                         style = SCE_P_WORD;
 426                                 } else if (kwLast == kwClass) {
 427                                         style = SCE_P_CLASSNAME;
 428                                 } else if (kwLast == kwDef) {
 429                                         style = SCE_P_DEFNAME;
 430                                 } else if (kwLast == kwCDef || kwLast == kwCPDef) {
 431                                         Sci_Position pos = sc.currentPos;
 432                                         unsigned char ch = styler.SafeGetCharAt(pos, '\0');
 433                                         while (ch != '\0') {
 434                                                 if (ch == '(') {
 435                                                         style = SCE_P_DEFNAME;
 436                                                         break;
 437                                                 } else if (ch == ':') {
 438                                                         style = SCE_P_CLASSNAME;
 439                                                         break;
 440                                                 } else if (ch == ' ' || ch == '\t' || ch == '\n' || ch == '\r') {
 441                                                         pos++;
 442                                                         ch = styler.SafeGetCharAt(pos, '\0');
 443                                                 } else {
 444                                                         break;
 445                                                 }
 446                                         }
 447                                 } else if (keywords2.InList(s)) {
 448                                         if (options.keywords2NoSubIdentifiers) {
 449                                                 // We don't want to highlight keywords2
 450                                                 // that are used as a sub-identifier,
 451                                                 // i.e. not open in "foo.open".
 452                                                 Sci_Position pos = styler.GetStartSegment() - 1;
 453                                                 if (pos < 0 || (styler.SafeGetCharAt(pos, '\0') != '.'))
 454                                                         style = SCE_P_WORD2;
 455                                         } else {
 456                                                 style = SCE_P_WORD2;
 457                                         }
 458                                 } else {
 459                                         int subStyle = classifierIdentifiers.ValueFor(s);
 460                                         if (subStyle >= 0) {
 461                                                 style = subStyle;
 462                                         }
 463                                 }
 464                                 sc.ChangeState(style);
 465                                 sc.SetState(SCE_P_DEFAULT);
 466                                 if (style == SCE_P_WORD) {
 467                                         if (0 == strcmp(s, "class"))
 468                                                 kwLast = kwClass;
 469                                         else if (0 == strcmp(s, "def"))
 470                                                 kwLast = kwDef;
 471                                         else if (0 == strcmp(s, "import"))
 472                                                 kwLast = kwImport;
 473                                         else if (0 == strcmp(s, "cdef"))
 474                                                 kwLast = kwCDef;
 475                                         else if (0 == strcmp(s, "cpdef"))
 476                                                 kwLast = kwCPDef;
 477                                         else if (0 == strcmp(s, "cimport"))
 478                                                 kwLast = kwImport;
 479                                         else if (kwLast != kwCDef && kwLast != kwCPDef)
 480                                                 kwLast = kwOther;
 481                                 } else if (kwLast != kwCDef && kwLast != kwCPDef) {
 482                                         kwLast = kwOther;
 483                                 }
 484                         }
 485                 } else if ((sc.state == SCE_P_COMMENTLINE) || (sc.state == SCE_P_COMMENTBLOCK)) {
 486                         if (sc.ch == '\r' || sc.ch == '\n') {
 487                                 sc.SetState(SCE_P_DEFAULT);
 488                         }
 489                 } else if (sc.state == SCE_P_DECORATOR) {
 490                         if (!IsAWordChar(sc.ch)) {
 491                                 sc.SetState(SCE_P_DEFAULT);
 492                         }
 493                 } else if ((sc.state == SCE_P_STRING) || (sc.state == SCE_P_CHARACTER)) {
 494                         if (sc.ch == '\\') {
 495                                 if ((sc.chNext == '\r') && (sc.GetRelative(2) == '\n')) {
 496                                         sc.Forward();
 497                                 }
 498                                 if (sc.chNext == '\n' || sc.chNext == '\r') {
 499                                         inContinuedString = true;
 500                                 } else {
 501                                         // Don't roll over the newline.
 502                                         sc.Forward();
 503                                 }
 504                         } else if ((sc.state == SCE_P_STRING) && (sc.ch == '\"')) {
 505                                 sc.ForwardSetState(SCE_P_DEFAULT);
 506                                 needEOLCheck = true;
 507                         } else if ((sc.state == SCE_P_CHARACTER) && (sc.ch == '\'')) {
 508                                 sc.ForwardSetState(SCE_P_DEFAULT);
 509                                 needEOLCheck = true;
 510                         }
 511                 } else if (sc.state == SCE_P_TRIPLE) {
 512                         if (sc.ch == '\\') {
 513                                 sc.Forward();
 514                         } else if (sc.Match("\'\'\'")) {
 515                                 sc.Forward();
 516                                 sc.Forward();
 517                                 sc.ForwardSetState(SCE_P_DEFAULT);
 518                                 needEOLCheck = true;
 519                         }
 520                 } else if (sc.state == SCE_P_TRIPLEDOUBLE) {
 521                         if (sc.ch == '\\') {
 522                                 sc.Forward();
 523                         } else if (sc.Match("\"\"\"")) {
 524                                 sc.Forward();
 525                                 sc.Forward();
 526                                 sc.ForwardSetState(SCE_P_DEFAULT);
 527                                 needEOLCheck = true;
 528                         }
 529                 }
 530
 531                 if (!indentGood && !IsASpaceOrTab(sc.ch)) {
 532                         styler.IndicatorFill(startIndicator, sc.currentPos, indicatorWhitespace, 1);
 533                         startIndicator = sc.currentPos;
 534                         indentGood = true;
 535                 }
 536
 537                 // One cdef or cpdef line, clear kwLast only at end of line
 538                 if ((kwLast == kwCDef || kwLast == kwCPDef) && sc.atLineEnd) {
 539                         kwLast = kwOther;
 540                 }
 541
 542                 // State exit code may have moved on to end of line
 543                 if (needEOLCheck && sc.atLineEnd) {
 544                         lineCurrent++;
 545                         styler.IndentAmount(lineCurrent, &spaceFlags, IsPyComment);
 546                         if (!sc.More())
 547                                 break;
 548                 }
 549
 550                 // Check for a new state starting character
 551                 if (sc.state == SCE_P_DEFAULT) {
 552                         if (IsADigit(sc.ch) || (sc.ch == '.' && IsADigit(sc.chNext))) {
 553                                 if (sc.ch == '0' && (sc.chNext == 'x' || sc.chNext == 'X')) {
 554                                         base_n_number = true;
 555                                         sc.SetState(SCE_P_NUMBER);
 556                                 } else if (sc.ch == '0' &&
 557                                         (sc.chNext == 'o' || sc.chNext == 'O' || sc.chNext == 'b' || sc.chNext == 'B')) {
 558                                         if (options.base2or8Literals) {
 559                                                 base_n_number = true;
 560                                                 sc.SetState(SCE_P_NUMBER);
 561                                         } else {
 562                                                 sc.SetState(SCE_P_NUMBER);
 563                                                 sc.ForwardSetState(SCE_P_IDENTIFIER);
 564                                         }
 565                                 } else {
 566                                         base_n_number = false;
 567                                         sc.SetState(SCE_P_NUMBER);
 568                                 }
 569                         } else if ((IsASCII(sc.ch) && isoperator(static_cast<char>(sc.ch))) || sc.ch == '`') {
 570                                 sc.SetState(SCE_P_OPERATOR);
 571                         } else if (sc.ch == '#') {
 572                                 sc.SetState(sc.chNext == '#' ? SCE_P_COMMENTBLOCK : SCE_P_COMMENTLINE);
 573                         } else if (sc.ch == '@') {
 574                                 if (IsFirstNonWhitespace(sc.currentPos, styler))
 575                                         sc.SetState(SCE_P_DECORATOR);
 576                                 else
 577                                         sc.SetState(SCE_P_OPERATOR);
 578                         } else if (IsPyStringStart(sc.ch, sc.chNext, sc.GetRelative(2), allowedLiterals)) {
 579                                 Sci_PositionU nextIndex = 0;
 580                                 sc.SetState(GetPyStringState(styler, sc.currentPos, &nextIndex, allowedLiterals));
 581                                 while (nextIndex > (sc.currentPos + 1) && sc.More()) {
 582                                         sc.Forward();
 583                                 }
 584                         } else if (IsAWordStart(sc.ch)) {
 585                                 sc.SetState(SCE_P_IDENTIFIER);
 586                         }
 587                 }
 588         }
 589         styler.IndicatorFill(startIndicator, sc.currentPos, indicatorWhitespace, 0);
 590         sc.Complete();
 591 }
 592
 593 static bool IsCommentLine(Sci_Position line, Accessor &styler) {
 594         Sci_Position pos = styler.LineStart(line);
 595         Sci_Position eol_pos = styler.LineStart(line + 1) - 1;
 596         for (Sci_Position i = pos; i < eol_pos; i++) {
 597                 char ch = styler[i];
 598                 if (ch == '#')
 599                         return true;
 600                 else if (ch != ' ' && ch != '\t')
 601                         return false;
 602         }
 603         return false;
 604 }
 605
 606 static bool IsQuoteLine(Sci_Position line, Accessor &styler) {
 607         int style = styler.StyleAt(styler.LineStart(line)) & 31;
 608         return ((style == SCE_P_TRIPLE) || (style == SCE_P_TRIPLEDOUBLE));
 609 }
 610
 611
 612 void SCI_METHOD LexerPython::Fold(Sci_PositionU startPos, Sci_Position length, int /*initStyle - unused*/, IDocument *pAccess) {
 613         if (!options.fold)
 614                 return;
 615
 616         Accessor styler(pAccess, NULL);
 617
 618         const Sci_Position maxPos = startPos + length;
 619         const Sci_Position maxLines = (maxPos == styler.Length()) ? styler.GetLine(maxPos) : styler.GetLine(maxPos - 1);        // Requested last line
 620         const Sci_Position docLines = styler.GetLine(styler.Length());  // Available last line
 621
 622         // Backtrack to previous non-blank line so we can determine indent level
 623         // for any white space lines (needed esp. within triple quoted strings)
 624         // and so we can fix any preceding fold level (which is why we go back
 625         // at least one line in all cases)
 626         int spaceFlags = 0;
 627         Sci_Position lineCurrent = styler.GetLine(startPos);
 628         int indentCurrent = styler.IndentAmount(lineCurrent, &spaceFlags, NULL);
 629         while (lineCurrent > 0) {
 630                 lineCurrent--;
 631                 indentCurrent = styler.IndentAmount(lineCurrent, &spaceFlags, NULL);
 632                 if (!(indentCurrent & SC_FOLDLEVELWHITEFLAG) &&
 633                         (!IsCommentLine(lineCurrent, styler)) &&
 634                         (!IsQuoteLine(lineCurrent, styler)))
 635                         break;
 636         }
 637         int indentCurrentLevel = indentCurrent & SC_FOLDLEVELNUMBERMASK;
 638
 639         // Set up initial loop state
 640         startPos = styler.LineStart(lineCurrent);
 641         int prev_state = SCE_P_DEFAULT & 31;
 642         if (lineCurrent >= 1)
 643                 prev_state = styler.StyleAt(startPos - 1) & 31;
 644         int prevQuote = options.foldQuotes && ((prev_state == SCE_P_TRIPLE) || (prev_state == SCE_P_TRIPLEDOUBLE));
 645
 646         // Process all characters to end of requested range or end of any triple quote
 647         //that hangs over the end of the range.  Cap processing in all cases
 648         // to end of document (in case of unclosed quote at end).
 649         while ((lineCurrent <= docLines) && ((lineCurrent <= maxLines) || prevQuote)) {
 650
 651                 // Gather info
 652                 int lev = indentCurrent;
 653                 Sci_Position lineNext = lineCurrent + 1;
 654                 int indentNext = indentCurrent;
 655                 int quote = false;
 656                 if (lineNext <= docLines) {
 657                         // Information about next line is only available if not at end of document
 658                         indentNext = styler.IndentAmount(lineNext, &spaceFlags, NULL);
 659                         Sci_Position lookAtPos = (styler.LineStart(lineNext) == styler.Length()) ? styler.Length() - 1 : styler.LineStart(lineNext);
 660                         int style = styler.StyleAt(lookAtPos) & 31;
 661                         quote = options.foldQuotes && ((style == SCE_P_TRIPLE) || (style == SCE_P_TRIPLEDOUBLE));
 662                 }
 663                 const int quote_start = (quote && !prevQuote);
 664                 const int quote_continue = (quote && prevQuote);
 665                 if (!quote || !prevQuote)
 666                         indentCurrentLevel = indentCurrent & SC_FOLDLEVELNUMBERMASK;
 667                 if (quote)
 668                         indentNext = indentCurrentLevel;
 669                 if (indentNext & SC_FOLDLEVELWHITEFLAG)
 670                         indentNext = SC_FOLDLEVELWHITEFLAG | indentCurrentLevel;
 671
 672                 if (quote_start) {
 673                         // Place fold point at start of triple quoted string
 674                         lev |= SC_FOLDLEVELHEADERFLAG;
 675                 } else if (quote_continue || prevQuote) {
 676                         // Add level to rest of lines in the string
 677                         lev = lev + 1;
 678                 }
 679
 680                 // Skip past any blank lines for next indent level info; we skip also
 681                 // comments (all comments, not just those starting in column 0)
 682                 // which effectively folds them into surrounding code rather
 683                 // than screwing up folding.
 684
 685                 while (!quote &&
 686                         (lineNext < docLines) &&
 687                         ((indentNext & SC_FOLDLEVELWHITEFLAG) ||
 688                          (lineNext <= docLines && IsCommentLine(lineNext, styler)))) {
 689
 690                         lineNext++;
 691                         indentNext = styler.IndentAmount(lineNext, &spaceFlags, NULL);
 692                 }
 693
 694                 const int levelAfterComments = indentNext & SC_FOLDLEVELNUMBERMASK;
 695                 const int levelBeforeComments = Maximum(indentCurrentLevel,levelAfterComments);
 696
 697                 // Now set all the indent levels on the lines we skipped
 698                 // Do this from end to start.  Once we encounter one line
 699                 // which is indented more than the line after the end of
 700                 // the comment-block, use the level of the block before
 701
 702                 Sci_Position skipLine = lineNext;
 703                 int skipLevel = levelAfterComments;
 704
 705                 while (--skipLine > lineCurrent) {
 706                         int skipLineIndent = styler.IndentAmount(skipLine, &spaceFlags, NULL);
 707
 708                         if (options.foldCompact) {
 709                                 if ((skipLineIndent & SC_FOLDLEVELNUMBERMASK) > levelAfterComments)
 710                                         skipLevel = levelBeforeComments;
 711
 712                                 int whiteFlag = skipLineIndent & SC_FOLDLEVELWHITEFLAG;
 713
 714                                 styler.SetLevel(skipLine, skipLevel | whiteFlag);
 715                         } else {
 716                                 if ((skipLineIndent & SC_FOLDLEVELNUMBERMASK) > levelAfterComments &&
 717                                         !(skipLineIndent & SC_FOLDLEVELWHITEFLAG) &&
 718                                         !IsCommentLine(skipLine, styler))
 719                                         skipLevel = levelBeforeComments;
 720
 721                                 styler.SetLevel(skipLine, skipLevel);
 722                         }
 723                 }
 724
 725                 // Set fold header on non-quote line
 726                 if (!quote && !(indentCurrent & SC_FOLDLEVELWHITEFLAG)) {
 727                         if ((indentCurrent & SC_FOLDLEVELNUMBERMASK) < (indentNext & SC_FOLDLEVELNUMBERMASK))
 728                                 lev |= SC_FOLDLEVELHEADERFLAG;
 729                 }
 730
 731                 // Keep track of triple quote state of previous line
 732                 prevQuote = quote;
 733
 734                 // Set fold level for this line and move to next line
 735                 styler.SetLevel(lineCurrent, options.foldCompact ? lev : lev & ~SC_FOLDLEVELWHITEFLAG);
 736                 indentCurrent = indentNext;
 737                 lineCurrent = lineNext;
 738         }
 739
 740         // NOTE: Cannot set level of last line here because indentCurrent doesn't have
 741         // header flag set; the loop above is crafted to take care of this case!
 742         //styler.SetLevel(lineCurrent, indentCurrent);
 743 }
 744
 745 LexerModule lmPython(SCLEX_PYTHON, LexerPython::LexerFactoryPython, "python",
 746                                          pythonWordListDesc);