ext/scintilla/src/LexPerl.cxx

   1 // Scintilla source code edit control
   2 /** @file LexPerl.cxx
   3  ** Lexer for Perl.
   4  **/
   5 // Copyright 1998-2008 by Neil Hodgson <neilh@scintilla.org>
   6 // Lexical analysis fixes by Kein-Hong Man <mkh@pl.jaring.my>
   7 // The License.txt file describes the conditions under which this software may be distributed.
   8
   9 #include <stdlib.h>
  10 #include <string.h>
  11 #include <ctype.h>
  12 #include <stdio.h>
  13 #include <stdarg.h>
  14
  15 #include "Platform.h"
  16
  17 #include "PropSet.h"
  18 #include "Accessor.h"
  19 #include "StyleContext.h"
  20 #include "KeyWords.h"
  21 #include "Scintilla.h"
  22 #include "SciLexer.h"
  23 #include "CharacterSet.h"
  24
  25 #ifdef SCI_NAMESPACE
  26 using namespace Scintilla;
  27 #endif
  28
  29 // Info for HERE document handling from perldata.pod (reformatted):
  30 // ----------------------------------------------------------------
  31 // A line-oriented form of quoting is based on the shell ``here-doc'' syntax.
  32 // Following a << you specify a string to terminate the quoted material, and
  33 // all lines following the current line down to the terminating string are
  34 // the value of the item.
  35 // * The terminating string may be either an identifier (a word), or some
  36 //   quoted text.
  37 // * If quoted, the type of quotes you use determines the treatment of the
  38 //   text, just as in regular quoting.
  39 // * An unquoted identifier works like double quotes.
  40 // * There must be no space between the << and the identifier.
  41 //   (If you put a space it will be treated as a null identifier,
  42 //    which is valid, and matches the first empty line.)
  43 //   (This is deprecated, -w warns of this syntax)
  44 // * The terminating string must appear by itself (unquoted and
  45 //   with no surrounding whitespace) on the terminating line.
  46
  47 #define HERE_DELIM_MAX 256              // maximum length of HERE doc delimiter
  48
  49 #define PERLNUM_BINARY          1       // order is significant: 1-4 cannot have a dot
  50 #define PERLNUM_HEX                     2
  51 #define PERLNUM_OCTAL           3
  52 #define PERLNUM_FLOAT_EXP       4       // exponent part only
  53 #define PERLNUM_DECIMAL         5       // 1-5 are numbers; 6-7 are strings
  54 #define PERLNUM_VECTOR          6
  55 #define PERLNUM_V_VECTOR        7
  56 #define PERLNUM_BAD                     8
  57
  58 #define BACK_NONE               0       // lookback state for bareword disambiguation:
  59 #define BACK_OPERATOR   1       // whitespace/comments are insignificant
  60 #define BACK_KEYWORD    2       // operators/keywords are needed for disambiguation
  61
  62 static bool isPerlKeyword(unsigned int start, unsigned int end, WordList &keywords, Accessor &styler)
  63 {
  64         // old-style keyword matcher; needed because GetCurrent() needs
  65         // current segment to be committed, but we may abandon early...
  66         char s[100];
  67         unsigned int i, len = end - start;
  68         if (len > 30) { len = 30; }
  69         for (i = 0; i < len; i++, start++) s[i] = styler[start];
  70         s[i] = '\0';
  71         return keywords.InList(s);
  72 }
  73
  74 static int disambiguateBareword(Accessor &styler, unsigned int bk, unsigned int fw,
  75                                 int backFlag, unsigned int backPos, unsigned int endPos)
  76 {
  77         // identifiers are recognized by Perl as barewords under some
  78         // conditions, the following attempts to do the disambiguation
  79         // by looking backward and forward; result in 2 LSB
  80         int result = 0;
  81         bool moreback = false;          // true if passed newline/comments
  82         bool brace = false;                     // true if opening brace found
  83         // if BACK_NONE, neither operator nor keyword, so skip test
  84         if (backFlag == BACK_NONE)
  85                 return result;
  86         // first look backwards past whitespace/comments to set EOL flag
  87         // (some disambiguation patterns must be on a single line)
  88         if (backPos <= static_cast<unsigned int>(styler.LineStart(styler.GetLine(bk))))
  89                 moreback = true;
  90         // look backwards at last significant lexed item for disambiguation
  91         bk = backPos - 1;
  92         int ch = static_cast<unsigned char>(styler.SafeGetCharAt(bk));
  93         if (ch == '{' && !moreback) {
  94                 // {bareword: possible variable spec
  95                 brace = true;
  96         } else if ((ch == '&' && styler.SafeGetCharAt(bk - 1) != '&')
  97                         // &bareword: subroutine call
  98                            || styler.Match(bk - 1, "->")
  99                         // ->bareword: part of variable spec
 100                            || styler.Match(bk - 2, "sub")) {
 101                         // sub bareword: subroutine declaration
 102                         // (implied BACK_KEYWORD, no keywords end in 'sub'!)
 103                 result |= 1;
 104         }
 105         // next, scan forward after word past tab/spaces only;
 106         // if ch isn't one of '[{(,' we can skip the test
 107         if ((ch == '{' || ch == '(' || ch == '['|| ch == ',')
 108                 && fw < endPos) {
 109                 while (ch = static_cast<unsigned char>(styler.SafeGetCharAt(fw)),
 110                            IsASpaceOrTab(ch) && fw < endPos) {
 111                         fw++;
 112                 }
 113                 if ((ch == '}' && brace)
 114                         // {bareword}: variable spec
 115                         || styler.Match(fw, "=>")) {
 116                         // [{(, bareword=>: hash literal
 117                         result |= 2;
 118                 }
 119         }
 120         return result;
 121 }
 122
 123 static void skipWhitespaceComment(Accessor &styler, unsigned int &p)
 124 {
 125         // when backtracking, we need to skip whitespace and comments
 126         int style;
 127         while ((p > 0) && (style = styler.StyleAt(p),
 128                    style == SCE_PL_DEFAULT || style == SCE_PL_COMMENTLINE))
 129                 p--;
 130 }
 131
 132 static int styleBeforeBracePair(Accessor &styler, unsigned int bk)
 133 {
 134         // backtrack to find open '{' corresponding to a '}', balanced
 135         // return significant style to be tested for '/' disambiguation
 136         int braceCount = 1;
 137         if (bk == 0)
 138                 return SCE_PL_DEFAULT;
 139         while (--bk > 0) {
 140                 if (styler.StyleAt(bk) == SCE_PL_OPERATOR) {
 141                         int bkch = static_cast<unsigned char>(styler.SafeGetCharAt(bk));
 142                         if (bkch == ';') {      // early out
 143                                 break;
 144                         } else if (bkch == '}') {
 145                                 braceCount++;
 146                         } else if (bkch == '{') {
 147                                 if (--braceCount == 0) break;
 148                         }
 149                 }
 150         }
 151         if (bk > 0 && braceCount == 0) {
 152                 // balanced { found, bk > 0, skip more whitespace/comments
 153                 bk--;
 154                 skipWhitespaceComment(styler, bk);
 155                 return styler.StyleAt(bk);
 156         }
 157         return SCE_PL_DEFAULT;
 158 }
 159
 160 static int styleCheckIdentifier(Accessor &styler, unsigned int bk)
 161 {
 162         // backtrack to classify sub-styles of identifier under test
 163         // return sub-style to be tested for '/' disambiguation
 164         if (styler.SafeGetCharAt(bk) == '>')    // inputsymbol, like <foo>
 165                 return 1;
 166         // backtrack to check for possible "->" or "::" before identifier
 167         while (bk > 0 && styler.StyleAt(bk) == SCE_PL_IDENTIFIER) {
 168                 bk--;
 169         }
 170         while (bk > 0) {
 171                 int bkstyle = styler.StyleAt(bk);
 172                 if (bkstyle == SCE_PL_DEFAULT
 173                         || bkstyle == SCE_PL_COMMENTLINE) {
 174                         // skip whitespace, comments
 175                 } else if (bkstyle == SCE_PL_OPERATOR) {
 176                         // test for "->" and "::"
 177                         if (styler.Match(bk - 1, "->") || styler.Match(bk - 1, "::"))
 178                                 return 2;
 179                 } else
 180                         return 3;       // bare identifier
 181                 bk--;
 182         }
 183         return 0;
 184 }
 185
 186 static int inputsymbolScan(Accessor &styler, unsigned int pos, unsigned int endPos)
 187 {
 188         // looks forward for matching > on same line; a bit ugly
 189         unsigned int fw = pos;
 190         while (++fw < endPos) {
 191                 int fwch = static_cast<unsigned char>(styler.SafeGetCharAt(fw));
 192                 if (fwch == '\r' || fwch == '\n') {
 193                         return 0;
 194                 } else if (fwch == '>') {
 195                         if (styler.Match(fw - 2, "<=>"))        // '<=>' case
 196                                 return 0;
 197                         return fw - pos;
 198                 }
 199         }
 200         return 0;
 201 }
 202
 203 static int podLineScan(Accessor &styler, unsigned int &pos, unsigned int endPos)
 204 {
 205         // forward scan the current line to classify line for POD style
 206         int state = -1;
 207         while (pos <= endPos) {
 208                 int ch = static_cast<unsigned char>(styler.SafeGetCharAt(pos));
 209                 if (ch == '\n' || ch == '\r' || pos >= endPos) {
 210                         if (ch == '\r' && styler.SafeGetCharAt(pos + 1) == '\n') pos++;
 211                         break;
 212                 }
 213                 if (IsASpaceOrTab(ch)) {        // whitespace, take note
 214                         if (state == -1)
 215                                 state = SCE_PL_DEFAULT;
 216                 } else if (state == SCE_PL_DEFAULT) {   // verbatim POD line
 217                         state = SCE_PL_POD_VERB;
 218                 } else if (state != SCE_PL_POD_VERB) {  // regular POD line
 219                         state = SCE_PL_POD;
 220                 }
 221                 pos++;
 222         }
 223         if (state == -1)
 224                 state = SCE_PL_DEFAULT;
 225         return state;
 226 }
 227
 228 static bool styleCheckSubPrototype(Accessor &styler, unsigned int bk)
 229 {
 230         // backtrack to identify if we're starting a subroutine prototype
 231         // we also need to ignore whitespace/comments:
 232         // 'sub' [whitespace|comment] <identifier> [whitespace|comment]
 233         styler.Flush();
 234         skipWhitespaceComment(styler, bk);
 235         if (bk == 0 || styler.StyleAt(bk) != SCE_PL_IDENTIFIER) // check identifier
 236                 return false;
 237         while (bk > 0 && (styler.StyleAt(bk) == SCE_PL_IDENTIFIER)) {
 238                 bk--;
 239         }
 240         skipWhitespaceComment(styler, bk);
 241         if (bk < 2 || styler.StyleAt(bk) != SCE_PL_WORD // check "sub" keyword
 242                 || !styler.Match(bk - 2, "sub"))        // assume suffix is unique!
 243                 return false;
 244         return true;
 245 }
 246
 247 static bool isMatch(const char *sref, char *s)
 248 {
 249         // match per-line delimiter - must kill trailing CR if CRLF
 250         if (s[strlen(s) - 1] == '\r')
 251                 s[strlen(s) - 1] = '\0';
 252         return (strcmp(sref, s) == 0);
 253 }
 254
 255 static int actualNumStyle(int numberStyle) {
 256         if (numberStyle == PERLNUM_VECTOR || numberStyle == PERLNUM_V_VECTOR) {
 257                 return SCE_PL_STRING;
 258         } else if (numberStyle == PERLNUM_BAD) {
 259                 return SCE_PL_ERROR;
 260         }
 261         return SCE_PL_NUMBER;
 262 }
 263
 264 static int opposite(int ch) {
 265         if (ch == '(') return ')';
 266         if (ch == '[') return ']';
 267         if (ch == '{') return '}';
 268         if (ch == '<') return '>';
 269         return ch;
 270 }
 271
 272 static void ColourisePerlDoc(unsigned int startPos, int length, int initStyle,
 273                              WordList *keywordlists[], Accessor &styler) {
 274
 275         WordList &keywords = *keywordlists[0];
 276
 277         // keywords that forces /PATTERN/ at all times; should track vim's behaviour
 278         WordList reWords;
 279         reWords.Set("elsif if split while");
 280
 281         // charset classes
 282         CharacterSet setWordStart(CharacterSet::setAlpha, "_", 0x80, true);
 283         CharacterSet setWord(CharacterSet::setAlphaNum, "_", 0x80, true);
 284         CharacterSet setSingleCharOp(CharacterSet::setNone, "rwxoRWXOezsfdlpSbctugkTBMAC");
 285         // lexing of "%*</" operators is non-trivial; these are missing in the set below
 286         CharacterSet setPerlOperator(CharacterSet::setNone, "^&\\()-+=|{}[]:;>,?!.~");
 287         CharacterSet setQDelim(CharacterSet::setNone, "qrwx");
 288         CharacterSet setModifiers(CharacterSet::setAlpha);
 289         CharacterSet setPreferRE(CharacterSet::setNone, "*/<%");
 290         // setArray and setHash also accepts chars for special vars like $_,
 291         // which are then truncated when the next char does not match setVar
 292         CharacterSet setVar(CharacterSet::setAlphaNum, "#$_'", 0x80, true);
 293         CharacterSet setArray(CharacterSet::setAlpha, "#$_+-", 0x80, true);
 294         CharacterSet setHash(CharacterSet::setAlpha, "#$_!^+-", 0x80, true);
 295         CharacterSet &setPOD = setModifiers;
 296         CharacterSet setNonHereDoc(CharacterSet::setDigits, "=$@");
 297         CharacterSet setHereDocDelim(CharacterSet::setAlphaNum, "_");
 298         CharacterSet setSubPrototype(CharacterSet::setNone, "\\[$@%&*];");
 299         // for format identifiers
 300         CharacterSet setFormatStart(CharacterSet::setAlpha, "_=");
 301         CharacterSet &setFormat = setHereDocDelim;
 302
 303         // Lexer for perl often has to backtrack to start of current style to determine
 304         // which characters are being used as quotes, how deeply nested is the
 305         // start position and what the termination string is for HERE documents.
 306
 307         class HereDocCls {      // Class to manage HERE doc sequence
 308         public:
 309                 int State;              // 0: '<<' encountered
 310                                                 // 1: collect the delimiter
 311                                                 // 2: here doc text (lines after the delimiter)
 312                 int Quote;              // the char after '<<'
 313                 bool Quoted;            // true if Quote in ('\'','"','`')
 314                 int DelimiterLength;    // strlen(Delimiter)
 315                 char *Delimiter;        // the Delimiter, 256: sizeof PL_tokenbuf
 316                 HereDocCls() {
 317                         State = 0;
 318                         Quote = 0;
 319                         Quoted = false;
 320                         DelimiterLength = 0;
 321                         Delimiter = new char[HERE_DELIM_MAX];
 322                         Delimiter[0] = '\0';
 323                 }
 324                 void Append(int ch) {
 325                         Delimiter[DelimiterLength++] = static_cast<char>(ch);
 326                         Delimiter[DelimiterLength] = '\0';
 327                 }
 328                 ~HereDocCls() {
 329                         delete []Delimiter;
 330                 }
 331         };
 332         HereDocCls HereDoc;             // TODO: FIFO for stacked here-docs
 333
 334         class QuoteCls {        // Class to manage quote pairs
 335                 public:
 336                 int Rep;
 337                 int Count;
 338                 int Up, Down;
 339                 QuoteCls() {
 340                         this->New(1);
 341                 }
 342                 void New(int r = 1) {
 343                         Rep   = r;
 344                         Count = 0;
 345                         Up    = '\0';
 346                         Down  = '\0';
 347                 }
 348                 void Open(int u) {
 349                         Count++;
 350                         Up    = u;
 351                         Down  = opposite(Up);
 352                 }
 353         };
 354         QuoteCls Quote;
 355
 356         // additional state for number lexing
 357         int numState = PERLNUM_DECIMAL;
 358         int dotCount = 0;
 359
 360         unsigned int endPos = startPos + length;
 361
 362         // Backtrack to beginning of style if required...
 363         // If in a long distance lexical state, backtrack to find quote characters.
 364         // Includes strings (may be multi-line), numbers (additional state), format
 365         // bodies, as well as POD sections.
 366         if (initStyle == SCE_PL_HERE_Q
 367                 || initStyle == SCE_PL_HERE_QQ
 368                 || initStyle == SCE_PL_HERE_QX
 369                 || initStyle == SCE_PL_FORMAT
 370         ) {
 371                 int delim = (initStyle == SCE_PL_FORMAT) ? SCE_PL_FORMAT_IDENT:SCE_PL_HERE_DELIM;
 372                 while ((startPos > 1) && (styler.StyleAt(startPos) != delim)) {
 373                         startPos--;
 374                 }
 375                 startPos = styler.LineStart(styler.GetLine(startPos));
 376                 initStyle = styler.StyleAt(startPos - 1);
 377         }
 378         if (initStyle == SCE_PL_STRING_Q
 379                 || initStyle == SCE_PL_STRING_QQ
 380                 || initStyle == SCE_PL_STRING_QX
 381                 || initStyle == SCE_PL_STRING_QR
 382                 || initStyle == SCE_PL_STRING_QW
 383                 || initStyle == SCE_PL_REGEX
 384                 || initStyle == SCE_PL_REGSUBST
 385                 || initStyle == SCE_PL_STRING
 386                 || initStyle == SCE_PL_BACKTICKS
 387                 || initStyle == SCE_PL_CHARACTER
 388                 || initStyle == SCE_PL_NUMBER
 389                 || initStyle == SCE_PL_IDENTIFIER
 390                 || initStyle == SCE_PL_ERROR
 391                 || initStyle == SCE_PL_SUB_PROTOTYPE
 392         ) {
 393                 while ((startPos > 1) && (styler.StyleAt(startPos - 1) == initStyle)) {
 394                         startPos--;
 395                 }
 396                 initStyle = SCE_PL_DEFAULT;
 397         } else if (initStyle == SCE_PL_POD
 398                            || initStyle == SCE_PL_POD_VERB
 399         ) {
 400                 // POD backtracking finds preceeding blank lines and goes back past them
 401                 int ln = styler.GetLine(startPos);
 402                 if (ln > 0) {
 403                         initStyle = styler.StyleAt(styler.LineStart(--ln));
 404                         if (initStyle == SCE_PL_POD || initStyle == SCE_PL_POD_VERB) {
 405                                 while (ln > 0 && styler.GetLineState(ln) == SCE_PL_DEFAULT)
 406                                         ln--;
 407                         }
 408                         startPos = styler.LineStart(++ln);
 409                         initStyle = styler.StyleAt(startPos - 1);
 410                 } else {
 411                         startPos = 0;
 412                         initStyle = SCE_PL_DEFAULT;
 413                 }
 414         }
 415
 416         // backFlag, backPos are additional state to aid identifier corner cases.
 417         // Look backwards past whitespace and comments in order to detect either
 418         // operator or keyword. Later updated as we go along.
 419         int backFlag = BACK_NONE;
 420         unsigned int backPos = startPos;
 421         if (backPos > 0) {
 422                 backPos--;
 423                 skipWhitespaceComment(styler, backPos);
 424                 if (styler.StyleAt(backPos) == SCE_PL_OPERATOR)
 425                         backFlag = BACK_OPERATOR;
 426                 else if (styler.StyleAt(backPos) == SCE_PL_WORD)
 427                         backFlag = BACK_KEYWORD;
 428                 backPos++;
 429         }
 430
 431         StyleContext sc(startPos, endPos - startPos, initStyle, styler, static_cast<char>(STYLE_MAX));
 432
 433         for (; sc.More(); sc.Forward()) {
 434
 435                 // Determine if the current state should terminate.
 436                 switch (sc.state) {
 437                         case SCE_PL_OPERATOR:
 438                                 sc.SetState(SCE_PL_DEFAULT);
 439                                 backFlag = BACK_OPERATOR;
 440                                 backPos = sc.currentPos;
 441                                 break;
 442                         case SCE_PL_IDENTIFIER:         // identifier, bareword, inputsymbol
 443                                 if ((!setWord.Contains(sc.ch) && sc.ch != '\'')
 444                                         || sc.Match('.', '.')
 445                                         || sc.chPrev == '>') {  // end of inputsymbol
 446                                         sc.SetState(SCE_PL_DEFAULT);
 447                                 }
 448                                 break;
 449                         case SCE_PL_WORD:               // keyword, plus special cases
 450                                 if (!setWord.Contains(sc.ch)) {
 451                                         char s[100];
 452                                         sc.GetCurrent(s, sizeof(s));
 453                                         if ((strcmp(s, "__DATA__") == 0) || (strcmp(s, "__END__") == 0)) {
 454                                                 sc.ChangeState(SCE_PL_DATASECTION);
 455                                         } else {
 456                                                 if ((strcmp(s, "format") == 0)) {
 457                                                         sc.SetState(SCE_PL_FORMAT_IDENT);
 458                                                         HereDoc.State = 0;
 459                                                 } else {
 460                                                         sc.SetState(SCE_PL_DEFAULT);
 461                                                 }
 462                                                 backFlag = BACK_KEYWORD;
 463                                                 backPos = sc.currentPos;
 464                                         }
 465                                 }
 466                                 break;
 467                         case SCE_PL_SCALAR:
 468                         case SCE_PL_ARRAY:
 469                         case SCE_PL_HASH:
 470                         case SCE_PL_SYMBOLTABLE:
 471                                 if (sc.Match(':', ':')) {       // skip ::
 472                                         sc.Forward();
 473                                 } else if (!setVar.Contains(sc.ch)) {
 474                                         if (sc.LengthCurrent() == 1) {
 475                                                 // Special variable: $(, $_ etc.
 476                                                 sc.Forward();
 477                                         }
 478                                         sc.SetState(SCE_PL_DEFAULT);
 479                                 }
 480                                 break;
 481                         case SCE_PL_NUMBER:
 482                                 // if no early break, number style is terminated at "(go through)"
 483                                 if (sc.ch == '.') {
 484                                         if (sc.chNext == '.') {
 485                                                 // double dot is always an operator (go through)
 486                                         } else if (numState <= PERLNUM_FLOAT_EXP) {
 487                                                 // non-decimal number or float exponent, consume next dot
 488                                                 sc.SetState(SCE_PL_OPERATOR);
 489                                                 break;
 490                                         } else {        // decimal or vectors allows dots
 491                                                 dotCount++;
 492                                                 if (numState == PERLNUM_DECIMAL) {
 493                                                         if (dotCount <= 1)      // number with one dot in it
 494                                                                 break;
 495                                                         if (IsADigit(sc.chNext)) {      // really a vector
 496                                                                 numState = PERLNUM_VECTOR;
 497                                                                 break;
 498                                                         }
 499                                                         // number then dot (go through)
 500                                                 } else if (IsADigit(sc.chNext)) // vectors
 501                                                         break;
 502                                                 // vector then dot (go through)
 503                                         }
 504                                 } else if (sc.ch == '_') {
 505                                         // permissive underscoring for number and vector literals
 506                                         break;
 507                                 } else if (numState == PERLNUM_DECIMAL) {
 508                                         if (sc.ch == 'E' || sc.ch == 'e') {     // exponent, sign
 509                                                 numState = PERLNUM_FLOAT_EXP;
 510                                                 if (sc.chNext == '+' || sc.chNext == '-') {
 511                                                         sc.Forward();
 512                                                 }
 513                                                 break;
 514                                         } else if (IsADigit(sc.ch))
 515                                                 break;
 516                                         // number then word (go through)
 517                                 } else if (numState == PERLNUM_HEX) {
 518                                         if (IsADigit(sc.ch, 16))
 519                                                 break;
 520                                 } else if (numState == PERLNUM_VECTOR || numState == PERLNUM_V_VECTOR) {
 521                                         if (IsADigit(sc.ch))    // vector
 522                                                 break;
 523                                         if (setWord.Contains(sc.ch) && dotCount == 0) { // change to word
 524                                                 sc.ChangeState(SCE_PL_IDENTIFIER);
 525                                                 break;
 526                                         }
 527                                         // vector then word (go through)
 528                                 } else if (IsADigit(sc.ch)) {
 529                                         if (numState == PERLNUM_FLOAT_EXP) {
 530                                                 break;
 531                                         } else if (numState == PERLNUM_OCTAL) {
 532                                                 if (sc.ch <= '7') break;
 533                                         } else if (numState == PERLNUM_BINARY) {
 534                                                 if (sc.ch <= '1') break;
 535                                         }
 536                                         // mark invalid octal, binary numbers (go through)
 537                                         numState = PERLNUM_BAD;
 538                                         break;
 539                                 }
 540                                 // complete current number or vector
 541                                 sc.ChangeState(actualNumStyle(numState));
 542                                 sc.SetState(SCE_PL_DEFAULT);
 543                                 break;
 544                         case SCE_PL_COMMENTLINE:
 545                                 if (sc.atLineEnd) {
 546                                         sc.SetState(SCE_PL_DEFAULT);
 547                                 }
 548                                 break;
 549                         case SCE_PL_HERE_DELIM:
 550                                 if (HereDoc.State == 0) { // '<<' encountered
 551                                         int delim_ch = sc.chNext;
 552                                         int ws_skip = 0;
 553                                         HereDoc.State = 1;      // pre-init HERE doc class
 554                                         HereDoc.Quote = sc.chNext;
 555                                         HereDoc.Quoted = false;
 556                                         HereDoc.DelimiterLength = 0;
 557                                         HereDoc.Delimiter[HereDoc.DelimiterLength] = '\0';
 558                                         if (IsASpaceOrTab(delim_ch)) {
 559                                                 // skip whitespace; legal only for quoted delimiters
 560                                                 unsigned int i = sc.currentPos + 1;
 561                                                 while ((i < endPos) && IsASpaceOrTab(delim_ch)) {
 562                                                         i++;
 563                                                         delim_ch = static_cast<unsigned char>(styler.SafeGetCharAt(i));
 564                                                 }
 565                                                 ws_skip = i - sc.currentPos - 1;
 566                                         }
 567                                         if (delim_ch == '\'' || delim_ch == '"' || delim_ch == '`') {
 568                                                 // a quoted here-doc delimiter; skip any whitespace
 569                                                 sc.Forward(ws_skip + 1);
 570                                                 HereDoc.Quote = delim_ch;
 571                                                 HereDoc.Quoted = true;
 572                                         } else if (ws_skip == 0 && setNonHereDoc.Contains(sc.chNext)
 573                                                            || ws_skip > 0) {
 574                                                 // left shift << or <<= operator cases
 575                                                 // restore position if operator
 576                                                 sc.ChangeState(SCE_PL_OPERATOR);
 577                                                 sc.ForwardSetState(SCE_PL_DEFAULT);
 578                                                 backFlag = BACK_OPERATOR;
 579                                                 backPos = sc.currentPos;
 580                                                 HereDoc.State = 0;
 581                                         } else {
 582                                                 // specially handle initial '\' for identifier
 583                                                 if (ws_skip == 0 && HereDoc.Quote == '\\')
 584                                                         sc.Forward();
 585                                                 // an unquoted here-doc delimiter, no special handling
 586                                                 // (cannot be prefixed by spaces/tabs), or
 587                                                 // symbols terminates; deprecated zero-length delimiter
 588                                         }
 589                                 } else if (HereDoc.State == 1) { // collect the delimiter
 590                                         backFlag = BACK_NONE;
 591                                         if (HereDoc.Quoted) { // a quoted here-doc delimiter
 592                                                 if (sc.ch == HereDoc.Quote) { // closing quote => end of delimiter
 593                                                         sc.ForwardSetState(SCE_PL_DEFAULT);
 594                                                 } else if (!sc.atLineEnd) {
 595                                                         if (sc.Match('\\', static_cast<char>(HereDoc.Quote))) { // escaped quote
 596                                                                 sc.Forward();
 597                                                         }
 598                                                         if (sc.ch != '\r') {    // skip CR if CRLF
 599                                                                 HereDoc.Append(sc.ch);
 600                                                         }
 601                                                 }
 602                                         } else { // an unquoted here-doc delimiter
 603                                                 if (setHereDocDelim.Contains(sc.ch)) {
 604                                                         HereDoc.Append(sc.ch);
 605                                                 } else {
 606                                                         sc.SetState(SCE_PL_DEFAULT);
 607                                                 }
 608                                         }
 609                                         if (HereDoc.DelimiterLength >= HERE_DELIM_MAX - 1) {
 610                                                 sc.SetState(SCE_PL_ERROR);
 611                                                 HereDoc.State = 0;
 612                                         }
 613                                 }
 614                                 break;
 615                         case SCE_PL_HERE_Q:
 616                         case SCE_PL_HERE_QQ:
 617                         case SCE_PL_HERE_QX: {
 618                                 // also implies HereDoc.State == 2
 619                                 sc.Complete();
 620                                 while (!sc.atLineEnd)
 621                                         sc.Forward();
 622                                 char s[HERE_DELIM_MAX];
 623                                 sc.GetCurrent(s, sizeof(s));
 624                                 if (isMatch(HereDoc.Delimiter, s)) {
 625                                         sc.SetState(SCE_PL_DEFAULT);
 626                                         backFlag = BACK_NONE;
 627                                         HereDoc.State = 0;
 628                                 }
 629                                 } break;
 630                         case SCE_PL_POD:
 631                         case SCE_PL_POD_VERB: {
 632                                 unsigned int fw = sc.currentPos;
 633                                 int ln = styler.GetLine(fw);
 634                                 if (sc.atLineStart && sc.Match("=cut")) {       // end of POD
 635                                         sc.SetState(SCE_PL_POD);
 636                                         sc.Forward(4);
 637                                         sc.SetState(SCE_PL_DEFAULT);
 638                                         styler.SetLineState(ln, SCE_PL_POD);
 639                                         break;
 640                                 }
 641                                 int pod = podLineScan(styler, fw, endPos);      // classify POD line
 642                                 styler.SetLineState(ln, pod);
 643                                 if (pod == SCE_PL_DEFAULT) {
 644                                         if (sc.state == SCE_PL_POD_VERB) {
 645                                                 unsigned int fw2 = fw;
 646                                                 while (fw2 <= endPos && pod == SCE_PL_DEFAULT) {
 647                                                         fw = fw2++;     // penultimate line (last blank line)
 648                                                         pod = podLineScan(styler, fw2, endPos);
 649                                                         styler.SetLineState(styler.GetLine(fw2), pod);
 650                                                 }
 651                                                 if (pod == SCE_PL_POD) {        // truncate verbatim POD early
 652                                                         sc.SetState(SCE_PL_POD);
 653                                                 } else
 654                                                         fw = fw2;
 655                                         } else
 656                                                 pod = SCE_PL_POD;
 657                                 } else {
 658                                         if (pod == SCE_PL_POD_VERB      // still part of current paragraph
 659                                             && (styler.GetLineState(ln - 1) == SCE_PL_POD)) {
 660                                                 pod = SCE_PL_POD;
 661                                                 styler.SetLineState(ln, pod);
 662                                         } else if (pod == SCE_PL_POD
 663                                                            && (styler.GetLineState(ln - 1) == SCE_PL_POD_VERB)) {
 664                                                 pod = SCE_PL_POD_VERB;
 665                                                 styler.SetLineState(ln, pod);
 666                                         }
 667                                         sc.SetState(pod);
 668                                 }
 669                                 sc.Forward(fw - sc.currentPos); // commit style
 670                                 } break;
 671                         case SCE_PL_REGEX:
 672                         case SCE_PL_STRING_QR:
 673                                 if (Quote.Rep <= 0) {
 674                                         if (!setModifiers.Contains(sc.ch))
 675                                                 sc.SetState(SCE_PL_DEFAULT);
 676                                 } else if (!Quote.Up && !IsASpace(sc.ch)) {
 677                                         Quote.Open(sc.ch);
 678                                 } else if (sc.ch == '\\' && Quote.Up != '\\') {
 679                                         sc.Forward();
 680                                 } else if (sc.ch == Quote.Down) {
 681                                         Quote.Count--;
 682                                         if (Quote.Count == 0)
 683                                                 Quote.Rep--;
 684                                 } else if (sc.ch == Quote.Up) {
 685                                         Quote.Count++;
 686                                 }
 687                                 break;
 688                         case SCE_PL_REGSUBST:
 689                                 if (Quote.Rep <= 0) {
 690                                         if (!setModifiers.Contains(sc.ch))
 691                                                 sc.SetState(SCE_PL_DEFAULT);
 692                                 } else if (!Quote.Up && !IsASpace(sc.ch)) {
 693                                         Quote.Open(sc.ch);
 694                                 } else if (sc.ch == '\\' && Quote.Up != '\\') {
 695                                         sc.Forward();
 696                                 } else if (Quote.Count == 0 && Quote.Rep == 1) {
 697                                         // We matched something like s(...) or tr{...}, Perl 5.10
 698                                         // appears to allow almost any character for use as the
 699                                         // next delimiters. Whitespace and comments are accepted in
 700                                         // between, but we'll limit to whitespace here.
 701                                         // For '#', if no whitespace in between, it's a delimiter.
 702                                         if (IsASpace(sc.ch)) {
 703                                                 // Keep going
 704                                         } else if (sc.ch == '#' && IsASpaceOrTab(sc.chPrev)) {
 705                                                 sc.SetState(SCE_PL_DEFAULT);
 706                                         } else {
 707                                                 Quote.Open(sc.ch);
 708                                         }
 709                                 } else if (sc.ch == Quote.Down) {
 710                                         Quote.Count--;
 711                                         if (Quote.Count == 0)
 712                                                 Quote.Rep--;
 713                                         if (Quote.Up == Quote.Down)
 714                                                 Quote.Count++;
 715                                 } else if (sc.ch == Quote.Up) {
 716                                         Quote.Count++;
 717                                 }
 718                                 break;
 719                         case SCE_PL_STRING_Q:
 720                         case SCE_PL_STRING_QQ:
 721                         case SCE_PL_STRING_QX:
 722                         case SCE_PL_STRING_QW:
 723                         case SCE_PL_STRING:
 724                         case SCE_PL_CHARACTER:
 725                         case SCE_PL_BACKTICKS:
 726                                 if (!Quote.Down && !IsASpace(sc.ch)) {
 727                                         Quote.Open(sc.ch);
 728                                 } else if (sc.ch == '\\' && Quote.Up != '\\') {
 729                                         sc.Forward();
 730                                 } else if (sc.ch == Quote.Down) {
 731                                         Quote.Count--;
 732                                         if (Quote.Count == 0)
 733                                                 sc.ForwardSetState(SCE_PL_DEFAULT);
 734                                 } else if (sc.ch == Quote.Up) {
 735                                         Quote.Count++;
 736                                 }
 737                                 break;
 738                         case SCE_PL_SUB_PROTOTYPE: {
 739                                 int i = 0;
 740                                 // forward scan; must all be valid proto characters
 741                                 while (setSubPrototype.Contains(sc.GetRelative(i)))
 742                                         i++;
 743                                 if (sc.GetRelative(i) == ')') { // valid sub prototype
 744                                         sc.Forward(i);
 745                                         sc.ForwardSetState(SCE_PL_DEFAULT);
 746                                 } else {
 747                                         // abandon prototype, restart from '('
 748                                         sc.ChangeState(SCE_PL_OPERATOR);
 749                                         sc.SetState(SCE_PL_DEFAULT);
 750                                 }
 751                                 } break;
 752                         case SCE_PL_FORMAT: {
 753                                 sc.Complete();
 754                                 while (!sc.atLineEnd)
 755                                         sc.Forward();
 756                                 char s[10];
 757                                 sc.GetCurrent(s, sizeof(s));
 758                                 if (isMatch(".", s))
 759                                         sc.SetState(SCE_PL_DEFAULT);
 760                                 } break;
 761                         case SCE_PL_ERROR:
 762                                 break;
 763                 }
 764                 // Needed for specific continuation styles (one follows the other)
 765                 switch (sc.state) {
 766                         // continued from SCE_PL_WORD
 767                         case SCE_PL_FORMAT_IDENT:
 768                                 // occupies HereDoc state 3 to avoid clashing with HERE docs
 769                                 if (IsASpaceOrTab(sc.ch)) {             // skip whitespace
 770                                         sc.ChangeState(SCE_PL_DEFAULT);
 771                                         while (IsASpaceOrTab(sc.ch) && !sc.atLineEnd)
 772                                                 sc.Forward();
 773                                         sc.SetState(SCE_PL_FORMAT_IDENT);
 774                                 }
 775                                 if (setFormatStart.Contains(sc.ch)) {   // identifier or '='
 776                                         if (sc.ch != '=') {
 777                                                 do {
 778                                                         sc.Forward();
 779                                                 } while (setFormat.Contains(sc.ch));
 780                                         }
 781                                         while (IsASpaceOrTab(sc.ch) && !sc.atLineEnd)
 782                                                 sc.Forward();
 783                                         if (sc.ch == '=') {
 784                                                 sc.ForwardSetState(SCE_PL_DEFAULT);
 785                                                 HereDoc.State = 3;
 786                                         } else {
 787                                                 // invalid indentifier; inexact fallback, but hey
 788                                                 sc.ChangeState(SCE_PL_IDENTIFIER);
 789                                                 sc.SetState(SCE_PL_DEFAULT);
 790                                         }
 791                                 } else {
 792                                         sc.ChangeState(SCE_PL_DEFAULT); // invalid indentifier
 793                                 }
 794                                 backFlag = BACK_NONE;
 795                                 break;
 796                 }
 797
 798                 // Must check end of HereDoc states here before default state is handled
 799                 if (HereDoc.State == 1 && sc.atLineEnd) {
 800                         // Begin of here-doc (the line after the here-doc delimiter):
 801                         // Lexically, the here-doc starts from the next line after the >>, but the
 802                         // first line of here-doc seem to follow the style of the last EOL sequence
 803                         int st_new = SCE_PL_HERE_QQ;
 804                         HereDoc.State = 2;
 805                         if (HereDoc.Quoted) {
 806                                 if (sc.state == SCE_PL_HERE_DELIM) {
 807                                         // Missing quote at end of string! We are stricter than perl.
 808                                         // Colour here-doc anyway while marking this bit as an error.
 809                                         sc.ChangeState(SCE_PL_ERROR);
 810                                 }
 811                                 switch (HereDoc.Quote) {
 812                                         case '\'': st_new = SCE_PL_HERE_Q ; break;
 813                                         case '"' : st_new = SCE_PL_HERE_QQ; break;
 814                                         case '`' : st_new = SCE_PL_HERE_QX; break;
 815                                 }
 816                         } else {
 817                                 if (HereDoc.Quote == '\\')
 818                                         st_new = SCE_PL_HERE_Q;
 819                         }
 820                         sc.SetState(st_new);
 821                 }
 822                 if (HereDoc.State == 3 && sc.atLineEnd) {
 823                         // Start of format body.
 824                         HereDoc.State = 0;
 825                         sc.SetState(SCE_PL_FORMAT);
 826                 }
 827
 828                 // Determine if a new state should be entered.
 829                 if (sc.state == SCE_PL_DEFAULT) {
 830                         if (IsADigit(sc.ch) ||
 831                                 (IsADigit(sc.chNext) && (sc.ch == '.' || sc.ch == 'v'))) {
 832                                 sc.SetState(SCE_PL_NUMBER);
 833                                 backFlag = BACK_NONE;
 834                                 numState = PERLNUM_DECIMAL;
 835                                 dotCount = 0;
 836                                 if (sc.ch == '0') {             // hex,bin,octal
 837                                         if (sc.chNext == 'x') {
 838                                                 numState = PERLNUM_HEX;
 839                                         } else if (sc.chNext == 'b') {
 840                                                 numState = PERLNUM_BINARY;
 841                                         } else if (IsADigit(sc.chNext)) {
 842                                                 numState = PERLNUM_OCTAL;
 843                                         }
 844                                         if (numState != PERLNUM_DECIMAL) {
 845                                                 sc.Forward();
 846                                         }
 847                                 } else if (sc.ch == 'v') {              // vector
 848                                         numState = PERLNUM_V_VECTOR;
 849                                 }
 850                         } else if (setWord.Contains(sc.ch)) {
 851                                 // if immediately prefixed by '::', always a bareword
 852                                 sc.SetState(SCE_PL_WORD);
 853                                 if (sc.chPrev == ':' && sc.GetRelative(-2) == ':') {
 854                                         sc.ChangeState(SCE_PL_IDENTIFIER);
 855                                 }
 856                                 unsigned int bk = sc.currentPos;
 857                                 unsigned int fw = sc.currentPos + 1;
 858                                 // first check for possible quote-like delimiter
 859                                 if (sc.ch == 's' && !setWord.Contains(sc.chNext)) {
 860                                         sc.ChangeState(SCE_PL_REGSUBST);
 861                                         Quote.New(2);
 862                                 } else if (sc.ch == 'm' && !setWord.Contains(sc.chNext)) {
 863                                         sc.ChangeState(SCE_PL_REGEX);
 864                                         Quote.New();
 865                                 } else if (sc.ch == 'q' && !setWord.Contains(sc.chNext)) {
 866                                         sc.ChangeState(SCE_PL_STRING_Q);
 867                                         Quote.New();
 868                                 } else if (sc.ch == 'y' && !setWord.Contains(sc.chNext)) {
 869                                         sc.ChangeState(SCE_PL_REGSUBST);
 870                                         Quote.New(2);
 871                                 } else if (sc.Match('t', 'r') && !setWord.Contains(sc.GetRelative(2))) {
 872                                         sc.ChangeState(SCE_PL_REGSUBST);
 873                                         Quote.New(2);
 874                                         sc.Forward();
 875                                         fw++;
 876                                 } else if (sc.ch == 'q' && setQDelim.Contains(sc.chNext)
 877                                                    && !setWord.Contains(sc.GetRelative(2))) {
 878                                         if      (sc.chNext == 'q') sc.ChangeState(SCE_PL_STRING_QQ);
 879                                         else if (sc.chNext == 'x') sc.ChangeState(SCE_PL_STRING_QX);
 880                                         else if (sc.chNext == 'r') sc.ChangeState(SCE_PL_STRING_QR);
 881                                         else sc.ChangeState(SCE_PL_STRING_QW);  // sc.chNext == 'w'
 882                                         Quote.New();
 883                                         sc.Forward();
 884                                         fw++;
 885                                 } else if (sc.ch == 'x' && (sc.chNext == '=' || // repetition
 886                                                    !setWord.Contains(sc.chNext) ||
 887                                                    (IsADigit(sc.chPrev) && IsADigit(sc.chNext)))) {
 888                                         sc.ChangeState(SCE_PL_OPERATOR);
 889                                 }
 890                                 // if potentially a keyword, scan forward and grab word, then check
 891                                 // if it's really one; if yes, disambiguation test is performed
 892                                 // otherwise it is always a bareword and we skip a lot of scanning
 893                                 if (sc.state == SCE_PL_WORD) {
 894                                         while (setWord.Contains(static_cast<unsigned char>(styler.SafeGetCharAt(fw))))
 895                                                 fw++;
 896                                         if (!isPerlKeyword(styler.GetStartSegment(), fw, keywords, styler)) {
 897                                                 sc.ChangeState(SCE_PL_IDENTIFIER);
 898                                         }
 899                                 }
 900                                 // if already SCE_PL_IDENTIFIER, then no ambiguity, skip this
 901                                 // for quote-like delimiters/keywords, attempt to disambiguate
 902                                 // to select for bareword, change state -> SCE_PL_IDENTIFIER
 903                                 if (sc.state != SCE_PL_IDENTIFIER && bk > 0) {
 904                                         if (disambiguateBareword(styler, bk, fw, backFlag, backPos, endPos))
 905                                                 sc.ChangeState(SCE_PL_IDENTIFIER);
 906                                 }
 907                                 backFlag = BACK_NONE;
 908                         } else if (sc.ch == '#') {
 909                                 sc.SetState(SCE_PL_COMMENTLINE);
 910                         } else if (sc.ch == '\"') {
 911                                 sc.SetState(SCE_PL_STRING);
 912                                 Quote.New();
 913                                 Quote.Open(sc.ch);
 914                                 backFlag = BACK_NONE;
 915                         } else if (sc.ch == '\'') {
 916                                 if (sc.chPrev == '&' && setWordStart.Contains(sc.chNext)) {
 917                                         // Archaic call
 918                                         sc.SetState(SCE_PL_IDENTIFIER);
 919                                 } else {
 920                                         sc.SetState(SCE_PL_CHARACTER);
 921                                         Quote.New();
 922                                         Quote.Open(sc.ch);
 923                                 }
 924                                 backFlag = BACK_NONE;
 925                         } else if (sc.ch == '`') {
 926                                 sc.SetState(SCE_PL_BACKTICKS);
 927                                 Quote.New();
 928                                 Quote.Open(sc.ch);
 929                                 backFlag = BACK_NONE;
 930                         } else if (sc.ch == '$') {
 931                                 sc.SetState(SCE_PL_SCALAR);
 932                                 if (sc.chNext == '{') {
 933                                         sc.ForwardSetState(SCE_PL_OPERATOR);
 934                                 } else if (IsASpace(sc.chNext)) {
 935                                         sc.ForwardSetState(SCE_PL_DEFAULT);
 936                                 } else {
 937                                         sc.Forward();
 938                                         if (sc.Match('`', '`') || sc.Match(':', ':')) {
 939                                                 sc.Forward();
 940                                         }
 941                                 }
 942                                 backFlag = BACK_NONE;
 943                         } else if (sc.ch == '@') {
 944                                 sc.SetState(SCE_PL_ARRAY);
 945                                 if (setArray.Contains(sc.chNext)) {
 946                                         // no special treatment
 947                                 } else if (sc.chNext == ':' && sc.GetRelative(2) == ':') {
 948                                         sc.Forward(2);
 949                                 } else if (sc.chNext == '{' || sc.chNext == '[') {
 950                                         sc.ForwardSetState(SCE_PL_OPERATOR);
 951                                 } else {
 952                                         sc.ChangeState(SCE_PL_OPERATOR);
 953                                 }
 954                                 backFlag = BACK_NONE;
 955                         } else if (setPreferRE.Contains(sc.ch)) {
 956                                 // Explicit backward peeking to set a consistent preferRE for
 957                                 // any slash found, so no longer need to track preferRE state.
 958                                 // Find first previous significant lexed element and interpret.
 959                                 // A few symbols shares this code for disambiguation.
 960                                 bool preferRE = false;
 961                                 bool isHereDoc = sc.Match('<', '<');
 962                                 bool hereDocSpace = false;              // for: SCALAR [whitespace] '<<'
 963                                 unsigned int bk = (sc.currentPos > 0) ? sc.currentPos - 1: 0;
 964                                 unsigned int bkend;
 965                                 styler.Flush();
 966                                 if (styler.StyleAt(bk) == SCE_PL_DEFAULT)
 967                                         hereDocSpace = true;
 968                                 skipWhitespaceComment(styler, bk);
 969                                 if (bk == 0) {
 970                                         // avoid backward scanning breakage
 971                                         preferRE = true;
 972                                 } else {
 973                                         int bkstyle = styler.StyleAt(bk);
 974                                         int bkch = static_cast<unsigned char>(styler.SafeGetCharAt(bk));
 975                                         switch(bkstyle) {
 976                                         case SCE_PL_OPERATOR:
 977                                                 preferRE = true;
 978                                                 if (bkch == ')' || bkch == ']') {
 979                                                         preferRE = false;
 980                                                 } else if (bkch == '}') {
 981                                                         // backtrack by counting balanced brace pairs
 982                                                         // needed to test for variables like ${}, @{} etc.
 983                                                         bkstyle = styleBeforeBracePair(styler, bk);
 984                                                         if (bkstyle == SCE_PL_SCALAR
 985                                                                 || bkstyle == SCE_PL_ARRAY
 986                                                                 || bkstyle == SCE_PL_HASH
 987                                                                 || bkstyle == SCE_PL_SYMBOLTABLE
 988                                                                 || bkstyle == SCE_PL_OPERATOR) {
 989                                                                 preferRE = false;
 990                                                         }
 991                                                 } else if (bkch == '+' || bkch == '-') {
 992                                                         if (bkch == static_cast<unsigned char>(styler.SafeGetCharAt(bk - 1))
 993                                                                 && bkch != static_cast<unsigned char>(styler.SafeGetCharAt(bk - 2)))
 994                                                         // exceptions for operators: unary suffixes ++, --
 995                                                         preferRE = false;
 996                                                 }
 997                                                 break;
 998                                         case SCE_PL_IDENTIFIER:
 999                                                 preferRE = true;
1000                                                 bkstyle = styleCheckIdentifier(styler, bk);
1001                                                 if ((bkstyle == 1) || (bkstyle == 2)) {
1002                                                         // inputsymbol or var with "->" or "::" before identifier
1003                                                         preferRE = false;
1004                                                 } else if (bkstyle == 3) {
1005                                                         // bare identifier, test cases follows:
1006                                                         if (sc.ch == '/') {
1007                                                                 // if '/', /PATTERN/ unless digit/space immediately after '/'
1008                                                                 // if '//', always expect defined-or operator to follow identifier
1009                                                                 if (IsASpace(sc.chNext) || IsADigit(sc.chNext) || sc.chNext == '/')
1010                                                                         preferRE = false;
1011                                                         } else if (sc.ch == '*' || sc.ch == '%') {
1012                                                                 if (IsASpace(sc.chNext) || IsADigit(sc.chNext) || sc.Match('*', '*'))
1013                                                                         preferRE = false;
1014                                                         } else if (sc.ch == '<') {
1015                                                                 if (IsASpace(sc.chNext) || sc.chNext == '=')
1016                                                                         preferRE = false;
1017                                                         }
1018                                                 }
1019                                                 break;
1020                                         case SCE_PL_SCALAR:             // for $var<< case:
1021                                                 if (isHereDoc && hereDocSpace)  // if SCALAR whitespace '<<', *always* a HERE doc
1022                                                         preferRE = true;
1023                                                 break;
1024                                         case SCE_PL_WORD:
1025                                                 preferRE = true;
1026                                                 // for HERE docs, always true
1027                                                 if (sc.ch == '/') {
1028                                                         // adopt heuristics similar to vim-style rules:
1029                                                         // keywords always forced as /PATTERN/: split, if, elsif, while
1030                                                         // everything else /PATTERN/ unless digit/space immediately after '/'
1031                                                         // for '//', defined-or favoured unless special keywords
1032                                                         bkend = bk + 1;
1033                                                         while (bk > 0 && styler.StyleAt(bk - 1) == SCE_PL_WORD) {
1034                                                                 bk--;
1035                                                         }
1036                                                         if (isPerlKeyword(bk, bkend, reWords, styler))
1037                                                                 break;
1038                                                         if (IsASpace(sc.chNext) || IsADigit(sc.chNext) || sc.chNext == '/')
1039                                                                 preferRE = false;
1040                                                 } else if (sc.ch == '*' || sc.ch == '%') {
1041                                                         if (IsASpace(sc.chNext) || IsADigit(sc.chNext) || sc.Match('*', '*'))
1042                                                                 preferRE = false;
1043                                                 } else if (sc.ch == '<') {
1044                                                         if (IsASpace(sc.chNext) || sc.chNext == '=')
1045                                                                 preferRE = false;
1046                                                 }
1047                                                 break;
1048                                         // other styles uses the default, preferRE=false
1049                                         case SCE_PL_POD:
1050                                         case SCE_PL_HERE_Q:
1051                                         case SCE_PL_HERE_QQ:
1052                                         case SCE_PL_HERE_QX:
1053                                                 preferRE = true;
1054                                                 break;
1055                                         }
1056                                 }
1057                                 backFlag = BACK_NONE;
1058                                 if (isHereDoc) {        // handle '<<', HERE doc
1059                                         if (preferRE) {
1060                                                 sc.SetState(SCE_PL_HERE_DELIM);
1061                                                 HereDoc.State = 0;
1062                                         } else {                // << operator
1063                                                 sc.SetState(SCE_PL_OPERATOR);
1064                                                 sc.Forward();
1065                                         }
1066                                 } else if (sc.ch == '*') {      // handle '*', typeglob
1067                                         if (preferRE) {
1068                                                 sc.SetState(SCE_PL_SYMBOLTABLE);
1069                                                 if (sc.chNext == ':' && sc.GetRelative(2) == ':') {
1070                                                         sc.Forward(2);
1071                                                 } else if (sc.chNext == '{') {
1072                                                         sc.ForwardSetState(SCE_PL_OPERATOR);
1073                                                 } else {
1074                                                         sc.Forward();
1075                                                 }
1076                                         } else {
1077                                                 sc.SetState(SCE_PL_OPERATOR);
1078                                                 if (sc.chNext == '*')   // exponentiation
1079                                                         sc.Forward();
1080                                         }
1081                                 } else if (sc.ch == '%') {      // handle '%', hash
1082                                         if (preferRE) {
1083                                                 sc.SetState(SCE_PL_HASH);
1084                                                 if (setHash.Contains(sc.chNext)) {
1085                                                         sc.Forward();
1086                                                 } else if (sc.chNext == ':' && sc.GetRelative(2) == ':') {
1087                                                         sc.Forward(2);
1088                                                 } else if (sc.chNext == '{') {
1089                                                         sc.ForwardSetState(SCE_PL_OPERATOR);
1090                                                 } else {
1091                                                         sc.ChangeState(SCE_PL_OPERATOR);
1092                                                 }
1093                                         } else {
1094                                                 sc.SetState(SCE_PL_OPERATOR);
1095                                         }
1096                                 } else if (sc.ch == '<') {      // handle '<', inputsymbol
1097                                         if (preferRE) {
1098                                                 // forward scan
1099                                                 int i = inputsymbolScan(styler, sc.currentPos, endPos);
1100                                                 if (i > 0) {
1101                                                         sc.SetState(SCE_PL_IDENTIFIER);
1102                                                         sc.Forward(i);
1103                                                 } else {
1104                                                         sc.SetState(SCE_PL_OPERATOR);
1105                                                 }
1106                                         } else {
1107                                                 sc.SetState(SCE_PL_OPERATOR);
1108                                         }
1109                                 } else {                        // handle '/', regexp
1110                                         if (preferRE) {
1111                                                 sc.SetState(SCE_PL_REGEX);
1112                                                 Quote.New();
1113                                                 Quote.Open(sc.ch);
1114                                         } else {                // / and // operators
1115                                                 sc.SetState(SCE_PL_OPERATOR);
1116                                                 if (sc.chNext == '/') {
1117                                                         sc.Forward();
1118                                                 }
1119                                         }
1120                                 }
1121                         } else if (sc.ch == '='         // POD
1122                                            && setPOD.Contains(sc.chNext)
1123                                            && sc.atLineStart) {
1124                                 sc.SetState(SCE_PL_POD);
1125                                 backFlag = BACK_NONE;
1126                         } else if (sc.ch == '-' && setWordStart.Contains(sc.chNext)) {  // extended '-' cases
1127                                 unsigned int bk = sc.currentPos;
1128                                 unsigned int fw = 2;
1129                                 if (setSingleCharOp.Contains(sc.chNext) &&      // file test operators
1130                                         !setWord.Contains(sc.GetRelative(2))) {
1131                                         sc.SetState(SCE_PL_WORD);
1132                                 } else {
1133                                         // nominally a minus and bareword; find extent of bareword
1134                                         while (setWord.Contains(sc.GetRelative(fw)))
1135                                                 fw++;
1136                                         sc.SetState(SCE_PL_OPERATOR);
1137                                 }
1138                                 // force to bareword for hash key => or {variable literal} cases
1139                                 if (disambiguateBareword(styler, bk, bk + fw, backFlag, backPos, endPos) & 2) {
1140                                         sc.ChangeState(SCE_PL_IDENTIFIER);
1141                                 }
1142                                 backFlag = BACK_NONE;
1143                         } else if (sc.ch == '(' && sc.currentPos > 0) { // '(' or subroutine prototype
1144                                 if (styleCheckSubPrototype(styler, sc.currentPos - 1)) {
1145                                         sc.SetState(SCE_PL_SUB_PROTOTYPE);
1146                                         backFlag = BACK_NONE;
1147                                 } else {
1148                                         sc.SetState(SCE_PL_OPERATOR);
1149                                 }
1150                         } else if (setPerlOperator.Contains(sc.ch)) {   // operators
1151                                 sc.SetState(SCE_PL_OPERATOR);
1152                                 if (sc.Match('.', '.')) {       // .. and ...
1153                                         sc.Forward();
1154                                         if (sc.chNext == '.') sc.Forward();
1155                                 }
1156                         } else if (sc.ch == 4 || sc.ch == 26) {         // ^D and ^Z ends valid perl source
1157                                 sc.SetState(SCE_PL_DATASECTION);
1158                         } else {
1159                                 // keep colouring defaults
1160                                 sc.Complete();
1161                         }
1162                 }
1163         }
1164         sc.Complete();
1165 }
1166
1167 static bool IsCommentLine(int line, Accessor &styler) {
1168         int pos = styler.LineStart(line);
1169         int eol_pos = styler.LineStart(line + 1) - 1;
1170         for (int i = pos; i < eol_pos; i++) {
1171                 char ch = styler[i];
1172                 int style = styler.StyleAt(i);
1173                 if (ch == '#' && style == SCE_PL_COMMENTLINE)
1174                         return true;
1175                 else if (!IsASpaceOrTab(ch))
1176                         return false;
1177         }
1178         return false;
1179 }
1180
1181 static void FoldPerlDoc(unsigned int startPos, int length, int, WordList *[],
1182                         Accessor &styler) {
1183         bool foldComment = styler.GetPropertyInt("fold.comment") != 0;
1184         bool foldCompact = styler.GetPropertyInt("fold.compact", 1) != 0;
1185         // Custom folding of POD and packages
1186         bool foldPOD = styler.GetPropertyInt("fold.perl.pod", 1) != 0;
1187         bool foldPackage = styler.GetPropertyInt("fold.perl.package", 1) != 0;
1188         unsigned int endPos = startPos + length;
1189         int visibleChars = 0;
1190         int lineCurrent = styler.GetLine(startPos);
1191         int levelPrev = SC_FOLDLEVELBASE;
1192         if (lineCurrent > 0)
1193                 levelPrev = styler.LevelAt(lineCurrent - 1) >> 16;
1194         int levelCurrent = levelPrev;
1195         char chNext = styler[startPos];
1196         char chPrev = styler.SafeGetCharAt(startPos - 1);
1197         int styleNext = styler.StyleAt(startPos);
1198         // Used at end of line to determine if the line was a package definition
1199         bool isPackageLine = false;
1200         bool isPodHeading = false;
1201         for (unsigned int i = startPos; i < endPos; i++) {
1202                 char ch = chNext;
1203                 chNext = styler.SafeGetCharAt(i + 1);
1204                 int style = styleNext;
1205                 styleNext = styler.StyleAt(i + 1);
1206                 bool atEOL = (ch == '\r' && chNext != '\n') || (ch == '\n');
1207                 bool atLineStart = ((chPrev == '\r') || (chPrev == '\n')) || i == 0;
1208                 // Comment folding
1209                 if (foldComment && atEOL && IsCommentLine(lineCurrent, styler))
1210                 {
1211                         if (!IsCommentLine(lineCurrent - 1, styler)
1212                                 && IsCommentLine(lineCurrent + 1, styler))
1213                                 levelCurrent++;
1214                         else if (IsCommentLine(lineCurrent - 1, styler)
1215                                          && !IsCommentLine(lineCurrent+1, styler))
1216                                 levelCurrent--;
1217                 }
1218                 if (style == SCE_PL_OPERATOR) {
1219                         if (ch == '{') {
1220                                 levelCurrent++;
1221                         } else if (ch == '}') {
1222                                 levelCurrent--;
1223                         }
1224                 }
1225                 // Custom POD folding
1226                 if (foldPOD && atLineStart) {
1227                         int stylePrevCh = (i) ? styler.StyleAt(i - 1):SCE_PL_DEFAULT;
1228                         if (style == SCE_PL_POD) {
1229                                 if (stylePrevCh != SCE_PL_POD && stylePrevCh != SCE_PL_POD_VERB)
1230                                         levelCurrent++;
1231                                 else if (styler.Match(i, "=cut"))
1232                                         levelCurrent--;
1233                                 else if (styler.Match(i, "=head"))
1234                                         isPodHeading = true;
1235                         } else if (style == SCE_PL_DATASECTION) {
1236                                 if (ch == '=' && isalpha(chNext) && levelCurrent == SC_FOLDLEVELBASE)
1237                                         levelCurrent++;
1238                                 else if (styler.Match(i, "=cut") && levelCurrent > SC_FOLDLEVELBASE)
1239                                         levelCurrent--;
1240                                 else if (styler.Match(i, "=head"))
1241                                         isPodHeading = true;
1242                                 // if package used or unclosed brace, level > SC_FOLDLEVELBASE!
1243                                 // reset needed as level test is vs. SC_FOLDLEVELBASE
1244                                 else if (styler.Match(i, "__END__"))
1245                                         levelCurrent = SC_FOLDLEVELBASE;
1246                         }
1247                 }
1248                 // Custom package folding
1249                 if (foldPackage && atLineStart) {
1250                         if (style == SCE_PL_WORD && styler.Match(i, "package")) {
1251                                 isPackageLine = true;
1252                         }
1253                 }
1254
1255                 if (atEOL) {
1256                         int lev = levelPrev;
1257                         if (isPodHeading) {
1258                                 lev = levelPrev - 1;
1259                                 lev |= SC_FOLDLEVELHEADERFLAG;
1260                                 isPodHeading = false;
1261                         }
1262                         // Check if line was a package declaration
1263                         // because packages need "special" treatment
1264                         if (isPackageLine) {
1265                                 lev = SC_FOLDLEVELBASE | SC_FOLDLEVELHEADERFLAG;
1266                                 levelCurrent = SC_FOLDLEVELBASE + 1;
1267                                 isPackageLine = false;
1268                         }
1269                         lev |= levelCurrent << 16;
1270                         if (visibleChars == 0 && foldCompact)
1271                                 lev |= SC_FOLDLEVELWHITEFLAG;
1272                         if ((levelCurrent > levelPrev) && (visibleChars > 0))
1273                                 lev |= SC_FOLDLEVELHEADERFLAG;
1274                         if (lev != styler.LevelAt(lineCurrent)) {
1275                                 styler.SetLevel(lineCurrent, lev);
1276                         }
1277                         lineCurrent++;
1278                         levelPrev = levelCurrent;
1279                         visibleChars = 0;
1280                 }
1281                 if (!isspacechar(ch))
1282                         visibleChars++;
1283                 chPrev = ch;
1284         }
1285         // Fill in the real level of the next line, keeping the current flags as they will be filled in later
1286         int flagsNext = styler.LevelAt(lineCurrent) & ~SC_FOLDLEVELNUMBERMASK;
1287         styler.SetLevel(lineCurrent, levelPrev | flagsNext);
1288 }
1289
1290 static const char * const perlWordListDesc[] = {
1291         "Keywords",
1292         0
1293 };
1294
1295 LexerModule lmPerl(SCLEX_PERL, ColourisePerlDoc, "perl", FoldPerlDoc, perlWordListDesc, 8);