scintilla/lexers/LexYAML.cxx

   1 // Scintilla source code edit control
   2 /** @file LexYAML.cxx
   3  ** Lexer for YAML.
   4  **/
   5 // Copyright 2003- by Sean O'Dell <sean@celsoft.com>
   6 // The License.txt file describes the conditions under which this software may be distributed.
   7
   8 #include <stdlib.h>
   9 #include <string.h>
  10 #include <stdio.h>
  11 #include <stdarg.h>
  12 #include <assert.h>
  13 #include <ctype.h>
  14
  15 #include "ILexer.h"
  16 #include "Scintilla.h"
  17 #include "SciLexer.h"
  18
  19 #include "WordList.h"
  20 #include "LexAccessor.h"
  21 #include "Accessor.h"
  22 #include "StyleContext.h"
  23 #include "CharacterSet.h"
  24 #include "LexerModule.h"
  25
  26 #ifdef SCI_NAMESPACE
  27 using namespace Scintilla;
  28 #endif
  29
  30 static const char * const yamlWordListDesc[] = {
  31         "Keywords",
  32         0
  33 };
  34
  35 static inline bool AtEOL(Accessor &styler, Sci_PositionU i) {
  36         return (styler[i] == '\n') ||
  37                 ((styler[i] == '\r') && (styler.SafeGetCharAt(i + 1) != '\n'));
  38 }
  39
  40 static unsigned int SpaceCount(char* lineBuffer) {
  41         if (lineBuffer == NULL)
  42                 return 0;
  43
  44         char* headBuffer = lineBuffer;
  45
  46         while (*headBuffer == ' ')
  47                 headBuffer++;
  48
  49         return static_cast<unsigned int>(headBuffer - lineBuffer);
  50 }
  51
  52 static bool KeywordAtChar(char* lineBuffer, char* startComment, const WordList &keywords) {
  53         if (lineBuffer == NULL || startComment <= lineBuffer)
  54                 return false;
  55         char* endValue = startComment - 1;
  56         while (endValue >= lineBuffer && *endValue == ' ')
  57                 endValue--;
  58         Sci_PositionU len = static_cast<Sci_PositionU>(endValue - lineBuffer) + 1;
  59         char s[100];
  60         if (len > (sizeof(s) / sizeof(s[0]) - 1))
  61                 return false;
  62         strncpy(s, lineBuffer, len);
  63         s[len] = '\0';
  64         return (keywords.InList(s));
  65 }
  66
  67 #define YAML_STATE_BITSIZE              16
  68 #define YAML_STATE_MASK                 (0xFFFF0000)
  69 #define YAML_STATE_DOCUMENT             (1 << YAML_STATE_BITSIZE)
  70 #define YAML_STATE_VALUE                (2 << YAML_STATE_BITSIZE)
  71 #define YAML_STATE_COMMENT              (3 << YAML_STATE_BITSIZE)
  72 #define YAML_STATE_TEXT_PARENT  (4 << YAML_STATE_BITSIZE)
  73 #define YAML_STATE_TEXT                 (5 << YAML_STATE_BITSIZE)
  74
  75 static void ColouriseYAMLLine(
  76         char *lineBuffer,
  77         Sci_PositionU currentLine,
  78         Sci_PositionU lengthLine,
  79         Sci_PositionU startLine,
  80         Sci_PositionU endPos,
  81         WordList &keywords,
  82         Accessor &styler) {
  83
  84         Sci_PositionU i = 0;
  85         bool bInQuotes = false;
  86         unsigned int indentAmount = SpaceCount(lineBuffer);
  87
  88         if (currentLine > 0) {
  89                 int parentLineState = styler.GetLineState(currentLine - 1);
  90
  91                 if ((parentLineState&YAML_STATE_MASK) == YAML_STATE_TEXT || (parentLineState&YAML_STATE_MASK) == YAML_STATE_TEXT_PARENT) {
  92                         unsigned int parentIndentAmount = parentLineState&(~YAML_STATE_MASK);
  93                         if (indentAmount > parentIndentAmount) {
  94                                 styler.SetLineState(currentLine, YAML_STATE_TEXT | parentIndentAmount);
  95                                 styler.ColourTo(endPos, SCE_YAML_TEXT);
  96                                 return;
  97                         }
  98                 }
  99         }
 100         styler.SetLineState(currentLine, 0);
 101         if (strncmp(lineBuffer, "---", 3) == 0 || strncmp(lineBuffer, "...", 3) == 0) { // Document marker
 102                 styler.SetLineState(currentLine, YAML_STATE_DOCUMENT);
 103                 styler.ColourTo(endPos, SCE_YAML_DOCUMENT);
 104                 return;
 105         }
 106         // Skip initial spaces
 107         while ((i < lengthLine) && lineBuffer[i] == ' ') { // YAML always uses space, never TABS or anything else
 108                 i++;
 109         }
 110         if (lineBuffer[i] == '\t') { // if we skipped all spaces, and we are NOT inside a text block, this is wrong
 111                 styler.ColourTo(endPos, SCE_YAML_ERROR);
 112                 return;
 113         }
 114         if (lineBuffer[i] == '#') {     // Comment
 115                 styler.SetLineState(currentLine, YAML_STATE_COMMENT);
 116                 styler.ColourTo(endPos, SCE_YAML_COMMENT);
 117                 return;
 118         }
 119         while (i < lengthLine) {
 120                 if (lineBuffer[i] == '\'' || lineBuffer[i] == '\"') {
 121                         bInQuotes = !bInQuotes;
 122                 } else if (lineBuffer[i] == '#' && isspacechar(lineBuffer[i - 1]) && !bInQuotes) {
 123                         styler.ColourTo(startLine + i - 1, SCE_YAML_DEFAULT);
 124                         styler.ColourTo(endPos, SCE_YAML_COMMENT);
 125                         return;
 126                 } else if (lineBuffer[i] == ':' && !bInQuotes) {
 127                         styler.ColourTo(startLine + i - 1, SCE_YAML_IDENTIFIER);
 128                         styler.ColourTo(startLine + i, SCE_YAML_OPERATOR);
 129                         // Non-folding scalar
 130                         i++;
 131                         while ((i < lengthLine) && isspacechar(lineBuffer[i]))
 132                                 i++;
 133                         Sci_PositionU endValue = lengthLine - 1;
 134                         while ((endValue >= i) && isspacechar(lineBuffer[endValue]))
 135                                 endValue--;
 136                         lineBuffer[endValue + 1] = '\0';
 137                         if (lineBuffer[i] == '|' || lineBuffer[i] == '>') {
 138                                 i++;
 139                                 if (lineBuffer[i] == '+' || lineBuffer[i] == '-')
 140                                         i++;
 141                                 while ((i < lengthLine) && isspacechar(lineBuffer[i]))
 142                                         i++;
 143                                 if (lineBuffer[i] == '\0') {
 144                                         styler.SetLineState(currentLine, YAML_STATE_TEXT_PARENT | indentAmount);
 145                                         styler.ColourTo(endPos, SCE_YAML_DEFAULT);
 146                                         return;
 147                                 } else if (lineBuffer[i] == '#') {
 148                                         styler.SetLineState(currentLine, YAML_STATE_TEXT_PARENT | indentAmount);
 149                                         styler.ColourTo(startLine + i - 1, SCE_YAML_DEFAULT);
 150                                         styler.ColourTo(endPos, SCE_YAML_COMMENT);
 151                                         return;
 152                                 } else {
 153                                         styler.ColourTo(endPos, SCE_YAML_ERROR);
 154                                         return;
 155                                 }
 156                         } else if (lineBuffer[i] == '#') {
 157                                 styler.ColourTo(startLine + i - 1, SCE_YAML_DEFAULT);
 158                                 styler.ColourTo(endPos, SCE_YAML_COMMENT);
 159                                 return;
 160                         }
 161                         Sci_PositionU startComment = i;
 162                         bInQuotes = false;
 163                         while (startComment < lengthLine) { // Comment must be space padded
 164                                 if (lineBuffer[startComment] == '\'' || lineBuffer[startComment] == '\"')
 165                                         bInQuotes = !bInQuotes;
 166                                 if (lineBuffer[startComment] == '#' && isspacechar(lineBuffer[startComment - 1]) && !bInQuotes)
 167                                         break;
 168                                 startComment++;
 169                         }
 170                         styler.SetLineState(currentLine, YAML_STATE_VALUE);
 171                         if (lineBuffer[i] == '&' || lineBuffer[i] == '*') {
 172                                 styler.ColourTo(startLine + startComment - 1, SCE_YAML_REFERENCE);
 173                                 if (startComment < lengthLine)
 174                                         styler.ColourTo(endPos, SCE_YAML_COMMENT);
 175                                 return;
 176                         }
 177                         if (KeywordAtChar(&lineBuffer[i], &lineBuffer[startComment], keywords)) { // Convertible value (true/false, etc.)
 178                                 styler.ColourTo(startLine + startComment - 1, SCE_YAML_KEYWORD);
 179                                 if (startComment < lengthLine)
 180                                         styler.ColourTo(endPos, SCE_YAML_COMMENT);
 181                                 return;
 182                         }
 183                         Sci_PositionU i2 = i;
 184                         while ((i < startComment) && lineBuffer[i]) {
 185                                 if (!(IsASCII(lineBuffer[i]) && isdigit(lineBuffer[i])) && lineBuffer[i] != '-'
 186                                         && lineBuffer[i] != '.' && lineBuffer[i] != ',' && lineBuffer[i] != ' ') {
 187                                         styler.ColourTo(startLine + startComment - 1, SCE_YAML_DEFAULT);
 188                                         if (startComment < lengthLine)
 189                                                 styler.ColourTo(endPos, SCE_YAML_COMMENT);
 190                                         return;
 191                                 }
 192                                 i++;
 193                         }
 194                         if (i > i2) {
 195                                 styler.ColourTo(startLine + startComment - 1, SCE_YAML_NUMBER);
 196                                 if (startComment < lengthLine)
 197                                         styler.ColourTo(endPos, SCE_YAML_COMMENT);
 198                                 return;
 199                         }
 200                         break; // shouldn't get here, but just in case, the rest of the line is coloured the default
 201                 }
 202                 i++;
 203         }
 204         styler.ColourTo(endPos, SCE_YAML_DEFAULT);
 205 }
 206
 207 static void ColouriseYAMLDoc(Sci_PositionU startPos, Sci_Position length, int, WordList *keywordLists[], Accessor &styler) {
 208         char lineBuffer[1024] = "";
 209         styler.StartAt(startPos);
 210         styler.StartSegment(startPos);
 211         Sci_PositionU linePos = 0;
 212         Sci_PositionU startLine = startPos;
 213         Sci_PositionU endPos = startPos + length;
 214         Sci_PositionU maxPos = styler.Length();
 215         Sci_PositionU lineCurrent = styler.GetLine(startPos);
 216
 217         for (Sci_PositionU i = startPos; i < maxPos && i < endPos; i++) {
 218                 lineBuffer[linePos++] = styler[i];
 219                 if (AtEOL(styler, i) || (linePos >= sizeof(lineBuffer) - 1)) {
 220                         // End of line (or of line buffer) met, colourise it
 221                         lineBuffer[linePos] = '\0';
 222                         ColouriseYAMLLine(lineBuffer, lineCurrent, linePos, startLine, i, *keywordLists[0], styler);
 223                         linePos = 0;
 224                         startLine = i + 1;
 225                         lineCurrent++;
 226                 }
 227         }
 228         if (linePos > 0) {      // Last line does not have ending characters
 229                 ColouriseYAMLLine(lineBuffer, lineCurrent, linePos, startLine, startPos + length - 1, *keywordLists[0], styler);
 230         }
 231 }
 232
 233 static bool IsCommentLine(Sci_Position line, Accessor &styler) {
 234         Sci_Position pos = styler.LineStart(line);
 235         if (styler[pos] == '#')
 236                 return true;
 237         return false;
 238 }
 239
 240 static void FoldYAMLDoc(Sci_PositionU startPos, Sci_Position length, int /*initStyle - unused*/,
 241                       WordList *[], Accessor &styler) {
 242         const Sci_Position maxPos = startPos + length;
 243         const Sci_Position maxLines = styler.GetLine(maxPos - 1);             // Requested last line
 244         const Sci_Position docLines = styler.GetLine(styler.Length() - 1);  // Available last line
 245         const bool foldComment = styler.GetPropertyInt("fold.comment.yaml") != 0;
 246
 247         // Backtrack to previous non-blank line so we can determine indent level
 248         // for any white space lines
 249         // and so we can fix any preceding fold level (which is why we go back
 250         // at least one line in all cases)
 251         int spaceFlags = 0;
 252         Sci_Position lineCurrent = styler.GetLine(startPos);
 253         int indentCurrent = styler.IndentAmount(lineCurrent, &spaceFlags, NULL);
 254         while (lineCurrent > 0) {
 255                 lineCurrent--;
 256                 indentCurrent = styler.IndentAmount(lineCurrent, &spaceFlags, NULL);
 257                 if (!(indentCurrent & SC_FOLDLEVELWHITEFLAG) &&
 258                         (!IsCommentLine(lineCurrent, styler)))
 259                         break;
 260         }
 261         int indentCurrentLevel = indentCurrent & SC_FOLDLEVELNUMBERMASK;
 262
 263         // Set up initial loop state
 264         int prevComment = 0;
 265         if (lineCurrent >= 1)
 266                 prevComment = foldComment && IsCommentLine(lineCurrent - 1, styler);
 267
 268         // Process all characters to end of requested range
 269         // or comment that hangs over the end of the range.  Cap processing in all cases
 270         // to end of document (in case of unclosed comment at end).
 271         while ((lineCurrent <= docLines) && ((lineCurrent <= maxLines) || prevComment)) {
 272
 273                 // Gather info
 274                 int lev = indentCurrent;
 275                 Sci_Position lineNext = lineCurrent + 1;
 276                 int indentNext = indentCurrent;
 277                 if (lineNext <= docLines) {
 278                         // Information about next line is only available if not at end of document
 279                         indentNext = styler.IndentAmount(lineNext, &spaceFlags, NULL);
 280                 }
 281                 const int comment = foldComment && IsCommentLine(lineCurrent, styler);
 282                 const int comment_start = (comment && !prevComment && (lineNext <= docLines) &&
 283                                            IsCommentLine(lineNext, styler) && (lev > SC_FOLDLEVELBASE));
 284                 const int comment_continue = (comment && prevComment);
 285                 if (!comment)
 286                         indentCurrentLevel = indentCurrent & SC_FOLDLEVELNUMBERMASK;
 287                 if (indentNext & SC_FOLDLEVELWHITEFLAG)
 288                         indentNext = SC_FOLDLEVELWHITEFLAG | indentCurrentLevel;
 289
 290                 if (comment_start) {
 291                         // Place fold point at start of a block of comments
 292                         lev |= SC_FOLDLEVELHEADERFLAG;
 293                 } else if (comment_continue) {
 294                         // Add level to rest of lines in the block
 295                         lev = lev + 1;
 296                 }
 297
 298                 // Skip past any blank lines for next indent level info; we skip also
 299                 // comments (all comments, not just those starting in column 0)
 300                 // which effectively folds them into surrounding code rather
 301                 // than screwing up folding.
 302
 303                 while ((lineNext < docLines) &&
 304                         ((indentNext & SC_FOLDLEVELWHITEFLAG) ||
 305                          (lineNext <= docLines && IsCommentLine(lineNext, styler)))) {
 306
 307                         lineNext++;
 308                         indentNext = styler.IndentAmount(lineNext, &spaceFlags, NULL);
 309                 }
 310
 311                 const int levelAfterComments = indentNext & SC_FOLDLEVELNUMBERMASK;
 312                 const int levelBeforeComments = Maximum(indentCurrentLevel,levelAfterComments);
 313
 314                 // Now set all the indent levels on the lines we skipped
 315                 // Do this from end to start.  Once we encounter one line
 316                 // which is indented more than the line after the end of
 317                 // the comment-block, use the level of the block before
 318
 319                 Sci_Position skipLine = lineNext;
 320                 int skipLevel = levelAfterComments;
 321
 322                 while (--skipLine > lineCurrent) {
 323                         int skipLineIndent = styler.IndentAmount(skipLine, &spaceFlags, NULL);
 324
 325                         if ((skipLineIndent & SC_FOLDLEVELNUMBERMASK) > levelAfterComments)
 326                                 skipLevel = levelBeforeComments;
 327
 328                         int whiteFlag = skipLineIndent & SC_FOLDLEVELWHITEFLAG;
 329
 330                         styler.SetLevel(skipLine, skipLevel | whiteFlag);
 331                 }
 332
 333                 // Set fold header on non-comment line
 334                 if (!comment && !(indentCurrent & SC_FOLDLEVELWHITEFLAG) ) {
 335                         if ((indentCurrent & SC_FOLDLEVELNUMBERMASK) < (indentNext & SC_FOLDLEVELNUMBERMASK))
 336                                 lev |= SC_FOLDLEVELHEADERFLAG;
 337                 }
 338
 339                 // Keep track of block comment state of previous line
 340                 prevComment = comment_start || comment_continue;
 341
 342                 // Set fold level for this line and move to next line
 343                 styler.SetLevel(lineCurrent, lev);
 344                 indentCurrent = indentNext;
 345                 lineCurrent = lineNext;
 346         }
 347
 348         // NOTE: Cannot set level of last line here because indentCurrent doesn't have
 349         // header flag set; the loop above is crafted to take care of this case!
 350         //styler.SetLevel(lineCurrent, indentCurrent);
 351 }
 352
 353 LexerModule lmYAML(SCLEX_YAML, ColouriseYAMLDoc, "yaml", FoldYAMLDoc, yamlWordListDesc);