scintilla/lexers/LexYAML.cxx

   1 // Scintilla source code edit control
   2 /** @file LexYAML.cxx
   3  ** Lexer for YAML.
   4  **/
   5 // Copyright 2003- by Sean O'Dell <sean@celsoft.com>
   6 // The License.txt file describes the conditions under which this software may be distributed.
   7
   8 #include <stdlib.h>
   9 #include <string.h>
  10 #include <stdio.h>
  11 #include <stdarg.h>
  12 #include <assert.h>
  13 #include <ctype.h>
  14
  15 #include "ILexer.h"
  16 #include "Scintilla.h"
  17 #include "SciLexer.h"
  18
  19 #include "WordList.h"
  20 #include "LexAccessor.h"
  21 #include "Accessor.h"
  22 #include "StyleContext.h"
  23 #include "CharacterSet.h"
  24 #include "LexerModule.h"
  25
  26 #ifdef SCI_NAMESPACE
  27 using namespace Scintilla;
  28 #endif
  29
  30 static const char * const yamlWordListDesc[] = {
  31         "Keywords",
  32         0
  33 };
  34
  35 static inline bool AtEOL(Accessor &styler, Sci_PositionU i) {
  36         return (styler[i] == '\n') ||
  37                 ((styler[i] == '\r') && (styler.SafeGetCharAt(i + 1) != '\n'));
  38 }
  39
  40 static unsigned int SpaceCount(char* lineBuffer) {
  41         if (lineBuffer == NULL)
  42                 return 0;
  43
  44         char* headBuffer = lineBuffer;
  45
  46         while (*headBuffer == ' ')
  47                 headBuffer++;
  48
  49         return static_cast<unsigned int>(headBuffer - lineBuffer);
  50 }
  51
  52 static bool KeywordAtChar(char* lineBuffer, char* startComment, const WordList &keywords) {
  53         if (lineBuffer == NULL || startComment <= lineBuffer)
  54                 return false;
  55         char* endValue = startComment - 1;
  56         while (endValue >= lineBuffer && *endValue == ' ')
  57                 endValue--;
  58         Sci_PositionU len = static_cast<Sci_PositionU>(endValue - lineBuffer) + 1;
  59         char s[100];
  60         if (len > (sizeof(s) / sizeof(s[0]) - 1))
  61                 return false;
  62         strncpy(s, lineBuffer, len);
  63         s[len] = '\0';
  64         return (keywords.InList(s));
  65 }
  66
  67 #define YAML_STATE_BITSIZE              16
  68 #define YAML_STATE_MASK                 (0xFFFF0000)
  69 #define YAML_STATE_DOCUMENT             (1 << YAML_STATE_BITSIZE)
  70 #define YAML_STATE_VALUE                (2 << YAML_STATE_BITSIZE)
  71 #define YAML_STATE_COMMENT              (3 << YAML_STATE_BITSIZE)
  72 #define YAML_STATE_TEXT_PARENT  (4 << YAML_STATE_BITSIZE)
  73 #define YAML_STATE_TEXT                 (5 << YAML_STATE_BITSIZE)
  74
  75 static void ColouriseYAMLLine(
  76         char *lineBuffer,
  77         Sci_PositionU currentLine,
  78         Sci_PositionU lengthLine,
  79         Sci_PositionU startLine,
  80         Sci_PositionU endPos,
  81         WordList &keywords,
  82         Accessor &styler) {
  83
  84         Sci_PositionU i = 0;
  85         bool bInQuotes = false;
  86         unsigned int indentAmount = SpaceCount(lineBuffer);
  87
  88         if (currentLine > 0) {
  89                 int parentLineState = styler.GetLineState(currentLine - 1);
  90
  91                 if ((parentLineState&YAML_STATE_MASK) == YAML_STATE_TEXT || (parentLineState&YAML_STATE_MASK) == YAML_STATE_TEXT_PARENT) {
  92                         unsigned int parentIndentAmount = parentLineState&(~YAML_STATE_MASK);
  93                         if (indentAmount > parentIndentAmount) {
  94                                 styler.SetLineState(currentLine, YAML_STATE_TEXT | parentIndentAmount);
  95                                 styler.ColourTo(endPos, SCE_YAML_TEXT);
  96                                 return;
  97                         }
  98                 }
  99         }
 100         styler.SetLineState(currentLine, 0);
 101         if (strncmp(lineBuffer, "---", 3) == 0) {       // Document marker
 102                 styler.SetLineState(currentLine, YAML_STATE_DOCUMENT);
 103                 styler.ColourTo(endPos, SCE_YAML_DOCUMENT);
 104                 return;
 105         }
 106         // Skip initial spaces
 107         while ((i < lengthLine) && lineBuffer[i] == ' ') { // YAML always uses space, never TABS or anything else
 108                 i++;
 109         }
 110         if (lineBuffer[i] == '\t') { // if we skipped all spaces, and we are NOT inside a text block, this is wrong
 111                 styler.ColourTo(endPos, SCE_YAML_ERROR);
 112                 return;
 113         }
 114         if (lineBuffer[i] == '#') {     // Comment
 115                 styler.SetLineState(currentLine, YAML_STATE_COMMENT);
 116                 styler.ColourTo(endPos, SCE_YAML_COMMENT);
 117                 return;
 118         }
 119         while (i < lengthLine) {
 120                 if (lineBuffer[i] == '\'' || lineBuffer[i] == '\"') {
 121                         bInQuotes = !bInQuotes;
 122                 } else if (lineBuffer[i] == ':' && !bInQuotes) {
 123                         styler.ColourTo(startLine + i - 1, SCE_YAML_IDENTIFIER);
 124                         styler.ColourTo(startLine + i, SCE_YAML_OPERATOR);
 125                         // Non-folding scalar
 126                         i++;
 127                         while ((i < lengthLine) && isspacechar(lineBuffer[i]))
 128                                 i++;
 129                         Sci_PositionU endValue = lengthLine - 1;
 130                         while ((endValue >= i) && isspacechar(lineBuffer[endValue]))
 131                                 endValue--;
 132                         lineBuffer[endValue + 1] = '\0';
 133                         if (lineBuffer[i] == '|' || lineBuffer[i] == '>') {
 134                                 i++;
 135                                 if (lineBuffer[i] == '+' || lineBuffer[i] == '-')
 136                                         i++;
 137                                 while ((i < lengthLine) && isspacechar(lineBuffer[i]))
 138                                         i++;
 139                                 if (lineBuffer[i] == '\0') {
 140                                         styler.SetLineState(currentLine, YAML_STATE_TEXT_PARENT | indentAmount);
 141                                         styler.ColourTo(endPos, SCE_YAML_DEFAULT);
 142                                         return;
 143                                 } else if (lineBuffer[i] == '#') {
 144                                         styler.SetLineState(currentLine, YAML_STATE_TEXT_PARENT | indentAmount);
 145                                         styler.ColourTo(startLine + i - 1, SCE_YAML_DEFAULT);
 146                                         styler.ColourTo(endPos, SCE_YAML_COMMENT);
 147                                         return;
 148                                 } else {
 149                                         styler.ColourTo(endPos, SCE_YAML_ERROR);
 150                                         return;
 151                                 }
 152                         } else if (lineBuffer[i] == '#') {
 153                                 styler.ColourTo(startLine + i - 1, SCE_YAML_DEFAULT);
 154                                 styler.ColourTo(endPos, SCE_YAML_COMMENT);
 155                                 return;
 156                         }
 157                         Sci_PositionU startComment = i;
 158                         bInQuotes = false;
 159                         while (startComment < lengthLine) { // Comment must be space padded
 160                                 if (lineBuffer[startComment] == '\'' || lineBuffer[startComment] == '\"')
 161                                         bInQuotes = !bInQuotes;
 162                                 if (lineBuffer[startComment] == '#' && isspacechar(lineBuffer[startComment - 1]) && !bInQuotes)
 163                                         break;
 164                                 startComment++;
 165                         }
 166                         styler.SetLineState(currentLine, YAML_STATE_VALUE);
 167                         if (lineBuffer[i] == '&' || lineBuffer[i] == '*') {
 168                                 styler.ColourTo(startLine + startComment - 1, SCE_YAML_REFERENCE);
 169                                 if (startComment < lengthLine)
 170                                         styler.ColourTo(endPos, SCE_YAML_COMMENT);
 171                                 return;
 172                         }
 173                         if (KeywordAtChar(&lineBuffer[i], &lineBuffer[startComment], keywords)) { // Convertible value (true/false, etc.)
 174                                 styler.ColourTo(startLine + startComment - 1, SCE_YAML_KEYWORD);
 175                                 if (startComment < lengthLine)
 176                                         styler.ColourTo(endPos, SCE_YAML_COMMENT);
 177                                 return;
 178                         }
 179                         Sci_PositionU i2 = i;
 180                         while ((i < startComment) && lineBuffer[i]) {
 181                                 if (!(IsASCII(lineBuffer[i]) && isdigit(lineBuffer[i])) && lineBuffer[i] != '-'
 182                                         && lineBuffer[i] != '.' && lineBuffer[i] != ',' && lineBuffer[i] != ' ') {
 183                                         styler.ColourTo(startLine + startComment - 1, SCE_YAML_DEFAULT);
 184                                         if (startComment < lengthLine)
 185                                                 styler.ColourTo(endPos, SCE_YAML_COMMENT);
 186                                         return;
 187                                 }
 188                                 i++;
 189                         }
 190                         if (i > i2) {
 191                                 styler.ColourTo(startLine + startComment - 1, SCE_YAML_NUMBER);
 192                                 if (startComment < lengthLine)
 193                                         styler.ColourTo(endPos, SCE_YAML_COMMENT);
 194                                 return;
 195                         }
 196                         break; // shouldn't get here, but just in case, the rest of the line is coloured the default
 197                 }
 198                 i++;
 199         }
 200         styler.ColourTo(endPos, SCE_YAML_DEFAULT);
 201 }
 202
 203 static void ColouriseYAMLDoc(Sci_PositionU startPos, Sci_Position length, int, WordList *keywordLists[], Accessor &styler) {
 204         char lineBuffer[1024] = "";
 205         styler.StartAt(startPos);
 206         styler.StartSegment(startPos);
 207         Sci_PositionU linePos = 0;
 208         Sci_PositionU startLine = startPos;
 209         Sci_PositionU endPos = startPos + length;
 210         Sci_PositionU maxPos = styler.Length();
 211         Sci_PositionU lineCurrent = styler.GetLine(startPos);
 212
 213         for (Sci_PositionU i = startPos; i < maxPos && i < endPos; i++) {
 214                 lineBuffer[linePos++] = styler[i];
 215                 if (AtEOL(styler, i) || (linePos >= sizeof(lineBuffer) - 1)) {
 216                         // End of line (or of line buffer) met, colourise it
 217                         lineBuffer[linePos] = '\0';
 218                         ColouriseYAMLLine(lineBuffer, lineCurrent, linePos, startLine, i, *keywordLists[0], styler);
 219                         linePos = 0;
 220                         startLine = i + 1;
 221                         lineCurrent++;
 222                 }
 223         }
 224         if (linePos > 0) {      // Last line does not have ending characters
 225                 ColouriseYAMLLine(lineBuffer, lineCurrent, linePos, startLine, startPos + length - 1, *keywordLists[0], styler);
 226         }
 227 }
 228
 229 static bool IsCommentLine(Sci_Position line, Accessor &styler) {
 230         Sci_Position pos = styler.LineStart(line);
 231         if (styler[pos] == '#')
 232                 return true;
 233         return false;
 234 }
 235
 236 static void FoldYAMLDoc(Sci_PositionU startPos, Sci_Position length, int /*initStyle - unused*/,
 237                       WordList *[], Accessor &styler) {
 238         const Sci_Position maxPos = startPos + length;
 239         const Sci_Position maxLines = styler.GetLine(maxPos - 1);             // Requested last line
 240         const Sci_Position docLines = styler.GetLine(styler.Length() - 1);  // Available last line
 241         const bool foldComment = styler.GetPropertyInt("fold.comment.yaml") != 0;
 242
 243         // Backtrack to previous non-blank line so we can determine indent level
 244         // for any white space lines
 245         // and so we can fix any preceding fold level (which is why we go back
 246         // at least one line in all cases)
 247         int spaceFlags = 0;
 248         Sci_Position lineCurrent = styler.GetLine(startPos);
 249         int indentCurrent = styler.IndentAmount(lineCurrent, &spaceFlags, NULL);
 250         while (lineCurrent > 0) {
 251                 lineCurrent--;
 252                 indentCurrent = styler.IndentAmount(lineCurrent, &spaceFlags, NULL);
 253                 if (!(indentCurrent & SC_FOLDLEVELWHITEFLAG) &&
 254                         (!IsCommentLine(lineCurrent, styler)))
 255                         break;
 256         }
 257         int indentCurrentLevel = indentCurrent & SC_FOLDLEVELNUMBERMASK;
 258
 259         // Set up initial loop state
 260         int prevComment = 0;
 261         if (lineCurrent >= 1)
 262                 prevComment = foldComment && IsCommentLine(lineCurrent - 1, styler);
 263
 264         // Process all characters to end of requested range
 265         // or comment that hangs over the end of the range.  Cap processing in all cases
 266         // to end of document (in case of unclosed comment at end).
 267         while ((lineCurrent <= docLines) && ((lineCurrent <= maxLines) || prevComment)) {
 268
 269                 // Gather info
 270                 int lev = indentCurrent;
 271                 Sci_Position lineNext = lineCurrent + 1;
 272                 int indentNext = indentCurrent;
 273                 if (lineNext <= docLines) {
 274                         // Information about next line is only available if not at end of document
 275                         indentNext = styler.IndentAmount(lineNext, &spaceFlags, NULL);
 276                 }
 277                 const int comment = foldComment && IsCommentLine(lineCurrent, styler);
 278                 const int comment_start = (comment && !prevComment && (lineNext <= docLines) &&
 279                                            IsCommentLine(lineNext, styler) && (lev > SC_FOLDLEVELBASE));
 280                 const int comment_continue = (comment && prevComment);
 281                 if (!comment)
 282                         indentCurrentLevel = indentCurrent & SC_FOLDLEVELNUMBERMASK;
 283                 if (indentNext & SC_FOLDLEVELWHITEFLAG)
 284                         indentNext = SC_FOLDLEVELWHITEFLAG | indentCurrentLevel;
 285
 286                 if (comment_start) {
 287                         // Place fold point at start of a block of comments
 288                         lev |= SC_FOLDLEVELHEADERFLAG;
 289                 } else if (comment_continue) {
 290                         // Add level to rest of lines in the block
 291                         lev = lev + 1;
 292                 }
 293
 294                 // Skip past any blank lines for next indent level info; we skip also
 295                 // comments (all comments, not just those starting in column 0)
 296                 // which effectively folds them into surrounding code rather
 297                 // than screwing up folding.
 298
 299                 while ((lineNext < docLines) &&
 300                         ((indentNext & SC_FOLDLEVELWHITEFLAG) ||
 301                          (lineNext <= docLines && IsCommentLine(lineNext, styler)))) {
 302
 303                         lineNext++;
 304                         indentNext = styler.IndentAmount(lineNext, &spaceFlags, NULL);
 305                 }
 306
 307                 const int levelAfterComments = indentNext & SC_FOLDLEVELNUMBERMASK;
 308                 const int levelBeforeComments = Maximum(indentCurrentLevel,levelAfterComments);
 309
 310                 // Now set all the indent levels on the lines we skipped
 311                 // Do this from end to start.  Once we encounter one line
 312                 // which is indented more than the line after the end of
 313                 // the comment-block, use the level of the block before
 314
 315                 Sci_Position skipLine = lineNext;
 316                 int skipLevel = levelAfterComments;
 317
 318                 while (--skipLine > lineCurrent) {
 319                         int skipLineIndent = styler.IndentAmount(skipLine, &spaceFlags, NULL);
 320
 321                         if ((skipLineIndent & SC_FOLDLEVELNUMBERMASK) > levelAfterComments)
 322                                 skipLevel = levelBeforeComments;
 323
 324                         int whiteFlag = skipLineIndent & SC_FOLDLEVELWHITEFLAG;
 325
 326                         styler.SetLevel(skipLine, skipLevel | whiteFlag);
 327                 }
 328
 329                 // Set fold header on non-comment line
 330                 if (!comment && !(indentCurrent & SC_FOLDLEVELWHITEFLAG) ) {
 331                         if ((indentCurrent & SC_FOLDLEVELNUMBERMASK) < (indentNext & SC_FOLDLEVELNUMBERMASK))
 332                                 lev |= SC_FOLDLEVELHEADERFLAG;
 333                 }
 334
 335                 // Keep track of block comment state of previous line
 336                 prevComment = comment_start || comment_continue;
 337
 338                 // Set fold level for this line and move to next line
 339                 styler.SetLevel(lineCurrent, lev);
 340                 indentCurrent = indentNext;
 341                 lineCurrent = lineNext;
 342         }
 343
 344         // NOTE: Cannot set level of last line here because indentCurrent doesn't have
 345         // header flag set; the loop above is crafted to take care of this case!
 346         //styler.SetLevel(lineCurrent, indentCurrent);
 347 }
 348
 349 LexerModule lmYAML(SCLEX_YAML, ColouriseYAMLDoc, "yaml", FoldYAMLDoc, yamlWordListDesc);