ext/scintilla/lexers/LexYAML.cxx

   1 // Scintilla source code edit control
   2 /** @file LexYAML.cxx
   3  ** Lexer for YAML.
   4  **/
   5 // Copyright 2003- by Sean O'Dell <sean@celsoft.com>
   6 // The License.txt file describes the conditions under which this software may be distributed.
   7
   8 #include <stdlib.h>
   9 #include <string.h>
  10 #include <stdio.h>
  11 #include <stdarg.h>
  12 #include <assert.h>
  13 #include <ctype.h>
  14
  15 #include "ILexer.h"
  16 #include "Scintilla.h"
  17 #include "SciLexer.h"
  18
  19 #include "WordList.h"
  20 #include "LexAccessor.h"
  21 #include "Accessor.h"
  22 #include "StyleContext.h"
  23 #include "CharacterSet.h"
  24 #include "LexerModule.h"
  25
  26 #ifdef SCI_NAMESPACE
  27 using namespace Scintilla;
  28 #endif
  29
  30 static const char * const yamlWordListDesc[] = {
  31         "Keywords",
  32         0
  33 };
  34
  35 static inline bool AtEOL(Accessor &styler, unsigned int i) {
  36         return (styler[i] == '\n') ||
  37                 ((styler[i] == '\r') && (styler.SafeGetCharAt(i + 1) != '\n'));
  38 }
  39
  40 static unsigned int SpaceCount(char* lineBuffer) {
  41         if (lineBuffer == NULL)
  42                 return 0;
  43
  44         char* headBuffer = lineBuffer;
  45
  46         while (*headBuffer == ' ')
  47                 headBuffer++;
  48
  49         return headBuffer - lineBuffer;
  50 }
  51
  52 #define YAML_STATE_BITSIZE 16
  53 #define YAML_STATE_MASK                 (0xFFFF0000)
  54 #define YAML_STATE_DOCUMENT             (1 << YAML_STATE_BITSIZE)
  55 #define YAML_STATE_VALUE                        (2 << YAML_STATE_BITSIZE)
  56 #define YAML_STATE_COMMENT              (3 << YAML_STATE_BITSIZE)
  57 #define YAML_STATE_TEXT_PARENT  (4 << YAML_STATE_BITSIZE)
  58 #define YAML_STATE_TEXT                 (5 << YAML_STATE_BITSIZE)
  59
  60 static void ColouriseYAMLLine(
  61         char *lineBuffer,
  62         unsigned int currentLine,
  63         unsigned int lengthLine,
  64         unsigned int startLine,
  65         unsigned int endPos,
  66         WordList &keywords,
  67         Accessor &styler) {
  68
  69         unsigned int i = 0;
  70         bool bInQuotes = false;
  71         unsigned int indentAmount = SpaceCount(lineBuffer);
  72
  73         if (currentLine > 0) {
  74                 int parentLineState = styler.GetLineState(currentLine - 1);
  75
  76                 if ((parentLineState&YAML_STATE_MASK) == YAML_STATE_TEXT || (parentLineState&YAML_STATE_MASK) == YAML_STATE_TEXT_PARENT) {
  77                         unsigned int parentIndentAmount = parentLineState&(~YAML_STATE_MASK);
  78                         if (indentAmount > parentIndentAmount) {
  79                                 styler.SetLineState(currentLine, YAML_STATE_TEXT | parentIndentAmount);
  80                                 styler.ColourTo(endPos, SCE_YAML_TEXT);
  81                                 return;
  82                         }
  83                 }
  84         }
  85         styler.SetLineState(currentLine, 0);
  86         if (strncmp(lineBuffer, "---", 3) == 0) {       // Document marker
  87                 styler.SetLineState(currentLine, YAML_STATE_DOCUMENT);
  88                 styler.ColourTo(endPos, SCE_YAML_DOCUMENT);
  89                 return;
  90         }
  91         // Skip initial spaces
  92         while ((i < lengthLine) && lineBuffer[i] == ' ') { // YAML always uses space, never TABS or anything else
  93                 i++;
  94         }
  95         if (lineBuffer[i] == '\t') { // if we skipped all spaces, and we are NOT inside a text block, this is wrong
  96                 styler.ColourTo(endPos, SCE_YAML_ERROR);
  97                 return;
  98         }
  99         if (lineBuffer[i] == '#') {     // Comment
 100                 styler.SetLineState(currentLine, YAML_STATE_COMMENT);
 101                 styler.ColourTo(endPos, SCE_YAML_COMMENT);
 102                 return;
 103         }
 104         while (i < lengthLine) {
 105                 if (lineBuffer[i] == '\'' || lineBuffer[i] == '\"') {
 106                         bInQuotes = !bInQuotes;
 107                 } else if (lineBuffer[i] == ':' && !bInQuotes) {
 108                         styler.ColourTo(startLine + i - 1, SCE_YAML_IDENTIFIER);
 109                         styler.ColourTo(startLine + i, SCE_YAML_OPERATOR);
 110                         // Non-folding scalar
 111                         i++;
 112                         while ((i < lengthLine) && isspacechar(lineBuffer[i]))
 113                                 i++;
 114                         unsigned int endValue = lengthLine - 1;
 115                         while ((endValue >= i) && isspacechar(lineBuffer[endValue]))
 116                                 endValue--;
 117                         lineBuffer[endValue + 1] = '\0';
 118                         if (lineBuffer[i] == '|' || lineBuffer[i] == '>') {
 119                                 i++;
 120                                 if (lineBuffer[i] == '+' || lineBuffer[i] == '-')
 121                                         i++;
 122                                 while ((i < lengthLine) && isspacechar(lineBuffer[i]))
 123                                         i++;
 124                                 if (lineBuffer[i] == '\0') {
 125                                         styler.SetLineState(currentLine, YAML_STATE_TEXT_PARENT | indentAmount);
 126                                         styler.ColourTo(endPos, SCE_YAML_DEFAULT);
 127                                         return;
 128                                 } else if (lineBuffer[i] == '#') {
 129                                         styler.SetLineState(currentLine, YAML_STATE_TEXT_PARENT | indentAmount);
 130                                         styler.ColourTo(startLine + i - 1, SCE_YAML_DEFAULT);
 131                                         styler.ColourTo(endPos, SCE_YAML_COMMENT);
 132                                         return;
 133                                 } else {
 134                                         styler.ColourTo(endPos, SCE_YAML_ERROR);
 135                                         return;
 136                                 }
 137                         } else if (lineBuffer[i] == '#') {
 138                                 styler.ColourTo(startLine + i - 1, SCE_YAML_DEFAULT);
 139                                 styler.ColourTo(endPos, SCE_YAML_COMMENT);
 140                                 return;
 141                         }
 142                         styler.SetLineState(currentLine, YAML_STATE_VALUE);
 143                         if (lineBuffer[i] == '&' || lineBuffer[i] == '*') {
 144                                 styler.ColourTo(endPos, SCE_YAML_REFERENCE);
 145                                 return;
 146                         }
 147                         if (keywords.InList(&lineBuffer[i])) { // Convertible value (true/false, etc.)
 148                                 styler.ColourTo(endPos, SCE_YAML_KEYWORD);
 149                                 return;
 150                         } else {
 151                                 unsigned int i2 = i;
 152                                 while ((i < lengthLine) && lineBuffer[i]) {
 153                                         if (!(isascii(lineBuffer[i]) && isdigit(lineBuffer[i])) && lineBuffer[i] != '-' && lineBuffer[i] != '.' && lineBuffer[i] != ',') {
 154                                                 styler.ColourTo(endPos, SCE_YAML_DEFAULT);
 155                                                 return;
 156                                         }
 157                                         i++;
 158                                 }
 159                                 if (i > i2) {
 160                                         styler.ColourTo(endPos, SCE_YAML_NUMBER);
 161                                         return;
 162                                 }
 163                         }
 164                         break; // shouldn't get here, but just in case, the rest of the line is coloured the default
 165                 }
 166                 i++;
 167         }
 168         styler.ColourTo(endPos, SCE_YAML_DEFAULT);
 169 }
 170
 171 static void ColouriseYAMLDoc(unsigned int startPos, int length, int, WordList *keywordLists[], Accessor &styler) {
 172         char lineBuffer[1024];
 173         styler.StartAt(startPos);
 174         styler.StartSegment(startPos);
 175         unsigned int linePos = 0;
 176         unsigned int startLine = startPos;
 177         unsigned int endPos = startPos + length;
 178         unsigned int maxPos = styler.Length();
 179         unsigned int lineCurrent = styler.GetLine(startPos);
 180
 181         for (unsigned int i = startPos; i < maxPos && i < endPos; i++) {
 182                 lineBuffer[linePos++] = styler[i];
 183                 if (AtEOL(styler, i) || (linePos >= sizeof(lineBuffer) - 1)) {
 184                         // End of line (or of line buffer) met, colourise it
 185                         lineBuffer[linePos] = '\0';
 186                         ColouriseYAMLLine(lineBuffer, lineCurrent, linePos, startLine, i, *keywordLists[0], styler);
 187                         linePos = 0;
 188                         startLine = i + 1;
 189                         lineCurrent++;
 190                 }
 191         }
 192         if (linePos > 0) {      // Last line does not have ending characters
 193                 ColouriseYAMLLine(lineBuffer, lineCurrent, linePos, startLine, startPos + length - 1, *keywordLists[0], styler);
 194         }
 195 }
 196
 197 static bool IsCommentLine(int line, Accessor &styler) {
 198         int pos = styler.LineStart(line);
 199         if (styler[pos] == '#')
 200                 return true;
 201         return false;
 202 }
 203
 204 static void FoldYAMLDoc(unsigned int startPos, int length, int /*initStyle - unused*/,
 205                       WordList *[], Accessor &styler) {
 206         const int maxPos = startPos + length;
 207         const int maxLines = styler.GetLine(maxPos - 1);             // Requested last line
 208         const int docLines = styler.GetLine(styler.Length() - 1);  // Available last line
 209         const bool foldComment = styler.GetPropertyInt("fold.comment.yaml") != 0;
 210
 211         // Backtrack to previous non-blank line so we can determine indent level
 212         // for any white space lines
 213         // and so we can fix any preceding fold level (which is why we go back
 214         // at least one line in all cases)
 215         int spaceFlags = 0;
 216         int lineCurrent = styler.GetLine(startPos);
 217         int indentCurrent = styler.IndentAmount(lineCurrent, &spaceFlags, NULL);
 218         while (lineCurrent > 0) {
 219                 lineCurrent--;
 220                 indentCurrent = styler.IndentAmount(lineCurrent, &spaceFlags, NULL);
 221                 if (!(indentCurrent & SC_FOLDLEVELWHITEFLAG) &&
 222                         (!IsCommentLine(lineCurrent, styler)))
 223                         break;
 224         }
 225         int indentCurrentLevel = indentCurrent & SC_FOLDLEVELNUMBERMASK;
 226
 227         // Set up initial loop state
 228         int prevComment = 0;
 229         if (lineCurrent >= 1)
 230                 prevComment = foldComment && IsCommentLine(lineCurrent - 1, styler);
 231
 232         // Process all characters to end of requested range
 233         // or comment that hangs over the end of the range.  Cap processing in all cases
 234         // to end of document (in case of unclosed comment at end).
 235         while ((lineCurrent <= docLines) && ((lineCurrent <= maxLines) || prevComment)) {
 236
 237                 // Gather info
 238                 int lev = indentCurrent;
 239                 int lineNext = lineCurrent + 1;
 240                 int indentNext = indentCurrent;
 241                 if (lineNext <= docLines) {
 242                         // Information about next line is only available if not at end of document
 243                         indentNext = styler.IndentAmount(lineNext, &spaceFlags, NULL);
 244         }
 245                 const int comment = foldComment && IsCommentLine(lineCurrent, styler);
 246                 const int comment_start = (comment && !prevComment && (lineNext <= docLines) &&
 247                                            IsCommentLine(lineNext, styler) && (lev > SC_FOLDLEVELBASE));
 248                 const int comment_continue = (comment && prevComment);
 249                 if (!comment)
 250                         indentCurrentLevel = indentCurrent & SC_FOLDLEVELNUMBERMASK;
 251                 if (indentNext & SC_FOLDLEVELWHITEFLAG)
 252                         indentNext = SC_FOLDLEVELWHITEFLAG | indentCurrentLevel;
 253
 254                 if (comment_start) {
 255                         // Place fold point at start of a block of comments
 256                         lev |= SC_FOLDLEVELHEADERFLAG;
 257                 } else if (comment_continue) {
 258                         // Add level to rest of lines in the block
 259                         lev = lev + 1;
 260                 }
 261
 262                 // Skip past any blank lines for next indent level info; we skip also
 263                 // comments (all comments, not just those starting in column 0)
 264                 // which effectively folds them into surrounding code rather
 265                 // than screwing up folding.
 266
 267                 while ((lineNext < docLines) &&
 268                         ((indentNext & SC_FOLDLEVELWHITEFLAG) ||
 269                          (lineNext <= docLines && IsCommentLine(lineNext, styler)))) {
 270
 271                         lineNext++;
 272                         indentNext = styler.IndentAmount(lineNext, &spaceFlags, NULL);
 273                 }
 274
 275                 const int levelAfterComments = indentNext & SC_FOLDLEVELNUMBERMASK;
 276                 const int levelBeforeComments = Maximum(indentCurrentLevel,levelAfterComments);
 277
 278                 // Now set all the indent levels on the lines we skipped
 279                 // Do this from end to start.  Once we encounter one line
 280                 // which is indented more than the line after the end of
 281                 // the comment-block, use the level of the block before
 282
 283                 int skipLine = lineNext;
 284                 int skipLevel = levelAfterComments;
 285
 286                 while (--skipLine > lineCurrent) {
 287                         int skipLineIndent = styler.IndentAmount(skipLine, &spaceFlags, NULL);
 288
 289                         if ((skipLineIndent & SC_FOLDLEVELNUMBERMASK) > levelAfterComments)
 290                                 skipLevel = levelBeforeComments;
 291
 292                         int whiteFlag = skipLineIndent & SC_FOLDLEVELWHITEFLAG;
 293
 294                         styler.SetLevel(skipLine, skipLevel | whiteFlag);
 295                 }
 296
 297                 // Set fold header on non-comment line
 298                 if (!comment && !(indentCurrent & SC_FOLDLEVELWHITEFLAG) ) {
 299                         if ((indentCurrent & SC_FOLDLEVELNUMBERMASK) < (indentNext & SC_FOLDLEVELNUMBERMASK))
 300                                 lev |= SC_FOLDLEVELHEADERFLAG;
 301                 }
 302
 303                 // Keep track of block comment state of previous line
 304                 prevComment = comment_start || comment_continue;
 305
 306                 // Set fold level for this line and move to next line
 307                 styler.SetLevel(lineCurrent, lev);
 308                 indentCurrent = indentNext;
 309                 lineCurrent = lineNext;
 310         }
 311
 312         // NOTE: Cannot set level of last line here because indentCurrent doesn't have
 313         // header flag set; the loop above is crafted to take care of this case!
 314         //styler.SetLevel(lineCurrent, indentCurrent);
 315 }
 316
 317 LexerModule lmYAML(SCLEX_YAML, ColouriseYAMLDoc, "yaml", FoldYAMLDoc, yamlWordListDesc);