mcs/class/System.XML/System.Xml.Query/XQueryTokenizer.cs

   1 //
   2 // XQueryTokenizer.cs
   3 //
   4 // Author:
   5 //      Atsushi Enomoto <atsushi@ximian.com>
   6 //
   7 // Copyright (C) 2004 Novell, Inc (http://www.novell.com)
   8 //
   9 // Permission is hereby granted, free of charge, to any person obtaining
  10 // a copy of this software and associated documentation files (the
  11 // "Software"), to deal in the Software without restriction, including
  12 // without limitation the rights to use, copy, modify, merge, publish,
  13 // distribute, sublicense, and/or sell copies of the Software, and to
  14 // permit persons to whom the Software is furnished to do so, subject to
  15 // the following conditions:
  16 //
  17 // The above copyright notice and this permission notice shall be
  18 // included in all copies or substantial portions of the Software.
  19 //
  20 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  21 // EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  22 // MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  23 // NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
  24 // LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  25 // OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  26 // WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  27 //
  28 #if NET_2_0
  29
  30 using System;
  31 using System.Collections;
  32 using System.Collections.Generic;
  33 using System.IO;
  34 using System.Security.Policy;
  35 using System.Xml;
  36 using System.Xml.Query;
  37 using System.Xml.Schema;
  38 using System.Xml.XPath;
  39 using Mono.Xml.XQuery;
  40 using Mono.Xml.XPath2;
  41
  42 namespace Mono.Xml.XQuery.Parser
  43 {
  44         // FIXME: make internal in the future
  45         public class XQueryTokenizer
  46                 : Mono.Xml.XQuery.Parser.yyParser.yyInput, IXmlLineInfo
  47         {
  48                 int line = 1;
  49                 int column = 0;
  50                 bool nextIncrementLine;
  51
  52                 // namespace resolver
  53                 XmlNamespaceManager nsResolver;
  54                 string defaultFunctionNamespace = XQueryFunction.Namespace;
  55
  56                 // input source
  57                 TextReader source;
  58                 int peekChar = -1;
  59
  60                 // token info
  61                 int currentToken;
  62                 string prefixName;
  63                 object tokenValue;
  64
  65                 int lookAheadToken = -1;
  66                 object lookAheadTokenValue;
  67
  68                 // state info
  69                 WhitespaceHandling ws = WhitespaceHandling.Arbitrary;
  70                 ParseState state = ParseState.Default;
  71                 Stack stateStack;
  72
  73                 char [] buffer = new char [30];
  74                 int bufferIndex;
  75
  76                 public XQueryTokenizer (TextReader reader)
  77                 {
  78                         this.source = reader;
  79
  80                         stateStack = new Stack ();
  81
  82                         nsResolver = new XmlNamespaceManager (new NameTable ());
  83                         nsResolver.AddNamespace ("xs", XmlSchema.Namespace);
  84                         nsResolver.AddNamespace ("xdt", XmlSchema.XdtNamespace);
  85                         // FIXME: Are they really predefined?
  86                         nsResolver.AddNamespace ("xsi", XmlSchema.InstanceNamespace);
  87                         nsResolver.AddNamespace ("fn", "http://www.w3.org/2003/11/xpath-functions");
  88                         nsResolver.AddNamespace ("local", "http://www.w3.org/2003/11/xquery-local-functions");
  89                 }
  90
  91                 internal IXmlNamespaceResolver NSResolver {
  92                         get { return nsResolver; }
  93                 }
  94
  95                 internal string DefaultFunctionNamespace {
  96                         get { return defaultFunctionNamespace; }
  97                         set { defaultFunctionNamespace = value; }
  98                 }
  99
 100                 public void AddNamespace (string prefix, string ns)
 101                 {
 102                         nsResolver.AddNamespace (prefix, ns);
 103                 }
 104
 105                 public bool advance ()
 106                 {
 107                         if (currentToken < 0)
 108                                 return false;
 109                         if (lookAheadToken >= 0) {
 110                                 tokenValue = lookAheadTokenValue;
 111                                 currentToken = lookAheadToken;
 112                                 lookAheadToken = -1;
 113                         }
 114                         else
 115                                 currentToken = ParseToken ();
 116                         return currentToken >= 0;
 117                 }
 118
 119                 public int token ()
 120                 {
 121                         return currentToken;
 122                 }
 123
 124                 public object value ()
 125                 {
 126                         return tokenValue;
 127                 }
 128
 129                 public bool HasLineInfo ()
 130                 {
 131                         return true;
 132                 }
 133
 134                 public int LineNumber {
 135                         get { return line; }
 136                 }
 137
 138                 public int LinePosition {
 139                         get { return column; }
 140                 }
 141
 142                 internal WhitespaceHandling Space {
 143                         get { return ws; }
 144                         set { ws = value; }
 145                 }
 146
 147                 internal ParseState State {
 148                         get { return state; }
 149                         set {
 150 //                              Console.Error.WriteLine ("**** eno **** state transition from {0} to {1}, stack count = {2}", state, value, stateStack.Count);
 151 //foreach (ParseState ps in stateStack.ToArray ()) Console.Error.WriteLine ("***** eno ***** " + ps);
 152                                 state = value;
 153                         }
 154                 }
 155
 156                 internal void PushState (ParseState newState)
 157                 {
 158                         stateStack.Push (newState);
 159 //                      Console.Error.WriteLine ("**** eno **** state pushed {0}, added stack count = {1}", newState, stateStack.Count);
 160 //foreach (ParseState ps in stateStack.ToArray ()) Console.Error.WriteLine ("***** eno ***** " + ps);
 161                 }
 162
 163                 internal void PopState ()
 164                 {
 165                         if (stateStack.Count == 0)
 166                                 throw Error ("Internal state transition error. State stack is empty.");
 167                         state = (ParseState) stateStack.Pop ();
 168 //                      Console.Error.WriteLine ("**** eno **** state pop, now as {0}, stack count = {1}", state, stateStack.Count);
 169 //foreach (ParseState ps in stateStack.ToArray ()) Console.Error.WriteLine ("***** eno ***** " + ps);
 170                 }
 171
 172                 private XmlQueryCompileException Error (string message)
 173                 {
 174                         return new XmlQueryCompileException (message, this, null, null);
 175                 }
 176
 177                 private int ParseToken ()
 178                 {
 179                         bufferIndex = 0;
 180
 181                         switch (state) {
 182                         case ParseState.StartTag:
 183                                 break;
 184                         default:
 185                                 SkipWhitespaces ();
 186                                 break;
 187                         }
 188                         /*
 189                         switch (ws) {
 190                         case WhitespaceHandling.Arbitrary:
 191                                 SkipWhitespaces ();
 192                                 break;
 193                         case WhitespaceHandling.Explicit:
 194                                 if (!XmlChar.IsWhitespace (PeekChar ()))
 195                                         throw Error ("Whitespace is required.");
 196                                 goto case WhitespaceHandling.Arbitrary;
 197                         }
 198                         */
 199
 200                         int c = PeekChar ();
 201                         if (c < 0)
 202                                 return -1;
 203
 204                         // FIXME: consider DOUBLE_LITERAL
 205                         if (Char.IsNumber ((char) c)) {
 206                                 tokenValue = ReadDecimal (false);
 207                                 return Token.DECIMAL_LITERAL;
 208                         }
 209
 210                         switch (state) {
 211                         case ParseState.OccurenceIndicator:
 212                                 return ParseOccurenceIndicator ();
 213                         case ParseState.XmlPIContent:
 214                                 return ParseXmlPIContent ();
 215                         case ParseState.XmlComment:
 216                                 return ParseXmlCommentContent ();
 217                         case ParseState.ElementContent:
 218                                 return ParseElementContent ();
 219                         case ParseState.StartTag:
 220                                 return ParseStartTag ();
 221                         case ParseState.QuotAttributeContent:
 222                                 return ParseAttributeContent ('"');
 223                         case ParseState.AposAttributeContent:
 224                                 return ParseAttributeContent ('\'');
 225                         default:
 226                                 return ParseDefault ();
 227                         }
 228                 }
 229
 230                 private int ParseXQueryComment ()
 231                 {
 232                         while (true) {
 233                                 int c = ReadChar ();
 234                                 if (c < 0)
 235                                         throw Error ("Unexpected end of query text inside XML processing instruction content");
 236                                 if (c == ':') {
 237                                         if (PeekChar () == ')') {
 238                                                 ReadChar ();
 239                                                 tokenValue = CreateValueString ();
 240                                                 return Token.XML_PI_TO_END;
 241                                         }
 242                                         else
 243                                                 AddValueChar (':');
 244                                 }
 245                                 else
 246                                         AddValueChar ((char) c);
 247                         }
 248                 }
 249
 250                 private int ParseXmlPIContent ()
 251                 {
 252                         while (true) {
 253                                 int c = ReadChar ();
 254                                 if (c < 0)
 255                                         throw Error ("Unexpected end of query text inside XML processing instruction content");
 256                                 if (c == '?') {
 257                                         if (PeekChar () == '>') {
 258                                                 ReadChar ();
 259                                                 tokenValue = CreateValueString ();
 260                                                 return Token.XML_PI_TO_END;
 261                                         }
 262                                         else
 263                                                 AddValueChar ('?');
 264                                 }
 265                                 else
 266                                         AddValueChar ((char) c);
 267                         }
 268                 }
 269
 270                 private int ParseXmlCommentContent ()
 271                 {
 272                         // FIXME: handle ---> correctly
 273                         while (true) {
 274                                 int c = ReadChar ();
 275                                 if (c < 0)
 276                                         throw Error ("Unexpected end of query text inside XML comment content");
 277                                 if (c == '-') {
 278                                         if (PeekChar () == '-') {
 279                                                 ReadChar ();
 280                                                 if (PeekChar () == '>') {
 281                                                         tokenValue = CreateValueString ();
 282                                                         return Token.XML_COMMENT_TO_END;
 283                                                 } else {
 284                                                         AddValueChar ('-');
 285                                                         AddValueChar ('-');
 286                                                 }
 287                                         }
 288                                         else
 289                                                 AddValueChar ('-');
 290                                 }
 291                                 else
 292                                         AddValueChar ((char) c);
 293                         }
 294                 }
 295
 296                 private int ParseXmlCDataContent ()
 297                 {
 298                         // FIXME: handle ]]]> correctly
 299                         while (true) {
 300                                 int c = ReadChar ();
 301                                 if (c < 0)
 302                                         throw Error ("Unexpected end of query text inside XML CDATA section content");
 303                                 if (c == ']') {
 304                                         ReadChar ();
 305                                         if (PeekChar () == ']') {
 306                                                 ReadChar ();
 307                                                 if (PeekChar () == '>') {
 308                                                         tokenValue = CreateValueString ();
 309                                                         return Token.XML_CDATA_TO_END;
 310                                                 } else {
 311                                                         AddValueChar (']');
 312                                                         AddValueChar (']');
 313                                                 }
 314                                         }
 315                                         else
 316                                                 AddValueChar (']');
 317                                 }
 318                                 else
 319                                         AddValueChar ((char) c);
 320                         }
 321                 }
 322
 323                 private int ParseElementContent ()
 324                 {
 325                         tokenValue = null;
 326                         int c = PeekChar ();
 327                         if (c < 0)
 328                                 throw Error ("Unexpected end of query text inside XML processing instruction content");
 329                         switch ((char) c) {
 330                         case '<':
 331                         case '{':
 332                                 return ParseDefault ();
 333                         }
 334
 335                         while (true) {
 336                                 c = PeekChar ();
 337                                 if (c < 0)
 338                                         throw Error ("Unexpected end of query text inside XML processing instruction content");
 339                                 switch ((char) c) {
 340                                 case '&':
 341                                         ReadChar ();
 342                                         ReadPredefinedEntity ();
 343                                         continue;
 344                                 case '<':
 345                                         tokenValue += CreateValueString ();
 346                                         return Token.ELEM_CONTENT_LITERAL;
 347                                 default:
 348                                         AddValueChar ((char) c);
 349                                         ReadChar ();
 350                                         continue;
 351                                 }
 352                         }
 353                 }
 354
 355                 private void ReadPredefinedEntity ()
 356                 {
 357                         string token = ReadOneToken ();
 358                         Expect (";");
 359                         switch (token) {
 360                         case "lt":
 361                                 AddValueChar ('<');
 362                                 return;
 363                         case "gt":
 364                                 AddValueChar ('>');
 365                                 return;
 366                         case "amp":
 367                                 AddValueChar ('&');
 368                                 return;
 369                         case "quot":
 370                                 AddValueChar ('"');
 371                                 return;
 372                         case "apos":
 373                                 AddValueChar ('\'');
 374                                 return;
 375                         default:
 376                                 throw Error (String.Format ("Unexpected general entity name: {0} .", token));
 377                         }
 378                 }
 379
 380                 // FIXME: not used as yet
 381                 private int ParseExtContent ()
 382                 {
 383                         // FIXME: handle :::) correctly
 384                         while (true) {
 385                                 int c = PeekChar ();
 386                                 if (c < 0)
 387                                         throw Error ("Unexpected end of query text inside external content");
 388                                 if (c == ':') {
 389                                         ReadChar ();
 390                                         if (PeekChar () == ':') {
 391                                                 ReadChar ();
 392                                                 if (PeekChar () == ')') {
 393                                                         tokenValue = CreateValueString ();
 394                                                         return Token.EXT_CONTENT;
 395                                                 } else {
 396                                                         AddValueChar (':');
 397                                                         AddValueChar (':');
 398                                                 }
 399                                         }
 400                                         else
 401                                                 AddValueChar (':');
 402                                 }
 403                                 else
 404                                         AddValueChar ((char) c);
 405                         }
 406                 }
 407
 408                 private int ParseOccurenceIndicator ()
 409                 {
 410                         state = ParseState.Operator;
 411                         switch (PeekChar ()) {
 412                         case '?':
 413                                 ReadChar ();
 414                                 return Token.QUESTION;
 415                         case '*':
 416                                 ReadChar ();
 417                                 return Token.ASTERISK;
 418                         case '+':
 419                                 ReadChar ();
 420                                 return Token.PLUS;
 421                         default:
 422                                 return ParseOperator ();
 423                         }
 424                 }
 425
 426                 private int ParseStartTag ()
 427                 {
 428                         int c = PeekChar ();
 429                         switch (c) {
 430                         case '\'':
 431                                 ReadChar ();
 432                                 return Token.APOS;
 433                         case '"':
 434                                 ReadChar ();
 435                                 return Token.QUOT;
 436                         case '>':
 437                                 ReadChar ();
 438                                 return Token.GREATER;
 439                         case '/':
 440                                 ReadChar ();
 441                                 Expect (">");
 442                                 return Token.EMPTY_TAG_CLOSE;
 443                         }
 444                         // FIXME: there seems a bug in the spec that StartTag
 445                         // state must accept QName without heading space for
 446                         // start tag name.
 447 //                      if (!XmlChar.IsWhitespace (PeekChar ()))
 448 //                              throw Error ("Whitespace is required.");
 449                         SkipWhitespaces ();
 450                         return ParseDefault (); // only QName is allowed here.
 451                 }
 452
 453                 private int ParseAttributeContent (char closeChar)
 454                 {
 455                         int t = Token.ATT_VALUE_LITERAL;
 456                         while (true) {
 457                                 int c = PeekChar ();
 458                                 if (c < 0)
 459                                         throw Error ("Unexpected end of attribute value content.");
 460                                 if (c == closeChar) {
 461                                         ReadChar ();
 462                                         c = PeekChar ();
 463                                         if (c == closeChar) {
 464                                                 ReadChar ();
 465                                                 AddValueChar (closeChar);
 466                                         }
 467                                         else
 468                                                 t = closeChar == '"' ? Token.QUOT : Token.APOS;
 469                                 }
 470                                 else if (c == '{') {
 471                                         ReadChar ();
 472                                         c = PeekChar ();
 473                                         if (c == '{') {
 474                                                 ReadChar ();
 475                                                 AddValueChar ('{');
 476                                         }
 477                                         else
 478                                                 t = Token.OPEN_CURLY;
 479                                 }
 480                                 else
 481                                         AddValueChar ((char) ReadChar ());
 482
 483                                 if (t != Token.ATT_VALUE_LITERAL) {
 484                                         if (bufferIndex > 0) {
 485                                                 lookAheadToken = t;
 486                                                 tokenValue = CreateValueString ();
 487                                                 return Token.ATT_VALUE_LITERAL;
 488                                         }
 489                                         else
 490                                                 return t;
 491                                 }
 492                         }
 493                 }
 494
 495                 private int ParseOperator ()
 496                 {
 497                         // TODO: implement
 498                         return ParseDefault ();
 499                 }
 500
 501                 private int ParseDefault ()
 502                 {
 503                         int c = ReadChar ();
 504                         switch (c) {
 505                         case '.':
 506                                 if (PeekChar () == '.') {
 507                                         ReadChar ();
 508                                         return Token.DOT2;
 509                                 }
 510                                 else if (Char.IsNumber ((char) PeekChar ())) {
 511                                         tokenValue = ReadDecimal (true);
 512                                 }
 513                                 return Token.DOT;
 514                         case ',':
 515                                 return Token.COMMA;
 516                         case ';':
 517                                 return Token.SEMICOLON;
 518                         case '(':
 519                                 if (PeekChar () == ':') {
 520                                         ReadChar ();
 521                                         if (PeekChar () == ':') {
 522                                                 ReadChar ();
 523                                                 return Token.PRAGMA_OPEN;
 524                                         }
 525                                         ParseXQueryComment ();
 526                                         return ParseToken (); // start again
 527                                 }
 528                                 return Token.OPEN_PAREN;
 529                         case ')':
 530                                 return Token.CLOSE_PAREN;
 531                         case ':':
 532                                 switch (PeekChar ()) {
 533                                 case ':':
 534                                         ReadChar ();
 535                                         if (PeekChar () == ')') {
 536                                                 ReadChar ();
 537                                                 return Token.PRAGMA_CLOSE;
 538                                         }
 539                                         return Token.COLON2;
 540                                 case ')':
 541                                         ReadChar ();
 542                                         return Token.CLOSE_PAREN_COLON;
 543                                 case '=':
 544                                         ReadChar ();
 545                                         return Token.COLON_EQUAL;
 546                                 }
 547                                 return Token.COLON;
 548                         case '[':
 549                                 return Token.OPEN_BRACKET;
 550                         case ']':
 551                                 return Token.CLOSE_BRACKET;
 552                         case '{':
 553                                 return Token.OPEN_CURLY;
 554                         case '}':
 555                                 return Token.CLOSE_CURLY;
 556                         case '$':
 557                                 return Token.DOLLAR;
 558                         case '\'':
 559                                 tokenValue = ReadQuoted ('\'');
 560                                 return Token.STRING_LITERAL;
 561                         case '"':
 562                                 tokenValue = ReadQuoted ('"');
 563                                 return Token.STRING_LITERAL;
 564                         case '=':
 565                                 return Token.EQUAL;
 566                         case '<':
 567                                 // only happens when state is ElementContent
 568                                 // (otherwise it might be "/foo</bar")
 569                                 if (state == ParseState.ElementContent) {
 570                                         switch ((char) PeekChar ()) {
 571                                         case '/':
 572                                                 ReadChar ();
 573                                                 return Token.END_TAG_START;
 574                                         case '!':
 575                                                 ReadChar ();
 576                                                 switch (PeekChar ()) {
 577                                                 case '-':
 578                                                         ReadChar ();
 579                                                         if (ReadChar () != '-')
 580                                                                 throw Error ("Invalid sequence of characters '<!-'.");
 581
 582                                                         return Token.XML_COMMENT_START;
 583                                                 case '[':
 584                                                         ReadChar ();
 585                                                         Expect ("CDATA[");
 586                                                         return Token.XML_CDATA_START;
 587                                                 }
 588                                                 throw Error ("Invalid sequence of characters '<!'.");
 589                                         case '?':
 590                                                 ReadChar ();
 591                                                 return Token.XML_PI_START;
 592                                         default:
 593                                                 return Token.LESSER;
 594                                         }
 595                                 }
 596
 597                                 switch (PeekChar ()) {
 598                                 case '<':
 599                                         ReadChar ();
 600                                         return Token.LESSER2;
 601                                 case '=':
 602                                         ReadChar ();
 603                                         return Token.LESSER_EQUAL;
 604                                 }
 605                                 return Token.LESSER;
 606                         case '>':
 607                                 switch (PeekChar ()) {
 608                                 case '>':
 609                                         ReadChar ();
 610                                         return Token.GREATER2;
 611                                 case '=':
 612                                         ReadChar ();
 613                                         return Token.GREATER_EQUAL;
 614                                 }
 615                                 return Token.GREATER;
 616                         case '|':
 617                                 return Token.BAR;
 618                         case '*':
 619                                 if (PeekChar () == ':') {
 620                                         ReadChar ();
 621                                         // FIXME: more check
 622                                         tokenValue = new XmlQualifiedName (ReadOneToken (), "*");
 623                                         return Token.WILD_PREFIX;
 624                                 }
 625                                 return Token.ASTERISK;
 626                         case '+':
 627                                 return Token.PLUS;
 628                         case '-':
 629                                 return Token.MINUS;
 630                         case '/':
 631                                 // only happens when state is StartTag
 632                                 // (otherwise it might be "/>$extvar")
 633                                 if (state == ParseState.StartTag && PeekChar () == '>') {
 634                                         ReadChar ();
 635                                         return Token.EMPTY_TAG_CLOSE;
 636                                 }
 637                                 if (PeekChar () == '/') {
 638                                         ReadChar ();
 639                                         return Token.SLASH2;
 640                                 }
 641                                 return Token.SLASH;
 642                         case '?':
 643                                 return Token.QUESTION;
 644                         case '@':
 645                                 return Token.AT;
 646                         }
 647
 648                         peekChar = c;
 649                         prefixName = null;
 650                         string name = ReadOneToken ();
 651
 652                         tokenValue = name;
 653                         bool validKeyword = false;
 654
 655                         switch (state) {
 656                         case ParseState.XmlSpaceDecl:
 657                                 switch (name) {
 658                                 case "preserve":
 659                                         return Token.PRESERVE;
 660                                 case "strip":
 661                                         return Token.STRIP;
 662                                 }
 663                                 break;
 664                         case ParseState.CloseKindTest:
 665                                 if (name == "nillable")
 666                                         return Token.NILLABLE;
 667                                 break;
 668                         case ParseState.ExtKey:
 669                                 switch (name) {
 670                                 case "pragma":
 671                                         return Token.PRAGMA;
 672                                 case "extension":
 673                                         return Token.EXTENSION;
 674                                 }
 675                                 break;
 676                         case ParseState.KindTest:
 677                                 switch (name) {
 678                                 case "context":
 679                                         return Token.CONTEXT;
 680                                 case "element":
 681                                         return Token.ELEMENT;
 682                                 case "global":
 683                                         return Token.GLOBAL;
 684                                 case "type":
 685                                         return Token.TYPE;
 686                                 }
 687                                 break;
 688                         case ParseState.ItemType:
 689                                 switch (name) {
 690                                 case "attribute":
 691                                         return Token.ATTRIBUTE;
 692                                 case "comment":
 693                                         return Token.COMMENT;
 694                                 case "document-node":
 695                                         return Token.DOCUMENT_NODE;
 696                                 case "element":
 697                                         return Token.ELEMENT;
 698                                 case "empty":
 699                                         return Token.EMPTY;
 700                                 case "item":
 701                                         return Token.ITEM;
 702                                 case "node":
 703                                         return Token.NODE;
 704                                 case "processing-instruction":
 705                                         return Token.PROCESSING_INSTRUCTION;
 706                                 case "text":
 707                                         return Token.TEXT;
 708                                 }
 709                                 break;
 710                         case ParseState.NamespaceKeyword:
 711                                 switch (name) {
 712                                 case "declare":
 713                                         return Token.DECLARE;
 714                                 case "default":
 715                                         return Token.DEFAULT;
 716                                 case "element":
 717                                         return Token.ELEMENT;
 718                                 case "function":
 719                                         return Token.FUNCTION;
 720                                 case "namespace":
 721                                         return Token.NAMESPACE;
 722                                 }
 723                                 break;
 724                         case ParseState.OccurenceIndicator:
 725                         case ParseState.Operator:
 726                                 switch (name) {
 727                                 case "and":
 728                                 case "as":
 729                                 case "ascending":
 730                                 case "at":
 731                                 case "base-uri":
 732                                 case "by":
 733                                 case "case":
 734                                 case "cast":
 735                                 case "castable":
 736                                 case "collation":
 737                                 case "declare":
 738                                 case "default":
 739                                 case "descending":
 740                                 case "div":
 741                                 case "element":
 742                                 case "else":
 743                                 case "empty":
 744                                 case "eq":
 745                                 case "every":
 746                                 case "except":
 747                                 case "external":
 748                                 case "for":
 749                                 case "function":
 750                                 case "ge":
 751                                 case "global":
 752                                 case "greatest":
 753                                 case "gt":
 754                                 case "idiv":
 755                                 case "import":
 756                                 case "in":
 757                                 case "instance":
 758                                 case "intersect":
 759                                 case "is":
 760                                 case "lax":
 761                                 case "le":
 762                                 case "least":
 763                                 case "let":
 764                                 case "lt":
 765                                 case "mod":
 766                                 case "module":
 767                                 case "namespace":
 768                                 case "ne":
 769                                 case "of":
 770                                 case "or":
 771                                 case "order":
 772                                 case "ordered":
 773                                 case "ordering":
 774                                 case "return":
 775                                 case "satisfies":
 776                                 case "schema":
 777                                 case "skip":
 778                                 case "some":
 779                                 case "stable":
 780                                 case "strict":
 781                                 case "then":
 782                                 case "to":
 783                                 case "treat":
 784                                 case "typwswitch":
 785                                 case "union":
 786                                 case "unordered":
 787                                 case "variable":
 788                                 case "where":
 789                                 case "xmlspace":
 790                                         validKeyword = true;
 791                                         break;
 792                                 }
 793                                 break;
 794                         case ParseState.Default:
 795                                 switch (name) {
 796                                 case "ancestor":
 797                                 case "ancestor-or-self":
 798                                 case "as":
 799                                 case "attribute":
 800                                 case "base-uri":
 801                                 case "child":
 802                                 case "collation":
 803                                 case "comment":
 804                                 case "construction":
 805                                 case "declare":
 806                                 case "default":
 807                                 case "descendant":
 808                                 case "descendant-or-self":
 809                                 case "document":
 810                                 case "document-node":
 811                                 case "element":
 812                                 case "every":
 813                                 case "following":
 814                                 case "following-sibling":
 815                                 case "for":
 816                                 case "function":
 817                                 case "global":
 818                                 case "if":
 819                                 case "import":
 820                                 case "lax":
 821                                 case "let":
 822                                 case "module":
 823                                 case "namespace":
 824                                 case "node":
 825                                 case "ordered":
 826                                 case "parent":
 827                                 case "preceding":
 828                                 case "preceding-sibling":
 829                                 case "processing-instruction":
 830                                 case "schema":
 831                                 case "self":
 832                                 case "some":
 833                                 case "strict":
 834                                 case "strip":
 835                                 case "text":
 836                                 case "typeswitch":
 837                                 case "unordered":
 838                                 case "validate":
 839                                 case "validation":
 840                                 case "version":
 841                                 case "xmlspace":
 842                                 case "xquery":
 843                                         validKeyword = true;
 844                                         break;
 845                                 }
 846                                 break;
 847                         }
 848
 849                         if (validKeyword) {
 850                                 switch (name) {
 851                                 case "xquery":
 852                                         return Token.XQUERY;
 853                                 case "version":
 854                                         return Token.VERSION;
 855                                 case "pragma":
 856                                         return Token.PRAGMA;
 857                                 case "extension":
 858                                         return Token.EXTENSION;
 859                                 case "module":
 860                                         return Token.MODULE;
 861                                 case "namespace":
 862                                         return Token.NAMESPACE;
 863                                 case "declare":
 864                                         return Token.DECLARE;
 865                                 case "xmlspace":
 866                                         return Token.XMLSPACE;
 867                                 case "preserve":
 868                                         return Token.PRESERVE;
 869                                 case "strip":
 870                                         return Token.STRIP;
 871                                 case "default":
 872                                         return Token.DEFAULT;
 873                                 case "construction":
 874                                         return Token.CONSTRUCTION;
 875                                 case "ordering":
 876                                         return Token.ORDERING;
 877                                 case "ordered":
 878                                         return Token.ORDERED;
 879                                 case "unordered":
 880                                         return Token.UNORDERED;
 881                                 case "document-node":
 882                                         return Token.DOCUMENT_NODE;
 883                                 case "document":
 884                                         return Token.DOCUMENT;
 885                                 case "element":
 886                                         return Token.ELEMENT;
 887                                 case "attribute":
 888                                         return Token.ATTRIBUTE;
 889                                 case "processing-instruction":
 890                                         return Token.PROCESSING_INSTRUCTION;
 891                                 case "comment":
 892                                         return Token.COMMENT;
 893                                 case "text":
 894                                         return Token.TEXT;
 895                                 case "node":
 896                                         return Token.NODE;
 897                                 case "function":
 898                                         return Token.FUNCTION;
 899                                 case "collation":
 900                                         return Token.COLLATION;
 901                                 case "base-uri":
 902                                         return Token.BASEURI;
 903                                 case "import":
 904                                         return Token.IMPORT;
 905                                 case "schema":
 906                                         return Token.SCHEMA;
 907                                 case "at":
 908                                         return Token.AT;
 909                                 case "variable":
 910                                         return Token.VARIABLE;
 911                                 case "as":
 912                                         return Token.AS;
 913                                 case "external":
 914                                         return Token.EXTERNAL;
 915                                 case "validation":
 916                                         return Token.VALIDATION;
 917                                 case "lax":
 918                                         return Token.LAX;
 919                                 case "strict":
 920                                         return Token.STRICT;
 921                                 case "skip":
 922                                         return Token.SKIP;
 923                                 case "return":
 924                                         return Token.RETURN;
 925                                 case "for":
 926                                         return Token.FOR;
 927                                 case "let":
 928                                         return Token.LET;
 929                                 case "in":
 930                                         return Token.IN;
 931                                 case "where":
 932                                         return Token.WHERE;
 933                                 case "order":
 934                                         return Token.ORDER;
 935                                 case "by":
 936                                         return Token.BY;
 937                                 case "stable":
 938                                         return Token.STABLE;
 939                                 case "ascending":
 940                                         return Token.ASCENDING;
 941                                 case "descending":
 942                                         return Token.DESCENDING;
 943                                 case "empty":
 944                                         return Token.EMPTY;
 945                                 case "greatest":
 946                                         return Token.GREATEST;
 947                                 case "least":
 948                                         return Token.LEAST;
 949                                 case "some":
 950                                         return Token.SOME;
 951                                 case "every":
 952                                         return Token.EVERY;
 953                                 case "satisfies":
 954                                         return Token.SATISFIES;
 955                                 case "is":
 956                                         return Token.IS;
 957                                 case "to":
 958                                         return Token.TO;
 959                                 case "eq":
 960                                         return Token.EQ;
 961                                 case "ne":
 962                                         return Token.NE;
 963                                 case "lt":
 964                                         return Token.LT;
 965                                 case "le":
 966                                         return Token.LE;
 967                                 case "gt":
 968                                         return Token.GT;
 969                                 case "ge":
 970                                         return Token.GE;
 971                                 case "and":
 972                                         return Token.AND;
 973                                 case "or":
 974                                         return Token.OR;
 975                                 case "instance":
 976                                         return Token.INSTANCE;
 977                                 case "of":
 978                                         return Token.OF;
 979                                 case "if":
 980                                         return Token.IF;
 981                                 case "then":
 982                                         return Token.THEN;
 983                                 case "else":
 984                                         return Token.ELSE;
 985                                 case "typeswitch":
 986                                         return Token.TYPESWITCH;
 987                                 case "case":
 988                                         return Token.CASE;
 989                                 case "treat":
 990                                         return Token.TREAT;
 991                                 case "castable":
 992                                         return Token.CASTABLE;
 993                                 case "cast":
 994                                         return Token.CAST;
 995                                 case "div":
 996                                         return Token.DIV;
 997                                 case "idiv":
 998                                         return Token.IDIV;
 999                                 case "mod":
1000                                         return Token.MOD;
1001                                 case "union":
1002                                         return Token.UNION;
1003                                 case "intersect":
1004                                         return Token.INTERSECT;
1005                                 case "except":
1006                                         return Token.EXCEPT;
1007                                 case "validate":
1008                                         return Token.VALIDATE;
1009                                 case "context":
1010                                         return Token.CONTEXT;
1011                                 case "nillable":
1012                                         return Token.NILLABLE;
1013                                 case "item":
1014                                         return Token.ITEM;
1015                                 case "global":
1016                                         return Token.GLOBAL;
1017                                 case "type":
1018                                         return Token.TYPE;
1019                                 case "child":
1020                                         return Token.CHILD;
1021                                 case "descendant":
1022                                         return Token.DESCENDANT;
1023                                 case "self":
1024                                         return Token.SELF;
1025                                 case "descendant-or-self":
1026                                         return Token.DESCENDANT_OR_SELF;
1027                                 case "following-sibling":
1028                                         return Token.FOLLOWING_SIBLING;
1029                                 case "following":
1030                                         return Token.FOLLOWING;
1031                                 case "parent":
1032                                         return Token.PARENT;
1033                                 case "ancestor":
1034                                         return Token.ANCESTOR;
1035                                 case "preceding":
1036                                         return Token.PRECEDING;
1037                                 case "preceding-sibling":
1038                                         return Token.PRECEDING_SIBLING;
1039                                 case "ancestor-or-self":
1040                                         return Token.ANCESTOR_OR_SELF;
1041                                 }
1042                         }
1043
1044                         switch (state) {
1045                         case ParseState.NamespaceDecl:
1046                         case ParseState.NamespaceKeyword:
1047                         case ParseState.XmlSpaceDecl:
1048                         case ParseState.KindTestForPI:
1049                         case ParseState.XmlPI:
1050                                 return Token.NCNAME;
1051                         }
1052
1053                         if (PeekChar () == ':') {
1054                                 ReadChar ();
1055                                 prefixName = name;
1056                                 switch (PeekChar ()) {
1057                                 case '*':
1058                                         ReadChar ();
1059                                         name = "*";
1060                                         break;
1061                                 case '=': // ex. let foo:= ...
1062                                         ReadChar ();
1063                                         tokenValue = new XmlQualifiedName (name, nsResolver.DefaultNamespace);
1064                                         lookAheadToken = Token.COLON_EQUAL;
1065                                         return Token.QNAME;
1066                                 default:
1067                                         name = ReadOneToken ();
1068                                         break;
1069                                 }
1070
1071                                 string ns = nsResolver.LookupNamespace (prefixName);
1072                                 if (ns == null)
1073                                         throw Error (String.Format ("Prefix '{0}' is not mapped to any namespace URI.", prefixName));
1074                                 tokenValue = new XmlQualifiedName (name, ns);
1075                                 prefixName = null;
1076                                 return name == "*" ? Token.WILD_LOCALNAME : Token.QNAME;
1077                         }
1078                         tokenValue = new XmlQualifiedName (name);
1079                         return Token.QNAME;
1080                 }
1081
1082                 private int PeekChar ()
1083                 {
1084                         if (peekChar == -1)
1085                                 peekChar = source.Read ();
1086                         return peekChar;
1087                 }
1088
1089                 private int ReadChar ()
1090                 {
1091                         int ret;
1092                         if (peekChar != -1) {
1093                                 ret = peekChar;
1094                                 peekChar = -1;
1095                         }
1096                         else
1097                                 ret = source.Read ();
1098
1099                         if (nextIncrementLine) {
1100                                 line++;
1101                                 column = 0;
1102                                 nextIncrementLine = false;
1103                         }
1104                         column++;
1105                         switch (ret) {
1106                         case '\r':
1107                                 break;
1108                         case '\n':
1109                                 nextIncrementLine = true;
1110                                 goto default;
1111                         default:
1112                                 break;
1113                         }
1114
1115                         return ret;
1116                 }
1117
1118                 private void SkipWhitespaces ()
1119                 {
1120                         while (true) {
1121                                 switch (PeekChar ()) {
1122                                 case ' ':
1123                                 case '\t':
1124                                 case '\r':
1125                                 case '\n':
1126                                         ReadChar ();
1127                                         continue;
1128                                 default:
1129                                         return;
1130                                 }
1131                         }
1132                 }
1133
1134                 private void AddValueChar (char c)
1135                 {
1136                         if (bufferIndex == buffer.Length) {
1137                                 char [] newBuf = new char [bufferIndex * 2];
1138                                 Array.Copy (buffer, newBuf, bufferIndex);
1139                                 buffer = newBuf;
1140                         }
1141                         buffer [bufferIndex++] = c;
1142                 }
1143
1144                 private string CreateValueString ()
1145                 {
1146                         return new string (buffer, 0, bufferIndex);
1147                 }
1148
1149                 private void Expect (string expected)
1150                 {
1151                         for (int i = 0; i < expected.Length; i++)
1152                                 if (ReadChar () != expected [i])
1153                                         throw Error (String.Format ("Expected token '{0}' did not appear.", expected));
1154                 }
1155
1156                 // TODO: parse three quoted
1157                 private string ReadQuoted (char quoteChar)
1158                 {
1159                         bufferIndex = 0;
1160                         bool loop = true;
1161                         do {
1162                                 int c = ReadChar ();
1163                                 switch (c) {
1164                                 case -1:
1165                                 case '"':
1166                                         if (quoteChar == '"')
1167                                                 loop = false;
1168                                         break;
1169                                 case '\'':
1170                                         if (quoteChar == '\'')
1171                                                 loop = false;
1172                                         break;
1173                                 default:
1174                                         AddValueChar ((char) c);
1175                                         break;
1176                                 }
1177                         } while (loop);
1178
1179                         return CreateValueString ();
1180                 }
1181
1182                 private decimal ReadDecimal (bool floatingPoint)
1183                 {
1184                         bufferIndex = 0;
1185                         bool cond = true;
1186                         do {
1187                                 int c = PeekChar ();
1188                                 if (c < 0) {
1189                                         cond = false;
1190                                 }
1191                                 // FIXME: more complex
1192                                 else if (Char.IsNumber ((char) c) || c == '.') {
1193                                         ReadChar ();
1194                                         AddValueChar ((char) c);
1195                                         continue;
1196                                 }
1197                                 else
1198                                         cond = false;
1199                         } while (cond);
1200                         string s = (floatingPoint ? "." : "") + CreateValueString ();
1201                         return decimal.Parse (s);
1202                 }
1203
1204                 private string ReadOneToken ()
1205                 {
1206                         bufferIndex = 0;
1207                         bool loop = true;
1208                         do {
1209                                 int c = PeekChar ();
1210                                 switch (c) {
1211                                 case -1:
1212                                 case ' ':
1213                                 case '\t':
1214                                 case '\r':
1215                                 case '\n':
1216                                         loop = false;
1217                                         break;
1218                                 default:
1219                                         if (!IsTokenContinuable (c)) {
1220                                                 if (c == ':') {
1221                                                         if (prefixName != null)
1222                                                                 throw new XmlQueryCompileException ("Invalid colon was found.");
1223                                                         prefixName = CreateValueString ();
1224                                                 }
1225                                                 loop = false;
1226                                                 break;
1227                                         }
1228
1229                                         ReadChar ();
1230                                         AddValueChar ((char) c);
1231                                         break;
1232                                 }
1233                         } while (loop);
1234
1235                         return CreateValueString ();
1236                 }
1237
1238                 private bool IsTokenContinuable (int c)
1239                 {
1240                         switch (c) {
1241                         case '-':
1242                         case '_':
1243                         case '.':
1244                                 return true;
1245                         }
1246                         return XmlChar.IsNCNameChar (c);
1247                 }
1248
1249         }
1250
1251         public enum WhitespaceHandling {
1252                 Arbitrary,
1253                 Explicit,
1254                 Significant
1255         }
1256
1257         public enum ParseState {
1258                 Default,
1259                 Operator,
1260                 NamespaceDecl,
1261                 NamespaceKeyword,
1262                 XmlSpaceDecl,
1263                 ItemType,
1264                 KindTest,
1265                 KindTestForPI,
1266                 CloseKindTest,
1267                 OccurenceIndicator,
1268                 SchemaContextStep,
1269                 VarName,
1270                 StartTag,
1271                 ElementContent,
1272                 EndTag,
1273                 XmlComment,
1274                 ExprComment,
1275                 ExtKey,
1276                 XmlPI,
1277                 XmlPIContent,
1278                 CDataSection,
1279                 QuotAttributeContent,
1280                 AposAttributeContent,
1281         }
1282
1283 }
1284 #endif