class/Mono.Xml.Ext/Mono.Xml.XPath2/TokenizerBase.cs

   1 //
   2 // XQueryTokenizer.cs
   3 //
   4 // Author:
   5 //      Atsushi Enomoto <atsushi@ximian.com>
   6 //
   7 // Copyright (C) 2004 Novell, Inc (http://www.novell.com)
   8 //
   9 // Permission is hereby granted, free of charge, to any person obtaining
  10 // a copy of this software and associated documentation files (the
  11 // "Software"), to deal in the Software without restriction, including
  12 // without limitation the rights to use, copy, modify, merge, publish,
  13 // distribute, sublicense, and/or sell copies of the Software, and to
  14 // permit persons to whom the Software is furnished to do so, subject to
  15 // the following conditions:
  16 //
  17 // The above copyright notice and this permission notice shall be
  18 // included in all copies or substantial portions of the Software.
  19 //
  20 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  21 // EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  22 // MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  23 // NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
  24 // LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  25 // OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  26 // WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  27 //
  28 #if NET_2_0
  29
  30 using System;
  31 using System.Collections;
  32 using System.Collections.Generic;
  33 using System.IO;
  34 using System.Security.Policy;
  35 using System.Xml;
  36 using System.Xml.Query;
  37 using System.Xml.Schema;
  38 using System.Xml.XPath;
  39 using Mono.Xml.XQuery;
  40 using Mono.Xml.XPath2;
  41 using Mono.Xml;
  42
  43 #if XPATH2_PARSER
  44 namespace Mono.Xml.XPath2.Parser
  45 #elif XQUERY_PARSER
  46 namespace Mono.Xml.XQuery.Parser
  47 #endif
  48 {
  49         // FIXME: make internal in the future
  50         public class XQueryTokenizer : yyParser.yyInput, IXmlLineInfo
  51         {
  52                 int line = 1;
  53                 int column = 0;
  54                 bool nextIncrementLine;
  55
  56                 // namespace resolver
  57                 XmlNamespaceManager nsResolver;
  58                 string defaultFunctionNamespace = XQueryFunction.Namespace;
  59
  60                 // input source
  61                 TextReader source;
  62                 int peekChar = -1;
  63
  64                 // token info
  65                 int currentToken;
  66                 string prefixName;
  67                 object tokenValue;
  68
  69                 int lookAheadToken = -1;
  70                 object lookAheadTokenValue;
  71
  72                 // state info
  73                 WhitespaceHandling ws = WhitespaceHandling.Arbitrary;
  74                 ParseState state = ParseState.Default;
  75                 Stack stateStack;
  76
  77                 char [] buffer = new char [30];
  78                 int bufferIndex;
  79
  80                 public XQueryTokenizer (TextReader reader)
  81                 {
  82                         this.source = reader;
  83
  84                         stateStack = new Stack ();
  85
  86                         nsResolver = new XmlNamespaceManager (new NameTable ());
  87                         nsResolver.AddNamespace ("xs", XmlSchema.Namespace);
  88                         nsResolver.AddNamespace ("xdt", InternalPool.XdtNamespace);
  89                         // FIXME: Are they really predefined?
  90                         nsResolver.AddNamespace ("xsi", XmlSchema.InstanceNamespace);
  91                         nsResolver.AddNamespace ("fn", "http://www.w3.org/2003/11/xpath-functions");
  92                         nsResolver.AddNamespace ("local", "http://www.w3.org/2003/11/xquery-local-functions");
  93                 }
  94
  95                 internal IXmlNamespaceResolver NSResolver {
  96                         get { return nsResolver; }
  97                 }
  98
  99                 internal string DefaultFunctionNamespace {
 100                         get { return defaultFunctionNamespace; }
 101                         set { defaultFunctionNamespace = value; }
 102                 }
 103
 104                 public void AddNamespace (string prefix, string ns)
 105                 {
 106                         nsResolver.AddNamespace (prefix, ns);
 107                 }
 108
 109                 public bool advance ()
 110                 {
 111                         if (currentToken < 0)
 112                                 return false;
 113                         if (lookAheadToken >= 0) {
 114                                 tokenValue = lookAheadTokenValue;
 115                                 currentToken = lookAheadToken;
 116                                 lookAheadToken = -1;
 117                         }
 118                         else
 119                                 currentToken = ParseToken ();
 120                         return currentToken >= 0;
 121                 }
 122
 123                 public int token ()
 124                 {
 125                         return currentToken;
 126                 }
 127
 128                 public object value ()
 129                 {
 130                         return tokenValue;
 131                 }
 132
 133                 public bool HasLineInfo ()
 134                 {
 135                         return true;
 136                 }
 137
 138                 public int LineNumber {
 139                         get { return line; }
 140                 }
 141
 142                 public int LinePosition {
 143                         get { return column; }
 144                 }
 145
 146                 internal WhitespaceHandling Space {
 147                         get { return ws; }
 148                         set { ws = value; }
 149                 }
 150
 151                 internal ParseState State {
 152                         get { return state; }
 153                         set {
 154 //                              Console.Error.WriteLine ("**** eno **** state transition from {0} to {1}, stack count = {2}", state, value, stateStack.Count);
 155 //foreach (ParseState ps in stateStack.ToArray ()) Console.Error.WriteLine ("***** eno ***** " + ps);
 156                                 state = value;
 157                         }
 158                 }
 159
 160                 internal void PushState (ParseState newState)
 161                 {
 162                         stateStack.Push (newState);
 163 //                      Console.Error.WriteLine ("**** eno **** state pushed {0}, added stack count = {1}", newState, stateStack.Count);
 164 //foreach (ParseState ps in stateStack.ToArray ()) Console.Error.WriteLine ("***** eno ***** " + ps);
 165                 }
 166
 167                 internal void PopState ()
 168                 {
 169                         if (stateStack.Count == 0)
 170                                 throw Error ("Internal state transition error. State stack is empty.");
 171                         state = (ParseState) stateStack.Pop ();
 172 //                      Console.Error.WriteLine ("**** eno **** state pop, now as {0}, stack count = {1}", state, stateStack.Count);
 173 //foreach (ParseState ps in stateStack.ToArray ()) Console.Error.WriteLine ("***** eno ***** " + ps);
 174                 }
 175
 176                 private XmlQueryCompileException Error (string message)
 177                 {
 178                         return new XmlQueryCompileException (message, this, null, null);
 179                 }
 180
 181                 private int ParseToken ()
 182                 {
 183                         bufferIndex = 0;
 184
 185                         switch (state) {
 186                         case ParseState.StartTag:
 187                                 break;
 188                         default:
 189                                 SkipWhitespaces ();
 190                                 break;
 191                         }
 192                         /*
 193                         switch (ws) {
 194                         case WhitespaceHandling.Arbitrary:
 195                                 SkipWhitespaces ();
 196                                 break;
 197                         case WhitespaceHandling.Explicit:
 198                                 if (!XmlChar.IsWhitespace (PeekChar ()))
 199                                         throw Error ("Whitespace is required.");
 200                                 goto case WhitespaceHandling.Arbitrary;
 201                         }
 202                         */
 203
 204                         int c = PeekChar ();
 205                         if (c < 0)
 206                                 return -1;
 207
 208                         // FIXME: consider DOUBLE_LITERAL
 209                         if (Char.IsNumber ((char) c)) {
 210                                 tokenValue = ReadDecimal (false);
 211                                 return Token.DECIMAL_LITERAL;
 212                         }
 213
 214                         switch (state) {
 215                         case ParseState.OccurenceIndicator:
 216                                 return ParseOccurenceIndicator ();
 217                         case ParseState.XmlPIContent:
 218                                 return ParseXmlPIContent ();
 219                         case ParseState.XmlComment:
 220                                 return ParseXmlCommentContent ();
 221                         case ParseState.ElementContent:
 222                                 return ParseElementContent ();
 223                         case ParseState.StartTag:
 224                                 return ParseStartTag ();
 225                         case ParseState.QuotAttributeContent:
 226                                 return ParseAttributeContent ('"');
 227                         case ParseState.AposAttributeContent:
 228                                 return ParseAttributeContent ('\'');
 229                         default:
 230                                 return ParseDefault ();
 231                         }
 232                 }
 233
 234                 private int ParseXQueryComment ()
 235                 {
 236                         while (true) {
 237                                 int c = ReadChar ();
 238                                 if (c < 0)
 239                                         throw Error ("Unexpected end of query text inside XML processing instruction content");
 240                                 if (c == ':') {
 241                                         if (PeekChar () == ')') {
 242                                                 ReadChar ();
 243                                                 tokenValue = CreateValueString ();
 244                                                 return Token.XML_PI_TO_END;
 245                                         }
 246                                         else
 247                                                 AddValueChar (':');
 248                                 }
 249                                 else
 250                                         AddValueChar ((char) c);
 251                         }
 252                 }
 253
 254                 private int ParseXmlPIContent ()
 255                 {
 256                         while (true) {
 257                                 int c = ReadChar ();
 258                                 if (c < 0)
 259                                         throw Error ("Unexpected end of query text inside XML processing instruction content");
 260                                 if (c == '?') {
 261                                         if (PeekChar () == '>') {
 262                                                 ReadChar ();
 263                                                 tokenValue = CreateValueString ();
 264                                                 return Token.XML_PI_TO_END;
 265                                         }
 266                                         else
 267                                                 AddValueChar ('?');
 268                                 }
 269                                 else
 270                                         AddValueChar ((char) c);
 271                         }
 272                 }
 273
 274                 private int ParseXmlCommentContent ()
 275                 {
 276                         // FIXME: handle ---> correctly
 277                         while (true) {
 278                                 int c = ReadChar ();
 279                                 if (c < 0)
 280                                         throw Error ("Unexpected end of query text inside XML comment content");
 281                                 if (c == '-') {
 282                                         if (PeekChar () == '-') {
 283                                                 ReadChar ();
 284                                                 if (PeekChar () == '>') {
 285                                                         tokenValue = CreateValueString ();
 286                                                         return Token.XML_COMMENT_TO_END;
 287                                                 } else {
 288                                                         AddValueChar ('-');
 289                                                         AddValueChar ('-');
 290                                                 }
 291                                         }
 292                                         else
 293                                                 AddValueChar ('-');
 294                                 }
 295                                 else
 296                                         AddValueChar ((char) c);
 297                         }
 298                 }
 299
 300                 private int ParseXmlCDataContent ()
 301                 {
 302                         // FIXME: handle ]]]> correctly
 303                         while (true) {
 304                                 int c = ReadChar ();
 305                                 if (c < 0)
 306                                         throw Error ("Unexpected end of query text inside XML CDATA section content");
 307                                 if (c == ']') {
 308                                         ReadChar ();
 309                                         if (PeekChar () == ']') {
 310                                                 ReadChar ();
 311                                                 if (PeekChar () == '>') {
 312                                                         tokenValue = CreateValueString ();
 313                                                         return Token.XML_CDATA_TO_END;
 314                                                 } else {
 315                                                         AddValueChar (']');
 316                                                         AddValueChar (']');
 317                                                 }
 318                                         }
 319                                         else
 320                                                 AddValueChar (']');
 321                                 }
 322                                 else
 323                                         AddValueChar ((char) c);
 324                         }
 325                 }
 326
 327                 private int ParseElementContent ()
 328                 {
 329                         tokenValue = null;
 330                         int c = PeekChar ();
 331                         if (c < 0)
 332                                 throw Error ("Unexpected end of query text inside XML processing instruction content");
 333                         switch ((char) c) {
 334                         case '<':
 335                         case '{':
 336                                 return ParseDefault ();
 337                         }
 338
 339                         while (true) {
 340                                 c = PeekChar ();
 341                                 if (c < 0)
 342                                         throw Error ("Unexpected end of query text inside XML processing instruction content");
 343                                 switch ((char) c) {
 344                                 case '&':
 345                                         ReadChar ();
 346                                         ReadPredefinedEntity ();
 347                                         continue;
 348                                 case '<':
 349                                         tokenValue += CreateValueString ();
 350                                         return Token.ELEM_CONTENT_LITERAL;
 351                                 default:
 352                                         AddValueChar ((char) c);
 353                                         ReadChar ();
 354                                         continue;
 355                                 }
 356                         }
 357                 }
 358
 359                 private void ReadPredefinedEntity ()
 360                 {
 361                         string token = ReadOneToken ();
 362                         Expect (";");
 363                         switch (token) {
 364                         case "lt":
 365                                 AddValueChar ('<');
 366                                 return;
 367                         case "gt":
 368                                 AddValueChar ('>');
 369                                 return;
 370                         case "amp":
 371                                 AddValueChar ('&');
 372                                 return;
 373                         case "quot":
 374                                 AddValueChar ('"');
 375                                 return;
 376                         case "apos":
 377                                 AddValueChar ('\'');
 378                                 return;
 379                         default:
 380                                 throw Error (String.Format ("Unexpected general entity name: {0} .", token));
 381                         }
 382                 }
 383
 384                 // FIXME: not used as yet
 385                 private int ParseExtContent ()
 386                 {
 387                         // FIXME: handle :::) correctly
 388                         while (true) {
 389                                 int c = PeekChar ();
 390                                 if (c < 0)
 391                                         throw Error ("Unexpected end of query text inside external content");
 392                                 if (c == ':') {
 393                                         ReadChar ();
 394                                         if (PeekChar () == ':') {
 395                                                 ReadChar ();
 396                                                 if (PeekChar () == ')') {
 397                                                         tokenValue = CreateValueString ();
 398                                                         return Token.EXT_CONTENT;
 399                                                 } else {
 400                                                         AddValueChar (':');
 401                                                         AddValueChar (':');
 402                                                 }
 403                                         }
 404                                         else
 405                                                 AddValueChar (':');
 406                                 }
 407                                 else
 408                                         AddValueChar ((char) c);
 409                         }
 410                 }
 411
 412                 private int ParseOccurenceIndicator ()
 413                 {
 414                         state = ParseState.Operator;
 415                         switch (PeekChar ()) {
 416                         case '?':
 417                                 ReadChar ();
 418                                 return Token.QUESTION;
 419                         case '*':
 420                                 ReadChar ();
 421                                 return Token.ASTERISK;
 422                         case '+':
 423                                 ReadChar ();
 424                                 return Token.PLUS;
 425                         default:
 426                                 return ParseOperator ();
 427                         }
 428                 }
 429
 430                 private int ParseStartTag ()
 431                 {
 432                         int c = PeekChar ();
 433                         switch (c) {
 434                         case '\'':
 435                                 ReadChar ();
 436                                 return Token.APOS;
 437                         case '"':
 438                                 ReadChar ();
 439                                 return Token.QUOT;
 440                         case '>':
 441                                 ReadChar ();
 442                                 return Token.GREATER;
 443                         case '/':
 444                                 ReadChar ();
 445                                 Expect (">");
 446                                 return Token.EMPTY_TAG_CLOSE;
 447                         }
 448                         // FIXME: there seems a bug in the spec that StartTag
 449                         // state must accept QName without heading space for
 450                         // start tag name.
 451 //                      if (!XmlChar.IsWhitespace (PeekChar ()))
 452 //                              throw Error ("Whitespace is required.");
 453                         SkipWhitespaces ();
 454                         return ParseDefault (); // only QName is allowed here.
 455                 }
 456
 457                 private int ParseAttributeContent (char closeChar)
 458                 {
 459                         int t = Token.ATT_VALUE_LITERAL;
 460                         while (true) {
 461                                 int c = PeekChar ();
 462                                 if (c < 0)
 463                                         throw Error ("Unexpected end of attribute value content.");
 464                                 if (c == closeChar) {
 465                                         ReadChar ();
 466                                         c = PeekChar ();
 467                                         if (c == closeChar) {
 468                                                 ReadChar ();
 469                                                 AddValueChar (closeChar);
 470                                         }
 471                                         else
 472                                                 t = closeChar == '"' ? Token.QUOT : Token.APOS;
 473                                 }
 474                                 else if (c == '{') {
 475                                         ReadChar ();
 476                                         c = PeekChar ();
 477                                         if (c == '{') {
 478                                                 ReadChar ();
 479                                                 AddValueChar ('{');
 480                                         }
 481                                         else
 482                                                 t = Token.OPEN_CURLY;
 483                                 }
 484                                 else
 485                                         AddValueChar ((char) ReadChar ());
 486
 487                                 if (t != Token.ATT_VALUE_LITERAL) {
 488                                         if (bufferIndex > 0) {
 489                                                 lookAheadToken = t;
 490                                                 tokenValue = CreateValueString ();
 491                                                 return Token.ATT_VALUE_LITERAL;
 492                                         }
 493                                         else
 494                                                 return t;
 495                                 }
 496                         }
 497                 }
 498
 499                 private int ParseOperator ()
 500                 {
 501                         // TODO: implement
 502                         return ParseDefault ();
 503                 }
 504
 505                 private int ParseDefault ()
 506                 {
 507                         int c = ReadChar ();
 508                         switch (c) {
 509                         case '.':
 510                                 if (PeekChar () == '.') {
 511                                         ReadChar ();
 512                                         return Token.DOT2;
 513                                 }
 514                                 else if (Char.IsNumber ((char) PeekChar ())) {
 515                                         tokenValue = ReadDecimal (true);
 516                                 }
 517                                 return Token.DOT;
 518                         case ',':
 519                                 return Token.COMMA;
 520                         case ';':
 521                                 return Token.SEMICOLON;
 522                         case '(':
 523                                 if (PeekChar () == ':') {
 524                                         ReadChar ();
 525                                         if (PeekChar () == ':') {
 526                                                 ReadChar ();
 527                                                 return Token.PRAGMA_OPEN;
 528                                         }
 529                                         ParseXQueryComment ();
 530                                         return ParseToken (); // start again
 531                                 }
 532                                 return Token.OPEN_PAREN;
 533                         case ')':
 534                                 return Token.CLOSE_PAREN;
 535                         case ':':
 536                                 switch (PeekChar ()) {
 537                                 case ':':
 538                                         ReadChar ();
 539                                         if (PeekChar () == ')') {
 540                                                 ReadChar ();
 541                                                 return Token.PRAGMA_CLOSE;
 542                                         }
 543                                         return Token.COLON2;
 544                                 case ')':
 545                                         ReadChar ();
 546                                         return Token.CLOSE_PAREN_COLON;
 547                                 case '=':
 548                                         ReadChar ();
 549                                         return Token.COLON_EQUAL;
 550                                 }
 551                                 return Token.COLON;
 552                         case '[':
 553                                 return Token.OPEN_BRACKET;
 554                         case ']':
 555                                 return Token.CLOSE_BRACKET;
 556                         case '{':
 557                                 return Token.OPEN_CURLY;
 558                         case '}':
 559                                 return Token.CLOSE_CURLY;
 560                         case '$':
 561                                 return Token.DOLLAR;
 562                         case '\'':
 563                                 tokenValue = ReadQuoted ('\'');
 564                                 return Token.STRING_LITERAL;
 565                         case '"':
 566                                 tokenValue = ReadQuoted ('"');
 567                                 return Token.STRING_LITERAL;
 568                         case '=':
 569                                 return Token.EQUAL;
 570                         case '<':
 571                                 // only happens when state is ElementContent
 572                                 // (otherwise it might be "/foo</bar")
 573                                 if (state == ParseState.ElementContent) {
 574                                         switch ((char) PeekChar ()) {
 575                                         case '/':
 576                                                 ReadChar ();
 577                                                 return Token.END_TAG_START;
 578                                         case '!':
 579                                                 ReadChar ();
 580                                                 switch (PeekChar ()) {
 581                                                 case '-':
 582                                                         ReadChar ();
 583                                                         if (ReadChar () != '-')
 584                                                                 throw Error ("Invalid sequence of characters '<!-'.");
 585
 586                                                         return Token.XML_COMMENT_START;
 587                                                 case '[':
 588                                                         ReadChar ();
 589                                                         Expect ("CDATA[");
 590                                                         return Token.XML_CDATA_START;
 591                                                 }
 592                                                 throw Error ("Invalid sequence of characters '<!'.");
 593                                         case '?':
 594                                                 ReadChar ();
 595                                                 return Token.XML_PI_START;
 596                                         default:
 597                                                 return Token.LESSER;
 598                                         }
 599                                 }
 600
 601                                 switch (PeekChar ()) {
 602                                 case '<':
 603                                         ReadChar ();
 604                                         return Token.LESSER2;
 605                                 case '=':
 606                                         ReadChar ();
 607                                         return Token.LESSER_EQUAL;
 608                                 }
 609                                 return Token.LESSER;
 610                         case '>':
 611                                 switch (PeekChar ()) {
 612                                 case '>':
 613                                         ReadChar ();
 614                                         return Token.GREATER2;
 615                                 case '=':
 616                                         ReadChar ();
 617                                         return Token.GREATER_EQUAL;
 618                                 }
 619                                 return Token.GREATER;
 620                         case '|':
 621                                 return Token.BAR;
 622                         case '*':
 623                                 if (PeekChar () == ':') {
 624                                         ReadChar ();
 625                                         // FIXME: more check
 626                                         tokenValue = new XmlQualifiedName (ReadOneToken (), "*");
 627                                         return Token.WILD_PREFIX;
 628                                 }
 629                                 return Token.ASTERISK;
 630                         case '+':
 631                                 return Token.PLUS;
 632                         case '-':
 633                                 return Token.MINUS;
 634                         case '/':
 635                                 // only happens when state is StartTag
 636                                 // (otherwise it might be "/>$extvar")
 637                                 if (state == ParseState.StartTag && PeekChar () == '>') {
 638                                         ReadChar ();
 639                                         return Token.EMPTY_TAG_CLOSE;
 640                                 }
 641                                 if (PeekChar () == '/') {
 642                                         ReadChar ();
 643                                         return Token.SLASH2;
 644                                 }
 645                                 return Token.SLASH;
 646                         case '?':
 647                                 return Token.QUESTION;
 648                         case '@':
 649                                 return Token.AT;
 650                         }
 651
 652                         peekChar = c;
 653                         prefixName = null;
 654                         string name = ReadOneToken ();
 655
 656                         tokenValue = name;
 657                         bool validKeyword = false;
 658
 659                         switch (state) {
 660                         case ParseState.XmlSpaceDecl:
 661                                 switch (name) {
 662                                 case "preserve":
 663                                         return Token.PRESERVE;
 664                                 case "strip":
 665                                         return Token.STRIP;
 666                                 }
 667                                 break;
 668                         case ParseState.CloseKindTest:
 669                                 if (name == "nillable")
 670                                         return Token.NILLABLE;
 671                                 break;
 672                         case ParseState.ExtKey:
 673                                 switch (name) {
 674                                 case "pragma":
 675                                         return Token.PRAGMA;
 676                                 case "extension":
 677                                         return Token.EXTENSION;
 678                                 }
 679                                 break;
 680                         case ParseState.KindTest:
 681                                 switch (name) {
 682                                 case "context":
 683                                         return Token.CONTEXT;
 684                                 case "element":
 685                                         return Token.ELEMENT;
 686                                 case "global":
 687                                         return Token.GLOBAL;
 688                                 case "type":
 689                                         return Token.TYPE;
 690                                 }
 691                                 break;
 692                         case ParseState.ItemType:
 693                                 switch (name) {
 694                                 case "attribute":
 695                                         return Token.ATTRIBUTE;
 696                                 case "comment":
 697                                         return Token.COMMENT;
 698                                 case "document-node":
 699                                         return Token.DOCUMENT_NODE;
 700                                 case "element":
 701                                         return Token.ELEMENT;
 702                                 case "empty":
 703                                         return Token.EMPTY;
 704                                 case "item":
 705                                         return Token.ITEM;
 706                                 case "node":
 707                                         return Token.NODE;
 708                                 case "processing-instruction":
 709                                         return Token.PROCESSING_INSTRUCTION;
 710                                 case "text":
 711                                         return Token.TEXT;
 712                                 }
 713                                 break;
 714                         case ParseState.NamespaceKeyword:
 715                                 switch (name) {
 716                                 case "declare":
 717                                         return Token.DECLARE;
 718                                 case "default":
 719                                         return Token.DEFAULT;
 720                                 case "element":
 721                                         return Token.ELEMENT;
 722                                 case "function":
 723                                         return Token.FUNCTION;
 724                                 case "namespace":
 725                                         return Token.NAMESPACE;
 726                                 }
 727                                 break;
 728                         case ParseState.OccurenceIndicator:
 729                         case ParseState.Operator:
 730                                 switch (name) {
 731                                 case "and":
 732                                 case "as":
 733                                 case "ascending":
 734                                 case "at":
 735                                 case "base-uri":
 736                                 case "by":
 737                                 case "case":
 738                                 case "cast":
 739                                 case "castable":
 740                                 case "collation":
 741                                 case "declare":
 742                                 case "default":
 743                                 case "descending":
 744                                 case "div":
 745                                 case "element":
 746                                 case "else":
 747                                 case "empty":
 748                                 case "eq":
 749                                 case "every":
 750                                 case "except":
 751                                 case "external":
 752                                 case "for":
 753                                 case "function":
 754                                 case "ge":
 755                                 case "global":
 756                                 case "greatest":
 757                                 case "gt":
 758                                 case "idiv":
 759                                 case "import":
 760                                 case "in":
 761                                 case "instance":
 762                                 case "intersect":
 763                                 case "is":
 764                                 case "lax":
 765                                 case "le":
 766                                 case "least":
 767                                 case "let":
 768                                 case "lt":
 769                                 case "mod":
 770                                 case "module":
 771                                 case "namespace":
 772                                 case "ne":
 773                                 case "of":
 774                                 case "or":
 775                                 case "order":
 776                                 case "ordered":
 777                                 case "ordering":
 778                                 case "return":
 779                                 case "satisfies":
 780                                 case "schema":
 781                                 case "skip":
 782                                 case "some":
 783                                 case "stable":
 784                                 case "strict":
 785                                 case "then":
 786                                 case "to":
 787                                 case "treat":
 788                                 case "typwswitch":
 789                                 case "union":
 790                                 case "unordered":
 791                                 case "variable":
 792                                 case "where":
 793                                 case "xmlspace":
 794                                         validKeyword = true;
 795                                         break;
 796                                 }
 797                                 break;
 798                         case ParseState.Default:
 799                                 switch (name) {
 800                                 case "ancestor":
 801                                 case "ancestor-or-self":
 802                                 case "as":
 803                                 case "attribute":
 804                                 case "base-uri":
 805                                 case "child":
 806                                 case "collation":
 807                                 case "comment":
 808                                 case "construction":
 809                                 case "declare":
 810                                 case "default":
 811                                 case "descendant":
 812                                 case "descendant-or-self":
 813                                 case "document":
 814                                 case "document-node":
 815                                 case "element":
 816                                 case "every":
 817                                 case "following":
 818                                 case "following-sibling":
 819                                 case "for":
 820                                 case "function":
 821                                 case "global":
 822                                 case "if":
 823                                 case "import":
 824                                 case "lax":
 825                                 case "let":
 826                                 case "module":
 827                                 case "namespace":
 828                                 case "node":
 829                                 case "ordered":
 830                                 case "parent":
 831                                 case "preceding":
 832                                 case "preceding-sibling":
 833                                 case "processing-instruction":
 834                                 case "schema":
 835                                 case "self":
 836                                 case "some":
 837                                 case "strict":
 838                                 case "strip":
 839                                 case "text":
 840                                 case "typeswitch":
 841                                 case "unordered":
 842                                 case "validate":
 843                                 case "validation":
 844                                 case "version":
 845                                 case "xmlspace":
 846                                 case "xquery":
 847                                         validKeyword = true;
 848                                         break;
 849                                 }
 850                                 break;
 851                         }
 852
 853                         if (validKeyword) {
 854                                 switch (name) {
 855                                 case "xquery":
 856                                         return Token.XQUERY;
 857                                 case "version":
 858                                         return Token.VERSION;
 859                                 case "pragma":
 860                                         return Token.PRAGMA;
 861                                 case "extension":
 862                                         return Token.EXTENSION;
 863                                 case "module":
 864                                         return Token.MODULE;
 865                                 case "namespace":
 866                                         return Token.NAMESPACE;
 867                                 case "declare":
 868                                         return Token.DECLARE;
 869                                 case "xmlspace":
 870                                         return Token.XMLSPACE;
 871                                 case "preserve":
 872                                         return Token.PRESERVE;
 873                                 case "strip":
 874                                         return Token.STRIP;
 875                                 case "default":
 876                                         return Token.DEFAULT;
 877                                 case "construction":
 878                                         return Token.CONSTRUCTION;
 879                                 case "ordering":
 880                                         return Token.ORDERING;
 881                                 case "ordered":
 882                                         return Token.ORDERED;
 883                                 case "unordered":
 884                                         return Token.UNORDERED;
 885                                 case "document-node":
 886                                         return Token.DOCUMENT_NODE;
 887                                 case "document":
 888                                         return Token.DOCUMENT;
 889                                 case "element":
 890                                         return Token.ELEMENT;
 891                                 case "attribute":
 892                                         return Token.ATTRIBUTE;
 893                                 case "processing-instruction":
 894                                         return Token.PROCESSING_INSTRUCTION;
 895                                 case "comment":
 896                                         return Token.COMMENT;
 897                                 case "text":
 898                                         return Token.TEXT;
 899                                 case "node":
 900                                         return Token.NODE;
 901                                 case "function":
 902                                         return Token.FUNCTION;
 903                                 case "collation":
 904                                         return Token.COLLATION;
 905                                 case "base-uri":
 906                                         return Token.BASEURI;
 907                                 case "import":
 908                                         return Token.IMPORT;
 909                                 case "schema":
 910                                         return Token.SCHEMA;
 911                                 case "at":
 912                                         return Token.AT;
 913                                 case "variable":
 914                                         return Token.VARIABLE;
 915                                 case "as":
 916                                         return Token.AS;
 917                                 case "external":
 918                                         return Token.EXTERNAL;
 919                                 case "validation":
 920                                         return Token.VALIDATION;
 921                                 case "lax":
 922                                         return Token.LAX;
 923                                 case "strict":
 924                                         return Token.STRICT;
 925                                 case "skip":
 926                                         return Token.SKIP;
 927                                 case "return":
 928                                         return Token.RETURN;
 929                                 case "for":
 930                                         return Token.FOR;
 931                                 case "let":
 932                                         return Token.LET;
 933                                 case "in":
 934                                         return Token.IN;
 935                                 case "where":
 936                                         return Token.WHERE;
 937                                 case "order":
 938                                         return Token.ORDER;
 939                                 case "by":
 940                                         return Token.BY;
 941                                 case "stable":
 942                                         return Token.STABLE;
 943                                 case "ascending":
 944                                         return Token.ASCENDING;
 945                                 case "descending":
 946                                         return Token.DESCENDING;
 947                                 case "empty":
 948                                         return Token.EMPTY;
 949                                 case "greatest":
 950                                         return Token.GREATEST;
 951                                 case "least":
 952                                         return Token.LEAST;
 953                                 case "some":
 954                                         return Token.SOME;
 955                                 case "every":
 956                                         return Token.EVERY;
 957                                 case "satisfies":
 958                                         return Token.SATISFIES;
 959                                 case "is":
 960                                         return Token.IS;
 961                                 case "to":
 962                                         return Token.TO;
 963                                 case "eq":
 964                                         return Token.EQ;
 965                                 case "ne":
 966                                         return Token.NE;
 967                                 case "lt":
 968                                         return Token.LT;
 969                                 case "le":
 970                                         return Token.LE;
 971                                 case "gt":
 972                                         return Token.GT;
 973                                 case "ge":
 974                                         return Token.GE;
 975                                 case "and":
 976                                         return Token.AND;
 977                                 case "or":
 978                                         return Token.OR;
 979                                 case "instance":
 980                                         return Token.INSTANCE;
 981                                 case "of":
 982                                         return Token.OF;
 983                                 case "if":
 984                                         return Token.IF;
 985                                 case "then":
 986                                         return Token.THEN;
 987                                 case "else":
 988                                         return Token.ELSE;
 989                                 case "typeswitch":
 990                                         return Token.TYPESWITCH;
 991                                 case "case":
 992                                         return Token.CASE;
 993                                 case "treat":
 994                                         return Token.TREAT;
 995                                 case "castable":
 996                                         return Token.CASTABLE;
 997                                 case "cast":
 998                                         return Token.CAST;
 999                                 case "div":
1000                                         return Token.DIV;
1001                                 case "idiv":
1002                                         return Token.IDIV;
1003                                 case "mod":
1004                                         return Token.MOD;
1005                                 case "union":
1006                                         return Token.UNION;
1007                                 case "intersect":
1008                                         return Token.INTERSECT;
1009                                 case "except":
1010                                         return Token.EXCEPT;
1011                                 case "validate":
1012                                         return Token.VALIDATE;
1013                                 case "context":
1014                                         return Token.CONTEXT;
1015                                 case "nillable":
1016                                         return Token.NILLABLE;
1017                                 case "item":
1018                                         return Token.ITEM;
1019                                 case "global":
1020                                         return Token.GLOBAL;
1021                                 case "type":
1022                                         return Token.TYPE;
1023                                 case "child":
1024                                         return Token.CHILD;
1025                                 case "descendant":
1026                                         return Token.DESCENDANT;
1027                                 case "self":
1028                                         return Token.SELF;
1029                                 case "descendant-or-self":
1030                                         return Token.DESCENDANT_OR_SELF;
1031                                 case "following-sibling":
1032                                         return Token.FOLLOWING_SIBLING;
1033                                 case "following":
1034                                         return Token.FOLLOWING;
1035                                 case "parent":
1036                                         return Token.PARENT;
1037                                 case "ancestor":
1038                                         return Token.ANCESTOR;
1039                                 case "preceding":
1040                                         return Token.PRECEDING;
1041                                 case "preceding-sibling":
1042                                         return Token.PRECEDING_SIBLING;
1043                                 case "ancestor-or-self":
1044                                         return Token.ANCESTOR_OR_SELF;
1045                                 }
1046                         }
1047
1048                         switch (state) {
1049                         case ParseState.NamespaceDecl:
1050                         case ParseState.NamespaceKeyword:
1051                         case ParseState.XmlSpaceDecl:
1052                         case ParseState.KindTestForPI:
1053                         case ParseState.XmlPI:
1054                                 return Token.NCNAME;
1055                         }
1056
1057                         if (PeekChar () == ':') {
1058                                 ReadChar ();
1059                                 prefixName = name;
1060                                 switch (PeekChar ()) {
1061                                 case '*':
1062                                         ReadChar ();
1063                                         name = "*";
1064                                         break;
1065                                 case '=': // ex. let foo:= ...
1066                                         ReadChar ();
1067                                         tokenValue = new XmlQualifiedName (name, nsResolver.DefaultNamespace);
1068                                         lookAheadToken = Token.COLON_EQUAL;
1069                                         return Token.QNAME;
1070                                 default:
1071                                         name = ReadOneToken ();
1072                                         break;
1073                                 }
1074
1075                                 string ns = nsResolver.LookupNamespace (prefixName);
1076                                 if (ns == null)
1077                                         throw Error (String.Format ("Prefix '{0}' is not mapped to any namespace URI.", prefixName));
1078                                 tokenValue = new XmlQualifiedName (name, ns);
1079                                 prefixName = null;
1080                                 return name == "*" ? Token.WILD_LOCALNAME : Token.QNAME;
1081                         }
1082                         tokenValue = new XmlQualifiedName (name);
1083                         return Token.QNAME;
1084                 }
1085
1086                 private int PeekChar ()
1087                 {
1088                         if (peekChar == -1)
1089                                 peekChar = source.Read ();
1090                         return peekChar;
1091                 }
1092
1093                 private int ReadChar ()
1094                 {
1095                         int ret;
1096                         if (peekChar != -1) {
1097                                 ret = peekChar;
1098                                 peekChar = -1;
1099                         }
1100                         else
1101                                 ret = source.Read ();
1102
1103                         if (nextIncrementLine) {
1104                                 line++;
1105                                 column = 0;
1106                                 nextIncrementLine = false;
1107                         }
1108                         column++;
1109                         switch (ret) {
1110                         case '\r':
1111                                 break;
1112                         case '\n':
1113                                 nextIncrementLine = true;
1114                                 goto default;
1115                         default:
1116                                 break;
1117                         }
1118
1119                         return ret;
1120                 }
1121
1122                 private void SkipWhitespaces ()
1123                 {
1124                         while (true) {
1125                                 switch (PeekChar ()) {
1126                                 case ' ':
1127                                 case '\t':
1128                                 case '\r':
1129                                 case '\n':
1130                                         ReadChar ();
1131                                         continue;
1132                                 default:
1133                                         return;
1134                                 }
1135                         }
1136                 }
1137
1138                 private void AddValueChar (char c)
1139                 {
1140                         if (bufferIndex == buffer.Length) {
1141                                 char [] newBuf = new char [bufferIndex * 2];
1142                                 Array.Copy (buffer, newBuf, bufferIndex);
1143                                 buffer = newBuf;
1144                         }
1145                         buffer [bufferIndex++] = c;
1146                 }
1147
1148                 private string CreateValueString ()
1149                 {
1150                         return new string (buffer, 0, bufferIndex);
1151                 }
1152
1153                 private void Expect (string expected)
1154                 {
1155                         for (int i = 0; i < expected.Length; i++)
1156                                 if (ReadChar () != expected [i])
1157                                         throw Error (String.Format ("Expected token '{0}' did not appear.", expected));
1158                 }
1159
1160                 // TODO: parse three quoted
1161                 private string ReadQuoted (char quoteChar)
1162                 {
1163                         bufferIndex = 0;
1164                         bool loop = true;
1165                         do {
1166                                 int c = ReadChar ();
1167                                 switch (c) {
1168                                 case -1:
1169                                 case '"':
1170                                         if (quoteChar == '"')
1171                                                 loop = false;
1172                                         break;
1173                                 case '\'':
1174                                         if (quoteChar == '\'')
1175                                                 loop = false;
1176                                         break;
1177                                 default:
1178                                         AddValueChar ((char) c);
1179                                         break;
1180                                 }
1181                         } while (loop);
1182
1183                         return CreateValueString ();
1184                 }
1185
1186                 private decimal ReadDecimal (bool floatingPoint)
1187                 {
1188                         bufferIndex = 0;
1189                         bool cond = true;
1190                         do {
1191                                 int c = PeekChar ();
1192                                 if (c < 0) {
1193                                         cond = false;
1194                                 }
1195                                 // FIXME: more complex
1196                                 else if (Char.IsNumber ((char) c) || c == '.') {
1197                                         ReadChar ();
1198                                         AddValueChar ((char) c);
1199                                         continue;
1200                                 }
1201                                 else
1202                                         cond = false;
1203                         } while (cond);
1204                         string s = (floatingPoint ? "." : "") + CreateValueString ();
1205                         return decimal.Parse (s);
1206                 }
1207
1208                 private string ReadOneToken ()
1209                 {
1210                         bufferIndex = 0;
1211                         bool loop = true;
1212                         do {
1213                                 int c = PeekChar ();
1214                                 switch (c) {
1215                                 case -1:
1216                                 case ' ':
1217                                 case '\t':
1218                                 case '\r':
1219                                 case '\n':
1220                                         loop = false;
1221                                         break;
1222                                 default:
1223                                         if (!IsTokenContinuable (c)) {
1224                                                 if (c == ':') {
1225                                                         if (prefixName != null)
1226                                                                 throw new XmlQueryCompileException ("Invalid colon was found.");
1227                                                         prefixName = CreateValueString ();
1228                                                 }
1229                                                 loop = false;
1230                                                 break;
1231                                         }
1232
1233                                         ReadChar ();
1234                                         AddValueChar ((char) c);
1235                                         break;
1236                                 }
1237                         } while (loop);
1238
1239                         return CreateValueString ();
1240                 }
1241
1242                 private bool IsTokenContinuable (int c)
1243                 {
1244                         switch (c) {
1245                         case '-':
1246                         case '_':
1247                         case '.':
1248                                 return true;
1249                         }
1250                         return XmlChar.IsNCNameChar (c);
1251                 }
1252
1253         }
1254
1255         public enum WhitespaceHandling {
1256                 Arbitrary,
1257                 Explicit,
1258                 Significant
1259         }
1260
1261         public enum ParseState {
1262                 Default,
1263                 Operator,
1264                 NamespaceDecl,
1265                 NamespaceKeyword,
1266                 XmlSpaceDecl,
1267                 ItemType,
1268                 KindTest,
1269                 KindTestForPI,
1270                 CloseKindTest,
1271                 OccurenceIndicator,
1272                 SchemaContextStep,
1273                 VarName,
1274                 StartTag,
1275                 ElementContent,
1276                 EndTag,
1277                 XmlComment,
1278                 ExprComment,
1279                 ExtKey,
1280                 XmlPI,
1281                 XmlPIContent,
1282                 CDataSection,
1283                 QuotAttributeContent,
1284                 AposAttributeContent,
1285         }
1286
1287 }
1288 #endif