parser/html/nsHtml5Tokenizer.h

   1 /*
   2  * Copyright (c) 2005-2007 Henri Sivonen
   3  * Copyright (c) 2007-2017 Mozilla Foundation
   4  * Portions of comments Copyright 2004-2010 Apple Computer, Inc., Mozilla
   5  * Foundation, and Opera Software ASA.
   6  *
   7  * Permission is hereby granted, free of charge, to any person obtaining a
   8  * copy of this software and associated documentation files (the "Software"),
   9  * to deal in the Software without restriction, including without limitation
  10  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  11  * and/or sell copies of the Software, and to permit persons to whom the
  12  * Software is furnished to do so, subject to the following conditions:
  13  *
  14  * The above copyright notice and this permission notice shall be included in
  15  * all copies or substantial portions of the Software.
  16  *
  17  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  18  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  19  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
  20  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  21  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  22  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  23  * DEALINGS IN THE SOFTWARE.
  24  */
  25
  26 /*
  27  * THIS IS A GENERATED FILE. PLEASE DO NOT EDIT.
  28  * Please edit Tokenizer.java instead and regenerate.
  29  */
  30
  31 #ifndef nsHtml5Tokenizer_h
  32 #define nsHtml5Tokenizer_h
  33
  34 #include "jArray.h"
  35 #include "nsAHtml5TreeBuilderState.h"
  36 #include "nsAtom.h"
  37 #include "nsGkAtoms.h"
  38 #include "nsHtml5ArrayCopy.h"
  39 #include "nsHtml5AtomTable.h"
  40 #include "nsHtml5DocumentMode.h"
  41 #include "nsHtml5Highlighter.h"
  42 #include "nsHtml5Macros.h"
  43 #include "nsHtml5NamedCharacters.h"
  44 #include "nsHtml5NamedCharactersAccel.h"
  45 #include "nsHtml5String.h"
  46 #include "nsIContent.h"
  47 #include "nsTraceRefcnt.h"
  48
  49 class nsHtml5StreamParser;
  50
  51 class nsHtml5AttributeName;
  52 class nsHtml5ElementName;
  53 class nsHtml5TreeBuilder;
  54 class nsHtml5UTF16Buffer;
  55 class nsHtml5StateSnapshot;
  56 class nsHtml5Portability;
  57
  58 class nsHtml5Tokenizer {
  59  private:
  60   static const int32_t DATA_AND_RCDATA_MASK = ~1;
  61
  62  public:
  63   static const int32_t DATA = 0;
  64
  65   static const int32_t RCDATA = 1;
  66
  67   static const int32_t SCRIPT_DATA = 2;
  68
  69   static const int32_t RAWTEXT = 3;
  70
  71   static const int32_t SCRIPT_DATA_ESCAPED = 4;
  72
  73   static const int32_t ATTRIBUTE_VALUE_DOUBLE_QUOTED = 5;
  74
  75   static const int32_t ATTRIBUTE_VALUE_SINGLE_QUOTED = 6;
  76
  77   static const int32_t ATTRIBUTE_VALUE_UNQUOTED = 7;
  78
  79   static const int32_t PLAINTEXT = 8;
  80
  81   static const int32_t TAG_OPEN = 9;
  82
  83   static const int32_t CLOSE_TAG_OPEN = 10;
  84
  85   static const int32_t TAG_NAME = 11;
  86
  87   static const int32_t BEFORE_ATTRIBUTE_NAME = 12;
  88
  89   static const int32_t ATTRIBUTE_NAME = 13;
  90
  91   static const int32_t AFTER_ATTRIBUTE_NAME = 14;
  92
  93   static const int32_t BEFORE_ATTRIBUTE_VALUE = 15;
  94
  95   static const int32_t AFTER_ATTRIBUTE_VALUE_QUOTED = 16;
  96
  97   static const int32_t BOGUS_COMMENT = 17;
  98
  99   static const int32_t MARKUP_DECLARATION_OPEN = 18;
 100
 101   static const int32_t DOCTYPE = 19;
 102
 103   static const int32_t BEFORE_DOCTYPE_NAME = 20;
 104
 105   static const int32_t DOCTYPE_NAME = 21;
 106
 107   static const int32_t AFTER_DOCTYPE_NAME = 22;
 108
 109   static const int32_t BEFORE_DOCTYPE_PUBLIC_IDENTIFIER = 23;
 110
 111   static const int32_t DOCTYPE_PUBLIC_IDENTIFIER_DOUBLE_QUOTED = 24;
 112
 113   static const int32_t DOCTYPE_PUBLIC_IDENTIFIER_SINGLE_QUOTED = 25;
 114
 115   static const int32_t AFTER_DOCTYPE_PUBLIC_IDENTIFIER = 26;
 116
 117   static const int32_t BEFORE_DOCTYPE_SYSTEM_IDENTIFIER = 27;
 118
 119   static const int32_t DOCTYPE_SYSTEM_IDENTIFIER_DOUBLE_QUOTED = 28;
 120
 121   static const int32_t DOCTYPE_SYSTEM_IDENTIFIER_SINGLE_QUOTED = 29;
 122
 123   static const int32_t AFTER_DOCTYPE_SYSTEM_IDENTIFIER = 30;
 124
 125   static const int32_t BOGUS_DOCTYPE = 31;
 126
 127   static const int32_t COMMENT_START = 32;
 128
 129   static const int32_t COMMENT_START_DASH = 33;
 130
 131   static const int32_t COMMENT = 34;
 132
 133   static const int32_t COMMENT_END_DASH = 35;
 134
 135   static const int32_t COMMENT_END = 36;
 136
 137   static const int32_t COMMENT_END_BANG = 37;
 138
 139   static const int32_t NON_DATA_END_TAG_NAME = 38;
 140
 141   static const int32_t MARKUP_DECLARATION_HYPHEN = 39;
 142
 143   static const int32_t MARKUP_DECLARATION_OCTYPE = 40;
 144
 145   static const int32_t DOCTYPE_UBLIC = 41;
 146
 147   static const int32_t DOCTYPE_YSTEM = 42;
 148
 149   static const int32_t AFTER_DOCTYPE_PUBLIC_KEYWORD = 43;
 150
 151   static const int32_t BETWEEN_DOCTYPE_PUBLIC_AND_SYSTEM_IDENTIFIERS = 44;
 152
 153   static const int32_t AFTER_DOCTYPE_SYSTEM_KEYWORD = 45;
 154
 155   static const int32_t CONSUME_CHARACTER_REFERENCE = 46;
 156
 157   static const int32_t CONSUME_NCR = 47;
 158
 159   static const int32_t CHARACTER_REFERENCE_TAIL = 48;
 160
 161   static const int32_t HEX_NCR_LOOP = 49;
 162
 163   static const int32_t DECIMAL_NRC_LOOP = 50;
 164
 165   static const int32_t HANDLE_NCR_VALUE = 51;
 166
 167   static const int32_t HANDLE_NCR_VALUE_RECONSUME = 52;
 168
 169   static const int32_t CHARACTER_REFERENCE_HILO_LOOKUP = 53;
 170
 171   static const int32_t SELF_CLOSING_START_TAG = 54;
 172
 173   static const int32_t CDATA_START = 55;
 174
 175   static const int32_t CDATA_SECTION = 56;
 176
 177   static const int32_t CDATA_RSQB = 57;
 178
 179   static const int32_t CDATA_RSQB_RSQB = 58;
 180
 181   static const int32_t SCRIPT_DATA_LESS_THAN_SIGN = 59;
 182
 183   static const int32_t SCRIPT_DATA_ESCAPE_START = 60;
 184
 185   static const int32_t SCRIPT_DATA_ESCAPE_START_DASH = 61;
 186
 187   static const int32_t SCRIPT_DATA_ESCAPED_DASH = 62;
 188
 189   static const int32_t SCRIPT_DATA_ESCAPED_DASH_DASH = 63;
 190
 191   static const int32_t BOGUS_COMMENT_HYPHEN = 64;
 192
 193   static const int32_t RAWTEXT_RCDATA_LESS_THAN_SIGN = 65;
 194
 195   static const int32_t SCRIPT_DATA_ESCAPED_LESS_THAN_SIGN = 66;
 196
 197   static const int32_t SCRIPT_DATA_DOUBLE_ESCAPE_START = 67;
 198
 199   static const int32_t SCRIPT_DATA_DOUBLE_ESCAPED = 68;
 200
 201   static const int32_t SCRIPT_DATA_DOUBLE_ESCAPED_LESS_THAN_SIGN = 69;
 202
 203   static const int32_t SCRIPT_DATA_DOUBLE_ESCAPED_DASH = 70;
 204
 205   static const int32_t SCRIPT_DATA_DOUBLE_ESCAPED_DASH_DASH = 71;
 206
 207   static const int32_t SCRIPT_DATA_DOUBLE_ESCAPE_END = 72;
 208
 209   static const int32_t PROCESSING_INSTRUCTION = 73;
 210
 211   static const int32_t PROCESSING_INSTRUCTION_QUESTION_MARK = 74;
 212
 213   static const int32_t COMMENT_LESSTHAN = 76;
 214
 215   static const int32_t COMMENT_LESSTHAN_BANG = 77;
 216
 217   static const int32_t COMMENT_LESSTHAN_BANG_DASH = 78;
 218
 219   static const int32_t COMMENT_LESSTHAN_BANG_DASH_DASH = 79;
 220
 221  private:
 222   static const int32_t LEAD_OFFSET = (0xD800 - (0x10000 >> 10));
 223
 224   static char16_t LT_GT[];
 225   static char16_t LT_SOLIDUS[];
 226   static char16_t RSQB_RSQB[];
 227   static char16_t REPLACEMENT_CHARACTER[];
 228   static char16_t LF[];
 229   static char16_t CDATA_LSQB[];
 230   static char16_t OCTYPE[];
 231   static char16_t UBLIC[];
 232   static char16_t YSTEM[];
 233   static staticJArray<char16_t, int32_t> TITLE_ARR;
 234   static staticJArray<char16_t, int32_t> SCRIPT_ARR;
 235   static staticJArray<char16_t, int32_t> STYLE_ARR;
 236   static staticJArray<char16_t, int32_t> PLAINTEXT_ARR;
 237   static staticJArray<char16_t, int32_t> XMP_ARR;
 238   static staticJArray<char16_t, int32_t> TEXTAREA_ARR;
 239   static staticJArray<char16_t, int32_t> IFRAME_ARR;
 240   static staticJArray<char16_t, int32_t> NOEMBED_ARR;
 241   static staticJArray<char16_t, int32_t> NOSCRIPT_ARR;
 242   static staticJArray<char16_t, int32_t> NOFRAMES_ARR;
 243
 244  protected:
 245   nsHtml5TreeBuilder* tokenHandler;
 246   nsHtml5StreamParser* encodingDeclarationHandler;
 247   bool lastCR;
 248   int32_t stateSave;
 249
 250  private:
 251   int32_t returnStateSave;
 252
 253  protected:
 254   int32_t index;
 255
 256  private:
 257   bool forceQuirks;
 258   char16_t additional;
 259   int32_t entCol;
 260   int32_t firstCharKey;
 261   int32_t lo;
 262   int32_t hi;
 263   int32_t candidate;
 264   int32_t charRefBufMark;
 265
 266  protected:
 267   int32_t value;
 268
 269  private:
 270   bool seenDigits;
 271   bool suspendAfterCurrentNonTextToken;
 272
 273  protected:
 274   int32_t cstart;
 275
 276  private:
 277   nsHtml5String publicId;
 278   nsHtml5String systemId;
 279   autoJArray<char16_t, int32_t> strBuf;
 280   int32_t strBufLen;
 281   autoJArray<char16_t, int32_t> charRefBuf;
 282   int32_t charRefBufLen;
 283   autoJArray<char16_t, int32_t> bmpChar;
 284   autoJArray<char16_t, int32_t> astralChar;
 285
 286  protected:
 287   nsHtml5ElementName* endTagExpectation;
 288
 289  private:
 290   jArray<char16_t, int32_t> endTagExpectationAsArray;
 291
 292  protected:
 293   bool endTag;
 294
 295  private:
 296   bool containsHyphen;
 297   nsHtml5ElementName* tagName;
 298   nsHtml5ElementName* nonInternedTagName;
 299
 300  protected:
 301   nsHtml5AttributeName* attributeName;
 302
 303  private:
 304   nsHtml5AttributeName* nonInternedAttributeName;
 305   RefPtr<nsAtom> doctypeName;
 306   nsHtml5String publicIdentifier;
 307   nsHtml5String systemIdentifier;
 308   nsHtml5HtmlAttributes* attributes;
 309   bool newAttributesEachTime;
 310   bool shouldSuspend;
 311
 312  protected:
 313   bool confident;
 314
 315  private:
 316   int32_t line;
 317   int32_t attributeLine;
 318   nsHtml5AtomTable* interner;
 319   bool viewingXmlSource;
 320
 321  public:
 322   nsHtml5Tokenizer(nsHtml5TreeBuilder* tokenHandler, bool viewingXmlSource);
 323   void setInterner(nsHtml5AtomTable* interner);
 324   void initLocation(nsHtml5String newPublicId, nsHtml5String newSystemId);
 325   bool isViewingXmlSource();
 326   void setState(int32_t specialTokenizerState);
 327   void setStateAndEndTagExpectation(int32_t specialTokenizerState,
 328                                     nsHtml5ElementName* endTagExpectation);
 329
 330  private:
 331   void endTagExpectationToArray();
 332
 333  public:
 334   void setLineNumber(int32_t line);
 335   inline int32_t getLineNumber() { return line; }
 336
 337   nsHtml5HtmlAttributes* emptyAttributes();
 338
 339  private:
 340   inline void appendCharRefBuf(char16_t c) {
 341     MOZ_RELEASE_ASSERT(charRefBufLen < charRefBuf.length,
 342                        "Attempted to overrun charRefBuf!");
 343     charRefBuf[charRefBufLen++] = c;
 344   }
 345
 346   void emitOrAppendCharRefBuf(int32_t returnState);
 347   inline void clearStrBufAfterUse() { strBufLen = 0; }
 348
 349   inline void clearStrBufBeforeUse() {
 350     MOZ_ASSERT(!strBufLen, "strBufLen not reset after previous use!");
 351     strBufLen = 0;
 352   }
 353
 354   inline void clearStrBufAfterOneHyphen() {
 355     MOZ_ASSERT(strBufLen == 1, "strBufLen length not one!");
 356     MOZ_ASSERT(strBuf[0] == '-', "strBuf does not start with a hyphen!");
 357     strBufLen = 0;
 358   }
 359
 360   inline void appendStrBuf(char16_t c) {
 361     MOZ_ASSERT(strBufLen < strBuf.length,
 362                "Previous buffer length insufficient.");
 363     if (MOZ_UNLIKELY(strBufLen == strBuf.length)) {
 364       if (MOZ_UNLIKELY(!EnsureBufferSpace(1))) {
 365         MOZ_CRASH("Unable to recover from buffer reallocation failure");
 366       }
 367     }
 368     strBuf[strBufLen++] = c;
 369   }
 370
 371  protected:
 372   nsHtml5String strBufToString();
 373
 374  private:
 375   void strBufToDoctypeName();
 376   void emitStrBuf();
 377   inline void appendSecondHyphenToBogusComment() { appendStrBuf('-'); }
 378
 379   inline void adjustDoubleHyphenAndAppendToStrBufAndErr(
 380       char16_t c, bool reportedConsecutiveHyphens) {
 381     appendStrBuf(c);
 382   }
 383
 384   void appendStrBuf(char16_t* buffer, int32_t offset, int32_t length);
 385   inline void appendCharRefBufToStrBuf() {
 386     appendStrBuf(charRefBuf, 0, charRefBufLen);
 387     charRefBufLen = 0;
 388   }
 389
 390   void emitComment(int32_t provisionalHyphens, int32_t pos);
 391
 392  protected:
 393   void flushChars(char16_t* buf, int32_t pos);
 394
 395  private:
 396   void strBufToElementNameString();
 397   int32_t emitCurrentTagToken(bool selfClosing, int32_t pos);
 398   void attributeNameComplete();
 399   void addAttributeWithoutValue();
 400   void addAttributeWithValue();
 401
 402  public:
 403   void start();
 404   bool tokenizeBuffer(nsHtml5UTF16Buffer* buffer);
 405
 406  private:
 407   template <class P>
 408   int32_t stateLoop(int32_t state, char16_t c, int32_t pos, char16_t* buf,
 409                     bool reconsume, int32_t returnState, int32_t endPos);
 410   void initDoctypeFields();
 411   template <class P>
 412   void adjustDoubleHyphenAndAppendToStrBufCarriageReturn();
 413   template <class P>
 414   void adjustDoubleHyphenAndAppendToStrBufLineFeed();
 415   template <class P>
 416   void appendStrBufLineFeed();
 417   template <class P>
 418   void appendStrBufCarriageReturn();
 419   template <class P>
 420   void emitCarriageReturn(char16_t* buf, int32_t pos);
 421   void emitReplacementCharacter(char16_t* buf, int32_t pos);
 422   void maybeEmitReplacementCharacter(char16_t* buf, int32_t pos);
 423   void emitPlaintextReplacementCharacter(char16_t* buf, int32_t pos);
 424   void setAdditionalAndRememberAmpersandLocation(char16_t add);
 425   void bogusDoctype();
 426   void bogusDoctypeWithoutQuirks();
 427   void handleNcrValue(int32_t returnState);
 428
 429  public:
 430   void eof();
 431
 432  private:
 433   void emitDoctypeToken(int32_t pos);
 434   void suspendIfRequestedAfterCurrentNonTextToken();
 435   void suspendAfterCurrentTokenIfNotInText();
 436   bool suspensionAfterCurrentNonTextTokenPending();
 437
 438  public:
 439   bool internalEncodingDeclaration(nsHtml5String internalCharset);
 440
 441  private:
 442   void emitOrAppendTwo(const char16_t* val, int32_t returnState);
 443   void emitOrAppendOne(const char16_t* val, int32_t returnState);
 444
 445  public:
 446   void end();
 447   void requestSuspension();
 448   bool isInDataState();
 449   void resetToDataState();
 450   void loadState(nsHtml5Tokenizer* other);
 451   void initializeWithoutStarting();
 452   void setEncodingDeclarationHandler(
 453       nsHtml5StreamParser* encodingDeclarationHandler);
 454   ~nsHtml5Tokenizer();
 455   static void initializeStatics();
 456   static void releaseStatics();
 457
 458 #include "nsHtml5TokenizerHSupplement.h"
 459 };
 460
 461 #endif