parser/html/nsHtml5Tokenizer.h

   1 /*
   2  * Copyright (c) 2005-2007 Henri Sivonen
   3  * Copyright (c) 2007-2017 Mozilla Foundation
   4  * Portions of comments Copyright 2004-2010 Apple Computer, Inc., Mozilla
   5  * Foundation, and Opera Software ASA.
   6  *
   7  * Permission is hereby granted, free of charge, to any person obtaining a
   8  * copy of this software and associated documentation files (the "Software"),
   9  * to deal in the Software without restriction, including without limitation
  10  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  11  * and/or sell copies of the Software, and to permit persons to whom the
  12  * Software is furnished to do so, subject to the following conditions:
  13  *
  14  * The above copyright notice and this permission notice shall be included in
  15  * all copies or substantial portions of the Software.
  16  *
  17  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  18  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  19  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
  20  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  21  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  22  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  23  * DEALINGS IN THE SOFTWARE.
  24  */
  25
  26 /*
  27  * THIS IS A GENERATED FILE. PLEASE DO NOT EDIT.
  28  * Please edit Tokenizer.java instead and regenerate.
  29  */
  30
  31 #ifndef nsHtml5Tokenizer_h
  32 #define nsHtml5Tokenizer_h
  33
  34 #include "nsAtom.h"
  35 #include "nsHtml5AtomTable.h"
  36 #include "nsHtml5String.h"
  37 #include "nsIContent.h"
  38 #include "nsTraceRefcnt.h"
  39 #include "jArray.h"
  40 #include "nsHtml5DocumentMode.h"
  41 #include "nsHtml5ArrayCopy.h"
  42 #include "nsHtml5NamedCharacters.h"
  43 #include "nsHtml5NamedCharactersAccel.h"
  44 #include "nsGkAtoms.h"
  45 #include "nsAHtml5TreeBuilderState.h"
  46 #include "nsHtml5Macros.h"
  47 #include "nsHtml5Highlighter.h"
  48 #include "nsHtml5TokenizerLoopPolicies.h"
  49
  50 class nsHtml5StreamParser;
  51
  52 class nsHtml5AttributeName;
  53 class nsHtml5ElementName;
  54 class nsHtml5TreeBuilder;
  55 class nsHtml5MetaScanner;
  56 class nsHtml5UTF16Buffer;
  57 class nsHtml5StateSnapshot;
  58 class nsHtml5Portability;
  59
  60 class nsHtml5Tokenizer {
  61  private:
  62   static const int32_t DATA_AND_RCDATA_MASK = ~1;
  63
  64  public:
  65   static const int32_t DATA = 0;
  66
  67   static const int32_t RCDATA = 1;
  68
  69   static const int32_t SCRIPT_DATA = 2;
  70
  71   static const int32_t RAWTEXT = 3;
  72
  73   static const int32_t SCRIPT_DATA_ESCAPED = 4;
  74
  75   static const int32_t ATTRIBUTE_VALUE_DOUBLE_QUOTED = 5;
  76
  77   static const int32_t ATTRIBUTE_VALUE_SINGLE_QUOTED = 6;
  78
  79   static const int32_t ATTRIBUTE_VALUE_UNQUOTED = 7;
  80
  81   static const int32_t PLAINTEXT = 8;
  82
  83   static const int32_t TAG_OPEN = 9;
  84
  85   static const int32_t CLOSE_TAG_OPEN = 10;
  86
  87   static const int32_t TAG_NAME = 11;
  88
  89   static const int32_t BEFORE_ATTRIBUTE_NAME = 12;
  90
  91   static const int32_t ATTRIBUTE_NAME = 13;
  92
  93   static const int32_t AFTER_ATTRIBUTE_NAME = 14;
  94
  95   static const int32_t BEFORE_ATTRIBUTE_VALUE = 15;
  96
  97   static const int32_t AFTER_ATTRIBUTE_VALUE_QUOTED = 16;
  98
  99   static const int32_t BOGUS_COMMENT = 17;
 100
 101   static const int32_t MARKUP_DECLARATION_OPEN = 18;
 102
 103   static const int32_t DOCTYPE = 19;
 104
 105   static const int32_t BEFORE_DOCTYPE_NAME = 20;
 106
 107   static const int32_t DOCTYPE_NAME = 21;
 108
 109   static const int32_t AFTER_DOCTYPE_NAME = 22;
 110
 111   static const int32_t BEFORE_DOCTYPE_PUBLIC_IDENTIFIER = 23;
 112
 113   static const int32_t DOCTYPE_PUBLIC_IDENTIFIER_DOUBLE_QUOTED = 24;
 114
 115   static const int32_t DOCTYPE_PUBLIC_IDENTIFIER_SINGLE_QUOTED = 25;
 116
 117   static const int32_t AFTER_DOCTYPE_PUBLIC_IDENTIFIER = 26;
 118
 119   static const int32_t BEFORE_DOCTYPE_SYSTEM_IDENTIFIER = 27;
 120
 121   static const int32_t DOCTYPE_SYSTEM_IDENTIFIER_DOUBLE_QUOTED = 28;
 122
 123   static const int32_t DOCTYPE_SYSTEM_IDENTIFIER_SINGLE_QUOTED = 29;
 124
 125   static const int32_t AFTER_DOCTYPE_SYSTEM_IDENTIFIER = 30;
 126
 127   static const int32_t BOGUS_DOCTYPE = 31;
 128
 129   static const int32_t COMMENT_START = 32;
 130
 131   static const int32_t COMMENT_START_DASH = 33;
 132
 133   static const int32_t COMMENT = 34;
 134
 135   static const int32_t COMMENT_END_DASH = 35;
 136
 137   static const int32_t COMMENT_END = 36;
 138
 139   static const int32_t COMMENT_END_BANG = 37;
 140
 141   static const int32_t NON_DATA_END_TAG_NAME = 38;
 142
 143   static const int32_t MARKUP_DECLARATION_HYPHEN = 39;
 144
 145   static const int32_t MARKUP_DECLARATION_OCTYPE = 40;
 146
 147   static const int32_t DOCTYPE_UBLIC = 41;
 148
 149   static const int32_t DOCTYPE_YSTEM = 42;
 150
 151   static const int32_t AFTER_DOCTYPE_PUBLIC_KEYWORD = 43;
 152
 153   static const int32_t BETWEEN_DOCTYPE_PUBLIC_AND_SYSTEM_IDENTIFIERS = 44;
 154
 155   static const int32_t AFTER_DOCTYPE_SYSTEM_KEYWORD = 45;
 156
 157   static const int32_t CONSUME_CHARACTER_REFERENCE = 46;
 158
 159   static const int32_t CONSUME_NCR = 47;
 160
 161   static const int32_t CHARACTER_REFERENCE_TAIL = 48;
 162
 163   static const int32_t HEX_NCR_LOOP = 49;
 164
 165   static const int32_t DECIMAL_NRC_LOOP = 50;
 166
 167   static const int32_t HANDLE_NCR_VALUE = 51;
 168
 169   static const int32_t HANDLE_NCR_VALUE_RECONSUME = 52;
 170
 171   static const int32_t CHARACTER_REFERENCE_HILO_LOOKUP = 53;
 172
 173   static const int32_t SELF_CLOSING_START_TAG = 54;
 174
 175   static const int32_t CDATA_START = 55;
 176
 177   static const int32_t CDATA_SECTION = 56;
 178
 179   static const int32_t CDATA_RSQB = 57;
 180
 181   static const int32_t CDATA_RSQB_RSQB = 58;
 182
 183   static const int32_t SCRIPT_DATA_LESS_THAN_SIGN = 59;
 184
 185   static const int32_t SCRIPT_DATA_ESCAPE_START = 60;
 186
 187   static const int32_t SCRIPT_DATA_ESCAPE_START_DASH = 61;
 188
 189   static const int32_t SCRIPT_DATA_ESCAPED_DASH = 62;
 190
 191   static const int32_t SCRIPT_DATA_ESCAPED_DASH_DASH = 63;
 192
 193   static const int32_t BOGUS_COMMENT_HYPHEN = 64;
 194
 195   static const int32_t RAWTEXT_RCDATA_LESS_THAN_SIGN = 65;
 196
 197   static const int32_t SCRIPT_DATA_ESCAPED_LESS_THAN_SIGN = 66;
 198
 199   static const int32_t SCRIPT_DATA_DOUBLE_ESCAPE_START = 67;
 200
 201   static const int32_t SCRIPT_DATA_DOUBLE_ESCAPED = 68;
 202
 203   static const int32_t SCRIPT_DATA_DOUBLE_ESCAPED_LESS_THAN_SIGN = 69;
 204
 205   static const int32_t SCRIPT_DATA_DOUBLE_ESCAPED_DASH = 70;
 206
 207   static const int32_t SCRIPT_DATA_DOUBLE_ESCAPED_DASH_DASH = 71;
 208
 209   static const int32_t SCRIPT_DATA_DOUBLE_ESCAPE_END = 72;
 210
 211   static const int32_t PROCESSING_INSTRUCTION = 73;
 212
 213   static const int32_t PROCESSING_INSTRUCTION_QUESTION_MARK = 74;
 214
 215  private:
 216   static const int32_t LEAD_OFFSET = (0xD800 - (0x10000 >> 10));
 217
 218   static char16_t LT_GT[];
 219   static char16_t LT_SOLIDUS[];
 220   static char16_t RSQB_RSQB[];
 221   static char16_t REPLACEMENT_CHARACTER[];
 222   static char16_t LF[];
 223   static char16_t CDATA_LSQB[];
 224   static char16_t OCTYPE[];
 225   static char16_t UBLIC[];
 226   static char16_t YSTEM[];
 227   static staticJArray<char16_t, int32_t> TITLE_ARR;
 228   static staticJArray<char16_t, int32_t> SCRIPT_ARR;
 229   static staticJArray<char16_t, int32_t> STYLE_ARR;
 230   static staticJArray<char16_t, int32_t> PLAINTEXT_ARR;
 231   static staticJArray<char16_t, int32_t> XMP_ARR;
 232   static staticJArray<char16_t, int32_t> TEXTAREA_ARR;
 233   static staticJArray<char16_t, int32_t> IFRAME_ARR;
 234   static staticJArray<char16_t, int32_t> NOEMBED_ARR;
 235   static staticJArray<char16_t, int32_t> NOSCRIPT_ARR;
 236   static staticJArray<char16_t, int32_t> NOFRAMES_ARR;
 237
 238  protected:
 239   nsHtml5TreeBuilder* tokenHandler;
 240   nsHtml5StreamParser* encodingDeclarationHandler;
 241   bool lastCR;
 242   int32_t stateSave;
 243
 244  private:
 245   int32_t returnStateSave;
 246
 247  protected:
 248   int32_t index;
 249
 250  private:
 251   bool forceQuirks;
 252   char16_t additional;
 253   int32_t entCol;
 254   int32_t firstCharKey;
 255   int32_t lo;
 256   int32_t hi;
 257   int32_t candidate;
 258   int32_t charRefBufMark;
 259
 260  protected:
 261   int32_t value;
 262
 263  private:
 264   bool seenDigits;
 265
 266  protected:
 267   int32_t cstart;
 268
 269  private:
 270   nsHtml5String publicId;
 271   nsHtml5String systemId;
 272   autoJArray<char16_t, int32_t> strBuf;
 273   int32_t strBufLen;
 274   autoJArray<char16_t, int32_t> charRefBuf;
 275   int32_t charRefBufLen;
 276   autoJArray<char16_t, int32_t> bmpChar;
 277   autoJArray<char16_t, int32_t> astralChar;
 278
 279  protected:
 280   nsHtml5ElementName* endTagExpectation;
 281
 282  private:
 283   jArray<char16_t, int32_t> endTagExpectationAsArray;
 284
 285  protected:
 286   bool endTag;
 287
 288  private:
 289   bool containsHyphen;
 290   nsHtml5ElementName* tagName;
 291   nsHtml5ElementName* nonInternedTagName;
 292
 293  protected:
 294   nsHtml5AttributeName* attributeName;
 295
 296  private:
 297   nsHtml5AttributeName* nonInternedAttributeName;
 298   RefPtr<nsAtom> doctypeName;
 299   nsHtml5String publicIdentifier;
 300   nsHtml5String systemIdentifier;
 301   nsHtml5HtmlAttributes* attributes;
 302   bool newAttributesEachTime;
 303   bool shouldSuspend;
 304
 305  protected:
 306   bool confident;
 307
 308  private:
 309   int32_t line;
 310   int32_t attributeLine;
 311   nsHtml5AtomTable* interner;
 312   bool viewingXmlSource;
 313
 314  public:
 315   nsHtml5Tokenizer(nsHtml5TreeBuilder* tokenHandler, bool viewingXmlSource);
 316   void setInterner(nsHtml5AtomTable* interner);
 317   void initLocation(nsHtml5String newPublicId, nsHtml5String newSystemId);
 318   bool isViewingXmlSource();
 319   void setState(int32_t specialTokenizerState);
 320   void setStateAndEndTagExpectation(int32_t specialTokenizerState,
 321                                     nsHtml5ElementName* endTagExpectation);
 322
 323  private:
 324   void endTagExpectationToArray();
 325
 326  public:
 327   void setLineNumber(int32_t line);
 328   inline int32_t getLineNumber() { return line; }
 329
 330   nsHtml5HtmlAttributes* emptyAttributes();
 331
 332  private:
 333   inline void appendCharRefBuf(char16_t c) {
 334     MOZ_RELEASE_ASSERT(charRefBufLen < charRefBuf.length,
 335                        "Attempted to overrun charRefBuf!");
 336     charRefBuf[charRefBufLen++] = c;
 337   }
 338
 339   void emitOrAppendCharRefBuf(int32_t returnState);
 340   inline void clearStrBufAfterUse() { strBufLen = 0; }
 341
 342   inline void clearStrBufBeforeUse() {
 343     MOZ_ASSERT(!strBufLen, "strBufLen not reset after previous use!");
 344     strBufLen = 0;
 345   }
 346
 347   inline void clearStrBufAfterOneHyphen() {
 348     MOZ_ASSERT(strBufLen == 1, "strBufLen length not one!");
 349     MOZ_ASSERT(strBuf[0] == '-', "strBuf does not start with a hyphen!");
 350     strBufLen = 0;
 351   }
 352
 353   inline void appendStrBuf(char16_t c) {
 354     MOZ_ASSERT(strBufLen < strBuf.length,
 355                "Previous buffer length insufficient.");
 356     if (MOZ_UNLIKELY(strBufLen == strBuf.length)) {
 357       if (MOZ_UNLIKELY(!EnsureBufferSpace(1))) {
 358         MOZ_CRASH("Unable to recover from buffer reallocation failure");
 359       }
 360     }
 361     strBuf[strBufLen++] = c;
 362   }
 363
 364  protected:
 365   nsHtml5String strBufToString();
 366
 367  private:
 368   void strBufToDoctypeName();
 369   void emitStrBuf();
 370   inline void appendSecondHyphenToBogusComment() { appendStrBuf('-'); }
 371
 372   inline void adjustDoubleHyphenAndAppendToStrBufAndErr(char16_t c) {
 373     errConsecutiveHyphens();
 374     appendStrBuf(c);
 375   }
 376
 377   void appendStrBuf(char16_t* buffer, int32_t offset, int32_t length);
 378   inline void appendCharRefBufToStrBuf() {
 379     appendStrBuf(charRefBuf, 0, charRefBufLen);
 380     charRefBufLen = 0;
 381   }
 382
 383   void emitComment(int32_t provisionalHyphens, int32_t pos);
 384
 385  protected:
 386   void flushChars(char16_t* buf, int32_t pos);
 387
 388  private:
 389   void strBufToElementNameString();
 390   int32_t emitCurrentTagToken(bool selfClosing, int32_t pos);
 391   void attributeNameComplete();
 392   void addAttributeWithoutValue();
 393   void addAttributeWithValue();
 394
 395  public:
 396   void start();
 397   bool tokenizeBuffer(nsHtml5UTF16Buffer* buffer);
 398
 399  private:
 400   template <class P>
 401   int32_t stateLoop(int32_t state, char16_t c, int32_t pos, char16_t* buf,
 402                     bool reconsume, int32_t returnState, int32_t endPos);
 403   void initDoctypeFields();
 404   inline void adjustDoubleHyphenAndAppendToStrBufCarriageReturn() {
 405     silentCarriageReturn();
 406     adjustDoubleHyphenAndAppendToStrBufAndErr('\n');
 407   }
 408
 409   inline void adjustDoubleHyphenAndAppendToStrBufLineFeed() {
 410     silentLineFeed();
 411     adjustDoubleHyphenAndAppendToStrBufAndErr('\n');
 412   }
 413
 414   inline void appendStrBufLineFeed() {
 415     silentLineFeed();
 416     appendStrBuf('\n');
 417   }
 418
 419   inline void appendStrBufCarriageReturn() {
 420     silentCarriageReturn();
 421     appendStrBuf('\n');
 422   }
 423
 424  protected:
 425   inline void silentCarriageReturn() {
 426     ++line;
 427     lastCR = true;
 428   }
 429
 430   inline void silentLineFeed() { ++line; }
 431
 432  private:
 433   void emitCarriageReturn(char16_t* buf, int32_t pos);
 434   void emitReplacementCharacter(char16_t* buf, int32_t pos);
 435   void emitPlaintextReplacementCharacter(char16_t* buf, int32_t pos);
 436   void setAdditionalAndRememberAmpersandLocation(char16_t add);
 437   void bogusDoctype();
 438   void bogusDoctypeWithoutQuirks();
 439   void handleNcrValue(int32_t returnState);
 440
 441  public:
 442   void eof();
 443
 444  private:
 445   void emitDoctypeToken(int32_t pos);
 446
 447  protected:
 448   inline char16_t checkChar(char16_t* buf, int32_t pos) { return buf[pos]; }
 449
 450  public:
 451   bool internalEncodingDeclaration(nsHtml5String internalCharset);
 452
 453  private:
 454   void emitOrAppendTwo(const char16_t* val, int32_t returnState);
 455   void emitOrAppendOne(const char16_t* val, int32_t returnState);
 456
 457  public:
 458   void end();
 459   void requestSuspension();
 460   bool isInDataState();
 461   void resetToDataState();
 462   void loadState(nsHtml5Tokenizer* other);
 463   void initializeWithoutStarting();
 464   void setEncodingDeclarationHandler(
 465       nsHtml5StreamParser* encodingDeclarationHandler);
 466   ~nsHtml5Tokenizer();
 467   static void initializeStatics();
 468   static void releaseStatics();
 469
 470 #include "nsHtml5TokenizerHSupplement.h"
 471 };
 472
 473 #endif