tests/HTMLPurifier/LexerTest.php

   1 <?php
   2
   3 class HTMLPurifier_LexerTest extends HTMLPurifier_Harness
   4 {
   5
   6     protected $_has_pear = false;
   7
   8     public function __construct()
   9     {
  10         parent::__construct();
  11         if ($GLOBALS['HTMLPurifierTest']['PH5P']) {
  12             require_once 'HTMLPurifier/Lexer/PH5P.php';
  13         }
  14     }
  15
  16     // HTMLPurifier_Lexer::create() --------------------------------------------
  17
  18     public function test_create()
  19     {
  20         $this->config->set('Core.MaintainLineNumbers', true);
  21         $lexer = HTMLPurifier_Lexer::create($this->config);
  22         $this->assertIsA($lexer, 'HTMLPurifier_Lexer_DirectLex');
  23     }
  24
  25     public function test_create_objectLexerImpl()
  26     {
  27         $this->config->set('Core.LexerImpl', new HTMLPurifier_Lexer_DirectLex());
  28         $lexer = HTMLPurifier_Lexer::create($this->config);
  29         $this->assertIsA($lexer, 'HTMLPurifier_Lexer_DirectLex');
  30     }
  31
  32     public function test_create_unknownLexer()
  33     {
  34         $this->config->set('Core.LexerImpl', 'AsdfAsdf');
  35         $this->expectException(new HTMLPurifier_Exception('Cannot instantiate unrecognized Lexer type AsdfAsdf'));
  36         HTMLPurifier_Lexer::create($this->config);
  37     }
  38
  39     public function test_create_incompatibleLexer()
  40     {
  41         $this->config->set('Core.LexerImpl', 'DOMLex');
  42         $this->config->set('Core.MaintainLineNumbers', true);
  43         $this->expectException(new HTMLPurifier_Exception('Cannot use lexer that does not support line numbers with Core.MaintainLineNumbers or Core.CollectErrors (use DirectLex instead)'));
  44         HTMLPurifier_Lexer::create($this->config);
  45     }
  46
  47     // HTMLPurifier_Lexer->parseData() -----------------------------------------
  48
  49     public function assertParseData($input, $expect = true)
  50     {
  51         if ($expect === true) $expect = $input;
  52         $lexer = new HTMLPurifier_Lexer();
  53         $this->assertIdentical($expect, $lexer->parseData($input));
  54     }
  55
  56     public function test_parseData_plainText()
  57     {
  58         $this->assertParseData('asdf');
  59     }
  60
  61     public function test_parseData_ampersandEntity()
  62     {
  63         $this->assertParseData('&amp;', '&');
  64     }
  65
  66     public function test_parseData_quotEntity()
  67     {
  68         $this->assertParseData('&quot;', '"');
  69     }
  70
  71     public function test_parseData_aposNumericEntity()
  72     {
  73         $this->assertParseData('&#039;', "'");
  74     }
  75
  76     public function test_parseData_aposCompactNumericEntity()
  77     {
  78         $this->assertParseData('&#39;', "'");
  79     }
  80
  81     public function test_parseData_adjacentAmpersandEntities()
  82     {
  83         $this->assertParseData('&amp;&amp;&amp;', '&&&');
  84     }
  85
  86     public function test_parseData_trailingUnescapedAmpersand()
  87     {
  88         $this->assertParseData('&amp;&', '&&');
  89     }
  90
  91     public function test_parseData_internalUnescapedAmpersand()
  92     {
  93         $this->assertParseData('Procter & Gamble');
  94     }
  95
  96     public function test_parseData_improperEntityFaultToleranceTest()
  97     {
  98         $this->assertParseData('&#x2D;');
  99     }
 100
 101     // HTMLPurifier_Lexer->extractBody() ---------------------------------------
 102
 103     public function assertExtractBody($text, $extract = true)
 104     {
 105         $lexer = new HTMLPurifier_Lexer();
 106         $result = $lexer->extractBody($text);
 107         if ($extract === true) $extract = $text;
 108         $this->assertIdentical($extract, $result);
 109     }
 110
 111     public function test_extractBody_noBodyTags()
 112     {
 113         $this->assertExtractBody('<b>Bold</b>');
 114     }
 115
 116     public function test_extractBody_lowercaseBodyTags()
 117     {
 118         $this->assertExtractBody('<html><body><b>Bold</b></body></html>', '<b>Bold</b>');
 119     }
 120
 121     public function test_extractBody_uppercaseBodyTags()
 122     {
 123         $this->assertExtractBody('<HTML><BODY><B>Bold</B></BODY></HTML>', '<B>Bold</B>');
 124     }
 125
 126     public function test_extractBody_realisticUseCase()
 127     {
 128         $this->assertExtractBody(
 129 '<?xml version="1.0"
 130 <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
 131     "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
 132 <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
 133    <head>
 134       <title>xyz</title>
 135    </head>
 136    <body>
 137       <form method="post" action="whatever1">
 138          <div>
 139             <input type="text" name="username" />
 140             <input type="text" name="password" />
 141             <input type="submit" />
 142          </div>
 143       </form>
 144    </body>
 145 </html>',
 146     '
 147       <form method="post" action="whatever1">
 148          <div>
 149             <input type="text" name="username" />
 150             <input type="text" name="password" />
 151             <input type="submit" />
 152          </div>
 153       </form>
 154    ');
 155     }
 156
 157     public function test_extractBody_bodyWithAttributes()
 158     {
 159         $this->assertExtractBody('<html><body bgcolor="#F00"><b>Bold</b></body></html>', '<b>Bold</b>');
 160     }
 161
 162     public function test_extractBody_preserveUnclosedBody()
 163     {
 164         $this->assertExtractBody('<body>asdf'); // not closed, don't accept
 165     }
 166
 167     public function test_extractBody_useLastBody()
 168     {
 169         $this->assertExtractBody('<body>foo</body>bar</body>', 'foo</body>bar');
 170     }
 171
 172     public function test_extractBody_ignoreCommented()
 173     {
 174         $this->assertExtractBody('$<!-- <body>foo</body> -->^');
 175     }
 176
 177     public function test_extractBody_butCanStillWork()
 178     {
 179         $this->assertExtractBody('<!-- b --><body>a</body>', 'a');
 180     }
 181
 182     // HTMLPurifier_Lexer->tokenizeHTML() --------------------------------------
 183
 184     public function assertTokenization($input, $expect, $alt_expect = array())
 185     {
 186         $lexers = array();
 187         $lexers['DirectLex']  = new HTMLPurifier_Lexer_DirectLex();
 188         if (class_exists('DOMDocument')) {
 189             $lexers['DOMLex'] = new HTMLPurifier_Lexer_DOMLex();
 190             $lexers['PH5P']   = new HTMLPurifier_Lexer_PH5P();
 191         }
 192         foreach ($lexers as $name => $lexer) {
 193             $result = $lexer->tokenizeHTML($input, $this->config, $this->context);
 194             if (isset($alt_expect[$name])) {
 195                 if ($alt_expect[$name] === false) continue;
 196                 $t_expect = $alt_expect[$name];
 197                 $this->assertIdentical($result, $alt_expect[$name], "$name: %s");
 198             } else {
 199                 $t_expect = $expect;
 200                 $this->assertIdentical($result, $expect, "$name: %s");
 201             }
 202             if ($t_expect != $result) {
 203                 printTokens($result);
 204             }
 205         }
 206     }
 207
 208     public function test_tokenizeHTML_emptyInput()
 209     {
 210         $this->assertTokenization('', array());
 211     }
 212
 213     public function test_tokenizeHTML_plainText()
 214     {
 215         $this->assertTokenization(
 216             'This is regular text.',
 217             array(
 218                 new HTMLPurifier_Token_Text('This is regular text.')
 219             )
 220         );
 221     }
 222
 223     public function test_tokenizeHTML_textAndTags()
 224     {
 225         $this->assertTokenization(
 226             'This is <b>bold</b> text',
 227             array(
 228                 new HTMLPurifier_Token_Text('This is '),
 229                 new HTMLPurifier_Token_Start('b', array()),
 230                 new HTMLPurifier_Token_Text('bold'),
 231                 new HTMLPurifier_Token_End('b'),
 232                 new HTMLPurifier_Token_Text(' text'),
 233             )
 234         );
 235     }
 236
 237     public function test_tokenizeHTML_normalizeCase()
 238     {
 239         $this->assertTokenization(
 240             '<DIV>Totally rad dude. <b>asdf</b></div>',
 241             array(
 242                 new HTMLPurifier_Token_Start('DIV', array()),
 243                 new HTMLPurifier_Token_Text('Totally rad dude. '),
 244                 new HTMLPurifier_Token_Start('b', array()),
 245                 new HTMLPurifier_Token_Text('asdf'),
 246                 new HTMLPurifier_Token_End('b'),
 247                 new HTMLPurifier_Token_End('div'),
 248             )
 249         );
 250     }
 251
 252     public function test_tokenizeHTML_notWellFormed()
 253     {
 254         $this->assertTokenization(
 255             '<asdf></asdf><d></d><poOloka><poolasdf><ds></asdf></ASDF>',
 256             array(
 257                 new HTMLPurifier_Token_Start('asdf'),
 258                 new HTMLPurifier_Token_End('asdf'),
 259                 new HTMLPurifier_Token_Start('d'),
 260                 new HTMLPurifier_Token_End('d'),
 261                 new HTMLPurifier_Token_Start('poOloka'),
 262                 new HTMLPurifier_Token_Start('poolasdf'),
 263                 new HTMLPurifier_Token_Start('ds'),
 264                 new HTMLPurifier_Token_End('asdf'),
 265                 new HTMLPurifier_Token_End('ASDF'),
 266             ),
 267             array(
 268                 'DOMLex' => $alt = array(
 269                     new HTMLPurifier_Token_Empty('asdf'),
 270                     new HTMLPurifier_Token_Empty('d'),
 271                     new HTMLPurifier_Token_Start('pooloka'),
 272                     new HTMLPurifier_Token_Start('poolasdf'),
 273                     new HTMLPurifier_Token_Empty('ds'),
 274                     new HTMLPurifier_Token_End('poolasdf'),
 275                     new HTMLPurifier_Token_End('pooloka'),
 276                 ),
 277                 // 20140831: Weird, but whatever...
 278                 'PH5P' => array(new HTMLPurifier_Token_Empty('asdf')),
 279             )
 280         );
 281     }
 282
 283     public function test_tokenizeHTML_whitespaceInTag()
 284     {
 285         $this->assertTokenization(
 286             '<a'."\t".'href="foobar.php"'."\n".'title="foo!">Link to <b id="asdf">foobar</b></a>',
 287             array(
 288                 new HTMLPurifier_Token_Start('a',array('href'=>'foobar.php','title'=>'foo!')),
 289                 new HTMLPurifier_Token_Text('Link to '),
 290                 new HTMLPurifier_Token_Start('b',array('id'=>'asdf')),
 291                 new HTMLPurifier_Token_Text('foobar'),
 292                 new HTMLPurifier_Token_End('b'),
 293                 new HTMLPurifier_Token_End('a'),
 294             )
 295         );
 296     }
 297
 298     public function test_tokenizeHTML_singleAttribute()
 299     {
 300         $this->assertTokenization(
 301             '<br style="&amp;" />',
 302             array(
 303                 new HTMLPurifier_Token_Empty('br', array('style' => '&'))
 304             )
 305         );
 306     }
 307
 308     public function test_tokenizeHTML_emptyTag()
 309     {
 310         $this->assertTokenization(
 311             '<br />',
 312             array( new HTMLPurifier_Token_Empty('br') )
 313         );
 314     }
 315
 316     public function test_tokenizeHTML_comment()
 317     {
 318         $this->assertTokenization(
 319             '<!-- Comment -->',
 320             array( new HTMLPurifier_Token_Comment(' Comment ') )
 321         );
 322     }
 323
 324     public function test_tokenizeHTML_malformedComment()
 325     {
 326         $this->assertTokenization(
 327             '<!-- not so well formed --->',
 328             array( new HTMLPurifier_Token_Comment(' not so well formed -') )
 329         );
 330     }
 331
 332     public function test_tokenizeHTML_unterminatedTag()
 333     {
 334         $this->assertTokenization(
 335             '<a href=""',
 336             array( new HTMLPurifier_Token_Text('<a href=""') ),
 337             array(
 338                 // I like our behavior better, but it's non-standard
 339                 'DOMLex'   => array( new HTMLPurifier_Token_Empty('a', array('href'=>'')) ),
 340                 'PH5P' => false, // total barfing, grabs scaffolding too
 341             )
 342         );
 343     }
 344
 345     public function test_tokenizeHTML_specialEntities()
 346     {
 347         $this->assertTokenization(
 348             '&lt;b&gt;',
 349             array(
 350                 new HTMLPurifier_Token_Text('<b>')
 351             ),
 352             array(
 353                 // some parsers will separate entities out
 354                 'PH5P' => array(
 355                     new HTMLPurifier_Token_Text('<'),
 356                     new HTMLPurifier_Token_Text('b'),
 357                     new HTMLPurifier_Token_Text('>'),
 358                 ),
 359             )
 360         );
 361     }
 362
 363     public function test_tokenizeHTML_earlyQuote()
 364     {
 365         $this->assertTokenization(
 366             '<a "=>',
 367             array( new HTMLPurifier_Token_Empty('a') ),
 368             array(
 369                 // we barf on this input
 370                 'DirectLex' => array(
 371                     new HTMLPurifier_Token_Start('a', array('"' => ''))
 372                 ),
 373                 'PH5P' => false, // behavior varies; handle this personally
 374             )
 375         );
 376     }
 377
 378     public function test_tokenizeHTML_earlyQuote_PH5P()
 379     {
 380         if (!class_exists('DOMDocument')) return;
 381         $lexer = new HTMLPurifier_Lexer_PH5P();
 382         $result = $lexer->tokenizeHTML('<a "=>', $this->config, $this->context);
 383         if ($this->context->get('PH5PError', true)) {
 384             $this->assertIdentical(array(
 385                 new HTMLPurifier_Token_Start('a', array('"' => ''))
 386             ), $result);
 387         } else {
 388             $this->assertIdentical(array(
 389                 new HTMLPurifier_Token_Empty('a', array('"' => ''))
 390             ), $result);
 391         }
 392     }
 393
 394     public function test_tokenizeHTML_unescapedQuote()
 395     {
 396         $this->assertTokenization(
 397             '"',
 398             array( new HTMLPurifier_Token_Text('"') )
 399         );
 400     }
 401
 402     public function test_tokenizeHTML_escapedQuote()
 403     {
 404         $this->assertTokenization(
 405             '&quot;',
 406             array( new HTMLPurifier_Token_Text('"') )
 407         );
 408     }
 409
 410     public function test_tokenizeHTML_cdata()
 411     {
 412         $this->assertTokenization(
 413             '<![CDATA[You <b>can&#39;t</b> get me!]]>',
 414             array( new HTMLPurifier_Token_Text('You <b>can&#39;t</b> get me!') ),
 415             array(
 416                 'PH5P' =>  array(
 417                     new HTMLPurifier_Token_Text('You '),
 418                     new HTMLPurifier_Token_Text('<'),
 419                     new HTMLPurifier_Token_Text('b'),
 420                     new HTMLPurifier_Token_Text('>'),
 421                     new HTMLPurifier_Token_Text('can'),
 422                     new HTMLPurifier_Token_Text('&'),
 423                     new HTMLPurifier_Token_Text('#39;t'),
 424                     new HTMLPurifier_Token_Text('<'),
 425                     new HTMLPurifier_Token_Text('/b'),
 426                     new HTMLPurifier_Token_Text('>'),
 427                     new HTMLPurifier_Token_Text(' get me!'),
 428                 ),
 429             )
 430         );
 431     }
 432
 433     public function test_tokenizeHTML_characterEntity()
 434     {
 435         $this->assertTokenization(
 436             '&theta;',
 437             array( new HTMLPurifier_Token_Text("\xCE\xB8") )
 438         );
 439     }
 440
 441     public function test_tokenizeHTML_characterEntityInCDATA()
 442     {
 443         $this->assertTokenization(
 444             '<![CDATA[&rarr;]]>',
 445             array( new HTMLPurifier_Token_Text("&rarr;") ),
 446             array(
 447                 'PH5P' => array(
 448                     new HTMLPurifier_Token_Text('&'),
 449                     new HTMLPurifier_Token_Text('rarr;'),
 450                 ),
 451             )
 452         );
 453     }
 454
 455     public function test_tokenizeHTML_entityInAttribute()
 456     {
 457         $this->assertTokenization(
 458             '<a href="index.php?title=foo&amp;id=bar">Link</a>',
 459             array(
 460                 new HTMLPurifier_Token_Start('a',array('href' => 'index.php?title=foo&id=bar')),
 461                 new HTMLPurifier_Token_Text('Link'),
 462                 new HTMLPurifier_Token_End('a'),
 463             )
 464         );
 465     }
 466
 467     public function test_tokenizeHTML_preserveUTF8()
 468     {
 469         $this->assertTokenization(
 470             "\xCE\xB8",
 471             array( new HTMLPurifier_Token_Text("\xCE\xB8") )
 472         );
 473     }
 474
 475     public function test_tokenizeHTML_specialEntityInAttribute()
 476     {
 477         $this->assertTokenization(
 478             '<br test="x &lt; 6" />',
 479             array( new HTMLPurifier_Token_Empty('br', array('test' => 'x < 6')) )
 480         );
 481     }
 482
 483     public function test_tokenizeHTML_emoticonProtection()
 484     {
 485         $this->assertTokenization(
 486             '<b>Whoa! <3 That\'s not good >.></b>',
 487             array(
 488                 new HTMLPurifier_Token_Start('b'),
 489                 new HTMLPurifier_Token_Text('Whoa! '),
 490                 new HTMLPurifier_Token_Text('<'),
 491                 new HTMLPurifier_Token_Text('3 That\'s not good >.>'),
 492                 new HTMLPurifier_Token_End('b')
 493             ),
 494             array(
 495                 // text is absorbed together
 496                 'DOMLex' => array(
 497                     new HTMLPurifier_Token_Start('b'),
 498                     new HTMLPurifier_Token_Text('Whoa! <3 That\'s not good >.>'),
 499                     new HTMLPurifier_Token_End('b'),
 500                 ),
 501                 'PH5P' => array( // interesting grouping
 502                     new HTMLPurifier_Token_Start('b'),
 503                     new HTMLPurifier_Token_Text('Whoa! '),
 504                     new HTMLPurifier_Token_Text('<'),
 505                     new HTMLPurifier_Token_Text('3 That\'s not good >.>'),
 506                     new HTMLPurifier_Token_End('b'),
 507                 ),
 508             )
 509         );
 510     }
 511
 512     public function test_tokenizeHTML_commentWithFunkyChars()
 513     {
 514         $this->assertTokenization(
 515             '<!-- This >< comment --><br />',
 516             array(
 517                 new HTMLPurifier_Token_Comment(' This >< comment '),
 518                 new HTMLPurifier_Token_Empty('br'),
 519             )
 520         );
 521     }
 522
 523     public function test_tokenizeHTML_unterminatedComment()
 524     {
 525         $this->assertTokenization(
 526             '<!-- This >< comment',
 527             array( new HTMLPurifier_Token_Comment(' This >< comment') ),
 528             array(
 529                 'DOMLex'   => false,
 530                 'PH5P'     => false,
 531             )
 532         );
 533     }
 534
 535     public function test_tokenizeHTML_scriptCDATAContents()
 536     {
 537         $this->config->set('HTML.Trusted', true);
 538         $this->assertTokenization(
 539             'Foo: <script>alert("<foo>");</script>',
 540             array(
 541                 new HTMLPurifier_Token_Text('Foo: '),
 542                 new HTMLPurifier_Token_Start('script'),
 543                 new HTMLPurifier_Token_Text('alert("<foo>");'),
 544                 new HTMLPurifier_Token_End('script'),
 545             ),
 546             array(
 547                 // PH5P, for some reason, bubbles the script to <head>
 548                 'PH5P' => false,
 549             )
 550         );
 551     }
 552
 553     public function test_tokenizeHTML_entitiesInComment()
 554     {
 555         $this->assertTokenization(
 556             '<!-- This comment < &lt; & -->',
 557             array( new HTMLPurifier_Token_Comment(' This comment < &lt; & ') )
 558         );
 559     }
 560
 561     public function test_tokenizeHTML_attributeWithSpecialCharacters()
 562     {
 563         $this->assertTokenization(
 564             '<a href="><>">',
 565             array( new HTMLPurifier_Token_Empty('a', array('href' => '><>')) ),
 566             array(
 567                 'DirectLex' => array(
 568                     new HTMLPurifier_Token_Start('a', array('href' => '')),
 569                     new HTMLPurifier_Token_Text('<'),
 570                     new HTMLPurifier_Token_Text('">'),
 571                 )
 572             )
 573         );
 574     }
 575
 576     public function test_tokenizeHTML_emptyTagWithSlashInAttribute()
 577     {
 578         $this->assertTokenization(
 579             '<param name="src" value="http://example.com/video.wmv" />',
 580             array( new HTMLPurifier_Token_Empty('param', array('name' => 'src', 'value' => 'http://example.com/video.wmv')) )
 581         );
 582     }
 583
 584     public function test_tokenizeHTML_style()
 585     {
 586         $extra = array(
 587                 // PH5P doesn't seem to like style tags
 588                 'PH5P' => false,
 589                 // DirectLex defers to RemoveForeignElements for textification
 590                 'DirectLex' => array(
 591                     new HTMLPurifier_Token_Start('style', array('type' => 'text/css')),
 592                     new HTMLPurifier_Token_Comment("\ndiv {}\n"),
 593                     new HTMLPurifier_Token_End('style'),
 594                 ),
 595             );
 596         if (!defined('LIBXML_VERSION')) {
 597             // LIBXML_VERSION is missing in early versions of PHP
 598             // prior to 1.30 of php-src/ext/libxml/libxml.c (version-wise,
 599             // this translates to 5.0.x. In such cases, punt the test entirely.
 600             return;
 601         } elseif (LIBXML_VERSION < 20628) {
 602             // libxml's behavior is wrong prior to this version, so make
 603             // appropriate accomodations
 604             $extra['DOMLex'] = $extra['DirectLex'];
 605         }
 606         $this->assertTokenization(
 607 '<style type="text/css"><!--
 608 div {}
 609 --></style>',
 610             array(
 611                 new HTMLPurifier_Token_Start('style', array('type' => 'text/css')),
 612                 new HTMLPurifier_Token_Text("\ndiv {}\n"),
 613                 new HTMLPurifier_Token_End('style'),
 614             ),
 615             $extra
 616         );
 617     }
 618
 619     public function test_tokenizeHTML_tagWithAtSignAndExtraGt()
 620     {
 621         $alt_expect = array(
 622             // Technically this is invalid, but it won't be a
 623             // problem with invalid element removal; also, this
 624             // mimics Mozilla's parsing of the tag.
 625             new HTMLPurifier_Token_Start('a@'),
 626             new HTMLPurifier_Token_Text('>'),
 627         );
 628         $this->assertTokenization(
 629             '<a@>>',
 630             array(
 631                 new HTMLPurifier_Token_Start('a'),
 632                 new HTMLPurifier_Token_Text('>'),
 633                 new HTMLPurifier_Token_End('a'),
 634             ),
 635             array(
 636                 'DirectLex' => $alt_expect,
 637             )
 638         );
 639     }
 640
 641     public function test_tokenizeHTML_emoticonHeart()
 642     {
 643         $this->assertTokenization(
 644             '<br /><3<br />',
 645             array(
 646                 new HTMLPurifier_Token_Empty('br'),
 647                 new HTMLPurifier_Token_Text('<'),
 648                 new HTMLPurifier_Token_Text('3'),
 649                 new HTMLPurifier_Token_Empty('br'),
 650             ),
 651             array(
 652                 'DOMLex' => array(
 653                     new HTMLPurifier_Token_Empty('br'),
 654                     new HTMLPurifier_Token_Text('<3'),
 655                     new HTMLPurifier_Token_Empty('br'),
 656                 ),
 657             )
 658         );
 659     }
 660
 661     public function test_tokenizeHTML_emoticonShiftyEyes()
 662     {
 663         $this->assertTokenization(
 664             '<b><<</b>',
 665             array(
 666                 new HTMLPurifier_Token_Start('b'),
 667                 new HTMLPurifier_Token_Text('<'),
 668                 new HTMLPurifier_Token_Text('<'),
 669                 new HTMLPurifier_Token_End('b'),
 670             ),
 671             array(
 672                 'DOMLex' => array(
 673                     new HTMLPurifier_Token_Start('b'),
 674                     new HTMLPurifier_Token_Text('<<'),
 675                     new HTMLPurifier_Token_End('b'),
 676                 ),
 677             )
 678         );
 679     }
 680
 681     public function test_tokenizeHTML_eon1996()
 682     {
 683         $this->assertTokenization(
 684             '< <b>test</b>',
 685             array(
 686                 new HTMLPurifier_Token_Text('<'),
 687                 new HTMLPurifier_Token_Text(' '),
 688                 new HTMLPurifier_Token_Start('b'),
 689                 new HTMLPurifier_Token_Text('test'),
 690                 new HTMLPurifier_Token_End('b'),
 691             ),
 692             array(
 693                 'DOMLex' => array(
 694                     new HTMLPurifier_Token_Text('< '),
 695                     new HTMLPurifier_Token_Start('b'),
 696                     new HTMLPurifier_Token_Text('test'),
 697                     new HTMLPurifier_Token_End('b'),
 698                 ),
 699             )
 700         );
 701     }
 702
 703     public function test_tokenizeHTML_bodyInCDATA()
 704     {
 705         $alt_tokens = array(
 706             new HTMLPurifier_Token_Text('<'),
 707             new HTMLPurifier_Token_Text('body'),
 708             new HTMLPurifier_Token_Text('>'),
 709             new HTMLPurifier_Token_Text('Foo'),
 710             new HTMLPurifier_Token_Text('<'),
 711             new HTMLPurifier_Token_Text('/body'),
 712             new HTMLPurifier_Token_Text('>'),
 713         );
 714         $this->assertTokenization(
 715             '<![CDATA[<body>Foo</body>]]>',
 716             array(
 717                 new HTMLPurifier_Token_Text('<body>Foo</body>'),
 718             ),
 719             array(
 720                 'PH5P' => $alt_tokens,
 721             )
 722         );
 723     }
 724
 725     public function test_tokenizeHTML_()
 726     {
 727         $this->assertTokenization(
 728             '<a><img /></a>',
 729             array(
 730                 new HTMLPurifier_Token_Start('a'),
 731                 new HTMLPurifier_Token_Empty('img'),
 732                 new HTMLPurifier_Token_End('a'),
 733             )
 734         );
 735     }
 736
 737     public function test_tokenizeHTML_ignoreIECondComment()
 738     {
 739         $this->assertTokenization(
 740             '<!--[if IE]>foo<a>bar<!-- baz --><![endif]-->',
 741             array()
 742         );
 743     }
 744
 745     public function test_tokenizeHTML_removeProcessingInstruction()
 746     {
 747         $this->config->set('Core.RemoveProcessingInstructions', true);
 748         $this->assertTokenization(
 749             '<?xml blah blah ?>',
 750             array()
 751         );
 752     }
 753
 754    public function test_tokenizeHTML_removeNewline()
 755    {
 756         $this->config->set('Core.NormalizeNewlines', true);
 757         $this->assertTokenization(
 758             "plain\rtext\r\n",
 759             array(
 760                 new HTMLPurifier_Token_Text("plain\ntext\n")
 761             )
 762         );
 763    }
 764
 765    public function test_tokenizeHTML_noRemoveNewline()
 766    {
 767         $this->config->set('Core.NormalizeNewlines', false);
 768         $this->assertTokenization(
 769             "plain\rtext\r\n",
 770             array(
 771                 new HTMLPurifier_Token_Text("plain\rtext\r\n")
 772             )
 773         );
 774      }
 775
 776     public function test_tokenizeHTML_conditionalCommentUngreedy()
 777     {
 778         $this->assertTokenization(
 779             '<!--[if gte mso 9]>a<![endif]-->b<!--[if gte mso 9]>c<![endif]-->',
 780             array(
 781                 new HTMLPurifier_Token_Text("b")
 782             )
 783         );
 784     }
 785
 786     public function test_tokenizeHTML_imgTag()
 787     {
 788         $start = array(
 789                         new HTMLPurifier_Token_Start('img',
 790                             array(
 791                                 'src' => 'img_11775.jpg',
 792                                 'alt' => '[Img #11775]',
 793                                 'id' => 'EMBEDDED_IMG_11775',
 794                             )
 795                         )
 796                     );
 797         $this->assertTokenization(
 798             '<img src="img_11775.jpg" alt="[Img #11775]" id="EMBEDDED_IMG_11775" >',
 799             array(
 800                 new HTMLPurifier_Token_Empty('img',
 801                     array(
 802                         'src' => 'img_11775.jpg',
 803                         'alt' => '[Img #11775]',
 804                         'id' => 'EMBEDDED_IMG_11775',
 805                     )
 806                 )
 807             ),
 808             array(
 809                 'DirectLex' => $start,
 810                 )
 811         );
 812     }
 813
 814     public function test_tokenizeHTML_prematureDivClose()
 815     {
 816         $this->assertTokenization(
 817             '</div>dontdie',
 818             array(
 819                 new HTMLPurifier_Token_End('div'),
 820                 new HTMLPurifier_Token_Text('dontdie')
 821             ),
 822             array(
 823                 'DOMLex' => $alt = array(new HTMLPurifier_Token_Text('dontdie')),
 824                 'PH5P' => $alt
 825             )
 826         );
 827     }
 828
 829
 830     /*
 831
 832     public function test_tokenizeHTML_()
 833     {
 834         $this->assertTokenization(
 835             ,
 836             array(
 837
 838             )
 839         );
 840     }
 841     */
 842
 843 }
 844
 845 // vim: et sw=4 sts=4