tests/HTMLPurifier/LexerTest.php

   1 <?php
   2
   3 class HTMLPurifier_LexerTest extends HTMLPurifier_Harness
   4 {
   5
   6     protected $_has_pear = false;
   7
   8     public function __construct()
   9     {
  10         parent::__construct();
  11         if ($GLOBALS['HTMLPurifierTest']['PH5P']) {
  12             require_once 'HTMLPurifier/Lexer/PH5P.php';
  13         }
  14     }
  15
  16     // HTMLPurifier_Lexer::create() --------------------------------------------
  17
  18     public function test_create()
  19     {
  20         $this->config->set('Core.MaintainLineNumbers', true);
  21         $lexer = HTMLPurifier_Lexer::create($this->config);
  22         $this->assertIsA($lexer, 'HTMLPurifier_Lexer_DirectLex');
  23     }
  24
  25     public function test_create_objectLexerImpl()
  26     {
  27         $this->config->set('Core.LexerImpl', new HTMLPurifier_Lexer_DirectLex());
  28         $lexer = HTMLPurifier_Lexer::create($this->config);
  29         $this->assertIsA($lexer, 'HTMLPurifier_Lexer_DirectLex');
  30     }
  31
  32     public function test_create_unknownLexer()
  33     {
  34         $this->config->set('Core.LexerImpl', 'AsdfAsdf');
  35         $this->expectException(new HTMLPurifier_Exception('Cannot instantiate unrecognized Lexer type AsdfAsdf'));
  36         HTMLPurifier_Lexer::create($this->config);
  37     }
  38
  39     public function test_create_incompatibleLexer()
  40     {
  41         $this->config->set('Core.LexerImpl', 'DOMLex');
  42         $this->config->set('Core.MaintainLineNumbers', true);
  43         $this->expectException(new HTMLPurifier_Exception('Cannot use lexer that does not support line numbers with Core.MaintainLineNumbers or Core.CollectErrors (use DirectLex instead)'));
  44         HTMLPurifier_Lexer::create($this->config);
  45     }
  46
  47     // HTMLPurifier_Lexer->parseData() -----------------------------------------
  48
  49     public function assertParseData($input, $expect = true)
  50     {
  51         if ($expect === true) $expect = $input;
  52         $lexer = new HTMLPurifier_Lexer();
  53         $this->assertIdentical($expect, $lexer->parseData($input));
  54     }
  55
  56     public function test_parseData_plainText()
  57     {
  58         $this->assertParseData('asdf');
  59     }
  60
  61     public function test_parseData_ampersandEntity()
  62     {
  63         $this->assertParseData('&amp;', '&');
  64     }
  65
  66     public function test_parseData_quotEntity()
  67     {
  68         $this->assertParseData('&quot;', '"');
  69     }
  70
  71     public function test_parseData_aposNumericEntity()
  72     {
  73         $this->assertParseData('&#039;', "'");
  74     }
  75
  76     public function test_parseData_aposCompactNumericEntity()
  77     {
  78         $this->assertParseData('&#39;', "'");
  79     }
  80
  81     public function test_parseData_adjacentAmpersandEntities()
  82     {
  83         $this->assertParseData('&amp;&amp;&amp;', '&&&');
  84     }
  85
  86     public function test_parseData_trailingUnescapedAmpersand()
  87     {
  88         $this->assertParseData('&amp;&', '&&');
  89     }
  90
  91     public function test_parseData_internalUnescapedAmpersand()
  92     {
  93         $this->assertParseData('Procter & Gamble');
  94     }
  95
  96     public function test_parseData_improperEntityFaultToleranceTest()
  97     {
  98         $this->assertParseData('&#x2D;');
  99     }
 100
 101     // HTMLPurifier_Lexer->extractBody() ---------------------------------------
 102
 103     public function assertExtractBody($text, $extract = true)
 104     {
 105         $lexer = new HTMLPurifier_Lexer();
 106         $result = $lexer->extractBody($text);
 107         if ($extract === true) $extract = $text;
 108         $this->assertIdentical($extract, $result);
 109     }
 110
 111     public function test_extractBody_noBodyTags()
 112     {
 113         $this->assertExtractBody('<b>Bold</b>');
 114     }
 115
 116     public function test_extractBody_lowercaseBodyTags()
 117     {
 118         $this->assertExtractBody('<html><body><b>Bold</b></body></html>', '<b>Bold</b>');
 119     }
 120
 121     public function test_extractBody_uppercaseBodyTags()
 122     {
 123         $this->assertExtractBody('<HTML><BODY><B>Bold</B></BODY></HTML>', '<B>Bold</B>');
 124     }
 125
 126     public function test_extractBody_realisticUseCase()
 127     {
 128         $this->assertExtractBody(
 129 '<?xml version="1.0"
 130 <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
 131     "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
 132 <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
 133    <head>
 134       <title>xyz</title>
 135    </head>
 136    <body>
 137       <form method="post" action="whatever1">
 138          <div>
 139             <input type="text" name="username" />
 140             <input type="text" name="password" />
 141             <input type="submit" />
 142          </div>
 143       </form>
 144    </body>
 145 </html>',
 146     '
 147       <form method="post" action="whatever1">
 148          <div>
 149             <input type="text" name="username" />
 150             <input type="text" name="password" />
 151             <input type="submit" />
 152          </div>
 153       </form>
 154    ');
 155     }
 156
 157     public function test_extractBody_bodyWithAttributes()
 158     {
 159         $this->assertExtractBody('<html><body bgcolor="#F00"><b>Bold</b></body></html>', '<b>Bold</b>');
 160     }
 161
 162     public function test_extractBody_preserveUnclosedBody()
 163     {
 164         $this->assertExtractBody('<body>asdf'); // not closed, don't accept
 165     }
 166
 167     public function test_extractBody_useLastBody()
 168     {
 169         $this->assertExtractBody('<body>foo</body>bar</body>', 'foo</body>bar');
 170     }
 171
 172     // HTMLPurifier_Lexer->tokenizeHTML() --------------------------------------
 173
 174     public function assertTokenization($input, $expect, $alt_expect = array())
 175     {
 176         $lexers = array();
 177         $lexers['DirectLex']  = new HTMLPurifier_Lexer_DirectLex();
 178         if (class_exists('DOMDocument')) {
 179             $lexers['DOMLex'] = new HTMLPurifier_Lexer_DOMLex();
 180             $lexers['PH5P']   = new HTMLPurifier_Lexer_PH5P();
 181         }
 182         foreach ($lexers as $name => $lexer) {
 183             $result = $lexer->tokenizeHTML($input, $this->config, $this->context);
 184             if (isset($alt_expect[$name])) {
 185                 if ($alt_expect[$name] === false) continue;
 186                 $t_expect = $alt_expect[$name];
 187                 $this->assertIdentical($result, $alt_expect[$name], "$name: %s");
 188             } else {
 189                 $t_expect = $expect;
 190                 $this->assertIdentical($result, $expect, "$name: %s");
 191             }
 192             if ($t_expect != $result) {
 193                 printTokens($result);
 194             }
 195         }
 196     }
 197
 198     public function test_tokenizeHTML_emptyInput()
 199     {
 200         $this->assertTokenization('', array());
 201     }
 202
 203     public function test_tokenizeHTML_plainText()
 204     {
 205         $this->assertTokenization(
 206             'This is regular text.',
 207             array(
 208                 new HTMLPurifier_Token_Text('This is regular text.')
 209             )
 210         );
 211     }
 212
 213     public function test_tokenizeHTML_textAndTags()
 214     {
 215         $this->assertTokenization(
 216             'This is <b>bold</b> text',
 217             array(
 218                 new HTMLPurifier_Token_Text('This is '),
 219                 new HTMLPurifier_Token_Start('b', array()),
 220                 new HTMLPurifier_Token_Text('bold'),
 221                 new HTMLPurifier_Token_End('b'),
 222                 new HTMLPurifier_Token_Text(' text'),
 223             )
 224         );
 225     }
 226
 227     public function test_tokenizeHTML_normalizeCase()
 228     {
 229         $this->assertTokenization(
 230             '<DIV>Totally rad dude. <b>asdf</b></div>',
 231             array(
 232                 new HTMLPurifier_Token_Start('DIV', array()),
 233                 new HTMLPurifier_Token_Text('Totally rad dude. '),
 234                 new HTMLPurifier_Token_Start('b', array()),
 235                 new HTMLPurifier_Token_Text('asdf'),
 236                 new HTMLPurifier_Token_End('b'),
 237                 new HTMLPurifier_Token_End('div'),
 238             )
 239         );
 240     }
 241
 242     public function test_tokenizeHTML_notWellFormed()
 243     {
 244         $this->assertTokenization(
 245             '<asdf></asdf><d></d><poOloka><poolasdf><ds></asdf></ASDF>',
 246             array(
 247                 new HTMLPurifier_Token_Start('asdf'),
 248                 new HTMLPurifier_Token_End('asdf'),
 249                 new HTMLPurifier_Token_Start('d'),
 250                 new HTMLPurifier_Token_End('d'),
 251                 new HTMLPurifier_Token_Start('poOloka'),
 252                 new HTMLPurifier_Token_Start('poolasdf'),
 253                 new HTMLPurifier_Token_Start('ds'),
 254                 new HTMLPurifier_Token_End('asdf'),
 255                 new HTMLPurifier_Token_End('ASDF'),
 256             ),
 257             array(
 258                 'DOMLex' => $alt = array(
 259                     new HTMLPurifier_Token_Empty('asdf'),
 260                     new HTMLPurifier_Token_Empty('d'),
 261                     new HTMLPurifier_Token_Start('pooloka'),
 262                     new HTMLPurifier_Token_Start('poolasdf'),
 263                     new HTMLPurifier_Token_Empty('ds'),
 264                     new HTMLPurifier_Token_End('poolasdf'),
 265                     new HTMLPurifier_Token_End('pooloka'),
 266                 ),
 267                 'PH5P' => $alt,
 268             )
 269         );
 270     }
 271
 272     public function test_tokenizeHTML_whitespaceInTag()
 273     {
 274         $this->assertTokenization(
 275             '<a'."\t".'href="foobar.php"'."\n".'title="foo!">Link to <b id="asdf">foobar</b></a>',
 276             array(
 277                 new HTMLPurifier_Token_Start('a',array('href'=>'foobar.php','title'=>'foo!')),
 278                 new HTMLPurifier_Token_Text('Link to '),
 279                 new HTMLPurifier_Token_Start('b',array('id'=>'asdf')),
 280                 new HTMLPurifier_Token_Text('foobar'),
 281                 new HTMLPurifier_Token_End('b'),
 282                 new HTMLPurifier_Token_End('a'),
 283             )
 284         );
 285     }
 286
 287     public function test_tokenizeHTML_singleAttribute()
 288     {
 289         $this->assertTokenization(
 290             '<br style="&amp;" />',
 291             array(
 292                 new HTMLPurifier_Token_Empty('br', array('style' => '&'))
 293             )
 294         );
 295     }
 296
 297     public function test_tokenizeHTML_emptyTag()
 298     {
 299         $this->assertTokenization(
 300             '<br />',
 301             array( new HTMLPurifier_Token_Empty('br') )
 302         );
 303     }
 304
 305     public function test_tokenizeHTML_comment()
 306     {
 307         $this->assertTokenization(
 308             '<!-- Comment -->',
 309             array( new HTMLPurifier_Token_Comment(' Comment ') )
 310         );
 311     }
 312
 313     public function test_tokenizeHTML_malformedComment()
 314     {
 315         $this->assertTokenization(
 316             '<!-- not so well formed --->',
 317             array( new HTMLPurifier_Token_Comment(' not so well formed -') )
 318         );
 319     }
 320
 321     public function test_tokenizeHTML_unterminatedTag()
 322     {
 323         $this->assertTokenization(
 324             '<a href=""',
 325             array( new HTMLPurifier_Token_Text('<a href=""') ),
 326             array(
 327                 // I like our behavior better, but it's non-standard
 328                 'DOMLex'   => array( new HTMLPurifier_Token_Empty('a', array('href'=>'')) ),
 329                 'PH5P' => false, // total barfing, grabs scaffolding too
 330             )
 331         );
 332     }
 333
 334     public function test_tokenizeHTML_specialEntities()
 335     {
 336         $this->assertTokenization(
 337             '&lt;b&gt;',
 338             array(
 339                 new HTMLPurifier_Token_Text('<b>')
 340             ),
 341             array(
 342                 // some parsers will separate entities out
 343                 'PH5P' => array(
 344                     new HTMLPurifier_Token_Text('<'),
 345                     new HTMLPurifier_Token_Text('b'),
 346                     new HTMLPurifier_Token_Text('>'),
 347                 ),
 348             )
 349         );
 350     }
 351
 352     public function test_tokenizeHTML_earlyQuote()
 353     {
 354         $this->assertTokenization(
 355             '<a "=>',
 356             array( new HTMLPurifier_Token_Empty('a') ),
 357             array(
 358                 // we barf on this input
 359                 'DirectLex' => array(
 360                     new HTMLPurifier_Token_Start('a', array('"' => ''))
 361                 ),
 362                 'PH5P' => false, // behavior varies; handle this personally
 363             )
 364         );
 365     }
 366
 367     public function test_tokenizeHTML_earlyQuote_PH5P()
 368     {
 369         if (!class_exists('DOMDocument')) return;
 370         $lexer = new HTMLPurifier_Lexer_PH5P();
 371         $result = $lexer->tokenizeHTML('<a "=>', $this->config, $this->context);
 372         if ($this->context->get('PH5PError', true)) {
 373             $this->assertIdentical(array(
 374                 new HTMLPurifier_Token_Start('a', array('"' => ''))
 375             ), $result);
 376         } else {
 377             $this->assertIdentical(array(
 378                 new HTMLPurifier_Token_Empty('a', array('"' => ''))
 379             ), $result);
 380         }
 381     }
 382
 383     public function test_tokenizeHTML_unescapedQuote()
 384     {
 385         $this->assertTokenization(
 386             '"',
 387             array( new HTMLPurifier_Token_Text('"') )
 388         );
 389     }
 390
 391     public function test_tokenizeHTML_escapedQuote()
 392     {
 393         $this->assertTokenization(
 394             '&quot;',
 395             array( new HTMLPurifier_Token_Text('"') )
 396         );
 397     }
 398
 399     public function test_tokenizeHTML_cdata()
 400     {
 401         $this->assertTokenization(
 402             '<![CDATA[You <b>can&#39;t</b> get me!]]>',
 403             array( new HTMLPurifier_Token_Text('You <b>can&#39;t</b> get me!') ),
 404             array(
 405                 'PH5P' =>  array(
 406                     new HTMLPurifier_Token_Text('You '),
 407                     new HTMLPurifier_Token_Text('<'),
 408                     new HTMLPurifier_Token_Text('b'),
 409                     new HTMLPurifier_Token_Text('>'),
 410                     new HTMLPurifier_Token_Text('can'),
 411                     new HTMLPurifier_Token_Text('&'),
 412                     new HTMLPurifier_Token_Text('#39;t'),
 413                     new HTMLPurifier_Token_Text('<'),
 414                     new HTMLPurifier_Token_Text('/b'),
 415                     new HTMLPurifier_Token_Text('>'),
 416                     new HTMLPurifier_Token_Text(' get me!'),
 417                 ),
 418             )
 419         );
 420     }
 421
 422     public function test_tokenizeHTML_characterEntity()
 423     {
 424         $this->assertTokenization(
 425             '&theta;',
 426             array( new HTMLPurifier_Token_Text("\xCE\xB8") )
 427         );
 428     }
 429
 430     public function test_tokenizeHTML_characterEntityInCDATA()
 431     {
 432         $this->assertTokenization(
 433             '<![CDATA[&rarr;]]>',
 434             array( new HTMLPurifier_Token_Text("&rarr;") ),
 435             array(
 436                 'PH5P' => array(
 437                     new HTMLPurifier_Token_Text('&'),
 438                     new HTMLPurifier_Token_Text('rarr;'),
 439                 ),
 440             )
 441         );
 442     }
 443
 444     public function test_tokenizeHTML_entityInAttribute()
 445     {
 446         $this->assertTokenization(
 447             '<a href="index.php?title=foo&amp;id=bar">Link</a>',
 448             array(
 449                 new HTMLPurifier_Token_Start('a',array('href' => 'index.php?title=foo&id=bar')),
 450                 new HTMLPurifier_Token_Text('Link'),
 451                 new HTMLPurifier_Token_End('a'),
 452             )
 453         );
 454     }
 455
 456     public function test_tokenizeHTML_preserveUTF8()
 457     {
 458         $this->assertTokenization(
 459             "\xCE\xB8",
 460             array( new HTMLPurifier_Token_Text("\xCE\xB8") )
 461         );
 462     }
 463
 464     public function test_tokenizeHTML_specialEntityInAttribute()
 465     {
 466         $this->assertTokenization(
 467             '<br test="x &lt; 6" />',
 468             array( new HTMLPurifier_Token_Empty('br', array('test' => 'x < 6')) )
 469         );
 470     }
 471
 472     public function test_tokenizeHTML_emoticonProtection()
 473     {
 474         $this->assertTokenization(
 475             '<b>Whoa! <3 That\'s not good >.></b>',
 476             array(
 477                 new HTMLPurifier_Token_Start('b'),
 478                 new HTMLPurifier_Token_Text('Whoa! '),
 479                 new HTMLPurifier_Token_Text('<'),
 480                 new HTMLPurifier_Token_Text('3 That\'s not good >.>'),
 481                 new HTMLPurifier_Token_End('b')
 482             ),
 483             array(
 484                 // text is absorbed together
 485                 'DOMLex' => array(
 486                     new HTMLPurifier_Token_Start('b'),
 487                     new HTMLPurifier_Token_Text('Whoa! <3 That\'s not good >.>'),
 488                     new HTMLPurifier_Token_End('b'),
 489                 ),
 490                 'PH5P' => array( // interesting grouping
 491                     new HTMLPurifier_Token_Start('b'),
 492                     new HTMLPurifier_Token_Text('Whoa! '),
 493                     new HTMLPurifier_Token_Text('<'),
 494                     new HTMLPurifier_Token_Text('3 That\'s not good >.>'),
 495                     new HTMLPurifier_Token_End('b'),
 496                 ),
 497             )
 498         );
 499     }
 500
 501     public function test_tokenizeHTML_commentWithFunkyChars()
 502     {
 503         $this->assertTokenization(
 504             '<!-- This >< comment --><br />',
 505             array(
 506                 new HTMLPurifier_Token_Comment(' This >< comment '),
 507                 new HTMLPurifier_Token_Empty('br'),
 508             )
 509         );
 510     }
 511
 512     public function test_tokenizeHTML_unterminatedComment()
 513     {
 514         $this->assertTokenization(
 515             '<!-- This >< comment',
 516             array( new HTMLPurifier_Token_Comment(' This >< comment') ),
 517             array(
 518                 'DOMLex'   => false,
 519                 'PH5P'     => false,
 520             )
 521         );
 522     }
 523
 524     public function test_tokenizeHTML_scriptCDATAContents()
 525     {
 526         $this->config->set('HTML.Trusted', true);
 527         $this->assertTokenization(
 528             'Foo: <script>alert("<foo>");</script>',
 529             array(
 530                 new HTMLPurifier_Token_Text('Foo: '),
 531                 new HTMLPurifier_Token_Start('script'),
 532                 new HTMLPurifier_Token_Text('alert("<foo>");'),
 533                 new HTMLPurifier_Token_End('script'),
 534             ),
 535             array(
 536                 // PH5P, for some reason, bubbles the script to <head>
 537                 'PH5P' => false,
 538             )
 539         );
 540     }
 541
 542     public function test_tokenizeHTML_entitiesInComment()
 543     {
 544         $this->assertTokenization(
 545             '<!-- This comment < &lt; & -->',
 546             array( new HTMLPurifier_Token_Comment(' This comment < &lt; & ') )
 547         );
 548     }
 549
 550     public function test_tokenizeHTML_attributeWithSpecialCharacters()
 551     {
 552         $this->assertTokenization(
 553             '<a href="><>">',
 554             array( new HTMLPurifier_Token_Empty('a', array('href' => '><>')) ),
 555             array(
 556                 'DirectLex' => array(
 557                     new HTMLPurifier_Token_Start('a', array('href' => '')),
 558                     new HTMLPurifier_Token_Text('<'),
 559                     new HTMLPurifier_Token_Text('">'),
 560                 )
 561             )
 562         );
 563     }
 564
 565     public function test_tokenizeHTML_emptyTagWithSlashInAttribute()
 566     {
 567         $this->assertTokenization(
 568             '<param name="src" value="http://example.com/video.wmv" />',
 569             array( new HTMLPurifier_Token_Empty('param', array('name' => 'src', 'value' => 'http://example.com/video.wmv')) )
 570         );
 571     }
 572
 573     public function test_tokenizeHTML_style()
 574     {
 575         $extra = array(
 576                 // PH5P doesn't seem to like style tags
 577                 'PH5P' => false,
 578                 // DirectLex defers to RemoveForeignElements for textification
 579                 'DirectLex' => array(
 580                     new HTMLPurifier_Token_Start('style', array('type' => 'text/css')),
 581                     new HTMLPurifier_Token_Comment("\ndiv {}\n"),
 582                     new HTMLPurifier_Token_End('style'),
 583                 ),
 584             );
 585         if (!defined('LIBXML_VERSION')) {
 586             // LIBXML_VERSION is missing in early versions of PHP
 587             // prior to 1.30 of php-src/ext/libxml/libxml.c (version-wise,
 588             // this translates to 5.0.x. In such cases, punt the test entirely.
 589             return;
 590         } elseif (LIBXML_VERSION < 20628) {
 591             // libxml's behavior is wrong prior to this version, so make
 592             // appropriate accomodations
 593             $extra['DOMLex'] = $extra['DirectLex'];
 594         }
 595         $this->assertTokenization(
 596 '<style type="text/css"><!--
 597 div {}
 598 --></style>',
 599             array(
 600                 new HTMLPurifier_Token_Start('style', array('type' => 'text/css')),
 601                 new HTMLPurifier_Token_Text("\ndiv {}\n"),
 602                 new HTMLPurifier_Token_End('style'),
 603             ),
 604             $extra
 605         );
 606     }
 607
 608     public function test_tokenizeHTML_tagWithAtSignAndExtraGt()
 609     {
 610         $alt_expect = array(
 611             // Technically this is invalid, but it won't be a
 612             // problem with invalid element removal; also, this
 613             // mimics Mozilla's parsing of the tag.
 614             new HTMLPurifier_Token_Start('a@'),
 615             new HTMLPurifier_Token_Text('>'),
 616         );
 617         $this->assertTokenization(
 618             '<a@>>',
 619             array(
 620                 new HTMLPurifier_Token_Start('a'),
 621                 new HTMLPurifier_Token_Text('>'),
 622                 new HTMLPurifier_Token_End('a'),
 623             ),
 624             array(
 625                 'DirectLex' => $alt_expect,
 626             )
 627         );
 628     }
 629
 630     public function test_tokenizeHTML_emoticonHeart()
 631     {
 632         $this->assertTokenization(
 633             '<br /><3<br />',
 634             array(
 635                 new HTMLPurifier_Token_Empty('br'),
 636                 new HTMLPurifier_Token_Text('<'),
 637                 new HTMLPurifier_Token_Text('3'),
 638                 new HTMLPurifier_Token_Empty('br'),
 639             ),
 640             array(
 641                 'DOMLex' => array(
 642                     new HTMLPurifier_Token_Empty('br'),
 643                     new HTMLPurifier_Token_Text('<3'),
 644                     new HTMLPurifier_Token_Empty('br'),
 645                 ),
 646             )
 647         );
 648     }
 649
 650     public function test_tokenizeHTML_emoticonShiftyEyes()
 651     {
 652         $this->assertTokenization(
 653             '<b><<</b>',
 654             array(
 655                 new HTMLPurifier_Token_Start('b'),
 656                 new HTMLPurifier_Token_Text('<'),
 657                 new HTMLPurifier_Token_Text('<'),
 658                 new HTMLPurifier_Token_End('b'),
 659             ),
 660             array(
 661                 'DOMLex' => array(
 662                     new HTMLPurifier_Token_Start('b'),
 663                     new HTMLPurifier_Token_Text('<<'),
 664                     new HTMLPurifier_Token_End('b'),
 665                 ),
 666             )
 667         );
 668     }
 669
 670     public function test_tokenizeHTML_eon1996()
 671     {
 672         $this->assertTokenization(
 673             '< <b>test</b>',
 674             array(
 675                 new HTMLPurifier_Token_Text('<'),
 676                 new HTMLPurifier_Token_Text(' '),
 677                 new HTMLPurifier_Token_Start('b'),
 678                 new HTMLPurifier_Token_Text('test'),
 679                 new HTMLPurifier_Token_End('b'),
 680             ),
 681             array(
 682                 'DOMLex' => array(
 683                     new HTMLPurifier_Token_Text('< '),
 684                     new HTMLPurifier_Token_Start('b'),
 685                     new HTMLPurifier_Token_Text('test'),
 686                     new HTMLPurifier_Token_End('b'),
 687                 ),
 688             )
 689         );
 690     }
 691
 692     public function test_tokenizeHTML_bodyInCDATA()
 693     {
 694         $alt_tokens = array(
 695             new HTMLPurifier_Token_Text('<'),
 696             new HTMLPurifier_Token_Text('body'),
 697             new HTMLPurifier_Token_Text('>'),
 698             new HTMLPurifier_Token_Text('Foo'),
 699             new HTMLPurifier_Token_Text('<'),
 700             new HTMLPurifier_Token_Text('/body'),
 701             new HTMLPurifier_Token_Text('>'),
 702         );
 703         $this->assertTokenization(
 704             '<![CDATA[<body>Foo</body>]]>',
 705             array(
 706                 new HTMLPurifier_Token_Text('<body>Foo</body>'),
 707             ),
 708             array(
 709                 'PH5P' => $alt_tokens,
 710             )
 711         );
 712     }
 713
 714     public function test_tokenizeHTML_()
 715     {
 716         $this->assertTokenization(
 717             '<a><img /></a>',
 718             array(
 719                 new HTMLPurifier_Token_Start('a'),
 720                 new HTMLPurifier_Token_Empty('img'),
 721                 new HTMLPurifier_Token_End('a'),
 722             )
 723         );
 724     }
 725
 726     public function test_tokenizeHTML_ignoreIECondComment()
 727     {
 728         $this->assertTokenization(
 729             '<!--[if IE]>foo<a>bar<!-- baz --><![endif]-->',
 730             array()
 731         );
 732     }
 733
 734     public function test_tokenizeHTML_removeProcessingInstruction()
 735     {
 736         $this->config->set('Core.RemoveProcessingInstructions', true);
 737         $this->assertTokenization(
 738             '<?xml blah blah ?>',
 739             array()
 740         );
 741     }
 742
 743    public function test_tokenizeHTML_removeNewline()
 744    {
 745         $this->config->set('Core.NormalizeNewlines', true);
 746         $this->assertTokenization(
 747             "plain\rtext\r\n",
 748             array(
 749                 new HTMLPurifier_Token_Text("plain\ntext\n")
 750             )
 751         );
 752    }
 753
 754    public function test_tokenizeHTML_noRemoveNewline()
 755    {
 756         $this->config->set('Core.NormalizeNewlines', false);
 757         $this->assertTokenization(
 758             "plain\rtext\r\n",
 759             array(
 760                 new HTMLPurifier_Token_Text("plain\rtext\r\n")
 761             )
 762         );
 763      }
 764
 765     public function test_tokenizeHTML_conditionalCommentUngreedy()
 766     {
 767         $this->assertTokenization(
 768             '<!--[if gte mso 9]>a<![endif]-->b<!--[if gte mso 9]>c<![endif]-->',
 769             array(
 770                 new HTMLPurifier_Token_Text("b")
 771             )
 772         );
 773     }
 774
 775     public function test_tokenizeHTML_imgTag()
 776     {
 777         $start = array(
 778                         new HTMLPurifier_Token_Start('img',
 779                             array(
 780                                 'src' => 'img_11775.jpg',
 781                                 'alt' => '[Img #11775]',
 782                                 'id' => 'EMBEDDED_IMG_11775',
 783                             )
 784                         )
 785                     );
 786         $this->assertTokenization(
 787             '<img src="img_11775.jpg" alt="[Img #11775]" id="EMBEDDED_IMG_11775" >',
 788             array(
 789                 new HTMLPurifier_Token_Empty('img',
 790                     array(
 791                         'src' => 'img_11775.jpg',
 792                         'alt' => '[Img #11775]',
 793                         'id' => 'EMBEDDED_IMG_11775',
 794                     )
 795                 )
 796             ),
 797             array(
 798                 'DirectLex' => $start,
 799                 )
 800         );
 801     }
 802
 803
 804     /*
 805
 806     public function test_tokenizeHTML_()
 807     {
 808         $this->assertTokenization(
 809             ,
 810             array(
 811
 812             )
 813         );
 814     }
 815     */
 816
 817 }
 818
 819 // vim: et sw=4 sts=4