tests/HTMLPurifier/LexerTest.php

   1 <?php
   2
   3 require_once 'HTMLPurifier/Lexer/DirectLex.php';
   4
   5 class HTMLPurifier_LexerTest extends HTMLPurifier_Harness
   6 {
   7
   8     var $Lexer;
   9     var $DirectLex, $PEARSax3, $DOMLex;
  10     var $_entity_lookup;
  11     var $_has_pear = false;
  12     var $_has_dom  = false;
  13
  14     function setUp() {
  15         $this->Lexer     = new HTMLPurifier_Lexer();
  16
  17         $this->DirectLex = new HTMLPurifier_Lexer_DirectLex();
  18
  19         // E_STRICT = 2048, int used for PHP4 compat
  20         if ( $GLOBALS['HTMLPurifierTest']['PEAR'] &&
  21              ((error_reporting() & 2048) != 2048)
  22         ) {
  23             $this->_has_pear = true;
  24             require_once 'HTMLPurifier/Lexer/PEARSax3.php';
  25             $this->PEARSax3  = new HTMLPurifier_Lexer_PEARSax3();
  26         }
  27
  28         $this->_has_dom = version_compare(PHP_VERSION, '5', '>=');
  29         if ($this->_has_dom) {
  30             require_once 'HTMLPurifier/Lexer/DOMLex.php';
  31             $this->DOMLex    = new HTMLPurifier_Lexer_DOMLex();
  32         }
  33
  34         $this->_entity_lookup = HTMLPurifier_EntityLookup::instance();
  35
  36     }
  37
  38     function test_create() {
  39         $config = HTMLPurifier_Config::create(array('Core.MaintainLineNumbers' => true));
  40         $lexer = HTMLPurifier_Lexer::create($config);
  41         $this->assertIsA($lexer, 'HTMLPurifier_Lexer_DirectLex');
  42     }
  43
  44     function assertExtractBody($text, $extract = true) {
  45         $result = $this->Lexer->extractBody($text);
  46         if ($extract === true) $extract = $text;
  47         $this->assertIdentical($extract, $result);
  48     }
  49
  50     function test_parseData() {
  51         $HP =& $this->Lexer;
  52
  53         $this->assertIdentical('asdf', $HP->parseData('asdf'));
  54         $this->assertIdentical('&', $HP->parseData('&amp;'));
  55         $this->assertIdentical('"', $HP->parseData('&quot;'));
  56         $this->assertIdentical("'", $HP->parseData('&#039;'));
  57         $this->assertIdentical("'", $HP->parseData('&#39;'));
  58         $this->assertIdentical('&&&', $HP->parseData('&amp;&amp;&amp;'));
  59         $this->assertIdentical('&&', $HP->parseData('&amp;&')); // [INVALID]
  60         $this->assertIdentical('Procter & Gamble',
  61                 $HP->parseData('Procter & Gamble')); // [INVALID]
  62
  63         // This is not special, thus not converted. Test of fault tolerance,
  64         // realistically speaking, this should never happen
  65         $this->assertIdentical('&#x2D;', $HP->parseData('&#x2D;'));
  66     }
  67
  68
  69     function test_extractBody() {
  70         $this->assertExtractBody('<b>Bold</b>');
  71         $this->assertExtractBody('<html><body><b>Bold</b></body></html>', '<b>Bold</b>');
  72         $this->assertExtractBody('<HTML><BODY><B>Bold</B></BODY></HTML>', '<B>Bold</B>');
  73         $this->assertExtractBody(
  74 '<?xml version="1.0"
  75 <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
  76     "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
  77 <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
  78    <head>
  79       <title>xyz</title>
  80    </head>
  81    <body>
  82       <form method="post" action="whatever1">
  83          <div>
  84             <input type="text" name="username" />
  85             <input type="text" name="password" />
  86             <input type="submit" />
  87          </div>
  88       </form>
  89    </body>
  90 </html>',
  91     '
  92       <form method="post" action="whatever1">
  93          <div>
  94             <input type="text" name="username" />
  95             <input type="text" name="password" />
  96             <input type="submit" />
  97          </div>
  98       </form>
  99    ');
 100         $this->assertExtractBody('<html><body bgcolor="#F00"><b>Bold</b></body></html>', '<b>Bold</b>');
 101         $this->assertExtractBody('<body>asdf'); // not closed, don't accept
 102
 103     }
 104
 105     function test_tokenizeHTML() {
 106
 107         $input = array();
 108         $expect = array();
 109         $sax_expect = array();
 110         $config = array();
 111
 112         $input[0] = '';
 113         $expect[0] = array();
 114
 115         $input[1] = 'This is regular text.';
 116         $expect[1] = array(
 117             new HTMLPurifier_Token_Text('This is regular text.')
 118             );
 119
 120         $input[2] = 'This is <b>bold</b> text';
 121         $expect[2] = array(
 122             new HTMLPurifier_Token_Text('This is ')
 123            ,new HTMLPurifier_Token_Start('b', array())
 124            ,new HTMLPurifier_Token_Text('bold')
 125            ,new HTMLPurifier_Token_End('b')
 126            ,new HTMLPurifier_Token_Text(' text')
 127             );
 128
 129         $input[3] = '<DIV>Totally rad dude. <b>asdf</b></div>';
 130         $expect[3] = array(
 131             new HTMLPurifier_Token_Start('DIV', array())
 132            ,new HTMLPurifier_Token_Text('Totally rad dude. ')
 133            ,new HTMLPurifier_Token_Start('b', array())
 134            ,new HTMLPurifier_Token_Text('asdf')
 135            ,new HTMLPurifier_Token_End('b')
 136            ,new HTMLPurifier_Token_End('div')
 137             );
 138
 139         // [XML-INVALID]
 140         $input[4] = '<asdf></asdf><d></d><poOloka><poolasdf><ds></asdf></ASDF>';
 141         $expect[4] = array(
 142             new HTMLPurifier_Token_Start('asdf')
 143            ,new HTMLPurifier_Token_End('asdf')
 144            ,new HTMLPurifier_Token_Start('d')
 145            ,new HTMLPurifier_Token_End('d')
 146            ,new HTMLPurifier_Token_Start('poOloka')
 147            ,new HTMLPurifier_Token_Start('poolasdf')
 148            ,new HTMLPurifier_Token_Start('ds')
 149            ,new HTMLPurifier_Token_End('asdf')
 150            ,new HTMLPurifier_Token_End('ASDF')
 151             );
 152         // DOM is different because it condenses empty tags into REAL empty ones
 153         // as well as makes it well-formed
 154         $dom_expect[4] = array(
 155             new HTMLPurifier_Token_Empty('asdf')
 156            ,new HTMLPurifier_Token_Empty('d')
 157            ,new HTMLPurifier_Token_Start('pooloka')
 158            ,new HTMLPurifier_Token_Start('poolasdf')
 159            ,new HTMLPurifier_Token_Empty('ds')
 160            ,new HTMLPurifier_Token_End('poolasdf')
 161            ,new HTMLPurifier_Token_End('pooloka')
 162             );
 163
 164         $input[5] = '<a'."\t".'href="foobar.php"'."\n".'title="foo!">Link to <b id="asdf">foobar</b></a>';
 165         $expect[5] = array(
 166             new HTMLPurifier_Token_Start('a',array('href'=>'foobar.php','title'=>'foo!'))
 167            ,new HTMLPurifier_Token_Text('Link to ')
 168            ,new HTMLPurifier_Token_Start('b',array('id'=>'asdf'))
 169            ,new HTMLPurifier_Token_Text('foobar')
 170            ,new HTMLPurifier_Token_End('b')
 171            ,new HTMLPurifier_Token_End('a')
 172             );
 173
 174         $input[6] = '<br />';
 175         $expect[6] = array(
 176             new HTMLPurifier_Token_Empty('br')
 177             );
 178
 179         // [SGML-INVALID] [RECOVERABLE]
 180         $input[7] = '<!-- Comment --> <!-- not so well formed --->';
 181         $expect[7] = array(
 182             new HTMLPurifier_Token_Comment(' Comment ')
 183            ,new HTMLPurifier_Token_Text(' ')
 184            ,new HTMLPurifier_Token_Comment(' not so well formed -')
 185             );
 186         $sax_expect[7] = false; // we need to figure out proper comment output
 187
 188         // [SGML-INVALID]
 189         $input[8] = '<a href=""';
 190         $expect[8] = array(
 191             new HTMLPurifier_Token_Text('<a href=""')
 192             );
 193         // SAX parses it into a tag
 194         $sax_expect[8] = array(
 195             new HTMLPurifier_Token_Start('a', array('href'=>''))
 196             );
 197         // DOM parses it into an empty tag
 198         $dom_expect[8] = array(
 199             new HTMLPurifier_Token_Empty('a', array('href'=>''))
 200             );
 201
 202         $input[9] = '&lt;b&gt;';
 203         $expect[9] = array(
 204             new HTMLPurifier_Token_Text('<b>')
 205             );
 206         $sax_expect[9] = array(
 207             new HTMLPurifier_Token_Text('<')
 208            ,new HTMLPurifier_Token_Text('b')
 209            ,new HTMLPurifier_Token_Text('>')
 210             );
 211         // note that SAX can clump text nodes together. We won't be
 212         // too picky though
 213
 214         // [SGML-INVALID]
 215         $input[10] = '<a "=>';
 216         // We barf on this, aim for no attributes
 217         $expect[10] = array(
 218             new HTMLPurifier_Token_Start('a', array('"' => ''))
 219             );
 220         // DOM correctly has no attributes, but also closes the tag
 221         $dom_expect[10] = array(
 222             new HTMLPurifier_Token_Empty('a')
 223             );
 224         // SAX barfs on this
 225         $sax_expect[10] = array(
 226             new HTMLPurifier_Token_Start('a', array('"' => ''))
 227             );
 228
 229         // [INVALID] [RECOVERABLE]
 230         $input[11] = '"';
 231         $expect[11] = array( new HTMLPurifier_Token_Text('"') );
 232
 233         // compare with this valid one:
 234         $input[12] = '&quot;';
 235         $expect[12] = array( new HTMLPurifier_Token_Text('"') );
 236         $sax_expect[12] = false; // choked!
 237
 238         // CDATA sections!
 239         $input[13] = '<![CDATA[You <b>can&#39;t</b> get me!]]>';
 240         $expect[13] = array( new HTMLPurifier_Token_Text(
 241             'You <b>can&#39;t</b> get me!' // raw
 242             ) );
 243         $sax_expect[13] = array( // SAX has a seperate call for each entity
 244                 new HTMLPurifier_Token_Text('You '),
 245                 new HTMLPurifier_Token_Text('<'),
 246                 new HTMLPurifier_Token_Text('b'),
 247                 new HTMLPurifier_Token_Text('>'),
 248                 new HTMLPurifier_Token_Text('can'),
 249                 new HTMLPurifier_Token_Text('&'),
 250                 new HTMLPurifier_Token_Text('#39;t'),
 251                 new HTMLPurifier_Token_Text('<'),
 252                 new HTMLPurifier_Token_Text('/b'),
 253                 new HTMLPurifier_Token_Text('>'),
 254                 new HTMLPurifier_Token_Text(' get me!')
 255             );
 256
 257         $char_theta = $this->_entity_lookup->table['theta'];
 258         $char_rarr  = $this->_entity_lookup->table['rarr'];
 259
 260         // test entity replacement
 261         $input[14] = '&theta;';
 262         $expect[14] = array( new HTMLPurifier_Token_Text($char_theta) );
 263
 264         // test that entities aren't replaced in CDATA sections
 265         $input[15] = '&theta; <![CDATA[&rarr;]]>';
 266         $expect[15] = array( new HTMLPurifier_Token_Text($char_theta . ' &rarr;') );
 267         $sax_expect[15] = array(
 268                 new HTMLPurifier_Token_Text($char_theta . ' '),
 269                 new HTMLPurifier_Token_Text('&'),
 270                 new HTMLPurifier_Token_Text('rarr;')
 271             );
 272
 273         // test entity resolution in attributes
 274         $input[16] = '<a href="index.php?title=foo&amp;id=bar">Link</a>';
 275         $expect[16] = array(
 276                 new HTMLPurifier_Token_Start('a',array('href' => 'index.php?title=foo&id=bar'))
 277                ,new HTMLPurifier_Token_Text('Link')
 278                ,new HTMLPurifier_Token_End('a')
 279             );
 280
 281         // test that UTF-8 is preserved
 282         $char_hearts = $this->_entity_lookup->table['hearts'];
 283         $input[17] = $char_hearts;
 284         $expect[17] = array( new HTMLPurifier_Token_Text($char_hearts) );
 285
 286         // test weird characters in attributes
 287         $input[18] = '<br test="x &lt; 6" />';
 288         $expect[18] = array( new HTMLPurifier_Token_Empty('br', array('test' => 'x < 6')) );
 289
 290         // test emoticon protection
 291         $input[19] = '<b>Whoa! <3 That\'s not good >.></b>';
 292         $expect[19] = array(
 293             new HTMLPurifier_Token_Start('b'),
 294             new HTMLPurifier_Token_Text('Whoa! '),
 295             new HTMLPurifier_Token_Text('<3 That\'s not good >'),
 296             new HTMLPurifier_Token_Text('.>'),
 297             new HTMLPurifier_Token_End('b'),
 298         );
 299         $dom_expect[19] = array(
 300             new HTMLPurifier_Token_Start('b'),
 301             new HTMLPurifier_Token_Text('Whoa! <3 That\'s not good >.>'),
 302             new HTMLPurifier_Token_End('b'),
 303         );
 304         $sax_expect[19] = false; // SAX drops the < character
 305         $config[19] = HTMLPurifier_Config::create(array('Core.AggressivelyFixLt' => true));
 306
 307         // test comment parsing with funky characters inside
 308         $input[20] = '<!-- This >< comment --><br />';
 309         $expect[20] = array(
 310             new HTMLPurifier_Token_Comment(' This >< comment '),
 311             new HTMLPurifier_Token_Empty('br')
 312         );
 313         $sax_expect[20] = false;
 314         $config[20] = HTMLPurifier_Config::create(array('Core.AggressivelyFixLt' => true));
 315
 316         // test comment parsing of missing end
 317         $input[21] = '<!-- This >< comment';
 318         $expect[21] = array(
 319             new HTMLPurifier_Token_Comment(' This >< comment')
 320         );
 321         $sax_expect[21] = false;
 322         $dom_expect[21] = false;
 323         $config[21] = HTMLPurifier_Config::create(array('Core.AggressivelyFixLt' => true));
 324
 325         // test CDATA tags
 326         $input[22] = '<script>alert("<foo>");</script>';
 327         $expect[22] = array(
 328             new HTMLPurifier_Token_Start('script')
 329            ,new HTMLPurifier_Token_Text('alert("<foo>");')
 330            ,new HTMLPurifier_Token_End('script')
 331         );
 332         $config[22] = HTMLPurifier_Config::create(array('HTML.Trusted' => true));
 333         $sax_expect[22] = false;
 334
 335         // test escaping
 336         $input[23] = '<!-- This comment < &lt; & -->';
 337         $expect[23] = array(
 338             new HTMLPurifier_Token_Comment(' This comment < &lt; & ') );
 339         $sax_expect[23] = false; $config[23] =
 340         HTMLPurifier_Config::create(array('Core.AggressivelyFixLt' =>
 341         true));
 342
 343         // more DirectLex edge-cases
 344         $input[24] = '<a href="><>">';
 345         $expect[24] = array(
 346             new HTMLPurifier_Token_Start('a', array('href' => '')),
 347             new HTMLPurifier_Token_Text('<">')
 348         );
 349         $sax_expect[24] = false;
 350         $dom_expect[24] = array(
 351             new HTMLPurifier_Token_Empty('a', array('href' => '><>'))
 352         );
 353
 354         // empty tag with attributes
 355         $input[25] = '<param name="src" value="http://example.com/video.wmv" />';
 356         $expect[25] = array(
 357             new HTMLPurifier_Token_Empty('param', array('name' => 'src', 'value' => 'http://example.com/video.wmv'))
 358         );
 359
 360         $default_config = HTMLPurifier_Config::createDefault();
 361         $default_context = new HTMLPurifier_Context();
 362         foreach($input as $i => $discard) {
 363             if (!isset($config[$i])) $config[$i] = $default_config;
 364
 365             $result = $this->DirectLex->tokenizeHTML($input[$i], $config[$i], $default_context);
 366             $this->assertIdentical($expect[$i], $result, 'DirectLexTest '.$i.': %s');
 367             paintIf($result, $expect[$i] != $result);
 368
 369             if ($this->_has_pear) {
 370                 // assert unless I say otherwise
 371                 $sax_result = $this->PEARSax3->tokenizeHTML($input[$i], $config[$i], $default_context);
 372                 if (!isset($sax_expect[$i])) {
 373                     // by default, assert with normal result
 374                     $this->assertIdentical($expect[$i], $sax_result, 'PEARSax3Test '.$i.': %s');
 375                     paintIf($sax_result, $expect[$i] != $sax_result);
 376                 } elseif ($sax_expect[$i] === false) {
 377                     // assertions were turned off, optionally dump
 378                     // paintIf($sax_expect, $i == NUMBER);
 379                 } else {
 380                     // match with a custom SAX result array
 381                     $this->assertIdentical($sax_expect[$i], $sax_result, 'PEARSax3Test (custom) '.$i.': %s');
 382                     paintIf($sax_result, $sax_expect[$i] != $sax_result);
 383                 }
 384             }
 385
 386             if ($this->_has_dom) {
 387                 $dom_result = $this->DOMLex->tokenizeHTML($input[$i], $config[$i], $default_context);
 388                 // same structure as SAX
 389                 if (!isset($dom_expect[$i])) {
 390                     $this->assertIdentical($expect[$i], $dom_result, 'DOMLexTest '.$i.': %s');
 391                     paintIf($dom_result, $expect[$i] != $dom_result);
 392                 } elseif ($dom_expect[$i] === false) {
 393                     // paintIf($dom_result, $i == NUMBER);
 394                 } else {
 395                     $this->assertIdentical($dom_expect[$i], $dom_result, 'DOMLexTest (custom) '.$i.': %s');
 396                     paintIf($dom_result, $dom_expect[$i] != $dom_result);
 397                 }
 398             }
 399
 400         }
 401
 402     }
 403
 404 }
 405