tests/HTMLPurifier/LexerTest.php

   1 <?php
   2
   3 require_once 'HTMLPurifier/Lexer/DirectLex.php';
   4
   5 class HTMLPurifier_LexerTest extends UnitTestCase
   6 {
   7
   8     var $Lexer;
   9     var $DirectLex, $PEARSax3, $DOMLex;
  10     var $_entity_lookup;
  11     var $_has_pear = false;
  12     var $_has_dom  = false;
  13
  14     function setUp() {
  15         $this->Lexer     = new HTMLPurifier_Lexer();
  16
  17         $this->DirectLex = new HTMLPurifier_Lexer_DirectLex();
  18
  19         if ( $GLOBALS['HTMLPurifierTest']['PEAR'] ) {
  20             $this->_has_pear = true;
  21             require_once 'HTMLPurifier/Lexer/PEARSax3.php';
  22             $this->PEARSax3  = new HTMLPurifier_Lexer_PEARSax3();
  23         }
  24
  25         $this->_has_dom = version_compare(PHP_VERSION, '5', '>=');
  26         if ($this->_has_dom) {
  27             require_once 'HTMLPurifier/Lexer/DOMLex.php';
  28             $this->DOMLex    = new HTMLPurifier_Lexer_DOMLex();
  29         }
  30
  31         $this->_entity_lookup = HTMLPurifier_EntityLookup::instance();
  32
  33     }
  34
  35     function assertCleanUTF8($string, $expect = null) {
  36         if ($expect === null) $expect = $string;
  37         $this->assertIdentical($this->Lexer->cleanUTF8($string), $expect);
  38     }
  39
  40     function test_cleanUTF8() {
  41         $this->assertCleanUTF8('Normal string.');
  42         $this->assertCleanUTF8("Test\tAllowed\nControl\rCharacters");
  43         $this->assertCleanUTF8("null byte: \0", 'null byte: ');
  44         $this->assertCleanUTF8("\1\2\3\4\5\6\7", '');
  45         $this->assertCleanUTF8("\x7F", ''); // one byte invalid SGML char
  46         $this->assertCleanUTF8("\xC2\x80", ''); // two byte invalid SGML
  47         $this->assertCleanUTF8("\xF3\xBF\xBF\xBF"); // valid four byte
  48         $this->assertCleanUTF8("\xDF\xFF", ''); // malformed UTF8
  49     }
  50
  51     function test_substituteNonSpecialEntities() {
  52         $char_theta = $this->_entity_lookup->table['theta'];
  53         $this->assertIdentical($char_theta,
  54             $this->Lexer->substituteNonSpecialEntities('&theta;') );
  55         $this->assertIdentical('"',
  56             $this->Lexer->substituteNonSpecialEntities('"') );
  57
  58         // numeric tests, adapted from Feyd
  59         $args = array();
  60         $args[] = array(1114112,false     );
  61         $args[] = array(1114111,'F48FBFBF'); // 0x0010FFFF
  62         $args[] = array(1048576,'F4808080'); // 0x00100000
  63         $args[] = array(1048575,'F3BFBFBF'); // 0x000FFFFF
  64         $args[] = array(262144, 'F1808080'); // 0x00040000
  65         $args[] = array(262143, 'F0BFBFBF'); // 0x0003FFFF
  66         $args[] = array(65536,  'F0908080'); // 0x00010000
  67         $args[] = array(65535,  'EFBFBF'  ); // 0x0000FFFF
  68         $args[] = array(57344,  'EE8080'  ); // 0x0000E000
  69         $args[] = array(57343,  false     ); // 0x0000DFFF  these are ill-formed
  70         $args[] = array(56040,  false     ); // 0x0000DAE8  these are ill-formed
  71         $args[] = array(55296,  false     ); // 0x0000D800  these are ill-formed
  72         $args[] = array(55295,  'ED9FBF'  ); // 0x0000D7FF
  73         $args[] = array(53248,  'ED8080'  ); // 0x0000D000
  74         $args[] = array(53247,  'ECBFBF'  ); // 0x0000CFFF
  75         $args[] = array(4096,   'E18080'  ); // 0x00001000
  76         $args[] = array(4095,   'E0BFBF'  ); // 0x00000FFF
  77         $args[] = array(2048,   'E0A080'  ); // 0x00000800
  78         $args[] = array(2047,   'DFBF'    ); // 0x000007FF
  79         $args[] = array(128,    'C280'    ); // 0x00000080  invalid SGML char
  80         $args[] = array(127,    '7F'      ); // 0x0000007F  invalid SGML char
  81         $args[] = array(0,      '00'      ); // 0x00000000  invalid SGML char
  82
  83         $args[] = array(20108,  'E4BA8C'  ); // 0x00004E8C
  84         $args[] = array(77,     '4D'      ); // 0x0000004D
  85         $args[] = array(66306,  'F0908C82'); // 0x00010302
  86         $args[] = array(1072,   'D0B0'    ); // 0x00000430
  87
  88         foreach ($args as $arg) {
  89             $string = '&#' . $arg[0] . ';' . // decimal
  90                       '&#x' . dechex($arg[0]) . ';'; // hex
  91             $expect = '';
  92             if ($arg[1] !== false) {
  93                 $chars = str_split($arg[1], 2);
  94                 foreach ($chars as $char) {
  95                     $expect .= chr(hexdec($char));
  96                 }
  97                 $expect .= $expect; // double it
  98             }
  99             $this->assertIdentical(
 100                 $this->Lexer->substituteNonSpecialEntities($string),
 101                 $expect,
 102                 $arg[0] . ': %s'
 103             );
 104         }
 105
 106     }
 107
 108     function assertExtractBody($text, $extract = true) {
 109         $result = $this->Lexer->extractBody($text);
 110         if ($extract === true) $extract = $text;
 111         $this->assertIdentical($extract, $result);
 112     }
 113
 114     function test_extractBody() {
 115         $this->assertExtractBody('<b>Bold</b>');
 116         $this->assertExtractBody('<html><body><b>Bold</b></body></html>', '<b>Bold</b>');
 117         $this->assertExtractBody('<HTML><BODY><B>Bold</B></BODY></HTML>', '<B>Bold</B>');
 118         $this->assertExtractBody(
 119 '<?xml version="1.0"?>
 120 <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
 121     "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
 122 <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
 123    <head>
 124       <title>xyz</title>
 125    </head>
 126    <body>
 127       <form method="post" action="whatever1">
 128          <div>
 129             <input type="text" name="username" />
 130             <input type="text" name="password" />
 131             <input type="submit" />
 132          </div>
 133       </form>
 134    </body>
 135 </html>',
 136     '
 137       <form method="post" action="whatever1">
 138          <div>
 139             <input type="text" name="username" />
 140             <input type="text" name="password" />
 141             <input type="submit" />
 142          </div>
 143       </form>
 144    ');
 145         $this->assertExtractBody('<html><body bgcolor="#F00"><b>Bold</b></body></html>', '<b>Bold</b>');
 146         $this->assertExtractBody('<body>asdf'); // not closed, don't accept
 147
 148     }
 149
 150     function test_tokenizeHTML() {
 151
 152         $input = array();
 153         $expect = array();
 154         $sax_expect = array();
 155         $config = array();
 156
 157         $input[0] = '';
 158         $expect[0] = array();
 159
 160         $input[1] = 'This is regular text.';
 161         $expect[1] = array(
 162             new HTMLPurifier_Token_Text('This is regular text.')
 163             );
 164
 165         $input[2] = 'This is <b>bold</b> text';
 166         $expect[2] = array(
 167             new HTMLPurifier_Token_Text('This is ')
 168            ,new HTMLPurifier_Token_Start('b', array())
 169            ,new HTMLPurifier_Token_Text('bold')
 170            ,new HTMLPurifier_Token_End('b')
 171            ,new HTMLPurifier_Token_Text(' text')
 172             );
 173
 174         $input[3] = '<DIV>Totally rad dude. <b>asdf</b></div>';
 175         $expect[3] = array(
 176             new HTMLPurifier_Token_Start('DIV', array())
 177            ,new HTMLPurifier_Token_Text('Totally rad dude. ')
 178            ,new HTMLPurifier_Token_Start('b', array())
 179            ,new HTMLPurifier_Token_Text('asdf')
 180            ,new HTMLPurifier_Token_End('b')
 181            ,new HTMLPurifier_Token_End('div')
 182             );
 183
 184         // [XML-INVALID]
 185         $input[4] = '<asdf></asdf><d></d><poOloka><poolasdf><ds></asdf></ASDF>';
 186         $expect[4] = array(
 187             new HTMLPurifier_Token_Start('asdf')
 188            ,new HTMLPurifier_Token_End('asdf')
 189            ,new HTMLPurifier_Token_Start('d')
 190            ,new HTMLPurifier_Token_End('d')
 191            ,new HTMLPurifier_Token_Start('poOloka')
 192            ,new HTMLPurifier_Token_Start('poolasdf')
 193            ,new HTMLPurifier_Token_Start('ds')
 194            ,new HTMLPurifier_Token_End('asdf')
 195            ,new HTMLPurifier_Token_End('ASDF')
 196             );
 197         // DOM is different because it condenses empty tags into REAL empty ones
 198         // as well as makes it well-formed
 199         $dom_expect[4] = array(
 200             new HTMLPurifier_Token_Empty('asdf')
 201            ,new HTMLPurifier_Token_Empty('d')
 202            ,new HTMLPurifier_Token_Start('pooloka')
 203            ,new HTMLPurifier_Token_Start('poolasdf')
 204            ,new HTMLPurifier_Token_Empty('ds')
 205            ,new HTMLPurifier_Token_End('poolasdf')
 206            ,new HTMLPurifier_Token_End('pooloka')
 207             );
 208
 209         $input[5] = '<a'."\t".'href="foobar.php"'."\n".'title="foo!">Link to <b id="asdf">foobar</b></a>';
 210         $expect[5] = array(
 211             new HTMLPurifier_Token_Start('a',array('href'=>'foobar.php','title'=>'foo!'))
 212            ,new HTMLPurifier_Token_Text('Link to ')
 213            ,new HTMLPurifier_Token_Start('b',array('id'=>'asdf'))
 214            ,new HTMLPurifier_Token_Text('foobar')
 215            ,new HTMLPurifier_Token_End('b')
 216            ,new HTMLPurifier_Token_End('a')
 217             );
 218
 219         $input[6] = '<br />';
 220         $expect[6] = array(
 221             new HTMLPurifier_Token_Empty('br')
 222             );
 223
 224         // [SGML-INVALID] [RECOVERABLE]
 225         $input[7] = '<!-- Comment --> <!-- not so well formed --->';
 226         $expect[7] = array(
 227             new HTMLPurifier_Token_Comment(' Comment ')
 228            ,new HTMLPurifier_Token_Text(' ')
 229            ,new HTMLPurifier_Token_Comment(' not so well formed -')
 230             );
 231         $sax_expect[7] = false; // we need to figure out proper comment output
 232
 233         // [SGML-INVALID]
 234         $input[8] = '<a href=""';
 235         $expect[8] = array(
 236             new HTMLPurifier_Token_Text('<a href=""')
 237             );
 238         // SAX parses it into a tag
 239         $sax_expect[8] = array(
 240             new HTMLPurifier_Token_Start('a', array('href'=>''))
 241             );
 242         // DOM parses it into an empty tag
 243         $dom_expect[8] = array(
 244             new HTMLPurifier_Token_Empty('a', array('href'=>''))
 245             );
 246
 247         $input[9] = '&lt;b&gt;';
 248         $expect[9] = array(
 249             new HTMLPurifier_Token_Text('<b>')
 250             );
 251         $sax_expect[9] = array(
 252             new HTMLPurifier_Token_Text('<')
 253            ,new HTMLPurifier_Token_Text('b')
 254            ,new HTMLPurifier_Token_Text('>')
 255             );
 256         // note that SAX can clump text nodes together. We won't be
 257         // too picky though
 258
 259         // [SGML-INVALID]
 260         $input[10] = '<a "=>';
 261         // We barf on this, aim for no attributes
 262         $expect[10] = array(
 263             new HTMLPurifier_Token_Start('a', array('"' => ''))
 264             );
 265         // DOM correctly has no attributes, but also closes the tag
 266         $dom_expect[10] = array(
 267             new HTMLPurifier_Token_Empty('a')
 268             );
 269         // SAX barfs on this
 270         $sax_expect[10] = array(
 271             new HTMLPurifier_Token_Start('a', array('"' => ''))
 272             );
 273
 274         // [INVALID] [RECOVERABLE]
 275         $input[11] = '"';
 276         $expect[11] = array( new HTMLPurifier_Token_Text('"') );
 277
 278         // compare with this valid one:
 279         $input[12] = '&quot;';
 280         $expect[12] = array( new HTMLPurifier_Token_Text('"') );
 281         $sax_expect[12] = false; // choked!
 282
 283         // CDATA sections!
 284         $input[13] = '<![CDATA[You <b>can&#39;t</b> get me!]]>';
 285         $expect[13] = array( new HTMLPurifier_Token_Text(
 286             'You <b>can&#39;t</b> get me!' // raw
 287             ) );
 288         $sax_expect[13] = array( // SAX has a seperate call for each entity
 289                 new HTMLPurifier_Token_Text('You '),
 290                 new HTMLPurifier_Token_Text('<'),
 291                 new HTMLPurifier_Token_Text('b'),
 292                 new HTMLPurifier_Token_Text('>'),
 293                 new HTMLPurifier_Token_Text('can'),
 294                 new HTMLPurifier_Token_Text('&'),
 295                 new HTMLPurifier_Token_Text('#39;t'),
 296                 new HTMLPurifier_Token_Text('<'),
 297                 new HTMLPurifier_Token_Text('/b'),
 298                 new HTMLPurifier_Token_Text('>'),
 299                 new HTMLPurifier_Token_Text(' get me!')
 300             );
 301
 302         $char_theta = $this->_entity_lookup->table['theta'];
 303         $char_rarr  = $this->_entity_lookup->table['rarr'];
 304
 305         // test entity replacement
 306         $input[14] = '&theta;';
 307         $expect[14] = array( new HTMLPurifier_Token_Text($char_theta) );
 308
 309         // test that entities aren't replaced in CDATA sections
 310         $input[15] = '&theta; <![CDATA[&rarr;]]>';
 311         $expect[15] = array( new HTMLPurifier_Token_Text($char_theta . ' &rarr;') );
 312         $sax_expect[15] = array(
 313                 new HTMLPurifier_Token_Text($char_theta . ' '),
 314                 new HTMLPurifier_Token_Text('&'),
 315                 new HTMLPurifier_Token_Text('rarr;')
 316             );
 317
 318         // test entity resolution in attributes
 319         $input[16] = '<a href="index.php?title=foo&amp;id=bar">Link</a>';
 320         $expect[16] = array(
 321                 new HTMLPurifier_Token_Start('a',array('href' => 'index.php?title=foo&id=bar'))
 322                ,new HTMLPurifier_Token_Text('Link')
 323                ,new HTMLPurifier_Token_End('a')
 324             );
 325         $sax_expect[16] = false; // PEARSax doesn't support it!
 326
 327         // test that UTF-8 is preserved
 328         $char_hearts = $this->_entity_lookup->table['hearts'];
 329         $input[17] = $char_hearts;
 330         $expect[17] = array( new HTMLPurifier_Token_Text($char_hearts) );
 331
 332         $default_config = HTMLPurifier_Config::createDefault();
 333         foreach($input as $i => $discard) {
 334             if (!isset($config[$i])) $config[$i] = $default_config;
 335
 336             $result = $this->DirectLex->tokenizeHTML($input[$i], $config[$i]);
 337             $this->assertEqual($expect[$i], $result, 'DirectLexTest '.$i.': %s');
 338             paintIf($result, $expect[$i] != $result);
 339
 340             if ($this->_has_pear) {
 341                 // assert unless I say otherwise
 342                 $sax_result = $this->PEARSax3->tokenizeHTML($input[$i], $config[$i]);
 343                 if (!isset($sax_expect[$i])) {
 344                     // by default, assert with normal result
 345                     $this->assertEqual($expect[$i], $sax_result, 'PEARSax3Test '.$i.': %s');
 346                     paintIf($sax_result, $expect[$i] != $sax_result);
 347                 } elseif ($sax_expect[$i] === false) {
 348                     // assertions were turned off, optionally dump
 349                     // paintIf($sax_expect, $i == NUMBER);
 350                 } else {
 351                     // match with a custom SAX result array
 352                     $this->assertEqual($sax_expect[$i], $sax_result, 'PEARSax3Test (custom) '.$i.': %s');
 353                     paintIf($sax_result, $sax_expect[$i] != $sax_result);
 354                 }
 355             }
 356
 357             if ($this->_has_dom) {
 358                 $dom_result = $this->DOMLex->tokenizeHTML($input[$i], $config[$i]);
 359                 // same structure as SAX
 360                 if (!isset($dom_expect[$i])) {
 361                     $this->assertEqual($expect[$i], $dom_result, 'DOMLexTest '.$i.': %s');
 362                     paintIf($dom_result, $expect[$i] != $dom_result);
 363                 } elseif ($dom_expect[$i] === false) {
 364                     // paintIf($dom_result, $i == NUMBER);
 365                 } else {
 366                     $this->assertEqual($dom_expect[$i], $dom_result, 'DOMLexTest (custom) '.$i.': %s');
 367                     paintIf($dom_result, $dom_expect[$i] != $dom_result);
 368                 }
 369             }
 370
 371         }
 372
 373     }
 374
 375 }
 376
 377 ?>