Fix iconv truncation bug.
[htmlpurifier.git] / tests / HTMLPurifier / EncoderTest.php
blob944ace9b93a03be4320c1294404d4836fb7a702b
1 <?php
3 class HTMLPurifier_EncoderTest extends HTMLPurifier_Harness
6 protected $_entity_lookup;
8 function setUp() {
9 $this->_entity_lookup = HTMLPurifier_EntityLookup::instance();
10 parent::setUp();
13 function assertCleanUTF8($string, $expect = null) {
14 if ($expect === null) $expect = $string;
15 $this->assertIdentical(HTMLPurifier_Encoder::cleanUTF8($string), $expect, 'iconv: %s');
16 $this->assertIdentical(HTMLPurifier_Encoder::cleanUTF8($string, true), $expect, 'PHP: %s');
19 function test_cleanUTF8() {
20 $this->assertCleanUTF8('Normal string.');
21 $this->assertCleanUTF8("Test\tAllowed\nControl\rCharacters");
22 $this->assertCleanUTF8("null byte: \0", 'null byte: ');
23 $this->assertCleanUTF8("\1\2\3\4\5\6\7", '');
24 $this->assertCleanUTF8("\x7F", ''); // one byte invalid SGML char
25 $this->assertCleanUTF8("\xC2\x80", ''); // two byte invalid SGML
26 $this->assertCleanUTF8("\xF3\xBF\xBF\xBF"); // valid four byte
27 $this->assertCleanUTF8("\xDF\xFF", ''); // malformed UTF8
28 // invalid codepoints
29 $this->assertCleanUTF8("\xED\xB0\x80", '');
32 function test_convertToUTF8_noConvert() {
33 // UTF-8 means that we don't touch it
34 $this->assertIdentical(
35 HTMLPurifier_Encoder::convertToUTF8("\xF6", $this->config, $this->context),
36 "\xF6", // this is invalid
37 'Expected identical [Binary: F6]'
41 function test_convertToUTF8_spuriousEncoding() {
42 $this->config->set('Core.Encoding', 'utf99');
43 $this->expectError('Invalid encoding utf99');
44 $this->assertIdentical(
45 HTMLPurifier_Encoder::convertToUTF8("\xF6", $this->config, $this->context),
50 function test_convertToUTF8_iso8859_1() {
51 $this->config->set('Core.Encoding', 'ISO-8859-1');
52 $this->assertIdentical(
53 HTMLPurifier_Encoder::convertToUTF8("\xF6", $this->config, $this->context),
54 "\xC3\xB6"
58 function test_convertToUTF8_withoutIconv() {
59 $this->config->set('Core.Encoding', 'ISO-8859-1');
60 $this->config->set('Test.ForceNoIconv', true);
61 $this->assertIdentical(
62 HTMLPurifier_Encoder::convertToUTF8("\xF6", $this->config, $this->context),
63 "\xC3\xB6"
68 function getZhongWen() {
69 return "\xE4\xB8\xAD\xE6\x96\x87 (Chinese)";
72 function test_convertFromUTF8_utf8() {
73 // UTF-8 means that we don't touch it
74 $this->assertIdentical(
75 HTMLPurifier_Encoder::convertFromUTF8("\xC3\xB6", $this->config, $this->context),
76 "\xC3\xB6"
80 function test_convertFromUTF8_iso8859_1() {
81 $this->config->set('Core.Encoding', 'ISO-8859-1');
82 $this->assertIdentical(
83 HTMLPurifier_Encoder::convertFromUTF8("\xC3\xB6", $this->config, $this->context),
84 "\xF6",
85 'Expected identical [Binary: F6]'
89 function test_convertFromUTF8_iconvNoChars() {
90 if (!function_exists('iconv')) return;
91 $this->config->set('Core.Encoding', 'ISO-8859-1');
92 $this->assertIdentical(
93 HTMLPurifier_Encoder::convertFromUTF8($this->getZhongWen(), $this->config, $this->context),
94 " (Chinese)"
98 function test_convertFromUTF8_phpNormal() {
99 // Plain PHP implementation has slightly different behavior
100 $this->config->set('Core.Encoding', 'ISO-8859-1');
101 $this->config->set('Test.ForceNoIconv', true);
102 $this->assertIdentical(
103 HTMLPurifier_Encoder::convertFromUTF8("\xC3\xB6", $this->config, $this->context),
104 "\xF6",
105 'Expected identical [Binary: F6]'
109 function test_convertFromUTF8_phpNoChars() {
110 $this->config->set('Core.Encoding', 'ISO-8859-1');
111 $this->config->set('Test.ForceNoIconv', true);
112 $this->assertIdentical(
113 HTMLPurifier_Encoder::convertFromUTF8($this->getZhongWen(), $this->config, $this->context),
114 "?? (Chinese)"
118 function test_convertFromUTF8_withProtection() {
119 // Preserve the characters!
120 $this->config->set('Core.Encoding', 'ISO-8859-1');
121 $this->config->set('Core.EscapeNonASCIICharacters', true);
122 $this->assertIdentical(
123 HTMLPurifier_Encoder::convertFromUTF8($this->getZhongWen(), $this->config, $this->context),
124 "&#20013;&#25991; (Chinese)"
129 function test_convertToASCIIDumbLossless() {
131 // Uppercase thorn letter
132 $this->assertIdentical(
133 HTMLPurifier_Encoder::convertToASCIIDumbLossless("\xC3\x9Eorn"),
134 "&#222;orn"
137 $this->assertIdentical(
138 HTMLPurifier_Encoder::convertToASCIIDumbLossless("an"),
139 "an"
142 // test up to four bytes
143 $this->assertIdentical(
144 HTMLPurifier_Encoder::convertToASCIIDumbLossless("\xF3\xA0\x80\xA0"),
145 "&#917536;"
150 function assertASCIISupportCheck($enc, $ret) {
151 $test = HTMLPurifier_Encoder::testEncodingSupportsASCII($enc, true);
152 if ($test === false) return;
153 $this->assertIdentical(
154 HTMLPurifier_Encoder::testEncodingSupportsASCII($enc),
155 $ret
157 $this->assertIdentical(
158 HTMLPurifier_Encoder::testEncodingSupportsASCII($enc, true),
159 $ret
163 function test_testEncodingSupportsASCII() {
164 $this->assertASCIISupportCheck('Shift_JIS', array("\xC2\xA5" => '\\', "\xE2\x80\xBE" => '~'));
165 $this->assertASCIISupportCheck('JOHAB', array("\xE2\x82\xA9" => '\\'));
166 $this->assertASCIISupportCheck('ISO-8859-1', array());
167 $this->assertASCIISupportCheck('dontexist', array()); // canary
170 function testShiftJIS() {
171 if (!function_exists('iconv')) return;
172 $this->config->set('Core.Encoding', 'Shift_JIS');
173 // This actually looks like a Yen, but we're going to treat it differently
174 $this->assertIdentical(
175 HTMLPurifier_Encoder::convertFromUTF8('\\~', $this->config, $this->context),
176 '\\~'
178 $this->assertIdentical(
179 HTMLPurifier_Encoder::convertToUTF8('\\~', $this->config, $this->context),
180 '\\~'
184 function testIconvTruncateBug() {
185 if (!function_exists('iconv')) return;
186 if (HTMLPurifier_Encoder::testIconvTruncateBug() !== HTMLPurifier_Encoder::ICONV_TRUNCATES) return;
187 $this->config->set('Core.Encoding', 'ISO-8859-1');
188 $this->assertIdentical(
189 HTMLPurifier_Encoder::convertFromUTF8("\xE4\xB8\xAD" . str_repeat('a', 10000), $this->config, $this->context),
190 str_repeat('a', 10000)
194 function testIconvChunking() {
195 if (!function_exists('iconv')) return;
196 if (HTMLPurifier_Encoder::testIconvTruncateBug() !== HTMLPurifier_Encoder::ICONV_TRUNCATES) return;
197 $this->assertIdentical(HTMLPurifier_Encoder::iconv('utf-8', 'iso-8859-1//IGNORE', "a\xF3\xA0\x80\xA0b", 4), 'ab');
198 $this->assertIdentical(HTMLPurifier_Encoder::iconv('utf-8', 'iso-8859-1//IGNORE', "aa\xE4\xB8\xADb", 4), 'aab');
199 $this->assertIdentical(HTMLPurifier_Encoder::iconv('utf-8', 'iso-8859-1//IGNORE', "aaa\xCE\xB1b", 4), 'aaab');
200 $this->assertIdentical(HTMLPurifier_Encoder::iconv('utf-8', 'iso-8859-1//IGNORE', "aaaa\xF3\xA0\x80\xA0b", 4), 'aaaab');
201 $this->assertIdentical(HTMLPurifier_Encoder::iconv('utf-8', 'iso-8859-1//IGNORE', "aaaa\xE4\xB8\xADb", 4), 'aaaab');
202 $this->assertIdentical(HTMLPurifier_Encoder::iconv('utf-8', 'iso-8859-1//IGNORE', "aaaa\xCE\xB1b", 4), 'aaaab');
207 // vim: et sw=4 sts=4