Merge branch 'MDL-64012' of https://github.com/timhunt/moodle
[moodle.git] / lib / tests / text_test.php
blob1c7400fbf9631f96edfca65ed54277f9a312e7f9
1 <?php
2 // This file is part of Moodle - http://moodle.org/
3 //
4 // Moodle is free software: you can redistribute it and/or modify
5 // it under the terms of the GNU General Public License as published by
6 // the Free Software Foundation, either version 3 of the License, or
7 // (at your option) any later version.
8 //
9 // Moodle is distributed in the hope that it will be useful,
10 // but WITHOUT ANY WARRANTY; without even the implied warranty of
11 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 // GNU General Public License for more details.
14 // You should have received a copy of the GNU General Public License
15 // along with Moodle. If not, see <http://www.gnu.org/licenses/>.
17 /**
18 * core_text unit tests.
20 * @package core
21 * @category phpunit
22 * @copyright 2012 Petr Skoda {@link http://skodak.org}
23 * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
26 defined('MOODLE_INTERNAL') || die();
29 /**
30 * Unit tests for our utf-8 aware text processing.
32 * @package core
33 * @category phpunit
34 * @copyright 2010 Petr Skoda (http://skodak.org)
35 * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
37 class core_text_testcase extends advanced_testcase {
39 /**
40 * Tests the static parse charset method.
42 public function test_parse_charset() {
43 $this->assertSame('windows-1250', core_text::parse_charset('Cp1250'));
44 // Does typo3 work? Some encoding moodle does not use.
45 $this->assertSame('windows-1252', core_text::parse_charset('ms-ansi'));
48 /**
49 * Tests the static convert method.
51 public function test_convert() {
52 $utf8 = "Žluťoučký koníček";
53 $iso2 = pack("H*", "ae6c75bb6f75e86bfd206b6f6eede8656b");
54 $win = pack("H*", "8e6c759d6f75e86bfd206b6f6eede8656b");
55 $this->assertSame($iso2, core_text::convert($utf8, 'utf-8', 'iso-8859-2'));
56 $this->assertSame($utf8, core_text::convert($iso2, 'iso-8859-2', 'utf-8'));
57 $this->assertSame($win, core_text::convert($utf8, 'utf-8', 'win-1250'));
58 $this->assertSame($utf8, core_text::convert($win, 'win-1250', 'utf-8'));
59 $this->assertSame($iso2, core_text::convert($win, 'win-1250', 'iso-8859-2'));
60 $this->assertSame($win, core_text::convert($iso2, 'iso-8859-2', 'win-1250'));
61 $this->assertSame($iso2, core_text::convert($iso2, 'iso-8859-2', 'iso-8859-2'));
62 $this->assertSame($win, core_text::convert($win, 'win-1250', 'cp1250'));
63 $this->assertSame($utf8, core_text::convert($utf8, 'utf-8', 'utf-8'));
65 $utf8 = '言語設定';
66 $str = pack("H*", "b8c0b8ecc0dfc4ea"); // EUC-JP
67 $this->assertSame($str, core_text::convert($utf8, 'utf-8', 'EUC-JP'));
68 $this->assertSame($utf8, core_text::convert($str, 'EUC-JP', 'utf-8'));
69 $this->assertSame($utf8, core_text::convert($utf8, 'utf-8', 'utf-8'));
71 $str = pack("H*", "1b24423840386c405f446a1b2842"); // ISO-2022-JP
72 $this->assertSame($str, core_text::convert($utf8, 'utf-8', 'ISO-2022-JP'));
73 $this->assertSame($utf8, core_text::convert($str, 'ISO-2022-JP', 'utf-8'));
74 $this->assertSame($utf8, core_text::convert($utf8, 'utf-8', 'utf-8'));
76 $str = pack("H*", "8cbe8cea90dd92e8"); // SHIFT-JIS
77 $this->assertSame($str, core_text::convert($utf8, 'utf-8', 'SHIFT-JIS'));
78 $this->assertSame($utf8, core_text::convert($str, 'SHIFT-JIS', 'utf-8'));
79 $this->assertSame($utf8, core_text::convert($utf8, 'utf-8', 'utf-8'));
81 $utf8 = '简体中文';
82 $str = pack("H*", "bcf2cce5d6d0cec4"); // GB2312
83 $this->assertSame($str, core_text::convert($utf8, 'utf-8', 'GB2312'));
84 $this->assertSame($utf8, core_text::convert($str, 'GB2312', 'utf-8'));
85 $this->assertSame($utf8, core_text::convert($utf8, 'utf-8', 'utf-8'));
87 $str = pack("H*", "bcf2cce5d6d0cec4"); // GB18030
88 $this->assertSame($str, core_text::convert($utf8, 'utf-8', 'GB18030'));
89 $this->assertSame($utf8, core_text::convert($str, 'GB18030', 'utf-8'));
90 $this->assertSame($utf8, core_text::convert($utf8, 'utf-8', 'utf-8'));
92 $utf8 = "Žluťoučký koníček";
93 $this->assertSame('Zlutoucky konicek', core_text::convert($utf8, 'utf-8', 'ascii'));
94 $this->assertSame($utf8, core_text::convert($utf8.chr(130), 'utf-8', 'utf-8'));
95 $utf8 = "Der eine stößt den Speer zum Mann";
96 $this->assertSame('Der eine stoesst den Speer zum Mann', core_text::convert($utf8, 'utf-8', 'ascii'));
97 $iso1 = core_text::convert($utf8, 'utf-8', 'iso-8859-1');
98 $this->assertSame('Der eine stoesst den Speer zum Mann', core_text::convert($iso1, 'iso-8859-1', 'ascii'));
102 * Tests the static sub string method.
104 public function test_substr() {
105 $str = "Žluťoučký koníček";
106 $this->assertSame($str, core_text::substr($str, 0));
107 $this->assertSame('luťoučký koníček', core_text::substr($str, 1));
108 $this->assertSame('luť', core_text::substr($str, 1, 3));
109 $this->assertSame($str, core_text::substr($str, 0, 100));
110 $this->assertSame('če', core_text::substr($str, -3, 2));
112 $iso2 = pack("H*", "ae6c75bb6f75e86bfd206b6f6eede8656b");
113 $this->assertSame(core_text::convert('luť', 'utf-8', 'iso-8859-2'), core_text::substr($iso2, 1, 3, 'iso-8859-2'));
114 $this->assertSame(core_text::convert($str, 'utf-8', 'iso-8859-2'), core_text::substr($iso2, 0, 100, 'iso-8859-2'));
115 $this->assertSame(core_text::convert('če', 'utf-8', 'iso-8859-2'), core_text::substr($iso2, -3, 2, 'iso-8859-2'));
117 $win = pack("H*", "8e6c759d6f75e86bfd206b6f6eede8656b");
118 $this->assertSame(core_text::convert('luť', 'utf-8', 'cp1250'), core_text::substr($win, 1, 3, 'cp1250'));
119 $this->assertSame(core_text::convert($str, 'utf-8', 'cp1250'), core_text::substr($win, 0, 100, 'cp1250'));
120 $this->assertSame(core_text::convert('če', 'utf-8', 'cp1250'), core_text::substr($win, -3, 2, 'cp1250'));
122 $str = pack("H*", "b8c0b8ecc0dfc4ea"); // EUC-JP
123 $s = pack("H*", "b8ec"); // EUC-JP
124 $this->assertSame($s, core_text::substr($str, 1, 1, 'EUC-JP'));
126 $str = pack("H*", "1b24423840386c405f446a1b2842"); // ISO-2022-JP
127 $s = pack("H*", "1b2442386c1b2842"); // ISO-2022-JP
128 $this->assertSame($s, core_text::substr($str, 1, 1, 'ISO-2022-JP'));
130 $str = pack("H*", "8cbe8cea90dd92e8"); // SHIFT-JIS
131 $s = pack("H*", "8cea"); // SHIFT-JIS
132 $this->assertSame($s, core_text::substr($str, 1, 1, 'SHIFT-JIS'));
134 $str = pack("H*", "bcf2cce5d6d0cec4"); // GB2312
135 $s = pack("H*", "cce5"); // GB2312
136 $this->assertSame($s, core_text::substr($str, 1, 1, 'GB2312'));
138 $str = pack("H*", "bcf2cce5d6d0cec4"); // GB18030
139 $s = pack("H*", "cce5"); // GB18030
140 $this->assertSame($s, core_text::substr($str, 1, 1, 'GB18030'));
144 * Tests the static string length method.
146 public function test_strlen() {
147 $str = "Žluťoučký koníček";
148 $this->assertSame(17, core_text::strlen($str));
150 $iso2 = pack("H*", "ae6c75bb6f75e86bfd206b6f6eede8656b");
151 $this->assertSame(17, core_text::strlen($iso2, 'iso-8859-2'));
153 $win = pack("H*", "8e6c759d6f75e86bfd206b6f6eede8656b");
154 $this->assertSame(17, core_text::strlen($win, 'cp1250'));
156 $str = pack("H*", "b8ec"); // EUC-JP
157 $this->assertSame(1, core_text::strlen($str, 'EUC-JP'));
158 $str = pack("H*", "b8c0b8ecc0dfc4ea"); // EUC-JP
159 $this->assertSame(4, core_text::strlen($str, 'EUC-JP'));
161 $str = pack("H*", "1b2442386c1b2842"); // ISO-2022-JP
162 $this->assertSame(1, core_text::strlen($str, 'ISO-2022-JP'));
163 $str = pack("H*", "1b24423840386c405f446a1b2842"); // ISO-2022-JP
164 $this->assertSame(4, core_text::strlen($str, 'ISO-2022-JP'));
166 $str = pack("H*", "8cea"); // SHIFT-JIS
167 $this->assertSame(1, core_text::strlen($str, 'SHIFT-JIS'));
168 $str = pack("H*", "8cbe8cea90dd92e8"); // SHIFT-JIS
169 $this->assertSame(4, core_text::strlen($str, 'SHIFT-JIS'));
171 $str = pack("H*", "cce5"); // GB2312
172 $this->assertSame(1, core_text::strlen($str, 'GB2312'));
173 $str = pack("H*", "bcf2cce5d6d0cec4"); // GB2312
174 $this->assertSame(4, core_text::strlen($str, 'GB2312'));
176 $str = pack("H*", "cce5"); // GB18030
177 $this->assertSame(1, core_text::strlen($str, 'GB18030'));
178 $str = pack("H*", "bcf2cce5d6d0cec4"); // GB18030
179 $this->assertSame(4, core_text::strlen($str, 'GB18030'));
183 * Test unicode safe string truncation.
185 public function test_str_max_bytes() {
186 // These are all 3 byte characters, so this is a 12-byte string.
187 $str = '言語設定';
189 $this->assertEquals(12, strlen($str));
191 // Step back, shortening the string 1 byte at a time. Should remove in 1 char chunks.
192 $conv = core_text::str_max_bytes($str, 12);
193 $this->assertEquals(12, strlen($conv));
194 $this->assertSame('言語設定', $conv);
195 $conv = core_text::str_max_bytes($str, 11);
196 $this->assertEquals(9, strlen($conv));
197 $this->assertSame('言語設', $conv);
198 $conv = core_text::str_max_bytes($str, 10);
199 $this->assertEquals(9, strlen($conv));
200 $this->assertSame('言語設', $conv);
201 $conv = core_text::str_max_bytes($str, 9);
202 $this->assertEquals(9, strlen($conv));
203 $this->assertSame('言語設', $conv);
204 $conv = core_text::str_max_bytes($str, 8);
205 $this->assertEquals(6, strlen($conv));
206 $this->assertSame('言語', $conv);
208 // Now try a mixed byte string.
209 $str = '言語設a定';
211 $this->assertEquals(13, strlen($str));
213 $conv = core_text::str_max_bytes($str, 11);
214 $this->assertEquals(10, strlen($conv));
215 $this->assertSame('言語設a', $conv);
216 $conv = core_text::str_max_bytes($str, 10);
217 $this->assertEquals(10, strlen($conv));
218 $this->assertSame('言語設a', $conv);
219 $conv = core_text::str_max_bytes($str, 9);
220 $this->assertEquals(9, strlen($conv));
221 $this->assertSame('言語設', $conv);
222 $conv = core_text::str_max_bytes($str, 8);
223 $this->assertEquals(6, strlen($conv));
224 $this->assertSame('言語', $conv);
226 // Test 0 byte case.
227 $conv = core_text::str_max_bytes($str, 0);
228 $this->assertEquals(0, strlen($conv));
229 $this->assertSame('', $conv);
233 * Tests the static strtolower method.
235 public function test_strtolower() {
236 $str = "Žluťoučký koníček";
237 $low = 'žluťoučký koníček';
238 $this->assertSame($low, core_text::strtolower($str));
240 $iso2 = pack("H*", "ae6c75bb6f75e86bfd206b6f6eede8656b");
241 $this->assertSame(core_text::convert($low, 'utf-8', 'iso-8859-2'), core_text::strtolower($iso2, 'iso-8859-2'));
243 $win = pack("H*", "8e6c759d6f75e86bfd206b6f6eede8656b");
244 $this->assertSame(core_text::convert($low, 'utf-8', 'cp1250'), core_text::strtolower($win, 'cp1250'));
246 $str = '言語設定';
247 $this->assertSame($str, core_text::strtolower($str));
249 $str = '简体中文';
250 $this->assertSame($str, core_text::strtolower($str));
252 $str = pack("H*", "1b24423840386c405f446a1b2842"); // ISO-2022-JP
253 $this->assertSame($str, core_text::strtolower($str, 'ISO-2022-JP'));
255 $str = pack("H*", "8cbe8cea90dd92e8"); // SHIFT-JIS
256 $this->assertSame($str, core_text::strtolower($str, 'SHIFT-JIS'));
258 $str = pack("H*", "bcf2cce5d6d0cec4"); // GB2312
259 $this->assertSame($str, core_text::strtolower($str, 'GB2312'));
261 $str = pack("H*", "bcf2cce5d6d0cec4"); // GB18030
262 $this->assertSame($str, core_text::strtolower($str, 'GB18030'));
264 // Typo3 has problems with integers.
265 $str = 1309528800;
266 $this->assertSame((string)$str, core_text::strtolower($str));
270 * Tests the static strtoupper.
272 public function test_strtoupper() {
273 $str = "Žluťoučký koníček";
274 $up = 'ŽLUŤOUČKÝ KONÍČEK';
275 $this->assertSame($up, core_text::strtoupper($str));
277 $iso2 = pack("H*", "ae6c75bb6f75e86bfd206b6f6eede8656b");
278 $this->assertSame(core_text::convert($up, 'utf-8', 'iso-8859-2'), core_text::strtoupper($iso2, 'iso-8859-2'));
280 $win = pack("H*", "8e6c759d6f75e86bfd206b6f6eede8656b");
281 $this->assertSame(core_text::convert($up, 'utf-8', 'cp1250'), core_text::strtoupper($win, 'cp1250'));
283 $str = '言語設定';
284 $this->assertSame($str, core_text::strtoupper($str));
286 $str = '简体中文';
287 $this->assertSame($str, core_text::strtoupper($str));
289 $str = pack("H*", "1b24423840386c405f446a1b2842"); // ISO-2022-JP
290 $this->assertSame($str, core_text::strtoupper($str, 'ISO-2022-JP'));
292 $str = pack("H*", "8cbe8cea90dd92e8"); // SHIFT-JIS
293 $this->assertSame($str, core_text::strtoupper($str, 'SHIFT-JIS'));
295 $str = pack("H*", "bcf2cce5d6d0cec4"); // GB2312
296 $this->assertSame($str, core_text::strtoupper($str, 'GB2312'));
298 $str = pack("H*", "bcf2cce5d6d0cec4"); // GB18030
299 $this->assertSame($str, core_text::strtoupper($str, 'GB18030'));
303 * Test the strrev method.
305 public function test_strrev() {
306 $strings = array(
307 "Žluťoučký koníček" => "kečínok ýkčuoťulŽ",
308 'ŽLUŤOUČKÝ KONÍČEK' => "KEČÍNOK ÝKČUOŤULŽ",
309 '言語設定' => '定設語言',
310 '简体中文' => '文中体简',
311 "Der eine stößt den Speer zum Mann" => "nnaM muz reepS ned tßöts enie reD"
313 foreach ($strings as $before => $after) {
314 // Make sure we can reverse it both ways and that it comes out the same.
315 $this->assertSame($after, core_text::strrev($before));
316 $this->assertSame($before, core_text::strrev($after));
317 // Reverse it twice to be doubly sure.
318 $this->assertSame($after, core_text::strrev(core_text::strrev($after)));
323 * Tests the static strpos method.
325 public function test_strpos() {
326 $str = "Žluťoučký koníček";
327 $this->assertSame(10, core_text::strpos($str, 'koníč'));
331 * Tests the static strrpos.
333 public function test_strrpos() {
334 $str = "Žluťoučký koníček";
335 $this->assertSame(11, core_text::strrpos($str, 'o'));
339 * Tests the static specialtoascii method.
341 public function test_specialtoascii() {
342 $str = "Žluťoučký koníček";
343 $this->assertSame('Zlutoucky konicek', core_text::specialtoascii($str));
347 * Tests the static encode_mimeheader method.
348 * This also tests method moodle_phpmailer::encodeHeader that calls core_text::encode_mimeheader
350 public function test_encode_mimeheader() {
351 global $CFG;
352 require_once($CFG->libdir.'/phpmailer/moodle_phpmailer.php');
353 $mailer = new moodle_phpmailer();
355 // Encode short string with non-latin characters.
356 $str = "Žluťoučký koníček";
357 $encodedstr = '=?utf-8?B?xb1sdcWlb3XEjWvDvSBrb27DrcSNZWs=?=';
358 $this->assertSame($encodedstr, core_text::encode_mimeheader($str));
359 $this->assertSame($encodedstr, $mailer->encodeHeader($str));
360 $this->assertSame('"' . $encodedstr . '"', $mailer->encodeHeader($str, 'phrase'));
362 // Encode short string without non-latin characters. Make sure the quotes are escaped in quoted email headers.
363 $latinstr = 'text"with quotes';
364 $this->assertSame($latinstr, core_text::encode_mimeheader($latinstr));
365 $this->assertSame($latinstr, $mailer->encodeHeader($latinstr));
366 $this->assertSame('"text\\"with quotes"', $mailer->encodeHeader($latinstr, 'phrase'));
368 // Encode long string without non-latin characters.
369 $longlatinstr = 'This is a very long text that still should not be split into several lines in the email headers because '.
370 'it does not have any non-latin characters. The "quotes" and \\backslashes should be escaped only if it\'s a part of email address';
371 $this->assertSame($longlatinstr, core_text::encode_mimeheader($longlatinstr));
372 $this->assertSame($longlatinstr, $mailer->encodeHeader($longlatinstr));
373 $longlatinstrwithslash = preg_replace(['/\\\\/', "/\"/"], ['\\\\\\', '\\"'], $longlatinstr);
374 $this->assertSame('"' . $longlatinstrwithslash . '"', $mailer->encodeHeader($longlatinstr, 'phrase'));
376 // Encode long string with non-latin characters.
377 $longstr = "Неопознанная ошибка в файле C:\\tmp\\: \"Не пользуйтесь виндоуз\"";
378 $encodedlongstr = "=?utf-8?B?0J3QtdC+0L/QvtC30L3QsNC90L3QsNGPINC+0YjQuNCx0LrQsCDQsiDRhNCw?=
379 =?utf-8?B?0LnQu9C1IEM6XHRtcFw6ICLQndC1INC/0L7Qu9GM0LfRg9C50YLQtdGB?=
380 =?utf-8?B?0Ywg0LLQuNC90LTQvtGD0Lci?=";
381 $this->assertSame($encodedlongstr, $mailer->encodeHeader($longstr));
382 $this->assertSame('"' . $encodedlongstr . '"', $mailer->encodeHeader($longstr, 'phrase'));
386 * Tests the static entities_to_utf8 method.
388 public function test_entities_to_utf8() {
389 $str = "&#x17d;lu&#x165;ou&#x10d;k&#xfd; kon&iacute;&#269;ek&copy;&quot;&amp;&lt;&gt;&sect;&laquo;";
390 $this->assertSame("Žluťoučký koníček©\"&<>§«", core_text::entities_to_utf8($str));
394 * Tests the static utf8_to_entities method.
396 public function test_utf8_to_entities() {
397 $str = "&#x17d;luťoučký kon&iacute;ček&copy;&quot;&amp;&lt;&gt;&sect;&laquo;";
398 $this->assertSame("&#x17d;lu&#x165;ou&#x10d;k&#xfd; kon&iacute;&#x10d;ek&copy;&quot;&amp;&lt;&gt;&sect;&laquo;", core_text::utf8_to_entities($str));
399 $this->assertSame("&#381;lu&#357;ou&#269;k&#253; kon&iacute;&#269;ek&copy;&quot;&amp;&lt;&gt;&sect;&laquo;", core_text::utf8_to_entities($str, true));
401 $str = "&#381;luťoučký kon&iacute;ček&copy;&quot;&amp;&lt;&gt;&sect;&laquo;";
402 $this->assertSame("&#x17d;lu&#x165;ou&#x10d;k&#xfd; kon&#xed;&#x10d;ek&#xa9;\"&<>&#xa7;&#xab;", core_text::utf8_to_entities($str, false, true));
403 $this->assertSame("&#381;lu&#357;ou&#269;k&#253; kon&#237;&#269;ek&#169;\"&<>&#167;&#171;", core_text::utf8_to_entities($str, true, true));
407 * Tests the static trim_utf8_bom method.
409 public function test_trim_utf8_bom() {
410 $bom = "\xef\xbb\xbf";
411 $str = "Žluťoučký koníček";
412 $this->assertSame($str.$bom, core_text::trim_utf8_bom($bom.$str.$bom));
416 * Tests the static remove_unicode_non_characters method.
418 public function test_remove_unicode_non_characters() {
419 // Confirm that texts which don't contain these characters are unchanged.
420 $this->assertSame('Frogs!', core_text::remove_unicode_non_characters('Frogs!'));
422 // Even if they contain some very scary characters.
423 $example = html_entity_decode('A&#xfffd;&#x1d15f;B');
424 $this->assertSame($example, core_text::remove_unicode_non_characters($example));
426 // Non-characters are removed wherever they may be, with other characters left.
427 $example = html_entity_decode('&#xfffe;A&#xffff;B&#x8fffe;C&#xfdd0;D&#xfffd;E&#xfdd5;');
428 $expected = html_entity_decode('ABCD&#xfffd;E');
429 $this->assertSame($expected, core_text::remove_unicode_non_characters($example));
431 // If you only have a non-character, you get empty string.
432 $example = html_entity_decode('&#xfffe;');
433 $this->assertSame('', core_text::remove_unicode_non_characters($example));
437 * Tests the static get_encodings method.
439 public function test_get_encodings() {
440 $encodings = core_text::get_encodings();
441 $this->assertTrue(is_array($encodings));
442 $this->assertTrue(count($encodings) > 1);
443 $this->assertTrue(isset($encodings['UTF-8']));
447 * Tests the static code2utf8 method.
449 public function test_code2utf8() {
450 $this->assertSame('Ž', core_text::code2utf8(381));
454 * Tests the static utf8ord method.
456 public function test_utf8ord() {
457 $this->assertSame(ord(''), core_text::utf8ord(''));
458 $this->assertSame(ord('f'), core_text::utf8ord('f'));
459 $this->assertSame(0x03B1, core_text::utf8ord('α'));
460 $this->assertSame(0x0439, core_text::utf8ord('й'));
461 $this->assertSame(0x2FA1F, core_text::utf8ord('𯨟'));
462 $this->assertSame(381, core_text::utf8ord('Ž'));
466 * Tests the static strtotitle method.
468 public function test_strtotitle() {
469 $str = "žluťoučký koníček";
470 $this->assertSame("Žluťoučký Koníček", core_text::strtotitle($str));
474 * Test strrchr.
476 public function test_strrchr() {
477 $str = "Žluťoučký koníček";
478 $this->assertSame('koníček', core_text::strrchr($str, 'koní'));
479 $this->assertSame('Žluťoučký ', core_text::strrchr($str, 'koní', true));
480 $this->assertFalse(core_text::strrchr($str, 'A'));
481 $this->assertFalse(core_text::strrchr($str, 'ç', true));