5 // Alexander Köplinger (alexander.koeplinger@xamarin.com)
7 // Copyright (C) 2017 Xamarin, Inc.
9 // Permission is hereby granted, free of charge, to any person obtaining
10 // a copy of this software and associated documentation files (the
11 // "Software"), to deal in the Software without restriction, including
12 // without limitation the rights to use, copy, modify, merge, publish,
13 // distribute, sublicense, and/or sell copies of the Software, and to
14 // permit persons to whom the Software is furnished to do so, subject to
15 // the following conditions:
17 // The above copyright notice and this permission notice shall be
18 // included in all copies or substantial portions of the Software.
20 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
21 // EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
22 // MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
23 // NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
24 // LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
25 // OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
26 // WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
30 using System
.Collections
.Generic
;
33 using NUnit
.Framework
;
34 using NUnit
.Framework
.Constraints
;
36 namespace MonoTests
.I18N
38 public class CodePageTestInfo
41 public bool IsBrowserDisplay
;
42 public bool IsBrowserSave
;
43 public bool IsMailNewsDisplay
;
44 public bool IsMailNewsSave
;
45 public byte SuperscriptFiveReplacementChar
;
46 public byte InfinityReplacementChar
;
47 public byte FFReplacementChar
;
51 public int? CharsWritten
;
52 public byte? SupplementChar
;
53 public bool SkipGetBytes7Test
;
54 public bool SkipEncoderFallbackTest
;
55 public bool SkipEncoderFallback2Test
;
56 public override string ToString () => "Codepage " + Convert
.ToString (CodePage
);
60 // NOTE: when adding/updating tests here consider updating
61 // the following files as well since they have similar tests:
63 // - mcs/class/corlib/Test/System.Text/ASCIIEncodingTest.cs
64 // - mcs/class/corlib/Test/System.Text/Latin1EncodingTest.cs
66 public abstract class EncodingTestBase
68 private char[] testchars
;
69 private byte[] testbytes
;
74 testchars
= new char[4];
79 testbytes
= new byte[4];
80 testbytes
[0] = (byte) 'T';
81 testbytes
[1] = (byte) 'e';
82 testbytes
[2] = (byte) 's';
83 testbytes
[3] = (byte) 't';
87 [TestCaseSource ("codepageTestInfos")]
88 public void IsBrowserDisplay (CodePageTestInfo cpInfo
)
90 Assert
.AreEqual (cpInfo
.IsBrowserDisplay
, Encoding
.GetEncoding (cpInfo
.CodePage
).IsBrowserDisplay
);
94 [TestCaseSource ("codepageTestInfos")]
95 public void IsBrowserSave (CodePageTestInfo cpInfo
)
97 Assert
.AreEqual (cpInfo
.IsBrowserSave
, Encoding
.GetEncoding (cpInfo
.CodePage
).IsBrowserSave
);
101 [TestCaseSource ("codepageTestInfos")]
102 public void IsMailNewsDisplay (CodePageTestInfo cpInfo
)
104 Assert
.AreEqual (cpInfo
.IsMailNewsDisplay
, Encoding
.GetEncoding (cpInfo
.CodePage
).IsMailNewsDisplay
);
108 [TestCaseSource ("codepageTestInfos")]
109 public void IsMailNewsSave (CodePageTestInfo cpInfo
)
111 Assert
.AreEqual (cpInfo
.IsMailNewsSave
, Encoding
.GetEncoding (cpInfo
.CodePage
).IsMailNewsSave
);
114 [Test
] // Test GetBytes(char[])
115 [TestCaseSource ("codepageTestInfos")]
116 public void TestGetBytes1 (CodePageTestInfo cpInfo
)
118 Encoding test_encoding
= Encoding
.GetEncoding (cpInfo
.CodePage
);
119 byte[] bytes
= test_encoding
.GetBytes(testchars
);
120 for (int i
= 0; i
< testchars
.Length
; i
++)
121 Assert
.AreEqual (testchars
[i
], (char) bytes
[i
]);
124 [Test
] // Test GetBytes(char[], int, int)
125 [TestCaseSource ("codepageTestInfos")]
126 public void TestGetBytes2 (CodePageTestInfo cpInfo
)
128 Encoding test_encoding
= Encoding
.GetEncoding (cpInfo
.CodePage
);
129 byte[] bytes
= test_encoding
.GetBytes(testchars
, 1, 1);
130 Assert
.AreEqual (1, bytes
.Length
, "#1");
131 Assert
.AreEqual (testchars
[1], (char) bytes
[0], "#2");
134 [Test
] // Test non-Latin1 char in char[]
135 [TestCaseSource ("codepageTestInfos")]
136 public void TestGetBytes3 (CodePageTestInfo cpInfo
)
138 Encoding test_encoding
= Encoding
.GetEncoding (cpInfo
.CodePage
);
139 testchars
[2] = (char) 0x100;
140 byte[] bytes
= test_encoding
.GetBytes(testchars
);
141 Assert
.AreEqual ('T', (char) bytes
[0], "#1");
142 Assert
.AreEqual ('e', (char) bytes
[1], "#2");
143 Assert
.AreEqual (cpInfo
.FFReplacementChar
, (char) bytes
[2], "#3");
144 Assert
.AreEqual ('t', (char) bytes
[3], "#4");
147 [Test
] // Test GetBytes(char[], int, int, byte[], int)
148 [TestCaseSource ("codepageTestInfos")]
149 public void TestGetBytes4 (CodePageTestInfo cpInfo
)
151 Encoding test_encoding
= Encoding
.GetEncoding (cpInfo
.CodePage
);
152 byte[] bytes
= new Byte
[1];
153 int cnt
= test_encoding
.GetBytes(testchars
, 1, 1, bytes
, 0);
154 Assert
.AreEqual (1, cnt
, "#1");
155 Assert
.AreEqual (testchars
[1], (char) bytes
[0], "#2");
158 [Test
] // Test GetBytes(string, int, int, byte[], int)
159 [TestCaseSource ("codepageTestInfos")]
160 public void TestGetBytes5 (CodePageTestInfo cpInfo
)
162 Encoding test_encoding
= Encoding
.GetEncoding (cpInfo
.CodePage
);
163 byte[] bytes
= new Byte
[1];
164 int cnt
= test_encoding
.GetBytes("Test", 1, 1, bytes
, 0);
165 Assert
.AreEqual ('e', (char) bytes
[0], "#1");
168 [Test
] // Test GetBytes(string)
169 [TestCaseSource ("codepageTestInfos")]
170 public void TestGetBytes6 (CodePageTestInfo cpInfo
)
172 Encoding test_encoding
= Encoding
.GetEncoding (cpInfo
.CodePage
);
173 byte[] bytes
= test_encoding
.GetBytes("Test");
174 for (int i
= 0; i
< testchars
.Length
; i
++)
175 Assert
.AreEqual (testchars
[i
], (char) bytes
[i
]);
178 [Test
] // Test GetBytes(string)
179 [TestCaseSource ("codepageTestInfos")]
180 public void TestGetBytes7 (CodePageTestInfo cpInfo
)
182 if (cpInfo
.SkipGetBytes7Test
)
183 Assert
.Ignore ("Codepage indicates this test should be skipped.");
185 var test_encoding
= Encoding
.GetEncoding (cpInfo
.CodePage
);
187 var expected
= new byte [] { 0x3F, 0x20, cpInfo.SuperscriptFiveReplacementChar, 0x20, cpInfo.InfinityReplacementChar }
;
188 if (cpInfo
.SupplementChar
.HasValue
) {
189 var expectedNew
= new byte [expected
.Length
+ 1];
190 expected
.CopyTo (expectedNew
, 0);
191 expectedNew
[expected
.Length
] = cpInfo
.SupplementChar
.Value
;
192 expected
= expectedNew
;
194 var actual
= test_encoding
.GetBytes("\u24c8 \u2075 \u221e"); // normal replacement
195 Assert
.AreEqual (expected
, actual
, "#1");
197 expected
= new byte [] { 0x3F, 0x3F }
;
198 actual
= test_encoding
.GetBytes("\ud83d\ude0a"); // surrogate pair replacement
199 Assert
.AreEqual (expected
, actual
, "#2");
201 expected
= new byte [] { 0x3F, 0x3F, 0x20 }
;
202 actual
= test_encoding
.GetBytes("\ud83d\ude0a "); // surrogate pair replacement
203 Assert
.AreEqual (expected
, actual
, "#3");
205 expected
= new byte [] { 0x20, 0x20, 0x3F, 0x3F, 0x20, 0x20 }
;
206 actual
= test_encoding
.GetBytes(" \ud83d\ude0a "); // surrogate pair replacement
207 Assert
.AreEqual (expected
, actual
, "#4");
209 expected
= new byte [] { 0x20, 0x20, 0x3F, 0x3F, 0x20, 0x20 }
;
210 actual
= test_encoding
.GetBytes(" \ud834\udd1e "); // surrogate pair replacement
211 Assert
.AreEqual (expected
, actual
, "#5");
213 expected
= new byte [] { 0x41, 0x42, 0x43, 0x00, 0x41, 0x42, 0x43 }
;
214 actual
= test_encoding
.GetBytes("ABC\0ABC"); // embedded zero byte not replaced
215 Assert
.AreEqual (expected
, actual
, "#6");
217 expected
= new byte [] { 0x20, 0x20, 0x3F, 0x20, 0x20 }
;
218 actual
= test_encoding
.GetBytes(" \ud834 "); // invalid surrogate pair replacement
219 Assert
.AreEqual (expected
, actual
, "#7");
222 [Test
] // Test GetChars(byte[])
223 [TestCaseSource ("codepageTestInfos")]
224 public void TestGetChars1 (CodePageTestInfo cpInfo
)
226 Encoding test_encoding
= Encoding
.GetEncoding (cpInfo
.CodePage
);
227 char[] chars
= test_encoding
.GetChars(testbytes
);
228 for (int i
= 0; i
< testbytes
.Length
; i
++)
229 Assert
.AreEqual (testbytes
[i
], (byte) chars
[i
]);
232 [Test
] // Test GetChars(byte[], int, int)
233 [TestCaseSource ("codepageTestInfos")]
234 public void TestGetChars2 (CodePageTestInfo cpInfo
)
236 Encoding test_encoding
= Encoding
.GetEncoding (cpInfo
.CodePage
);
237 char[] chars
= test_encoding
.GetChars(testbytes
, 1, 1);
238 Assert
.AreEqual (1, chars
.Length
, "#1");
239 Assert
.AreEqual (testbytes
[1], (byte) chars
[0], "#2");
242 [Test
] // Test GetChars(byte[], int, int, char[], int)
243 [TestCaseSource ("codepageTestInfos")]
244 public void TestGetChars4 (CodePageTestInfo cpInfo
)
246 Encoding test_encoding
= Encoding
.GetEncoding (cpInfo
.CodePage
);
247 char[] chars
= new char[1];
248 int cnt
= test_encoding
.GetChars(testbytes
, 1, 1, chars
, 0);
249 Assert
.AreEqual (1, cnt
, "#1");
250 Assert
.AreEqual (testbytes
[1], (byte) chars
[0], "#2");
253 [Test
] // Test GetString(char[])
254 [TestCaseSource ("codepageTestInfos")]
255 public void TestGetString1 (CodePageTestInfo cpInfo
)
257 Encoding test_encoding
= Encoding
.GetEncoding (cpInfo
.CodePage
);
258 string str
= test_encoding
.GetString(testbytes
);
259 Assert
.AreEqual ("Test", str
);
262 [Test
] // Test GetString(char[], int, int)
263 [TestCaseSource ("codepageTestInfos")]
264 public void TestGetString2 (CodePageTestInfo cpInfo
)
266 Encoding test_encoding
= Encoding
.GetEncoding (cpInfo
.CodePage
);
267 string str
= test_encoding
.GetString(testbytes
, 1, 2);
268 Assert
.AreEqual ("es", str
);
271 [Test
] // Test Decoder
272 [TestCaseSource ("codepageTestInfos")]
273 public void TestDecoder (CodePageTestInfo cpInfo
)
275 Encoding test_encoding
= Encoding
.GetEncoding (cpInfo
.CodePage
);
276 char[] chars
= new char[1];
277 int cnt
= test_encoding
.GetDecoder().GetChars(testbytes
, 1, 1, chars
, 0);
278 Assert
.AreEqual (1, cnt
, "#1");
279 Assert
.AreEqual (testbytes
[1], (byte) chars
[0], "#2");
282 [Test
] // Test Decoder
283 [TestCaseSource ("codepageTestInfos")]
284 public void TestEncoder (CodePageTestInfo cpInfo
)
286 Encoding test_encoding
= Encoding
.GetEncoding (cpInfo
.CodePage
);
287 byte[] bytes
= new Byte
[1];
288 int cnt
= test_encoding
.GetEncoder().GetBytes(testchars
, 1, 1, bytes
, 0, false);
289 Assert
.AreEqual (1, cnt
, "#1");
290 Assert
.AreEqual (testchars
[1], (char) bytes
[0], "#2");
294 [TestCaseSource ("codepageTestInfos")]
295 public void TestZero (CodePageTestInfo cpInfo
)
297 Encoding encoding
= Encoding
.GetEncoding (cpInfo
.CodePage
);
298 Assert
.AreEqual (string.Empty
, encoding
.GetString (new byte [0]), "#1");
299 Assert
.AreEqual (string.Empty
, encoding
.GetString (new byte [0], 0, 0), "#2");
303 [ExpectedException (typeof (EncoderFallbackException
))]
304 [TestCaseSource ("codepageTestInfos")]
305 public void EncoderFallback (CodePageTestInfo cpInfo
)
307 if (cpInfo
.SkipEncoderFallbackTest
)
308 Assert
.Ignore ("Codepage indicates this test should be skipped.");
310 Encoding e
= Encoding
.GetEncoding (cpInfo
.CodePage
).Clone () as Encoding
;
311 e
.EncoderFallback
= new EncoderExceptionFallback ();
312 e
.GetBytes ("\u24c8");
316 [TestCaseSource ("codepageTestInfos")]
317 public void EncoderFallback2 (CodePageTestInfo cpInfo
)
319 if (cpInfo
.SkipEncoderFallback2Test
)
320 Assert
.Ignore ("Codepage indicates this test should be skipped.");
322 Encoding e
= Encoding
.GetEncoding (cpInfo
.CodePage
).Clone () as Encoding
;
323 e
.EncoderFallback
= new BackslashEncoderReplaceFallback ();
325 byte[] bytes
= e
.GetBytes ("a\xac\u1234\u20ac\u8000");
326 var expected
= new byte[] { 0x61, 0xAC, 0x5C, 0x75, 0x31, 0x32, 0x33, 0x34, 0x5C, 0x75, 0x32, 0x30, 0x61, 0x63, 0x5C, 0x75, 0x38, 0x30, 0x30, 0x30 }
;
327 // FIXME: some codepages do have U+00AC (logical not sign) or U+20AC (euro sign), we need to adapt the tests
328 // Assert.AreEqual (expected, bytes);
330 bytes
= e
.GetBytes ("1\u04d92");
331 expected
= new byte[] { 0x31, 0x5C, 0x75, 0x30, 0x34, 0x64, 0x39, 0x32 }
;
332 Assert
.AreEqual (expected
, bytes
);
334 e
.EncoderFallback
= new EncoderExceptionOnWrongIndexFallback ('\u04d9', 1);
335 bytes
= e
.GetBytes ("1\u04d92");
336 expected
= new byte[] { 0x31, 0x21, 0x32 }
;
337 Assert
.AreEqual (expected
, bytes
);
339 e
.EncoderFallback
= new EncoderExceptionOnWrongIndexFallback ('\u04d9', 0);
340 bytes
= e
.GetBytes ("\u04d921");
341 expected
= new byte[] { 0x21, 0x32, 0x31 }
;
342 Assert
.AreEqual (expected
, bytes
);
346 // [ExpectedException (typeof (ArgumentException))]
347 [TestCaseSource ("codepageTestInfos")]
348 public void DecoderFallback2 (CodePageTestInfo cpInfo
)
350 var bytes
= new byte[] {
351 0x30, 0xa0, 0x31, 0xa8
353 var enc
= (Encoding
)Encoding
.GetEncoding (cpInfo
.CodePage
).Clone ();
354 enc
.DecoderFallback
= new TestFallbackDecoder ();
356 var chars
= new char [7];
357 var ret
= enc
.GetChars (bytes
, 0, bytes
.Length
, chars
, 0);
361 [TestCaseSource ("codepageTestInfos")]
362 public void DecoderFallback3 (CodePageTestInfo cpInfo
)
364 var bytes
= new byte[] {
365 0x30, 0xa0, 0x31, 0xa8
367 var enc
= (Encoding
)Encoding
.GetEncoding (cpInfo
.CodePage
).Clone ();
368 enc
.DecoderFallback
= new TestFallbackDecoder ();
370 var chars
= new char[] { '9', '8', '7', '6', '5' }
;
371 var ret
= enc
.GetChars (bytes
, 0, bytes
.Length
, chars
, 0);
373 Assert
.That (ret
, Is
.EqualTo (cpInfo
.CharsWritten
?? 4), "ret");
374 Assert
.That (chars
[0], Is
.EqualTo ('0'), "chars[0]");
375 Assert
.That (chars
[1], Is
.EqualTo (cpInfo
.A0Char
), "chars[1]");
376 Assert
.That (chars
[2], Is
.EqualTo ((cpInfo
).OneChar
?? '1'), "chars[2]");
377 Assert
.That (chars
[3], Is
.EqualTo ((char)cpInfo
.A8Char
), "chars[3]");
378 Assert
.That (chars
[4], Is
.EqualTo ('5'), "chars[4]");
381 class TestFallbackDecoder
: DecoderFallback
{
384 public override int MaxCharCount
{
385 get { return count; }
388 public override DecoderFallbackBuffer
CreateFallbackBuffer ()
390 return new Buffer ();
393 class Buffer
: DecoderFallbackBuffer
{
397 public override int Remaining
{
399 return queue
.Length
- index
;
403 public override char GetNextChar ()
405 return index
< queue
.Length
? queue
[index
++] : '\0';
408 public override bool Fallback (byte[] bytes
, int unused
)
410 queue
= new char[bytes
.Length
* count
];
412 for (int i
= 0; i
< bytes
.Length
; i
++) {
413 for (int j
= 0; j
< count
; j
++)
414 queue
[index
++] = (char)(bytes
[i
]+j
);
419 public override bool MovePrevious ()
421 throw new NotImplementedException ();
424 public override void Reset ()
431 class BackslashEncoderReplaceFallback
: EncoderFallback
433 class BackslashReplaceFallbackBuffer
: EncoderFallbackBuffer
435 List
<char> _buffer
= new List
<char> ();
438 public override bool Fallback (char charUnknownHigh
, char charUnknownLow
, int index
)
440 throw new NotImplementedException ();
444 public override bool Fallback (char charUnknown
, int index
)
447 int val
= (int)charUnknown
;
450 AddCharacter (val
>> 8);
451 AddCharacter (val
& 0xFF);
454 AddCharacter (charUnknown
);
459 private void AddCharacter (int val
)
461 AddOneDigit (((val
) & 0xF0) >> 4);
462 AddOneDigit (val
& 0x0F);
465 private void AddOneDigit (int val
)
468 _buffer
.Add ((char)('a' + val
- 0x0A));
470 _buffer
.Add ((char)('0' + val
));
474 public override char GetNextChar ()
476 if (_index
== _buffer
.Count
)
477 return Char
.MinValue
;
479 return _buffer
[_index
++];
482 public override bool MovePrevious ()
491 public override int Remaining
493 get { return _buffer.Count - _index; }
497 public override EncoderFallbackBuffer
CreateFallbackBuffer ()
499 return new BackslashReplaceFallbackBuffer ();
502 public override int MaxCharCount
504 get { throw new NotImplementedException (); }
508 class EncoderExceptionOnWrongIndexFallback
: EncoderFallback
510 char _expectedCharUnknown
;
513 public EncoderExceptionOnWrongIndexFallback (char expectedCharUnknown
, int expectedIndex
)
515 _expectedCharUnknown
= expectedCharUnknown
;
516 _expectedIndex
= expectedIndex
;
519 public override EncoderFallbackBuffer
CreateFallbackBuffer ()
521 return new EncoderExceptionOnWrongIndexFallbackBuffer (_expectedCharUnknown
, _expectedIndex
);
524 public override int MaxCharCount
=> 1;
526 class EncoderExceptionOnWrongIndexFallbackBuffer
: EncoderFallbackBuffer
528 char _expectedCharUnknown
;
532 public EncoderExceptionOnWrongIndexFallbackBuffer (char expectedCharUnknown
, int expectedIndex
)
534 _expectedCharUnknown
= expectedCharUnknown
;
535 _expectedIndex
= expectedIndex
;
538 public override int Remaining
=> read
? 0 : 1;
540 public override bool Fallback (char charUnknown
, int index
)
542 Assert
.AreEqual (_expectedCharUnknown
, charUnknown
);
543 Assert
.AreEqual (_expectedIndex
, index
);
547 public override bool Fallback (char charUnknownHigh
, char charUnknownLow
, int index
)
549 throw new NotImplementedException ();
553 public override char GetNextChar ()
562 public override bool MovePrevious ()