tests/test_encodings.c

   1 /*
   2  *      Copyright 2023 The Geany contributors
   3  *
   4  *      This program is free software; you can redistribute it and/or modify
   5  *      it under the terms of the GNU General Public License as published by
   6  *      the Free Software Foundation; either version 2 of the License, or
   7  *      (at your option) any later version.
   8  *
   9  *      This program is distributed in the hope that it will be useful,
  10  *      but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  *      MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12  *      GNU General Public License for more details.
  13  *
  14  *      You should have received a copy of the GNU General Public License along
  15  *      with this program; if not, write to the Free Software Foundation, Inc.,
  16  *      51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  17  */
  18
  19 #include "encodingsprivate.h"
  20 #include "main.h"
  21
  22
  23 /* Asserts 2 bytes buffers are identical, trying to provide a somewhat useful
  24  * error if not. */
  25 static void assert_cmpmem_eq_impl(const char *p1, const char *p2, gsize len,
  26                 const char *domain, const char *file, int line, const char *func,
  27                 const char *expr)
  28 {
  29         gchar *msg;
  30         gsize i;
  31
  32         for (i = 0; i < len && p1[i] == p2[i]; i++)
  33                 ;
  34         if (i == len)
  35                 return;
  36
  37         msg = g_strdup_printf("assertion failed (%s): bytes %#x and %#x differ at offset %lu (at \"%s\" and \"%s\")",
  38                         expr, (guint) (guchar) p1[i], (guint) (guchar) p2[i], i, p1 + i, p2 + i);
  39         g_assertion_message(domain, file, line, func, msg);
  40         g_free(msg);
  41 }
  42
  43 #define assert_cmpmem_eq_with_caller(p1, p2, len, domain, file, line, func) \
  44         assert_cmpmem_eq_impl(p1, p2, len, domain, file, line, func, #p1 " == " #p2)
  45
  46 #define assert_cmpmem_eq(p1, p2, len) assert_cmpmem_eq_impl(p1, p2, len, \
  47                 G_LOG_DOMAIN, __FILE__, __LINE__, G_STRFUNC, #p1 " == " #p2)
  48
  49 /*
  50  * @brief More convenient test API for encodings_convert_to_utf8_auto()
  51  * @param input Input buffer, NUL-terminated (well, at least there should be a
  52  *        trailing NUL).
  53  * @param input_size Actual size of @p input buffer, without the trailing NUL
  54  * @param disk_size Size on disk (as reported by e.g stat -- that may be 0 for
  55  *                  virtual files, otherwise should be input_size)
  56  * @param forced_enc Forced encoding, or NULL
  57  * @param expected_output Expected output data
  58  * @param expected_size Expected output size
  59  * @param expected_encoding Expected output encoding
  60  * @param expected_has_bom Whether the input contains a BOM
  61  * @param expected_partial Whether the output is expected to be truncated
  62  * @returns Whether the conversion succeeded and followed the parameters
  63  */
  64 static gboolean assert_convert_to_utf8_auto_impl(
  65                 const char *domain, const char *file, int line, const char *func,
  66                 const gchar *input, gsize input_size,
  67                 const gsize disk_size, const gchar *forced_enc,
  68                 const gchar *expected_output, gsize expected_size, const gchar *expected_encoding,
  69                 gboolean expected_has_bom, gboolean expected_partial)
  70 {
  71         gchar *buf = g_memdup(input, input_size + 1);
  72         gsize size = disk_size;
  73         gchar *used_encoding = NULL;
  74         gboolean has_bom = FALSE;
  75         gboolean partial = FALSE;
  76         gboolean ret;
  77
  78         g_log(domain, G_LOG_LEVEL_INFO, "%s:%d:%s: converting %lu bytes", file, line, func, input_size);
  79         ret = encodings_convert_to_utf8_auto(&buf, &size, forced_enc, &used_encoding, &has_bom, &partial);
  80         fflush(stdout);
  81         if (ret)
  82         {
  83                 assert_cmpmem_eq_with_caller(buf, expected_output, MIN(size, expected_size),
  84                                 domain, file, line, func);
  85                 g_assert_cmpuint(size, ==, expected_size);
  86                 if (expected_encoding)
  87                         g_assert_cmpstr(expected_encoding, ==, used_encoding);
  88                 g_assert_cmpint(has_bom, ==, expected_has_bom);
  89                 g_assert_cmpint(partial, ==, expected_partial);
  90
  91                 g_free(used_encoding);
  92         }
  93
  94         g_free(buf);
  95
  96         return ret;
  97 }
  98
  99
 100 #define assert_convert_to_utf8_auto(input, input_size, disk_size, forced_enc, \
 101                 expected_output, expected_size, expected_encoding, expected_has_bom, expected_partial) \
 102         assert_convert_to_utf8_auto_impl(G_LOG_DOMAIN, __FILE__, __LINE__, G_STRFUNC, \
 103                         input, input_size, disk_size, forced_enc, \
 104                         expected_output, expected_size, expected_encoding, expected_has_bom, expected_partial)
 105
 106
 107 static void test_encodings_convert_ascii_to_utf8_auto(void)
 108 {
 109 #define TEST_ASCII(success, str, forced_enc) \
 110                 g_assert(success == assert_convert_to_utf8_auto(str, G_N_ELEMENTS(str) - 1, G_N_ELEMENTS(str) - 1, \
 111                                 forced_enc, str, G_N_ELEMENTS(str) - 1, forced_enc, FALSE, \
 112                                 strlen(str) != G_N_ELEMENTS(str) - 1))
 113
 114         TEST_ASCII(TRUE, "This is a very basic ASCII test", NULL);
 115         TEST_ASCII(TRUE, "This is a very basic ASCII test", "None");
 116         TEST_ASCII(TRUE, "This is a very basic ASCII test", "ASCII");
 117         TEST_ASCII(TRUE, "This is a very basic ASCII test", "UTF-8");
 118         TEST_ASCII(TRUE, "S\till ve\ry \b\asic", NULL);
 119         TEST_ASCII(FALSE, "With\0some\0NULs\0", NULL);
 120         TEST_ASCII(TRUE, "With\0some\0NULs\0", "None");
 121         TEST_ASCII(FALSE, "With\0some\0NULs\0", "UTF-8");
 122
 123 #undef TEST_ASCII
 124 }
 125
 126
 127 static void test_encodings_convert_utf8_to_utf8_auto(void)
 128 {
 129 #define UTF8_BOM "\xef\xbb\xbf"
 130 #define TEST_UTF8(success, str, forced_enc)                                                                                                                                     \
 131         G_STMT_START {                                                                                                                                                                                  \
 132                 gboolean has_bom = strncmp(str, UTF8_BOM, 3) == 0;                                                                                                      \
 133                 g_assert(success == assert_convert_to_utf8_auto(str, G_N_ELEMENTS(str) - 1, G_N_ELEMENTS(str) - 1,      \
 134                                 forced_enc, str + (has_bom ? 3 : 0), G_N_ELEMENTS(str) - 1 - (has_bom ? 3 : 0),                         \
 135                                 forced_enc, has_bom, strlen(str) != G_N_ELEMENTS(str) - 1));                                                            \
 136         } G_STMT_END
 137
 138         TEST_UTF8(TRUE, "Thĩs îs å véry basìč ÅSÇǏÍ test", NULL);
 139         TEST_UTF8(TRUE, "Thĩs îs å véry basìč ÅSÇǏÍ test", "None");
 140         TEST_UTF8(TRUE, "Thĩs îs å véry basìč ÅSÇǏÍ test", "UTF-8");
 141         TEST_UTF8(FALSE, "Wíťh\0søme\0NÙLs\0", NULL);
 142         TEST_UTF8(FALSE, "Wíťh\0søme\0NÙLs\0", "UTF-8"); /* the NUL doesn't pass the UTF-8 check */
 143         TEST_UTF8(TRUE, "Wíťh\0søme\0NÙLs\0", "None"); /* with None we do no data validation, but report partial output */
 144
 145         /* with the inline hint */
 146         TEST_UTF8(TRUE, "coding:utf-8 bãśïč", NULL);
 147         TEST_UTF8(FALSE, "coding:utf-8 Wíťh\0søme\0NÙLs", NULL);
 148
 149         TEST_UTF8(TRUE, UTF8_BOM"With BOM", NULL);
 150         /* These won't pass the UTF-8 validation despite the BOM, so we fallback to
 151          * testing other options, and it will succeed with UTF-16 so there's no real
 152          * point in verifying this */
 153         /*TEST_UTF8(FALSE, UTF8_BOM"With BOM\0and NULs", NULL);*/
 154         /*TEST_UTF8(FALSE, UTF8_BOM"Wíth BØM\0añd NÙLs", NULL);*/
 155
 156         /* non-UTF-8 */
 157         TEST_UTF8(FALSE, "Th\xec""s", "UTF-8");
 158         TEST_UTF8(FALSE, "Th\xec""s\0", "UTF-8");
 159         TEST_UTF8(FALSE, "\0Th\xec""s", "UTF-8");
 160
 161 #undef TEST_UTF8
 162 #undef UTF8_BOM
 163 }
 164
 165
 166 static void test_encodings_convert_utf_other_to_utf8_auto(void)
 167 {
 168 #define UTF16_LE_BOM "\xff\xfe"
 169 #define UTF16_BE_BOM "\xfe\xff"
 170 #define UTF32_LE_BOM "\xff\xfe\x00\x00"
 171 #define UTF32_BE_BOM "\x00\x00\xfe\xff"
 172 #define TEST_ENC(success, input, output, has_bom, forced_enc, expected_encoding) \
 173                 g_assert(success == assert_convert_to_utf8_auto(input, G_N_ELEMENTS(input) - 1, G_N_ELEMENTS(input) - 1, \
 174                                 forced_enc, output, G_N_ELEMENTS(output) - 1, expected_encoding, has_bom, \
 175                                 strlen(output) != G_N_ELEMENTS(output) - 1))
 176 #define TEST(success, input, output, has_bom, forced_enc) \
 177                 TEST_ENC(success, input, output, has_bom, forced_enc, forced_enc)
 178
 179         TEST(TRUE, "N\000o\000 \000B\000O\000M\000", "No BOM", FALSE, NULL);
 180         TEST(TRUE, "N\000o\000 \000B\000\330\000M\000", "No BØM", FALSE, NULL);
 181         /* doesn't accept the NULs */
 182         TEST(FALSE, "N\000o\000 \000B\000O\000M\000\000\000a\000n\000d\000 \000N\000U\000L\000s\000", "No BOM\0and NULs", FALSE, NULL);
 183         TEST(FALSE, "N\000o\000 \000B\000\330\000M\000\000\000a\000\361\000d\000 \000N\000\331\000L\000s\000", "No BØM\0añd NÙLs", FALSE, NULL);
 184
 185         TEST(TRUE, UTF16_LE_BOM"W\000i\000t\000h\000 \000B\000O\000M\000", "With BOM", TRUE, NULL);
 186         TEST(TRUE, UTF16_LE_BOM"W\000i\000t\000h\000 \000B\000\330\000M\000", "With BØM", TRUE, NULL);
 187         /* doesn't accept the NULs */
 188         TEST(FALSE, UTF16_LE_BOM"W\000i\000t\000h\000 \000B\000O\000M\000\000\000a\000n\000d\000 \000N\000U\000L\000s\000", "With BOM\0and NULs", TRUE, NULL);
 189         TEST(FALSE, UTF16_LE_BOM"W\000\355\000t\000h\000 \000B\000\330\000M\000\000\000a\000\361\000d\000 \000N\000\331\000L\000s\000", "Wíth BØM\0añd NÙLs", TRUE, NULL);
 190
 191         /* We should actually be smarter in our selection of encoding introducing
 192          * probability scores, because this loads as UTF-16LE but is "圀椀琀栀 䈀伀䴀"
 193          * which doesn't seem to be real Chinese */
 194         TEST(TRUE, "\000N\000o\000 \000B\000O\000M", "No BOM", FALSE, "UTF-16BE");
 195         TEST(TRUE, "\000N\000o\000 \000B\000\330\000M", "No BØM", FALSE, NULL);
 196         /* doesn't accept the NULs -- and see above for the encoding choice */
 197         TEST(FALSE, "\000N\000o\000 \000B\000O\000M\000\000\000a\000n\000d\000 \000N\000U\000L\000s", "No BOM\0and NULs", FALSE, "UTF-16BE");
 198         TEST(FALSE, "\000N\000o\000 \000B\000\330\000M\000\000\000a\000\361\000d\000 \000N\000\331\000L\000s", "No BØM\0añd NÙLs", FALSE, NULL);
 199
 200         TEST(TRUE, UTF16_BE_BOM"\000W\000i\000t\000h\000 \000B\000O\000M", "With BOM", TRUE, NULL);
 201         TEST(TRUE, UTF16_BE_BOM"\000W\000i\000t\000h\000 \000B\000\330\000M", "With BØM", TRUE, NULL);
 202         /* doesn't accept the NULs */
 203         TEST(FALSE, UTF16_BE_BOM"\000W\000i\000t\000h\000 \000B\000O\000M\000\000\000a\000n\000d\000 \000N\000U\000L\000s", "With BOM\0and NULs", TRUE, NULL);
 204         TEST(FALSE, UTF16_BE_BOM"\000W\000\355\000t\000h\000 \000B\000\330\000M\000\000\000a\000\361\000d\000 \000N\000\331\000L\000s", "Wíth BØM\0añd NÙLs", TRUE, NULL);
 205
 206         TEST(TRUE, UTF32_LE_BOM"W\000\000\000i\000\000\000t\000\000\000h\000\000\000 \000\000\000B\000\000\000O\000\000\000M\000\000\000", "With BOM", TRUE, NULL);
 207         TEST(TRUE, UTF32_LE_BOM"W\000\000\000i\000\000\000t\000\000\000h\000\000\000 \000\000\000B\000\000\000\330\000\000\000M\000\000\000", "With BØM", TRUE, NULL);
 208         /* doesn't accept the NULs */
 209         TEST(FALSE, UTF32_LE_BOM"W\000\000\000i\000\000\000t\000\000\000h\000\000\000 \000\000\000B\000\000\000O\000\000\000M\000\000\000\000\000\000\000a\000\000\000n\000\000\000d\000\000\000 \000\000\000N\000\000\000U\000\000\000L\000\000\000s\000\000\000", "With BOM\0and NULs", TRUE, NULL);
 210         TEST(FALSE, UTF32_LE_BOM"W\000\000\000\355\000\000\000t\000\000\000h\000\000\000 \000\000\000B\000\000\000\330\000\000\000M\000\000\000\000\000\000\000a\000\000\000\361\000\000\000d\000\000\000 \000\000\000N\000\000\000\331\000\000\000L\000\000\000s\000\000\000", "Wíth BØM\0añd NÙLs", TRUE, NULL);
 211
 212         TEST(TRUE, UTF32_BE_BOM"\000\000\000W\000\000\000i\000\000\000t\000\000\000h\000\000\000 \000\000\000B\000\000\000O\000\000\000M", "With BOM", TRUE, NULL);
 213         TEST(TRUE, UTF32_BE_BOM"\000\000\000W\000\000\000i\000\000\000t\000\000\000h\000\000\000 \000\000\000B\000\000\000\330\000\000\000M", "With BØM", TRUE, NULL);
 214         /* doesn't accept the NULs */
 215         TEST(FALSE, UTF32_BE_BOM"\000\000\000W\000\000\000i\000\000\000t\000\000\000h\000\000\000 \000\000\000B\000\000\000O\000\000\000M\000\000\000\000\000\000\000a\000\000\000n\000\000\000d\000\000\000 \000\000\000N\000\000\000U\000\000\000L\000\000\000s", "With BOM\0and NULs", TRUE, NULL);
 216         TEST(FALSE, UTF32_BE_BOM"\000\000\000W\000\000\000\355\000\000\000t\000\000\000h\000\000\000 \000\000\000B\000\000\000\330\000\000\000M\000\000\000\000\000\000\000a\000\000\000\361\000\000\000d\000\000\000 \000\000\000N\000\000\000\331\000\000\000L\000\000\000s", "Wíth BØM\0añd NÙLs", TRUE, NULL);
 217
 218         /* meh, UTF-7 */
 219         TEST(TRUE, "No B+ANg-M", "No BØM", FALSE, "UTF-7");
 220         TEST(TRUE, "+/v8-With B+ANg-M", "With BØM", TRUE, NULL);
 221         TEST(FALSE, "No B+ANg-M+AAA-but NULs", "No BØM\0but NULs", FALSE, "UTF-7");
 222         /* Fails to load as UTF-7 because of the NUL, and succeeds as UTF-8 but
 223          * obviously doesn't match expectations */
 224         /*TEST(FALSE, "+/v8-With B+ANg-M+AAA-and NULs", "With BØM\0and NULs", TRUE, NULL);*/
 225
 226         /* empty data with BOMs */
 227         TEST_ENC(TRUE, "+/v8-", "", TRUE, NULL, "UTF-7"); /* UTF-7 */
 228         TEST_ENC(TRUE, UTF16_BE_BOM, "", TRUE, NULL, "UTF-16BE");
 229         TEST_ENC(TRUE, UTF16_LE_BOM, "", TRUE, NULL, "UTF-16LE");
 230         TEST_ENC(TRUE, UTF32_BE_BOM, "", TRUE, NULL, "UTF-32BE");
 231         TEST_ENC(TRUE, UTF32_LE_BOM, "", TRUE, NULL, "UTF-32LE");
 232
 233 #undef TEST
 234 #undef TEST_ENC
 235 #undef UTF32_BE_BOM
 236 #undef UTF32_LE_BOM
 237 #undef UTF16_BE_BOM
 238 #undef UTF16_LE_BOM
 239 }
 240
 241
 242 static void test_encodings_convert_iso8859_to_utf8_auto(void)
 243 {
 244 #define TEST(success, input, output, forced_enc) \
 245                 g_assert(success == assert_convert_to_utf8_auto(input, G_N_ELEMENTS(input) - 1, G_N_ELEMENTS(input) - 1, \
 246                                 forced_enc, output, G_N_ELEMENTS(output) - 1, forced_enc, FALSE, \
 247                                 strlen(output) != G_N_ELEMENTS(output) - 1))
 248
 249         TEST(TRUE, "Th\xec""s", "Thìs", NULL);
 250         TEST(TRUE, "Th\xec""s", "Thìs", "ISO-8859-1");
 251         TEST(TRUE, "Th\xec""s", "Thìs", "ISO-8859-15");
 252         TEST(TRUE, "\xa4""uro", "¤uro", "ISO-8859-1");
 253         TEST(TRUE, "\xa4""uro", "€uro", "ISO-8859-15");
 254         TEST(TRUE, "\xd8""ed", "Řed", "ISO-8859-2");
 255         /* make-believe UTF-8 BOM followed by non-UTF-8 data */
 256         TEST(TRUE, "\xef\xbb\xbf""not B\xd3M", "ï»¿not BÓM", NULL);
 257         TEST(TRUE, "coding:iso-8859-2 \xd8""ed", "coding:iso-8859-2 Řed", NULL);
 258         /* with NULs */
 259         TEST(FALSE, "W\xec""th\0z\xe9""r\xf8""s", "Wìth\0zérøs", "ISO-8859-1");
 260         TEST(FALSE, "W\xec""th\0z\xe9""r\xf8""s", "Wìth\0zérøs", "ISO-8859-15");
 261         /* This parses as UTF-16, but that's not really what we'd expect */
 262         /*TEST(FALSE, "W\xec""th\0z\xe9""r\xf8""s", "Wìth\0zérøs", NULL);*/
 263
 264         /* UTF-8 BOM with non-UTF-8 data, we should fallback */
 265         TEST(TRUE, "\xef\xbb\xbfW\xec""th\xf8""ut BOM", "ï»¿Wìthøut BOM", NULL);
 266
 267 #undef TEST
 268 }
 269
 270
 271 int main(int argc, char **argv)
 272 {
 273         g_test_init(&argc, &argv, NULL);
 274         gtk_init_check(&argc, &argv);
 275         main_init_headless();
 276
 277         g_test_add_func("/encodings/ascii/convert_to_utf8_auto", test_encodings_convert_ascii_to_utf8_auto);
 278         g_test_add_func("/encodings/utf8/convert_to_utf8_auto", test_encodings_convert_utf8_to_utf8_auto);
 279         g_test_add_func("/encodings/utf_other/convert_to_utf_other_auto", test_encodings_convert_utf_other_to_utf8_auto);
 280         g_test_add_func("/encodings/iso8859/convert_to_utf8_auto", test_encodings_convert_iso8859_to_utf8_auto);
 281
 282         return g_test_run();
 283 }