xapian-applications/omega/utf8converttest.cc

   1 /** @file
   2  * @brief test convert_to_utf8()
   3  */
   4 /* Copyright (C) 2008,2009,2013,2019 Olly Betts
   5  *
   6  * This program is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU General Public License as
   8  * published by the Free Software Foundation; either version 2 of the
   9  * License, or (at your option) any later version.
  10  *
  11  * This program is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14  * GNU General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU General Public License
  17  * along with this program; if not, write to the Free Software
  18  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301
  19  * USA
  20  */
  21
  22 #include <config.h>
  23
  24 #include <cstdlib>
  25 #include <iostream>
  26 #include <string>
  27
  28 #include "utf8convert.h"
  29
  30 using namespace std;
  31
  32 struct testcase {
  33     const char * charset;
  34     const char * dump;
  35     size_t len;
  36     const char * utf8;
  37 };
  38
  39 static const testcase tests[] = {
  40     { "utf8", "Hello world", 0, "Hello world" },
  41     { "iso-8859-1", "Hello world", 0, "Hello world" },
  42     { "us-ascii", "Hello world", 0, "Hello world" },
  43     { "iso-8859-1", "Hello\xa0world", 0, "Hello\xc2\xa0world" },
  44     { "ISO-8859-1", "Hello\xa0world", 0, "Hello\xc2\xa0world" },
  45     { "ISO8859-1", "Hello\xa0world", 0, "Hello\xc2\xa0world" },
  46 #if !defined HAVE_ICONV || defined __GNU_LIBRARY__
  47     // "8859_1" is not understood by Solaris iconv, for example.
  48     { "8859_1", "Hello\xa0world", 0, "Hello\xc2\xa0world" },
  49 #endif
  50     { "UTF16BE", "\0T\0e\0s\0t", 8, "Test" },
  51     { "UTF16", "\xfe\xff\0T\0e\0s\0t", 10, "Test" },
  52     { "UTF_16BE", "\0T\0e\0s\0t", 8, "Test" },
  53     { "UTF 16BE", "\0T\0e\0s\0t", 8, "Test" },
  54     { "UTF16LE", "T\0e\0s\0t\0", 8, "Test" },
  55     { "UTF16", "\xff\xfeT\0e\0s\0t\0", 10, "Test" },
  56     { "UCS-2BE", "\0T\0e\0s\0t", 8, "Test" },
  57     { "UCS2BE", "\0T\0e\0s\0t", 8, "Test" },
  58     { "UCS_2BE", "\0T\0e\0s\0t", 8, "Test" },
  59     { "UCS 2BE", "\0T\0e\0s\0t", 8, "Test" },
  60     { "UCS-2LE", "T\0e\0s\0t\0", 8, "Test" },
  61     { "UCS2LE", "T\0e\0s\0t\0", 8, "Test" },
  62     { "UTF16BE", "\xdb\xff\xdf\xfd", 0, "\xf4\x8f\xbf\xbd" },
  63     { "UTF16", "\xfe\xff\xdb\xff\xdf\xfd", 0, "\xf4\x8f\xbf\xbd" },
  64     { "UTF-16", "\xfe\xff\xdb\xff\xdf\xfd", 0, "\xf4\x8f\xbf\xbd" },
  65     { "UTF16LE", "\xff\xdb\xfd\xdf", 0, "\xf4\x8f\xbf\xbd" },
  66     { "UTF16", "\xff\xfe\xff\xdb\xfd\xdf", 0, "\xf4\x8f\xbf\xbd" },
  67     { "UCS-2", "\xfe\xff\0T\0e\0s\0t", 10, "Test" },
  68     { "UCS-2", "\xff\xfeT\0e\0s\0t\0", 10, "Test" },
  69     { "UCS2", "\xfe\xff\0T\0e\0s\0t", 10, "Test" },
  70     { "UCS2", "\xff\xfeT\0e\0s\0t\0", 10, "Test" },
  71     // If there's no BOM, we're supposed to assume BE.
  72     { "UTF16", "\xdb\xff\xdf\xfd", 0, "\xf4\x8f\xbf\xbd" },
  73     // Test "promoting" charset to windows-1252:
  74     { "iso-8859-1", "Price: \x80""20", 0, "Price: \xe2\x82\xac""20" },
  75     { "ISO-8859-1", "\x80\x81\x82\x83\x84\x85\x86\x87", 0, "\xe2\x82\xac\xc2\x81\xe2\x80\x9a\xc6\x92\xe2\x80\x9e\xe2\x80\xa6\xe2\x80\xa0\xe2\x80\xa1" },
  76     { "ISO-8859-1", "\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f", 0, "\xcb\x86\xe2\x80\xb0\xc5\xa0\xe2\x80\xb9\xc5\x92\xc2\x8d\xc5\xbd\xc2\x8f" },
  77     { "ISO-8859-1", "\x90\x91\x92\x93\x94\x95\x96\x97", 0, "\xc2\x90\xe2\x80\x98\xe2\x80\x99\xe2\x80\x9c\xe2\x80\x9d\xe2\x80\xa2\xe2\x80\x93\xe2\x80\x94" },
  78     { "ISO-8859-1", "\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f", 0, "\xcb\x9c\xe2\x84\xa2\xc5\xa1\xe2\x80\xba\xc5\x93\xc2\x9d\xc5\xbe\xc5\xb8" },
  79     { "ISO-8859-1", "\x7e\x7f\xa0\xa1", 0, "\x7e\x7f\xc2\xa0\xc2\xa1" },
  80     // Test iso-8859-15:
  81     { "iso-8859-15", "\xa3\xa4\xa5\xa6\xa7\xa8\xa9\xb4\xb8\xbc\xbd\xbe\xbf", 0, "\xc2\xa3\xe2\x82\xac\xc2\xa5\xc5\xa0\xc2\xa7\xc5\xa1\xc2\xa9\xc5\xbd\xc5\xbe\xc5\x92\xc5\x93\xc5\xb8\xc2\xbf" },
  82     { 0, 0, 0, 0 }
  83 };
  84
  85 int
  86 main()
  87 {
  88     for (size_t i = 0; tests[i].charset; ++i) {
  89         size_t len = tests[i].len;
  90         string dump;
  91         if (len) {
  92             dump.assign(tests[i].dump, len);
  93         } else {
  94             dump.assign(tests[i].dump);
  95         }
  96         convert_to_utf8(dump, tests[i].charset);
  97         if (tests[i].utf8 != dump) {
  98             cout << "Converting from " << tests[i].charset << "\n"
  99                     "Expected [" << tests[i].utf8 << "]\n"
 100                     "Got      [" << dump << "]" << endl;
 101             exit(1);
 102         }
 103     }
 104 }