xapian-core/tests/api_unicode.cc

   1 /** @file api_unicode.cc
   2  * @brief Test the Unicode and UTF-8 classes and functions.
   3  */
   4 /* Copyright (C) 2006,2007,2008,2009,2010,2011,2012,2013,2014,2015,2016 Olly Betts
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License as published by
   8  * the Free Software Foundation; either version 2 of the License, or
   9  * (at your option) any later version.
  10  *
  11  * This program is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14  * GNU General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU General Public License
  17  * along with this program; if not, write to the Free Software
  18  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
  19  */
  20
  21 #include <config.h>
  22
  23 #include "api_unicode.h"
  24
  25 #include <xapian.h>
  26
  27 #include "apitest.h"
  28 #include "testutils.h"
  29
  30 #include <cctype>
  31
  32 using namespace std;
  33
  34 struct testcase {
  35     const char * a, * b;
  36 };
  37
  38 static const testcase testcases[] = {
  39     { "abcd", "abcd" }, // Sanity check!
  40     { "a\x80""bcd", "a\xc2\x80""bcd" },
  41     { "a\xa0", "a\xc2\xa0" },
  42     { "a\xa0z", "a\xc2\xa0z" },
  43     { "x\xc1yz", "x\xc3\x81yz" },
  44     { "\xc2z", "\xc3\x82z" },
  45     { "\xc2", "\xc3\x82" },
  46     { "xy\xc3z", "xy\xc3\x83z" },
  47     { "xy\xc3\xc3z", "xy\xc3\x83\xc3\x83z" },
  48     { "xy\xc3\xc3", "xy\xc3\x83\xc3\x83" },
  49     { "\xe0", "\xc3\xa0" },
  50     { "\xe0\x80", "\xc3\xa0\xc2\x80" },
  51     { "\xe0\xc0", "\xc3\xa0\xc3\x80" },
  52     { "\xe0\xc0z", "\xc3\xa0\xc3\x80z" },
  53     { "\xe0\xc0zz", "\xc3\xa0\xc3\x80zz" },
  54     { "\xe0\xc0\x81", "\xc3\xa0\xc3\x80\xc2\x81" },
  55     { "\xe0\x82\xc1", "\xc3\xa0\xc2\x82\xc3\x81" },
  56     { "\xe0\xc5\xc7", "\xc3\xa0\xc3\x85\xc3\x87" },
  57     { "\xf0", "\xc3\xb0" },
  58     { "\xf0\x80", "\xc3\xb0\xc2\x80" },
  59     { "\xf0\xc0", "\xc3\xb0\xc3\x80" },
  60     { "\xf0\xc0z", "\xc3\xb0\xc3\x80z" },
  61     { "\xf0\xc0zz", "\xc3\xb0\xc3\x80zz" },
  62     { "\xf0\xc0\x81", "\xc3\xb0\xc3\x80\xc2\x81" },
  63     { "\xf0\x82\xc1", "\xc3\xb0\xc2\x82\xc3\x81" },
  64     { "\xf0\xc5\xc7", "\xc3\xb0\xc3\x85\xc3\x87" },
  65     { "\xf0\xc0\x81\xc9", "\xc3\xb0\xc3\x80\xc2\x81\xc3\x89" },
  66     { "\xf0\x82\xc1\xc8", "\xc3\xb0\xc2\x82\xc3\x81\xc3\x88" },
  67     { "\xf0\xc5\xc7\xc6", "\xc3\xb0\xc3\x85\xc3\x87\xc3\x86" },
  68     { "\xf0\xc0\x81\x89", "\xc3\xb0\xc3\x80\xc2\x81\xc2\x89" },
  69     { "\xf0\x82\xc1\x88", "\xc3\xb0\xc2\x82\xc3\x81\xc2\x88" },
  70     { "\xf0\xc5\xc7\xc6", "\xc3\xb0\xc3\x85\xc3\x87\xc3\x86" },
  71     { "\xf4P\x80\x80", "\xc3\xb4P\xc2\x80\xc2\x80" },
  72     { "\xf4\x80P\x80", "\xc3\xb4\xc2\x80P\xc2\x80" },
  73     { "\xf4\x80\x80P", "\xc3\xb4\xc2\x80\xc2\x80P" },
  74     { "\xfe\xffxyzzy", "\xc3\xbe\xc3\xbfxyzzy" },
  75     // Overlong encodings:
  76     { "\xc0\x80", "\xc3\x80\xc2\x80" },
  77     { "\xc0\xbf", "\xc3\x80\xc2\xbf" },
  78     { "\xc1\x80", "\xc3\x81\xc2\x80" },
  79     { "\xc1\xbf", "\xc3\x81\xc2\xbf" },
  80     { "\xe0\x80\x80", "\xc3\xa0\xc2\x80\xc2\x80" },
  81     { "\xe0\x9f\xbf", "\xc3\xa0\xc2\x9f\xc2\xbf" },
  82     { "\xf0\x80\x80\x80", "\xc3\xb0\xc2\x80\xc2\x80\xc2\x80" },
  83     { "\xf0\x8f\xbf\xbf", "\xc3\xb0\xc2\x8f\xc2\xbf\xc2\xbf" },
  84     // Above Unicode:
  85     { "\xf4\x90\x80\x80", "\xc3\xb4\xc2\x90\xc2\x80\xc2\x80" },
  86     { 0, 0 }
  87 };
  88
  89 // Test handling of invalid UTF-8 is as desired.
  90 DEFINE_TESTCASE(utf8iterator1, !backend) {
  91     const testcase * p;
  92     for (p = testcases; p->a; ++p) {
  93         tout.str(string());
  94         tout << '"' << p->a << "\" and \"" << p->b << '"' << endl;
  95         size_t a_len = strlen(p->a);
  96         Xapian::Utf8Iterator a(p->a, a_len);
  97
  98         size_t b_len = strlen(p->b);
  99         Xapian::Utf8Iterator b(p->b, b_len);
 100
 101         while (a != Xapian::Utf8Iterator() && b != Xapian::Utf8Iterator()) {
 102             TEST_EQUAL(*a, *b);
 103             ++a;
 104             ++b;
 105         }
 106
 107         // Test that we don't reach the end of one before the other.
 108         TEST(a == Xapian::Utf8Iterator());
 109         TEST(b == Xapian::Utf8Iterator());
 110     }
 111     return true;
 112 }
 113
 114 struct testcase2 {
 115     const char * a;
 116     unsigned long n;
 117 };
 118
 119 static const testcase2 testcases2[] = {
 120     { "a", 97 },
 121     { "\x80", 128 },
 122     { "\xa0", 160 },
 123     { "\xc2\x80", 128 },
 124     { "\xc2\xa0", 160 },
 125     { "\xe0\xa0\x80", 0x0800 },
 126     { "\xe1\x80\x80", 0x1000 },
 127     { "\xf0\xa8\xa8\x8f", 166415 },
 128     { "\xf3\x80\x80\x80", 0x0c0000 },
 129     { "\xf4\x80\x80\x80", 0x100000 },
 130     { 0, 0 }
 131 };
 132
 133 // Test decoding of UTF-8.
 134 DEFINE_TESTCASE(utf8iterator2, !backend) {
 135     const testcase2 * p;
 136     for (p = testcases2; p->a; ++p) {
 137         Xapian::Utf8Iterator a(p->a);
 138
 139         TEST(a != Xapian::Utf8Iterator());
 140         TEST_EQUAL(*a, p->n);
 141         TEST(++a == Xapian::Utf8Iterator());
 142     }
 143     return true;
 144 }
 145
 146 // Test Unicode categorisation.
 147 DEFINE_TESTCASE(unicode1, !backend) {
 148     using namespace Xapian;
 149     TEST_EQUAL(Unicode::get_category('a'), Unicode::LOWERCASE_LETTER);
 150     TEST_EQUAL(Unicode::get_category('0'), Unicode::DECIMAL_DIGIT_NUMBER);
 151     TEST_EQUAL(Unicode::get_category('$'), Unicode::CURRENCY_SYMBOL);
 152     TEST_EQUAL(Unicode::get_category(0xa3), Unicode::CURRENCY_SYMBOL);
 153     // U+0242 was added in Unicode 5.0.0.
 154     TEST_EQUAL(Unicode::get_category(0x242), Unicode::LOWERCASE_LETTER);
 155     // U+0526 was added in Unicode 6.0.0.
 156     TEST_EQUAL(Unicode::get_category(0x0526), Unicode::UPPERCASE_LETTER);
 157     // U+0527 was added in Unicode 6.0.0.
 158     TEST_EQUAL(Unicode::get_category(0x0527), Unicode::LOWERCASE_LETTER);
 159     // U+0620 was added in Unicode 6.0.0.
 160     TEST_EQUAL(Unicode::get_category(0x0620), Unicode::OTHER_LETTER);
 161     // U+065F was added in Unicode 6.0.0.
 162     TEST_EQUAL(Unicode::get_category(0x065F), Unicode::NON_SPACING_MARK);
 163     // U+06DE changed category in Unicode 6.0.0.
 164     TEST_EQUAL(Unicode::get_category(0x06DE), Unicode::OTHER_SYMBOL);
 165     // U+0840 was added in Unicode 6.0.0.
 166     TEST_EQUAL(Unicode::get_category(0x0840), Unicode::OTHER_LETTER);
 167     // U+093A was added in Unicode 6.0.0.
 168     TEST_EQUAL(Unicode::get_category(0x093A), Unicode::NON_SPACING_MARK);
 169     // U+093B was added in Unicode 6.0.0.
 170     TEST_EQUAL(Unicode::get_category(0x093B), Unicode::COMBINING_SPACING_MARK);
 171     // U+0CF1 changed category in Unicode 6.0.0.
 172     TEST_EQUAL(Unicode::get_category(0x0CF1), Unicode::OTHER_LETTER);
 173     // U+0CF2 changed category in Unicode 6.0.0.
 174     TEST_EQUAL(Unicode::get_category(0x0CF2), Unicode::OTHER_LETTER);
 175     // U+11A7 was added in Unicode 5.2.0.
 176     TEST_EQUAL(Unicode::get_category(0x11A7), Unicode::OTHER_LETTER);
 177     // U+9FCB was added in Unicode 5.2.0.
 178     TEST_EQUAL(Unicode::get_category(0x9FCB), Unicode::OTHER_LETTER);
 179     // U+FA6C was added in Unicode 5.2.0.
 180     TEST_EQUAL(Unicode::get_category(0xFA6C), Unicode::OTHER_LETTER);
 181     TEST_EQUAL(Unicode::get_category(0xFFFF), Unicode::UNASSIGNED);
 182     // Test characters outside BMP.
 183     TEST_EQUAL(Unicode::get_category(0x10345), Unicode::OTHER_LETTER);
 184     TEST_EQUAL(Unicode::get_category(0x10FFFD), Unicode::PRIVATE_USE);
 185     TEST_EQUAL(Unicode::get_category(0x10FFFF), Unicode::UNASSIGNED);
 186     // U+1109A was added in Unicode 5.2.0.
 187     TEST_EQUAL(Unicode::get_category(0x1109a), Unicode::OTHER_LETTER);
 188     // U+1F773 was added in Unicode 6.0.0.
 189     TEST_EQUAL(Unicode::get_category(0x1F773), Unicode::OTHER_SYMBOL);
 190     // U+2B740 was added in Unicode 6.0.0.
 191     TEST_EQUAL(Unicode::get_category(0x2B740), Unicode::OTHER_LETTER);
 192     // U+2B81D was added in Unicode 6.0.0.
 193     TEST_EQUAL(Unicode::get_category(0x2B81D), Unicode::OTHER_LETTER);
 194     // U+00A7 changed category in Unicode 6.1.0 (was OTHER_SYMBOL).
 195     TEST_EQUAL(Unicode::get_category(0xA7), Unicode::OTHER_PUNCTUATION);
 196     // U+00AA changed category in Unicode 6.1.0 (was LOWERCASE_LETTER).
 197     TEST_EQUAL(Unicode::get_category(0xAA), Unicode::OTHER_LETTER);
 198     // U+00B6 changed category in Unicode 6.1.0 (was OTHER_SYMBOL).
 199     TEST_EQUAL(Unicode::get_category(0xB6), Unicode::OTHER_PUNCTUATION);
 200     // U+00BA changed category in Unicode 6.1.0 (was LOWERCASE_LETTER).
 201     TEST_EQUAL(Unicode::get_category(0xBA), Unicode::OTHER_LETTER);
 202     // U+058F was added in Unicode 6.1.0.
 203     TEST_EQUAL(Unicode::get_category(0x058F), Unicode::CURRENCY_SYMBOL);
 204     // U+0604 was added in Unicode 6.1.0.
 205     TEST_EQUAL(Unicode::get_category(0x0604), Unicode::FORMAT);
 206     // U+08A0 was added in Unicode 6.1.0.
 207     TEST_EQUAL(Unicode::get_category(0x08A0), Unicode::OTHER_LETTER);
 208     // U+08E4 was added in Unicode 6.1.0.
 209     TEST_EQUAL(Unicode::get_category(0x08E4), Unicode::NON_SPACING_MARK);
 210     // U+0AF0 was added in Unicode 6.1.0.
 211     TEST_EQUAL(Unicode::get_category(0x0AF0), Unicode::OTHER_PUNCTUATION);
 212     // U+9FCC was added in Unicode 6.1.0.
 213     TEST_EQUAL(Unicode::get_category(0x9FCC), Unicode::OTHER_LETTER);
 214     // U+A7F9 was added in Unicode 6.1.0.
 215     TEST_EQUAL(Unicode::get_category(0xA7F9), Unicode::MODIFIER_LETTER);
 216     // U+110F0 was added in Unicode 6.1.0.
 217     TEST_EQUAL(Unicode::get_category(0x110F0), Unicode::DECIMAL_DIGIT_NUMBER);
 218     // U+11100 was added in Unicode 6.1.0.
 219     TEST_EQUAL(Unicode::get_category(0x11100), Unicode::NON_SPACING_MARK);
 220     // U+1EEF0 was added in Unicode 6.1.0.
 221     TEST_EQUAL(Unicode::get_category(0x1EEF0), Unicode::MATH_SYMBOL);
 222     // U+1F634 was added in Unicode 6.1.0.
 223     TEST_EQUAL(Unicode::get_category(0x1F634), Unicode::OTHER_SYMBOL);
 224     // U+20BA was added in Unicode 6.2.0.
 225     TEST_EQUAL(Unicode::get_category(0x20BA), Unicode::CURRENCY_SYMBOL);
 226     // U+061C was added in Unicode 6.3.0.
 227     TEST_EQUAL(Unicode::get_category(0x61C), Unicode::FORMAT);
 228     // U+037F "GREEK CAPITAL LETTER YOT" was added in Unicode 7.0.0.
 229     TEST_EQUAL(Unicode::get_category(0x37F), Unicode::UPPERCASE_LETTER);
 230
 231     // Added or changed in Unicode 8.0.0:
 232     // U+08B3 "ARABIC LETTER AIN WITH THREE DOTS BELOW".
 233     TEST_EQUAL(Unicode::get_category(0x8B3), Unicode::OTHER_LETTER);
 234     // U+0AF9 "GUJARATI LETTER ZHA".
 235     TEST_EQUAL(Unicode::get_category(0xAF9), Unicode::OTHER_LETTER);
 236     // U+0C5A "TELUGU LETTER RRRA".
 237     TEST_EQUAL(Unicode::get_category(0xC5A), Unicode::OTHER_LETTER);
 238     // U+0D5F "MALAYALAM LETTER ARCHAIC II".
 239     TEST_EQUAL(Unicode::get_category(0xD5F), Unicode::OTHER_LETTER);
 240     // U+13F5 "CHEROKEE LETTER MV".
 241     TEST_EQUAL(Unicode::get_category(0x13F5), Unicode::UPPERCASE_LETTER);
 242     // U+13F8 "CHEROKEE SMALL LETTER YE".
 243     TEST_EQUAL(Unicode::get_category(0x13F8), Unicode::LOWERCASE_LETTER);
 244     // U+19B7 "NEW TAI LUE VOWEL SIGN O" changed to be OTHER_LETTER in 8.0.0.
 245     TEST_EQUAL(Unicode::get_category(0x19B7), Unicode::OTHER_LETTER);
 246     // U+20BE "LARI SIGN".
 247     TEST_EQUAL(Unicode::get_category(0x20BE), Unicode::CURRENCY_SYMBOL);
 248     // U+218A "TURNED DIGIT TWO".
 249     TEST_EQUAL(Unicode::get_category(0x218A), Unicode::OTHER_SYMBOL);
 250     // U+10C9C "OLD HUNGARIAN CAPITAL LETTER OO".
 251     TEST_EQUAL(Unicode::get_category(0x10C9C), Unicode::UPPERCASE_LETTER);
 252     // U+12399 "CUNEIFORM SIGN U U".
 253     TEST_EQUAL(Unicode::get_category(0x12399), Unicode::OTHER_LETTER);
 254     // U+1D800 "SIGNWRITING HAND-FIST INDEX".
 255     TEST_EQUAL(Unicode::get_category(0x1D800), Unicode::OTHER_SYMBOL);
 256
 257     // Added or changed in Unicode 9.0.0:
 258     // U+08B6 "ARABIC LETTER BEH WITH SMALL MEEM ABOVE"
 259     TEST_EQUAL(Unicode::get_category(0x8B6), Unicode::OTHER_LETTER);
 260     // U+08E2 "ARABIC DISPUTED END OF AYAH"
 261     TEST_EQUAL(Unicode::get_category(0x8E2), Unicode::FORMAT);
 262     // U+0C80 "KANNADA SIGN SPACING CANDRABINDU"
 263     TEST_EQUAL(Unicode::get_category(0xC80), Unicode::OTHER_LETTER);
 264     // U+0D56 "MALAYALAM LETTER CHILLU LLL"
 265     TEST_EQUAL(Unicode::get_category(0xD56), Unicode::OTHER_LETTER);
 266     // U+0D58 "MALAYALAM FRACTION ONE ONE-HUNDRED-AND-SIXTIETH"
 267     TEST_EQUAL(Unicode::get_category(0xD58), Unicode::OTHER_NUMBER);
 268     // U+1885 "MONGOLIAN LETTER ALI GALI BALUDA"
 269     TEST_EQUAL(Unicode::get_category(0x1885), Unicode::NON_SPACING_MARK);
 270     // U+1886 "MONGOLIAN LETTER ALI GALI THREE BALUDA"
 271     TEST_EQUAL(Unicode::get_category(0x1886), Unicode::NON_SPACING_MARK);
 272     // U+104FB "OSAGE SMALL LETTER ZHA"
 273     TEST_EQUAL(Unicode::get_category(0x104FB), Unicode::LOWERCASE_LETTER);
 274     // U+1141F "NEWA LETTER TA"
 275     TEST_EQUAL(Unicode::get_category(0x1141F), Unicode::OTHER_LETTER);
 276     // U+1F989 "OWL"
 277     TEST_EQUAL(Unicode::get_category(0x1F989), Unicode::OTHER_SYMBOL);
 278
 279     // Test some invalid Unicode values.
 280     TEST_EQUAL(Unicode::get_category(0x110000), Unicode::UNASSIGNED);
 281     TEST_EQUAL(Unicode::get_category(0xFFFFFFFF), Unicode::UNASSIGNED);
 282     return true;
 283 }
 284
 285 DEFINE_TESTCASE(caseconvert1, !backend) {
 286     using namespace Xapian;
 287     for (unsigned ch = 0; ch < 128; ++ch) {
 288         TEST_EQUAL(Unicode::tolower(ch), unsigned(tolower(ch)));
 289         TEST_EQUAL(Unicode::toupper(ch), unsigned(toupper(ch)));
 290     }
 291
 292     // U+0242 was added in Unicode 5.0.0 as a lowercase form of U+0241.
 293     TEST_EQUAL(Unicode::tolower(0x242), 0x242);
 294     TEST_EQUAL(Unicode::toupper(0x242), 0x241);
 295     TEST_EQUAL(Unicode::toupper(0x241), 0x241);
 296     TEST_EQUAL(Unicode::tolower(0x241), 0x242);
 297
 298     // Regression test for bug fixed in 1.2.17.
 299     TEST_EQUAL(Unicode::tolower(0x1c5), 0x1c6);
 300     TEST_EQUAL(Unicode::tolower(0x1c8), 0x1c9);
 301     TEST_EQUAL(Unicode::tolower(0x1cb), 0x1cc);
 302     TEST_EQUAL(Unicode::tolower(0x1f2), 0x1f3);
 303
 304     // Pound currency symbol:
 305     TEST_EQUAL(Unicode::tolower(0xa3), 0xa3);
 306     TEST_EQUAL(Unicode::toupper(0xa3), 0xa3);
 307     // Unassigned:
 308     TEST_EQUAL(Unicode::tolower(0xFFFF), 0xFFFF);
 309     TEST_EQUAL(Unicode::toupper(0xFFFF), 0xFFFF);
 310     // Test characters outside BMP.
 311     TEST_EQUAL(Unicode::tolower(0x10345), 0x10345);
 312     TEST_EQUAL(Unicode::toupper(0x10345), 0x10345);
 313     TEST_EQUAL(Unicode::tolower(0x10FFFD), 0x10FFFD);
 314     TEST_EQUAL(Unicode::toupper(0x10FFFD), 0x10FFFD);
 315     TEST_EQUAL(Unicode::tolower(0x10FFFF), 0x10FFFF);
 316     TEST_EQUAL(Unicode::toupper(0x10FFFF), 0x10FFFF);
 317     // Test some invalid Unicode values.
 318     TEST_EQUAL(Unicode::tolower(0x110000), 0x110000);
 319     TEST_EQUAL(Unicode::toupper(0x110000), 0x110000);
 320     TEST_EQUAL(Unicode::tolower(0xFFFFFFFF), 0xFFFFFFFF);
 321     TEST_EQUAL(Unicode::toupper(0xFFFFFFFF), 0xFFFFFFFF);
 322
 323     return true;
 324 }
 325
 326 /// Test Unicode 5.1 and later support.
 327 DEFINE_TESTCASE(caseconvert2, !backend) {
 328     using namespace Xapian;
 329
 330     TEST_EQUAL(Unicode::toupper(0x250), 0x2c6f);
 331     TEST_EQUAL(Unicode::toupper(0x251), 0x2c6d);
 332     TEST_EQUAL(Unicode::toupper(0x271), 0x2c6e);
 333
 334     TEST_EQUAL(Unicode::get_category(0x2ec), Unicode::MODIFIER_LETTER);
 335     TEST_EQUAL(Unicode::get_category(0x374), Unicode::MODIFIER_LETTER);
 336     TEST_EQUAL(Unicode::get_category(0x487), Unicode::NON_SPACING_MARK);
 337     TEST_EQUAL(Unicode::get_category(0x5be), Unicode::DASH_PUNCTUATION);
 338     TEST_EQUAL(Unicode::get_category(0x1f093), Unicode::OTHER_SYMBOL);
 339
 340     // U+0526, U+0527 and U+A78D were added in Unicode 6.0.0:
 341     TEST_EQUAL(Unicode::toupper(0x265), 0xa78d);
 342     TEST_EQUAL(Unicode::tolower(0xa78d), 0x265);
 343     TEST_EQUAL(Unicode::tolower(0x526), 0x527);
 344     TEST_EQUAL(Unicode::toupper(0x527), 0x526);
 345
 346     // U+A7AA was added in Unicode 6.1.0:
 347     TEST_EQUAL(Unicode::toupper(0x266), 0xa7aa);
 348     TEST_EQUAL(Unicode::tolower(0xa7aa), 0x266);
 349     TEST_EQUAL(Unicode::tolower(0x526), 0x527);
 350     TEST_EQUAL(Unicode::toupper(0x527), 0x526);
 351
 352     TEST_EQUAL(Unicode::tolower(0x370), 0x371);
 353     TEST_EQUAL(Unicode::toupper(0x371), 0x370);
 354     TEST_EQUAL(Unicode::tolower(0x372), 0x373);
 355     TEST_EQUAL(Unicode::toupper(0x373), 0x372);
 356     TEST_EQUAL(Unicode::tolower(0x376), 0x377);
 357     TEST_EQUAL(Unicode::toupper(0x377), 0x376);
 358     TEST_EQUAL(Unicode::tolower(0x3cf), 0x3d7);
 359     TEST_EQUAL(Unicode::toupper(0x3d7), 0x3cf);
 360
 361     // U+20BA was added in Unicode 6.2.0:
 362     TEST_EQUAL(Unicode::toupper(0x20ba), 0x20ba);
 363     TEST_EQUAL(Unicode::tolower(0x20ba), 0x20ba);
 364
 365     // U+061C was added in Unicode 6.3.0:
 366     TEST_EQUAL(Unicode::toupper(0x61c), 0x61c);
 367     TEST_EQUAL(Unicode::tolower(0x61c), 0x61c);
 368
 369     unsigned u;
 370     for (u = 0x514; u < 0x524; u += 2) {
 371         TEST_EQUAL(Unicode::get_category(u), Unicode::UPPERCASE_LETTER);
 372         TEST_EQUAL(Unicode::get_category(u + 1), Unicode::LOWERCASE_LETTER);
 373         TEST_EQUAL(Unicode::tolower(u), u + 1);
 374         TEST_EQUAL(Unicode::toupper(u + 1), u);
 375     }
 376
 377     // U+A7B1 was added in Unicode 8.0.0 as an uppercase form of U+0287.
 378     TEST_EQUAL(Unicode::tolower(0xA7B1), 0x0287);
 379     TEST_EQUAL(Unicode::toupper(0xA7B1), 0xA7B1);
 380     TEST_EQUAL(Unicode::tolower(0x0287), 0x0287);
 381     TEST_EQUAL(Unicode::toupper(0x0287), 0xA7B1);
 382
 383     // U+A7B4 (capital) and U+A7B5 (small) added in Unicode 8.0.0
 384     TEST_EQUAL(Unicode::tolower(0xA7B4), 0xA7B5);
 385     TEST_EQUAL(Unicode::toupper(0xA7B4), 0xA7B4);
 386     TEST_EQUAL(Unicode::tolower(0xA7B5), 0xA7B5);
 387     TEST_EQUAL(Unicode::toupper(0xA7B5), 0xA7B4);
 388
 389     // U+A7AE was added in Unicode 9.0.0 as an uppercase form of U+026A.
 390     TEST_EQUAL(Unicode::tolower(0xA7AE), 0x026A);
 391     TEST_EQUAL(Unicode::toupper(0xA7AE), 0xA7AE);
 392     TEST_EQUAL(Unicode::tolower(0x026A), 0x026A);
 393     TEST_EQUAL(Unicode::toupper(0x026A), 0xA7AE);
 394
 395     return true;
 396 }
 397
 398 DEFINE_TESTCASE(utf8convert1, !backend) {
 399     string s;
 400     Xapian::Unicode::append_utf8(s, 'a');
 401     Xapian::Unicode::append_utf8(s, 128);
 402     Xapian::Unicode::append_utf8(s, 160);
 403     Xapian::Unicode::append_utf8(s, 0xFFFF);
 404     Xapian::Unicode::append_utf8(s, 166415);
 405     Xapian::Unicode::append_utf8(s, 0x10345);
 406     Xapian::Unicode::append_utf8(s, 0x10FFFD);
 407     Xapian::Unicode::append_utf8(s, 0xFFFFFFFF);
 408     Xapian::Unicode::append_utf8(s, 'z');
 409     TEST_STRINGS_EQUAL(s, "a"
 410                           "\xc2\x80"
 411                           "\xc2\xa0"
 412                           "\xef\xbf\xbf"
 413                           "\xf0\xa8\xa8\x8f"
 414                           "\xf0\x90\x8d\x85"
 415                           "\xf4\x8f\xbf\xbd"
 416                           ""
 417                           "z"
 418                           );
 419
 420     return true;
 421 }
 422
 423 DEFINE_TESTCASE(unicodepredicates1, !backend) {
 424     const unsigned wordchars[] = {
 425         // DECIMAL_DIGIT_NUMER
 426         '0', '7', '9',
 427         // LOWERCASE_LETTER
 428         'a', 'z', 0x250, 0x251, 0x271, 0x3d7,
 429         0x242, // (added in Unicode 5.0.0)
 430         // LOWERCASE_LETTER (added in Unicode 5.1.0)
 431         0x371, 0x373, 0x377, 0x514, 0x516, 0x518, 0x51a, 0x51c, 0x51e,
 432         0x520, 0x522,
 433         // UPPERCASE_LETTER
 434         'A', 'Z', 0x241,
 435         // UPPERCASE_LETTER (added in Unicode 5.1.0)
 436         0x370, 0x372, 0x376, 0x3cf, 0x515, 0x517, 0x519, 0x51b, 0x51d, 0x51f,
 437         0x521, 0x523, 0x2c6d, 0x2c6e, 0x2c6f,
 438         // OTHER_LETTER
 439         0x8bb, // Added in Unicode 9.0.0
 440         0xc80, // Added in Unicode 9.0.0
 441         0x10345,
 442         // MODIFIER_LETTER (added in Unicode 5.1.0)
 443         0x2ec, 0x374,
 444         // NON_SPACING_MARK (added to is_wordchar() in 1.1.0)
 445         0x651,
 446         0x487, // Added in Unicode 5.1.0
 447         0x8db, // Added in Unicode 9.0.0
 448         0
 449     };
 450     const unsigned currency[] = {
 451         // CURRENCY_SYMBOL
 452         '$', 0xa3,
 453         // CURRENCY_SYMBOL (added in Unicode 6.2.0)
 454         0x20ba,
 455         // CURRENCY_SYMBOL (added in Unicode 8.0.0)
 456         0x20be,
 457         0
 458     };
 459     const unsigned whitespace[] = {
 460         // CONTROL
 461         '\t', '\n', '\f', '\r',
 462         // SPACE_SEPARATOR
 463         ' ',
 464         0
 465     };
 466     const unsigned other[] = {
 467         // DASH_PUNCTUATION (added in Unicode 5.1.0)
 468         0x5be,
 469         // OTHER_SYMBOL
 470         0xd4f, // Added in Unicode 9.0.0
 471         0x1f093, // Added in Unicode 5.1.0
 472         // FORMAT
 473         0x61c, // Added in Unicode 6.3.0
 474         0x8e2, // Added in Unicode 9.0.0
 475         // UNASSIGNED
 476         0xffff, 0x10ffff, 0x110000, 0xFFFFFFFF,
 477         // PRIVATE_USE
 478         0x10fffd,
 479         0
 480     };
 481
 482     for (const unsigned * p = wordchars; *p; ++p) {
 483         TEST(Xapian::Unicode::is_wordchar(*p));
 484         TEST(!Xapian::Unicode::is_currency(*p));
 485         TEST(!Xapian::Unicode::is_whitespace(*p));
 486     }
 487
 488     for (const unsigned * p = currency; *p; ++p) {
 489         TEST(!Xapian::Unicode::is_wordchar(*p));
 490         TEST(Xapian::Unicode::is_currency(*p));
 491         TEST(!Xapian::Unicode::is_whitespace(*p));
 492     }
 493
 494     for (const unsigned * p = whitespace; *p; ++p) {
 495         TEST(!Xapian::Unicode::is_wordchar(*p));
 496         TEST(!Xapian::Unicode::is_currency(*p));
 497         TEST(Xapian::Unicode::is_whitespace(*p));
 498     }
 499
 500     for (const unsigned * p = other; *p; ++p) {
 501         TEST(!Xapian::Unicode::is_wordchar(*p));
 502         TEST(!Xapian::Unicode::is_currency(*p));
 503         TEST(!Xapian::Unicode::is_whitespace(*p));
 504     }
 505
 506     return true;
 507 }