hphp/runtime/ext/icu/ext_icu_collator.cpp

   1 #include "hphp/runtime/ext/icu/ext_icu_collator.h"
   2 #include "hphp/runtime/base/builtin-functions.h"
   3 #include "hphp/runtime/base/zend-collator.h"
   4 #include "hphp/runtime/base/zend-qsort.h"
   5
   6 namespace HPHP { namespace Intl {
   7 /////////////////////////////////////////////////////////////////////////////
   8 // class Collator
   9
  10 enum CollatorSort {
  11   SORT_REGULAR = 0,
  12   SORT_STRING = 1,
  13   SORT_NUMERIC = 2,
  14 };
  15
  16 const StaticString s_Collator("Collator");
  17
  18 #define FETCH_COL(dest, src, ret) \
  19   auto dest = Collator::Get(src); \
  20   if (!dest) { \
  21     raise_recoverable_error("Collator not initialized"); \
  22     return ret; \
  23   }
  24
  25 static void HHVM_METHOD(Collator, __construct, const String& locale) {
  26   auto data = Native::data<Collator>(this_);
  27   data->clearError();
  28   if (!locale.empty()) {
  29     UErrorCode error = U_ZERO_ERROR;
  30     data->setCollator(ucol_open(locale.c_str(), &error));
  31     if (U_SUCCESS(error)) {
  32       return;
  33     }
  34     /* Fallthrough and use default collator */
  35   }
  36   data->setError(U_USING_FALLBACK_WARNING);
  37   UErrorCode error = U_ZERO_ERROR;
  38   data->setCollator(ucol_open(uloc_getDefault(), &error));
  39   if (U_FAILURE(error)) {
  40     data->setError(error, "collator_create: unable to open ICU collator");
  41     data->setCollator(nullptr);
  42     return;
  43   }
  44 }
  45
  46 static bool HHVM_METHOD(Collator, asort, Variant& arr, int64_t flag) {
  47   FETCH_COL(data, this_, false);
  48   if (!arr.isArray()) {
  49     throw_expected_array_exception("Collator::asort");
  50     return false;
  51   }
  52   data->clearError();
  53   bool ret = collator_asort(arr, flag, true, data->collator(), data);
  54   if (U_FAILURE(data->getErrorCode())) {
  55     return false;
  56   }
  57   return ret;
  58 }
  59
  60 static Variant HHVM_METHOD(Collator, compare,
  61                            const Variant& str1, const Variant& str2) {
  62   FETCH_COL(data, this_, false);
  63   data->clearError();
  64   UErrorCode error = U_ZERO_ERROR;
  65   icu::UnicodeString ustr1(u16(str1.toString(), error));
  66   if (U_FAILURE(error)) {
  67     data->setError(error);
  68     return false;
  69   }
  70   error = U_ZERO_ERROR;
  71   icu::UnicodeString ustr2(u16(str2.toString(), error));
  72   if (U_FAILURE(error)) {
  73     data->setError(error);
  74     return false;
  75   }
  76   return (int64_t)ucol_strcoll(data->collator(),
  77                                ustr1.getBuffer(), ustr1.length(),
  78                                ustr2.getBuffer(), ustr2.length());
  79 }
  80
  81 static int64_t HHVM_METHOD(Collator, getAttribute, int64_t attr) {
  82   FETCH_COL(data, this_, 0);
  83   data->clearError();
  84   UErrorCode error = U_ZERO_ERROR;
  85   int64_t ret = (int64_t)ucol_getAttribute(data->collator(),
  86                                            (UColAttribute)attr,
  87                                            &error);
  88   if (U_FAILURE(error)) {
  89     data->setError(error, "Error getting attribute value");
  90     return 0;
  91   }
  92   return ret;
  93 }
  94
  95 static int64_t HHVM_METHOD(Collator, getErrorCode) {
  96   FETCH_COL(data, this_, 0);
  97   return data->getErrorCode();
  98 }
  99
 100 static String HHVM_METHOD(Collator, getErrorMessage) {
 101   FETCH_COL(data, this_, "");
 102   return data->getErrorMessage();
 103 }
 104
 105 static String HHVM_METHOD(Collator, getLocale, int64_t type) {
 106   FETCH_COL(data, this_, "");
 107   data->clearError();
 108   UErrorCode error = U_ZERO_ERROR;
 109   auto loc = ucol_getLocaleByType(data->collator(), (ULocDataLocaleType)type,
 110                                   &error);
 111   if (U_FAILURE(error)) {
 112     data->setError(error, "Error getting locale by type");
 113   }
 114   return String(loc, CopyString);
 115 }
 116
 117 static int64_t HHVM_METHOD(Collator, getStrength) {
 118   FETCH_COL(data, this_, false);
 119   return ucol_getStrength(data->collator());
 120 }
 121
 122 static bool HHVM_METHOD(Collator, setAttribute, int64_t attr, int64_t val) {
 123   FETCH_COL(data, this_, false);
 124   data->clearError();
 125   UErrorCode error = U_ZERO_ERROR;
 126   ucol_setAttribute(data->collator(), (UColAttribute)attr,
 127                                       (UColAttributeValue)val, &error);
 128   if (U_FAILURE(error)) {
 129     data->setError(error, "Error setting attribute value");
 130     return false;
 131   }
 132   return true;
 133 }
 134
 135 static Variant HHVM_METHOD(Collator, getSortKey, const String& val) {
 136   FETCH_COL(data, this_, false);
 137   UErrorCode error = U_ZERO_ERROR;
 138   icu::UnicodeString strval(u16(val, error));
 139   if (U_FAILURE(error)) {
 140     return false;
 141   }
 142
 143   int sortkey_len = ucol_getSortKey(data->collator(),
 144                                     strval.getBuffer(), strval.length(),
 145                                     nullptr,
 146                                     0);
 147   if (sortkey_len <= 0) {
 148     return false;
 149   }
 150
 151   String ret(sortkey_len, ReserveString);
 152   sortkey_len = ucol_getSortKey(data->collator(),
 153                                 strval.getBuffer(), strval.length(),
 154                                 (uint8_t*) ret.get()->mutableData(),
 155                                 ret.capacity());
 156   if (sortkey_len <= 0) {
 157     return false;
 158   }
 159
 160   ret.setSize(sortkey_len -1);
 161   return ret;
 162 }
 163
 164 static bool HHVM_METHOD(Collator, setStrength, int64_t strength) {
 165   FETCH_COL(data, this_, false);
 166   ucol_setStrength(data->collator(), (UCollationStrength)strength);
 167   return true;
 168 }
 169
 170 typedef struct _collator_sort_key_index {
 171   char* key;       /* pointer to sort key */
 172   ssize_t valPos;  /* position of the original array element */
 173 } collator_sort_key_index_t;
 174
 175 /* Bytes to reserve for sort keys */
 176 static const int32_t DEF_SORT_KEYS_BUF_SIZE = 1048576;
 177 static const int32_t DEF_SORT_KEYS_BUF_INCREMENT = 1048576;
 178
 179 /* Number of keys position to allocate */
 180 static const int32_t DEF_SORT_KEYS_INDX_BUF_SIZE = 512;
 181 static const int32_t DEF_SORT_KEYS_INDX_BUF_INCREMENT = 64;
 182
 183 static int collator_cmp_sort_keys(const void* p1, const void* p2, const void*) {
 184   char* key1 = ((collator_sort_key_index_t*)p1)->key;
 185   char* key2 = ((collator_sort_key_index_t*)p2)->key;
 186   return strcmp( key1, key2 );
 187 }
 188
 189 static bool HHVM_METHOD(Collator, sortWithSortKeys, Variant& arr) {
 190   FETCH_COL(data, this_, false);
 191   data->clearError();
 192
 193   if (!arr.isArray()) {
 194     return true;
 195   }
 196
 197   Array hash = arr.toArray();
 198   if (hash.size() == 0) {
 199     return true;
 200   }
 201
 202   // Preallocate sort keys buffer
 203   size_t sortKeysOffset = 0;
 204   size_t sortKeysLength = DEF_SORT_KEYS_BUF_SIZE;
 205   char*  sortKeys = (char*)req::malloc_noptrs(sortKeysLength);
 206
 207   if (!sortKeys) {
 208     throw Exception("Out of memory");
 209   }
 210   SCOPE_EXIT{ req::free(sortKeys); };
 211
 212   // Preallocate index buffer
 213   size_t sortIndexPos = 0;
 214   size_t sortIndexLength = DEF_SORT_KEYS_INDX_BUF_SIZE;
 215   auto   sortIndex =
 216     req::make_raw_array<collator_sort_key_index_t>(sortIndexLength);
 217   if (!sortIndex) {
 218     throw Exception("Out of memory");
 219   }
 220   SCOPE_EXIT{ req::destroy_raw_array(sortIndex, sortIndexLength); };
 221
 222   // Translate input hash to sortable index
 223   auto pos_limit = hash->iter_end();
 224   for (ssize_t pos = hash->iter_begin(); pos != pos_limit;
 225        pos = hash->iter_advance(pos)) {
 226     Variant val(hash->getValue(pos));
 227
 228     // Convert to UTF16
 229     icu::UnicodeString strval;
 230     if (val.isString()) {
 231       UErrorCode error = U_ZERO_ERROR;
 232       strval = u16(val.toString(), error);
 233       if (U_FAILURE(error)) {
 234         return false;
 235       }
 236      }
 237
 238     // Generate sort key
 239     int sortkey_len =
 240       ucol_getSortKey(data->collator(),
 241                       strval.getBuffer(), strval.length(),
 242                       (uint8_t*)(sortKeys + sortKeysOffset),
 243                       sortKeysLength - sortKeysOffset);
 244
 245     // Check for key buffer overflow
 246     if (sortkey_len > (sortKeysLength - sortKeysOffset)) {
 247       int32_t inc = (sortkey_len > DEF_SORT_KEYS_BUF_INCREMENT)
 248                   ?  sortkey_len : DEF_SORT_KEYS_BUF_INCREMENT;
 249       sortKeysLength += inc;
 250       sortKeys = (char*)req::realloc_noptrs(sortKeys, sortKeysLength);
 251       if (!sortKeys) {
 252         throw Exception("Out of memory");
 253       }
 254       sortkey_len =
 255         ucol_getSortKey(data->collator(),
 256                         strval.getBuffer(), strval.length(),
 257                         (uint8_t*)(sortKeys + sortKeysOffset),
 258                         sortKeysLength - sortKeysOffset);
 259       assertx(sortkey_len <= (sortKeysLength - sortKeysOffset));
 260     }
 261
 262     // Check for index buffer overflow
 263     if ((sortIndexPos + 1) > sortIndexLength) {
 264       sortIndexLength += DEF_SORT_KEYS_INDX_BUF_INCREMENT;
 265       sortIndex = (collator_sort_key_index_t*)req::realloc_untyped(
 266         sortIndex,
 267         sortIndexLength * sizeof(collator_sort_key_index_t)
 268       );
 269       if (!sortIndex) {
 270         throw Exception("Out of memory");
 271       }
 272     }
 273
 274     // Initially store offset into buffer, update later to deal with reallocs
 275     sortIndex[sortIndexPos].key = (char*)sortKeysOffset;
 276     sortKeysOffset += sortkey_len;
 277
 278     sortIndex[sortIndexPos].valPos = pos;
 279     ++sortIndexPos;
 280   }
 281
 282   // Update keys to location in realloc'd buffer
 283   for (int i = 0; i < sortIndexPos; ++i) {
 284     sortIndex[i].key = sortKeys + (ptrdiff_t)sortIndex[i].key;
 285   }
 286
 287   zend_qsort(sortIndex, sortIndexPos,
 288              sizeof(collator_sort_key_index_t),
 289              collator_cmp_sort_keys, nullptr);
 290
 291   Array ret = Array::Create();
 292   for (int i = 0; i < sortIndexPos; ++i) {
 293     ret.append(hash->getValue(sortIndex[i].valPos));
 294   }
 295   arr = ret;
 296   return true;
 297 }
 298
 299 static bool HHVM_METHOD(Collator, sort, Variant& arr,
 300                         int64_t sort_flag /* = Collator::SORT_REGULAR */) {
 301   FETCH_COL(data, this_, false);
 302   if (!arr.isArray()) {
 303     throw_expected_array_exception("Collator::sort");
 304     return false;
 305   }
 306   data->clearError();
 307   bool ret = collator_sort(arr, sort_flag, true, data->collator(), data);
 308   if (U_FAILURE(data->getErrorCode())) {
 309     return false;
 310   }
 311   return ret;
 312 }
 313
 314 //////////////////////////////////////////////////////////////////////////////
 315
 316 void IntlExtension::initCollator() {
 317   HHVM_ME(Collator, __construct);
 318   HHVM_ME(Collator, asort);
 319   HHVM_ME(Collator, compare);
 320   HHVM_ME(Collator, getAttribute);
 321   HHVM_ME(Collator, getErrorCode);
 322   HHVM_ME(Collator, getErrorMessage);
 323   HHVM_ME(Collator, getLocale);
 324   HHVM_ME(Collator, getSortKey);
 325   HHVM_ME(Collator, getStrength);
 326   HHVM_ME(Collator, setAttribute);
 327   HHVM_ME(Collator, setStrength);
 328   HHVM_ME(Collator, sortWithSortKeys);
 329   HHVM_ME(Collator, sort);
 330
 331   HHVM_RCC_INT(Collator, SORT_REGULAR, SORT_REGULAR);
 332   HHVM_RCC_INT(Collator, SORT_STRING, SORT_STRING);
 333   HHVM_RCC_INT(Collator, SORT_NUMERIC, SORT_NUMERIC);
 334
 335   HHVM_RCC_INT(Collator, FRENCH_COLLATION, UCOL_FRENCH_COLLATION);
 336   HHVM_RCC_INT(Collator, ALTERNATE_HANDLING, UCOL_ALTERNATE_HANDLING);
 337   HHVM_RCC_INT(Collator, CASE_FIRST, UCOL_CASE_FIRST);
 338   HHVM_RCC_INT(Collator, CASE_LEVEL, UCOL_CASE_LEVEL);
 339   HHVM_RCC_INT(Collator, NORMALIZATION_MODE, UCOL_NORMALIZATION_MODE);
 340   HHVM_RCC_INT(Collator, STRENGTH, UCOL_STRENGTH);
 341   HHVM_RCC_INT(Collator, HIRAGANA_QUATERNARY_MODE,
 342                UCOL_HIRAGANA_QUATERNARY_MODE);
 343   HHVM_RCC_INT(Collator, NUMERIC_COLLATION, UCOL_NUMERIC_COLLATION);
 344   HHVM_RCC_INT(Collator, PRIMARY, UCOL_PRIMARY);
 345   HHVM_RCC_INT(Collator, SECONDARY, UCOL_SECONDARY);
 346   HHVM_RCC_INT(Collator, TERTIARY, UCOL_TERTIARY);
 347   HHVM_RCC_INT(Collator, DEFAULT_STRENGTH, UCOL_DEFAULT_STRENGTH);
 348   HHVM_RCC_INT(Collator, QUATERNARY, UCOL_QUATERNARY);
 349   HHVM_RCC_INT(Collator, IDENTICAL, UCOL_IDENTICAL);
 350   HHVM_RCC_INT(Collator, OFF, UCOL_OFF);
 351   HHVM_RCC_INT(Collator, ON, UCOL_ON);
 352   HHVM_RCC_INT(Collator, SHIFTED, UCOL_SHIFTED);
 353   HHVM_RCC_INT(Collator, NON_IGNORABLE, UCOL_NON_IGNORABLE);
 354   HHVM_RCC_INT(Collator, LOWER_FIRST, UCOL_LOWER_FIRST);
 355   HHVM_RCC_INT(Collator, UPPER_FIRST, UCOL_UPPER_FIRST);
 356
 357   HHVM_RCC_INT(Collator, DEFAULT_VALUE, UCOL_DEFAULT);
 358
 359   Native::registerNativeDataInfo<Collator>(s_Collator.get());
 360
 361   loadSystemlib("icu_collator");
 362 }
 363
 364 //////////////////////////////////////////////////////////////////////////////
 365 }} // namespace HPHP::Intl