Modules/cjkcodecs/cjkcodecs.h

   1 /*
   2  * cjkcodecs.h: common header for cjkcodecs
   3  *
   4  * Written by Hye-Shik Chang <perky@FreeBSD.org>
   5  */
   6
   7 #ifndef _CJKCODECS_H_
   8 #define _CJKCODECS_H_
   9
  10 #define PY_SSIZE_T_CLEAN
  11 #include "Python.h"
  12 #include "multibytecodec.h"
  13
  14
  15 /* a unicode "undefined" codepoint */
  16 #define UNIINV  0xFFFE
  17
  18 /* internal-use DBCS codepoints which aren't used by any charsets */
  19 #define NOCHAR  0xFFFF
  20 #define MULTIC  0xFFFE
  21 #define DBCINV  0xFFFD
  22
  23 /* shorter macros to save source size of mapping tables */
  24 #define U UNIINV
  25 #define N NOCHAR
  26 #define M MULTIC
  27 #define D DBCINV
  28
  29 struct dbcs_index {
  30         const ucs2_t *map;
  31         unsigned char bottom, top;
  32 };
  33 typedef struct dbcs_index decode_map;
  34
  35 struct widedbcs_index {
  36         const ucs4_t *map;
  37         unsigned char bottom, top;
  38 };
  39 typedef struct widedbcs_index widedecode_map;
  40
  41 struct unim_index {
  42         const DBCHAR *map;
  43         unsigned char bottom, top;
  44 };
  45 typedef struct unim_index encode_map;
  46
  47 struct unim_index_bytebased {
  48         const unsigned char *map;
  49         unsigned char bottom, top;
  50 };
  51
  52 struct dbcs_map {
  53         const char *charset;
  54         const struct unim_index *encmap;
  55         const struct dbcs_index *decmap;
  56 };
  57
  58 struct pair_encodemap {
  59         ucs4_t uniseq;
  60         DBCHAR code;
  61 };
  62
  63 static const MultibyteCodec *codec_list;
  64 static const struct dbcs_map *mapping_list;
  65
  66 #define CODEC_INIT(encoding)                                            \
  67         static int encoding##_codec_init(const void *config)
  68
  69 #define ENCODER_INIT(encoding)                                          \
  70         static int encoding##_encode_init(                              \
  71                 MultibyteCodec_State *state, const void *config)
  72 #define ENCODER(encoding)                                               \
  73         static Py_ssize_t encoding##_encode(                            \
  74                 MultibyteCodec_State *state, const void *config,        \
  75                 const Py_UNICODE **inbuf, Py_ssize_t inleft,            \
  76                 unsigned char **outbuf, Py_ssize_t outleft, int flags)
  77 #define ENCODER_RESET(encoding)                                         \
  78         static Py_ssize_t encoding##_encode_reset(                      \
  79                 MultibyteCodec_State *state, const void *config,        \
  80                 unsigned char **outbuf, Py_ssize_t outleft)
  81
  82 #define DECODER_INIT(encoding)                                          \
  83         static int encoding##_decode_init(                              \
  84                 MultibyteCodec_State *state, const void *config)
  85 #define DECODER(encoding)                                               \
  86         static Py_ssize_t encoding##_decode(                            \
  87                 MultibyteCodec_State *state, const void *config,        \
  88                 const unsigned char **inbuf, Py_ssize_t inleft,         \
  89                 Py_UNICODE **outbuf, Py_ssize_t outleft)
  90 #define DECODER_RESET(encoding)                                         \
  91         static Py_ssize_t encoding##_decode_reset(                      \
  92                 MultibyteCodec_State *state, const void *config)
  93
  94 #if Py_UNICODE_SIZE == 4
  95 #define UCS4INVALID(code)       \
  96         if ((code) > 0xFFFF)    \
  97         return 1;
  98 #else
  99 #define UCS4INVALID(code)       \
 100         if (0) ;
 101 #endif
 102
 103 #define NEXT_IN(i)                              \
 104         (*inbuf) += (i);                        \
 105         (inleft) -= (i);
 106 #define NEXT_OUT(o)                             \
 107         (*outbuf) += (o);                       \
 108         (outleft) -= (o);
 109 #define NEXT(i, o)                              \
 110         NEXT_IN(i) NEXT_OUT(o)
 111
 112 #define REQUIRE_INBUF(n)                        \
 113         if (inleft < (n))                       \
 114                 return MBERR_TOOFEW;
 115 #define REQUIRE_OUTBUF(n)                       \
 116         if (outleft < (n))                      \
 117                 return MBERR_TOOSMALL;
 118
 119 #define IN1 ((*inbuf)[0])
 120 #define IN2 ((*inbuf)[1])
 121 #define IN3 ((*inbuf)[2])
 122 #define IN4 ((*inbuf)[3])
 123
 124 #define OUT1(c) ((*outbuf)[0]) = (c);
 125 #define OUT2(c) ((*outbuf)[1]) = (c);
 126 #define OUT3(c) ((*outbuf)[2]) = (c);
 127 #define OUT4(c) ((*outbuf)[3]) = (c);
 128
 129 #define WRITE1(c1)              \
 130         REQUIRE_OUTBUF(1)       \
 131         (*outbuf)[0] = (c1);
 132 #define WRITE2(c1, c2)          \
 133         REQUIRE_OUTBUF(2)       \
 134         (*outbuf)[0] = (c1);    \
 135         (*outbuf)[1] = (c2);
 136 #define WRITE3(c1, c2, c3)      \
 137         REQUIRE_OUTBUF(3)       \
 138         (*outbuf)[0] = (c1);    \
 139         (*outbuf)[1] = (c2);    \
 140         (*outbuf)[2] = (c3);
 141 #define WRITE4(c1, c2, c3, c4)  \
 142         REQUIRE_OUTBUF(4)       \
 143         (*outbuf)[0] = (c1);    \
 144         (*outbuf)[1] = (c2);    \
 145         (*outbuf)[2] = (c3);    \
 146         (*outbuf)[3] = (c4);
 147
 148 #if Py_UNICODE_SIZE == 2
 149 # define WRITEUCS4(c)                                           \
 150         REQUIRE_OUTBUF(2)                                       \
 151         (*outbuf)[0] = 0xd800 + (((c) - 0x10000) >> 10);        \
 152         (*outbuf)[1] = 0xdc00 + (((c) - 0x10000) & 0x3ff);      \
 153         NEXT_OUT(2)
 154 #else
 155 # define WRITEUCS4(c)                                           \
 156         REQUIRE_OUTBUF(1)                                       \
 157         **outbuf = (Py_UNICODE)(c);                             \
 158         NEXT_OUT(1)
 159 #endif
 160
 161 #define _TRYMAP_ENC(m, assi, val)                               \
 162         ((m)->map != NULL && (val) >= (m)->bottom &&            \
 163             (val)<= (m)->top && ((assi) = (m)->map[(val) -      \
 164             (m)->bottom]) != NOCHAR)
 165 #define TRYMAP_ENC_COND(charset, assi, uni)                     \
 166         _TRYMAP_ENC(&charset##_encmap[(uni) >> 8], assi, (uni) & 0xff)
 167 #define TRYMAP_ENC(charset, assi, uni)                          \
 168         if TRYMAP_ENC_COND(charset, assi, uni)
 169
 170 #define _TRYMAP_DEC(m, assi, val)                               \
 171         ((m)->map != NULL && (val) >= (m)->bottom &&            \
 172             (val)<= (m)->top && ((assi) = (m)->map[(val) -      \
 173             (m)->bottom]) != UNIINV)
 174 #define TRYMAP_DEC(charset, assi, c1, c2)                       \
 175         if _TRYMAP_DEC(&charset##_decmap[c1], assi, c2)
 176
 177 #define _TRYMAP_ENC_MPLANE(m, assplane, asshi, asslo, val)      \
 178         ((m)->map != NULL && (val) >= (m)->bottom &&            \
 179             (val)<= (m)->top &&                                 \
 180             ((assplane) = (m)->map[((val) - (m)->bottom)*3]) != 0 && \
 181             (((asshi) = (m)->map[((val) - (m)->bottom)*3 + 1]), 1) && \
 182             (((asslo) = (m)->map[((val) - (m)->bottom)*3 + 2]), 1))
 183 #define TRYMAP_ENC_MPLANE(charset, assplane, asshi, asslo, uni) \
 184         if _TRYMAP_ENC_MPLANE(&charset##_encmap[(uni) >> 8], \
 185                            assplane, asshi, asslo, (uni) & 0xff)
 186 #define TRYMAP_DEC_MPLANE(charset, assi, plane, c1, c2)         \
 187         if _TRYMAP_DEC(&charset##_decmap[plane][c1], assi, c2)
 188
 189 #if Py_UNICODE_SIZE == 2
 190 #define DECODE_SURROGATE(c)                                     \
 191         if (c >> 10 == 0xd800 >> 10) { /* high surrogate */     \
 192                 REQUIRE_INBUF(2)                                \
 193                 if (IN2 >> 10 == 0xdc00 >> 10) { /* low surrogate */ \
 194                     c = 0x10000 + ((ucs4_t)(c - 0xd800) << 10) + \
 195                         ((ucs4_t)(IN2) - 0xdc00);               \
 196                 }                                               \
 197         }
 198 #define GET_INSIZE(c)   ((c) > 0xffff ? 2 : 1)
 199 #else
 200 #define DECODE_SURROGATE(c) {;}
 201 #define GET_INSIZE(c)   1
 202 #endif
 203
 204 #define BEGIN_MAPPINGS_LIST static const struct dbcs_map _mapping_list[] = {
 205 #define MAPPING_ENCONLY(enc) {#enc, (void*)enc##_encmap, NULL},
 206 #define MAPPING_DECONLY(enc) {#enc, NULL, (void*)enc##_decmap},
 207 #define MAPPING_ENCDEC(enc) {#enc, (void*)enc##_encmap, (void*)enc##_decmap},
 208 #define END_MAPPINGS_LIST                               \
 209         {"", NULL, NULL} };                             \
 210         static const struct dbcs_map *mapping_list =    \
 211                 (const struct dbcs_map *)_mapping_list;
 212
 213 #define BEGIN_CODECS_LIST static const MultibyteCodec _codec_list[] = {
 214 #define _STATEFUL_METHODS(enc)          \
 215         enc##_encode,                   \
 216         enc##_encode_init,              \
 217         enc##_encode_reset,             \
 218         enc##_decode,                   \
 219         enc##_decode_init,              \
 220         enc##_decode_reset,
 221 #define _STATELESS_METHODS(enc)         \
 222         enc##_encode, NULL, NULL,       \
 223         enc##_decode, NULL, NULL,
 224 #define CODEC_STATEFUL(enc) {           \
 225         #enc, NULL, NULL,               \
 226         _STATEFUL_METHODS(enc)          \
 227 },
 228 #define CODEC_STATELESS(enc) {          \
 229         #enc, NULL, NULL,               \
 230         _STATELESS_METHODS(enc)         \
 231 },
 232 #define CODEC_STATELESS_WINIT(enc) {    \
 233         #enc, NULL,                     \
 234         enc##_codec_init,               \
 235         _STATELESS_METHODS(enc)         \
 236 },
 237 #define END_CODECS_LIST                                 \
 238         {"", NULL,} };                                  \
 239         static const MultibyteCodec *codec_list =       \
 240                 (const MultibyteCodec *)_codec_list;
 241
 242 static PyObject *
 243 getmultibytecodec(void)
 244 {
 245         static PyObject *cofunc = NULL;
 246
 247         if (cofunc == NULL) {
 248                 PyObject *mod = PyImport_ImportModuleNoBlock("_multibytecodec");
 249                 if (mod == NULL)
 250                         return NULL;
 251                 cofunc = PyObject_GetAttrString(mod, "__create_codec");
 252                 Py_DECREF(mod);
 253         }
 254         return cofunc;
 255 }
 256
 257 static PyObject *
 258 getcodec(PyObject *self, PyObject *encoding)
 259 {
 260         PyObject *codecobj, *r, *cofunc;
 261         const MultibyteCodec *codec;
 262         const char *enc;
 263
 264         if (!PyString_Check(encoding)) {
 265                 PyErr_SetString(PyExc_TypeError,
 266                                 "encoding name must be a string.");
 267                 return NULL;
 268         }
 269
 270         cofunc = getmultibytecodec();
 271         if (cofunc == NULL)
 272                 return NULL;
 273
 274         enc = PyString_AS_STRING(encoding);
 275         for (codec = codec_list; codec->encoding[0]; codec++)
 276                 if (strcmp(codec->encoding, enc) == 0)
 277                         break;
 278
 279         if (codec->encoding[0] == '\0') {
 280                 PyErr_SetString(PyExc_LookupError,
 281                                 "no such codec is supported.");
 282                 return NULL;
 283         }
 284
 285         codecobj = PyCObject_FromVoidPtr((void *)codec, NULL);
 286         if (codecobj == NULL)
 287                 return NULL;
 288
 289         r = PyObject_CallFunctionObjArgs(cofunc, codecobj, NULL);
 290         Py_DECREF(codecobj);
 291
 292         return r;
 293 }
 294
 295 static struct PyMethodDef __methods[] = {
 296         {"getcodec", (PyCFunction)getcodec, METH_O, ""},
 297         {NULL, NULL},
 298 };
 299
 300 static int
 301 register_maps(PyObject *module)
 302 {
 303         const struct dbcs_map *h;
 304
 305         for (h = mapping_list; h->charset[0] != '\0'; h++) {
 306                 char mhname[256] = "__map_";
 307                 int r;
 308                 strcpy(mhname + sizeof("__map_") - 1, h->charset);
 309                 r = PyModule_AddObject(module, mhname,
 310                                 PyCObject_FromVoidPtr((void *)h, NULL));
 311                 if (r == -1)
 312                         return -1;
 313         }
 314         return 0;
 315 }
 316
 317 #ifdef USING_BINARY_PAIR_SEARCH
 318 static DBCHAR
 319 find_pairencmap(ucs2_t body, ucs2_t modifier,
 320                 const struct pair_encodemap *haystack, int haystacksize)
 321 {
 322         int pos, min, max;
 323         ucs4_t value = body << 16 | modifier;
 324
 325         min = 0;
 326         max = haystacksize;
 327
 328         for (pos = haystacksize >> 1; min != max; pos = (min + max) >> 1)
 329                 if (value < haystack[pos].uniseq) {
 330                         if (max == pos) break;
 331                         else max = pos;
 332                 }
 333                 else if (value > haystack[pos].uniseq) {
 334                         if (min == pos) break;
 335                         else min = pos;
 336                 }
 337                 else
 338                         break;
 339
 340                 if (value == haystack[pos].uniseq)
 341                         return haystack[pos].code;
 342                 else
 343                         return DBCINV;
 344 }
 345 #endif
 346
 347 #ifdef USING_IMPORTED_MAPS
 348 #define IMPORT_MAP(locale, charset, encmap, decmap) \
 349         importmap("_codecs_" #locale, "__map_" #charset, \
 350                   (const void**)encmap, (const void**)decmap)
 351
 352 static int
 353 importmap(const char *modname, const char *symbol,
 354           const void **encmap, const void **decmap)
 355 {
 356         PyObject *o, *mod;
 357
 358         mod = PyImport_ImportModule((char *)modname);
 359         if (mod == NULL)
 360                 return -1;
 361
 362         o = PyObject_GetAttrString(mod, (char*)symbol);
 363         if (o == NULL)
 364                 goto errorexit;
 365         else if (!PyCObject_Check(o)) {
 366                 PyErr_SetString(PyExc_ValueError,
 367                                 "map data must be a CObject.");
 368                 goto errorexit;
 369         }
 370         else {
 371                 struct dbcs_map *map;
 372                 map = PyCObject_AsVoidPtr(o);
 373                 if (encmap != NULL)
 374                         *encmap = map->encmap;
 375                 if (decmap != NULL)
 376                         *decmap = map->decmap;
 377                 Py_DECREF(o);
 378         }
 379
 380         Py_DECREF(mod);
 381         return 0;
 382
 383 errorexit:
 384         Py_DECREF(mod);
 385         return -1;
 386 }
 387 #endif
 388
 389 #define I_AM_A_MODULE_FOR(loc)                                          \
 390         void                                                            \
 391         init_codecs_##loc(void)                                         \
 392         {                                                               \
 393                 PyObject *m = Py_InitModule("_codecs_" #loc, __methods);\
 394                 if (m != NULL)                                          \
 395                         (void)register_maps(m);                         \
 396         }
 397
 398 #endif