Modules/_codecsmodule.c

   1 /* ------------------------------------------------------------------------
   2
   3    _codecs -- Provides access to the codec registry and the builtin
   4               codecs.
   5
   6    This module should never be imported directly. The standard library
   7    module "codecs" wraps this builtin module for use within Python.
   8
   9    The codec registry is accessible via:
  10
  11      register(search_function) -> None
  12
  13      lookup(encoding) -> CodecInfo object
  14
  15    The builtin Unicode codecs use the following interface:
  16
  17      <encoding>_encode(Unicode_object[,errors='strict']) ->
  18         (string object, bytes consumed)
  19
  20      <encoding>_decode(char_buffer_obj[,errors='strict']) ->
  21         (Unicode object, bytes consumed)
  22
  23    <encoding>_encode() interfaces also accept non-Unicode object as
  24    input. The objects are then converted to Unicode using
  25    PyUnicode_FromObject() prior to applying the conversion.
  26
  27    These <encoding>s are available: utf_8, unicode_escape,
  28    raw_unicode_escape, unicode_internal, latin_1, ascii (7-bit),
  29    mbcs (on win32).
  30
  31
  32 Written by Marc-Andre Lemburg (mal@lemburg.com).
  33
  34 Copyright (c) Corporation for National Research Initiatives.
  35
  36    ------------------------------------------------------------------------ */
  37
  38 #define PY_SSIZE_T_CLEAN
  39 #include "Python.h"
  40
  41 /* --- Registry ----------------------------------------------------------- */
  42
  43 PyDoc_STRVAR(register__doc__,
  44 "register(search_function)\n\
  45 \n\
  46 Register a codec search function. Search functions are expected to take\n\
  47 one argument, the encoding name in all lower case letters, and return\n\
  48 a tuple of functions (encoder, decoder, stream_reader, stream_writer)\n\
  49 (or a CodecInfo object).");
  50
  51 static
  52 PyObject *codec_register(PyObject *self, PyObject *search_function)
  53 {
  54     if (PyCodec_Register(search_function))
  55         return NULL;
  56
  57     Py_RETURN_NONE;
  58 }
  59
  60 PyDoc_STRVAR(lookup__doc__,
  61 "lookup(encoding) -> CodecInfo\n\
  62 \n\
  63 Looks up a codec tuple in the Python codec registry and returns\n\
  64 a CodecInfo object.");
  65
  66 static
  67 PyObject *codec_lookup(PyObject *self, PyObject *args)
  68 {
  69     char *encoding;
  70
  71     if (!PyArg_ParseTuple(args, "s:lookup", &encoding))
  72         return NULL;
  73
  74     return _PyCodec_Lookup(encoding);
  75 }
  76
  77 PyDoc_STRVAR(encode__doc__,
  78 "encode(obj, [encoding[,errors]]) -> object\n\
  79 \n\
  80 Encodes obj using the codec registered for encoding. encoding defaults\n\
  81 to the default encoding. errors may be given to set a different error\n\
  82 handling scheme. Default is 'strict' meaning that encoding errors raise\n\
  83 a ValueError. Other possible values are 'ignore', 'replace' and\n\
  84 'xmlcharrefreplace' as well as any other name registered with\n\
  85 codecs.register_error that can handle ValueErrors.");
  86
  87 static PyObject *
  88 codec_encode(PyObject *self, PyObject *args)
  89 {
  90     const char *encoding = NULL;
  91     const char *errors = NULL;
  92     PyObject *v;
  93
  94     if (!PyArg_ParseTuple(args, "O|ss:encode", &v, &encoding, &errors))
  95         return NULL;
  96
  97 #ifdef Py_USING_UNICODE
  98     if (encoding == NULL)
  99         encoding = PyUnicode_GetDefaultEncoding();
 100 #else
 101     if (encoding == NULL) {
 102         PyErr_SetString(PyExc_ValueError, "no encoding specified");
 103         return NULL;
 104     }
 105 #endif
 106
 107     /* Encode via the codec registry */
 108     return PyCodec_Encode(v, encoding, errors);
 109 }
 110
 111 PyDoc_STRVAR(decode__doc__,
 112 "decode(obj, [encoding[,errors]]) -> object\n\
 113 \n\
 114 Decodes obj using the codec registered for encoding. encoding defaults\n\
 115 to the default encoding. errors may be given to set a different error\n\
 116 handling scheme. Default is 'strict' meaning that encoding errors raise\n\
 117 a ValueError. Other possible values are 'ignore' and 'replace'\n\
 118 as well as any other name registered with codecs.register_error that is\n\
 119 able to handle ValueErrors.");
 120
 121 static PyObject *
 122 codec_decode(PyObject *self, PyObject *args)
 123 {
 124     const char *encoding = NULL;
 125     const char *errors = NULL;
 126     PyObject *v;
 127
 128     if (!PyArg_ParseTuple(args, "O|ss:decode", &v, &encoding, &errors))
 129         return NULL;
 130
 131 #ifdef Py_USING_UNICODE
 132     if (encoding == NULL)
 133         encoding = PyUnicode_GetDefaultEncoding();
 134 #else
 135     if (encoding == NULL) {
 136         PyErr_SetString(PyExc_ValueError, "no encoding specified");
 137         return NULL;
 138     }
 139 #endif
 140
 141     /* Decode via the codec registry */
 142     return PyCodec_Decode(v, encoding, errors);
 143 }
 144
 145 /* --- Helpers ------------------------------------------------------------ */
 146
 147 static
 148 PyObject *codec_tuple(PyObject *unicode,
 149                       Py_ssize_t len)
 150 {
 151     PyObject *v;
 152     if (unicode == NULL)
 153         return NULL;
 154     v = Py_BuildValue("On", unicode, len);
 155     Py_DECREF(unicode);
 156     return v;
 157 }
 158
 159 /* --- String codecs ------------------------------------------------------ */
 160 static PyObject *
 161 escape_decode(PyObject *self,
 162               PyObject *args)
 163 {
 164     const char *errors = NULL;
 165     const char *data;
 166     Py_ssize_t size;
 167
 168     if (!PyArg_ParseTuple(args, "s#|z:escape_decode",
 169                           &data, &size, &errors))
 170         return NULL;
 171     return codec_tuple(PyString_DecodeEscape(data, size, errors, 0, NULL),
 172                        size);
 173 }
 174
 175 static PyObject *
 176 escape_encode(PyObject *self,
 177               PyObject *args)
 178 {
 179         PyObject *str;
 180         const char *errors = NULL;
 181         char *buf;
 182         Py_ssize_t len;
 183
 184         if (!PyArg_ParseTuple(args, "O!|z:escape_encode",
 185                               &PyString_Type, &str, &errors))
 186                 return NULL;
 187
 188         str = PyString_Repr(str, 0);
 189         if (!str)
 190                 return NULL;
 191
 192         /* The string will be quoted. Unquote, similar to unicode-escape. */
 193         buf = PyString_AS_STRING (str);
 194         len = PyString_GET_SIZE (str);
 195         memmove(buf, buf+1, len-2);
 196         if (_PyString_Resize(&str, len-2) < 0)
 197                 return NULL;
 198
 199         return codec_tuple(str, PyString_Size(str));
 200 }
 201
 202 #ifdef Py_USING_UNICODE
 203 /* --- Decoder ------------------------------------------------------------ */
 204
 205 static PyObject *
 206 unicode_internal_decode(PyObject *self,
 207                         PyObject *args)
 208 {
 209     PyObject *obj;
 210     const char *errors = NULL;
 211     const char *data;
 212     Py_ssize_t size;
 213
 214     if (!PyArg_ParseTuple(args, "O|z:unicode_internal_decode",
 215                           &obj, &errors))
 216         return NULL;
 217
 218     if (PyUnicode_Check(obj)) {
 219         Py_INCREF(obj);
 220         return codec_tuple(obj, PyUnicode_GET_SIZE(obj));
 221     }
 222     else {
 223         if (PyObject_AsReadBuffer(obj, (const void **)&data, &size))
 224             return NULL;
 225
 226         return codec_tuple(_PyUnicode_DecodeUnicodeInternal(data, size, errors),
 227                            size);
 228     }
 229 }
 230
 231 static PyObject *
 232 utf_7_decode(PyObject *self,
 233              PyObject *args)
 234 {
 235         Py_buffer pbuf;
 236     const char *errors = NULL;
 237     int final = 0;
 238     Py_ssize_t consumed;
 239     PyObject *decoded = NULL;
 240
 241     if (!PyArg_ParseTuple(args, "s*|zi:utf_7_decode",
 242                           &pbuf, &errors, &final))
 243         return NULL;
 244     consumed = pbuf.len;
 245
 246     decoded = PyUnicode_DecodeUTF7Stateful(pbuf.buf, pbuf.len, errors,
 247                                            final ? NULL : &consumed);
 248         PyBuffer_Release(&pbuf);
 249     if (decoded == NULL)
 250         return NULL;
 251     return codec_tuple(decoded, consumed);
 252 }
 253
 254 static PyObject *
 255 utf_8_decode(PyObject *self,
 256             PyObject *args)
 257 {
 258         Py_buffer pbuf;
 259     const char *errors = NULL;
 260     int final = 0;
 261     Py_ssize_t consumed;
 262     PyObject *decoded = NULL;
 263
 264     if (!PyArg_ParseTuple(args, "s*|zi:utf_8_decode",
 265                           &pbuf, &errors, &final))
 266         return NULL;
 267     consumed = pbuf.len;
 268
 269     decoded = PyUnicode_DecodeUTF8Stateful(pbuf.buf, pbuf.len, errors,
 270                                            final ? NULL : &consumed);
 271         PyBuffer_Release(&pbuf);
 272     if (decoded == NULL)
 273         return NULL;
 274     return codec_tuple(decoded, consumed);
 275 }
 276
 277 static PyObject *
 278 utf_16_decode(PyObject *self,
 279             PyObject *args)
 280 {
 281         Py_buffer pbuf;
 282     const char *errors = NULL;
 283     int byteorder = 0;
 284     int final = 0;
 285     Py_ssize_t consumed;
 286     PyObject *decoded;
 287
 288     if (!PyArg_ParseTuple(args, "s*|zi:utf_16_decode",
 289                           &pbuf, &errors, &final))
 290         return NULL;
 291     consumed = pbuf.len; /* This is overwritten unless final is true. */
 292     decoded = PyUnicode_DecodeUTF16Stateful(pbuf.buf, pbuf.len, errors,
 293                                         &byteorder, final ? NULL : &consumed);
 294         PyBuffer_Release(&pbuf);
 295     if (decoded == NULL)
 296         return NULL;
 297     return codec_tuple(decoded, consumed);
 298 }
 299
 300 static PyObject *
 301 utf_16_le_decode(PyObject *self,
 302                  PyObject *args)
 303 {
 304         Py_buffer pbuf;
 305     const char *errors = NULL;
 306     int byteorder = -1;
 307     int final = 0;
 308     Py_ssize_t consumed;
 309     PyObject *decoded = NULL;
 310
 311     if (!PyArg_ParseTuple(args, "s*|zi:utf_16_le_decode",
 312                           &pbuf, &errors, &final))
 313         return NULL;
 314
 315     consumed = pbuf.len; /* This is overwritten unless final is true. */
 316     decoded = PyUnicode_DecodeUTF16Stateful(pbuf.buf, pbuf.len, errors,
 317         &byteorder, final ? NULL : &consumed);
 318         PyBuffer_Release(&pbuf);
 319     if (decoded == NULL)
 320         return NULL;
 321     return codec_tuple(decoded, consumed);
 322 }
 323
 324 static PyObject *
 325 utf_16_be_decode(PyObject *self,
 326                  PyObject *args)
 327 {
 328         Py_buffer pbuf;
 329     const char *errors = NULL;
 330     int byteorder = 1;
 331     int final = 0;
 332     Py_ssize_t consumed;
 333     PyObject *decoded = NULL;
 334
 335     if (!PyArg_ParseTuple(args, "s*|zi:utf_16_be_decode",
 336                           &pbuf, &errors, &final))
 337         return NULL;
 338
 339     consumed = pbuf.len; /* This is overwritten unless final is true. */
 340     decoded = PyUnicode_DecodeUTF16Stateful(pbuf.buf, pbuf.len, errors,
 341         &byteorder, final ? NULL : &consumed);
 342         PyBuffer_Release(&pbuf);
 343     if (decoded == NULL)
 344         return NULL;
 345     return codec_tuple(decoded, consumed);
 346 }
 347
 348 /* This non-standard version also provides access to the byteorder
 349    parameter of the builtin UTF-16 codec.
 350
 351    It returns a tuple (unicode, bytesread, byteorder) with byteorder
 352    being the value in effect at the end of data.
 353
 354 */
 355
 356 static PyObject *
 357 utf_16_ex_decode(PyObject *self,
 358                  PyObject *args)
 359 {
 360         Py_buffer pbuf;
 361     const char *errors = NULL;
 362     int byteorder = 0;
 363     PyObject *unicode, *tuple;
 364     int final = 0;
 365     Py_ssize_t consumed;
 366
 367     if (!PyArg_ParseTuple(args, "s*|zii:utf_16_ex_decode",
 368                           &pbuf, &errors, &byteorder, &final))
 369         return NULL;
 370     consumed = pbuf.len; /* This is overwritten unless final is true. */
 371     unicode = PyUnicode_DecodeUTF16Stateful(pbuf.buf, pbuf.len, errors,
 372                                         &byteorder, final ? NULL : &consumed);
 373         PyBuffer_Release(&pbuf);
 374     if (unicode == NULL)
 375         return NULL;
 376     tuple = Py_BuildValue("Oni", unicode, consumed, byteorder);
 377     Py_DECREF(unicode);
 378     return tuple;
 379 }
 380
 381 static PyObject *
 382 utf_32_decode(PyObject *self,
 383             PyObject *args)
 384 {
 385         Py_buffer pbuf;
 386     const char *errors = NULL;
 387     int byteorder = 0;
 388     int final = 0;
 389     Py_ssize_t consumed;
 390     PyObject *decoded;
 391
 392     if (!PyArg_ParseTuple(args, "s*|zi:utf_32_decode",
 393                           &pbuf, &errors, &final))
 394         return NULL;
 395     consumed = pbuf.len; /* This is overwritten unless final is true. */
 396     decoded = PyUnicode_DecodeUTF32Stateful(pbuf.buf, pbuf.len, errors,
 397                                         &byteorder, final ? NULL : &consumed);
 398         PyBuffer_Release(&pbuf);
 399     if (decoded == NULL)
 400         return NULL;
 401     return codec_tuple(decoded, consumed);
 402 }
 403
 404 static PyObject *
 405 utf_32_le_decode(PyObject *self,
 406                  PyObject *args)
 407 {
 408         Py_buffer pbuf;
 409     const char *errors = NULL;
 410     int byteorder = -1;
 411     int final = 0;
 412     Py_ssize_t consumed;
 413     PyObject *decoded;
 414
 415     if (!PyArg_ParseTuple(args, "s*|zi:utf_32_le_decode",
 416                           &pbuf, &errors, &final))
 417         return NULL;
 418     consumed = pbuf.len; /* This is overwritten unless final is true. */
 419     decoded = PyUnicode_DecodeUTF32Stateful(pbuf.buf, pbuf.len, errors,
 420                                         &byteorder, final ? NULL : &consumed);
 421         PyBuffer_Release(&pbuf);
 422     if (decoded == NULL)
 423         return NULL;
 424     return codec_tuple(decoded, consumed);
 425 }
 426
 427 static PyObject *
 428 utf_32_be_decode(PyObject *self,
 429                  PyObject *args)
 430 {
 431         Py_buffer pbuf;
 432     const char *errors = NULL;
 433     int byteorder = 1;
 434     int final = 0;
 435     Py_ssize_t consumed;
 436     PyObject *decoded;
 437
 438     if (!PyArg_ParseTuple(args, "s*|zi:utf_32_be_decode",
 439                           &pbuf, &errors, &final))
 440         return NULL;
 441     consumed = pbuf.len; /* This is overwritten unless final is true. */
 442     decoded = PyUnicode_DecodeUTF32Stateful(pbuf.buf, pbuf.len, errors,
 443                                         &byteorder, final ? NULL : &consumed);
 444         PyBuffer_Release(&pbuf);
 445     if (decoded == NULL)
 446         return NULL;
 447     return codec_tuple(decoded, consumed);
 448 }
 449
 450 /* This non-standard version also provides access to the byteorder
 451    parameter of the builtin UTF-32 codec.
 452
 453    It returns a tuple (unicode, bytesread, byteorder) with byteorder
 454    being the value in effect at the end of data.
 455
 456 */
 457
 458 static PyObject *
 459 utf_32_ex_decode(PyObject *self,
 460                  PyObject *args)
 461 {
 462         Py_buffer pbuf;
 463     const char *errors = NULL;
 464     int byteorder = 0;
 465     PyObject *unicode, *tuple;
 466     int final = 0;
 467     Py_ssize_t consumed;
 468
 469     if (!PyArg_ParseTuple(args, "s*|zii:utf_32_ex_decode",
 470                           &pbuf, &errors, &byteorder, &final))
 471         return NULL;
 472     consumed = pbuf.len; /* This is overwritten unless final is true. */
 473     unicode = PyUnicode_DecodeUTF32Stateful(pbuf.buf, pbuf.len, errors,
 474                                         &byteorder, final ? NULL : &consumed);
 475         PyBuffer_Release(&pbuf);
 476     if (unicode == NULL)
 477         return NULL;
 478     tuple = Py_BuildValue("Oni", unicode, consumed, byteorder);
 479     Py_DECREF(unicode);
 480     return tuple;
 481 }
 482
 483 static PyObject *
 484 unicode_escape_decode(PyObject *self,
 485                      PyObject *args)
 486 {
 487         Py_buffer pbuf;
 488     const char *errors = NULL;
 489         PyObject *unicode;
 490
 491     if (!PyArg_ParseTuple(args, "s*|z:unicode_escape_decode",
 492                           &pbuf, &errors))
 493         return NULL;
 494
 495         unicode = PyUnicode_DecodeUnicodeEscape(pbuf.buf, pbuf.len, errors);
 496         PyBuffer_Release(&pbuf);
 497         return codec_tuple(unicode, pbuf.len);
 498 }
 499
 500 static PyObject *
 501 raw_unicode_escape_decode(PyObject *self,
 502                         PyObject *args)
 503 {
 504         Py_buffer pbuf;
 505     const char *errors = NULL;
 506         PyObject *unicode;
 507
 508     if (!PyArg_ParseTuple(args, "s*|z:raw_unicode_escape_decode",
 509                           &pbuf, &errors))
 510         return NULL;
 511
 512         unicode = PyUnicode_DecodeRawUnicodeEscape(pbuf.buf, pbuf.len, errors);
 513         PyBuffer_Release(&pbuf);
 514         return codec_tuple(unicode, pbuf.len);
 515 }
 516
 517 static PyObject *
 518 latin_1_decode(PyObject *self,
 519                PyObject *args)
 520 {
 521         Py_buffer pbuf;
 522         PyObject *unicode;
 523     const char *errors = NULL;
 524
 525     if (!PyArg_ParseTuple(args, "s*|z:latin_1_decode",
 526                           &pbuf, &errors))
 527         return NULL;
 528
 529         unicode = PyUnicode_DecodeLatin1(pbuf.buf, pbuf.len, errors);
 530         PyBuffer_Release(&pbuf);
 531         return codec_tuple(unicode, pbuf.len);
 532 }
 533
 534 static PyObject *
 535 ascii_decode(PyObject *self,
 536              PyObject *args)
 537 {
 538         Py_buffer pbuf;
 539         PyObject *unicode;
 540     const char *errors = NULL;
 541
 542     if (!PyArg_ParseTuple(args, "s*|z:ascii_decode",
 543                           &pbuf, &errors))
 544         return NULL;
 545
 546         unicode = PyUnicode_DecodeASCII(pbuf.buf, pbuf.len, errors);
 547         PyBuffer_Release(&pbuf);
 548         return codec_tuple(unicode, pbuf.len);
 549 }
 550
 551 static PyObject *
 552 charmap_decode(PyObject *self,
 553                PyObject *args)
 554 {
 555         Py_buffer pbuf;
 556         PyObject *unicode;
 557     const char *errors = NULL;
 558     PyObject *mapping = NULL;
 559
 560     if (!PyArg_ParseTuple(args, "s*|zO:charmap_decode",
 561                           &pbuf, &errors, &mapping))
 562         return NULL;
 563     if (mapping == Py_None)
 564         mapping = NULL;
 565
 566         unicode = PyUnicode_DecodeCharmap(pbuf.buf, pbuf.len, mapping, errors);
 567         PyBuffer_Release(&pbuf);
 568         return codec_tuple(unicode, pbuf.len);
 569 }
 570
 571 #if defined(MS_WINDOWS) && defined(HAVE_USABLE_WCHAR_T)
 572
 573 static PyObject *
 574 mbcs_decode(PyObject *self,
 575             PyObject *args)
 576 {
 577         Py_buffer pbuf;
 578     const char *errors = NULL;
 579     int final = 0;
 580     Py_ssize_t consumed;
 581     PyObject *decoded = NULL;
 582
 583     if (!PyArg_ParseTuple(args, "s*|zi:mbcs_decode",
 584                           &pbuf, &errors, &final))
 585         return NULL;
 586     consumed = pbuf.len;
 587
 588     decoded = PyUnicode_DecodeMBCSStateful(pbuf.buf, pbuf.len, errors,
 589                                            final ? NULL : &consumed);
 590         PyBuffer_Release(&pbuf);
 591     if (decoded == NULL)
 592         return NULL;
 593     return codec_tuple(decoded, consumed);
 594 }
 595
 596 #endif /* MS_WINDOWS */
 597
 598 /* --- Encoder ------------------------------------------------------------ */
 599
 600 static PyObject *
 601 readbuffer_encode(PyObject *self,
 602                   PyObject *args)
 603 {
 604     const char *data;
 605     Py_ssize_t size;
 606     const char *errors = NULL;
 607
 608     if (!PyArg_ParseTuple(args, "s#|z:readbuffer_encode",
 609                           &data, &size, &errors))
 610         return NULL;
 611
 612     return codec_tuple(PyString_FromStringAndSize(data, size),
 613                        size);
 614 }
 615
 616 static PyObject *
 617 charbuffer_encode(PyObject *self,
 618                   PyObject *args)
 619 {
 620     const char *data;
 621     Py_ssize_t size;
 622     const char *errors = NULL;
 623
 624     if (!PyArg_ParseTuple(args, "t#|z:charbuffer_encode",
 625                           &data, &size, &errors))
 626         return NULL;
 627
 628     return codec_tuple(PyString_FromStringAndSize(data, size),
 629                        size);
 630 }
 631
 632 static PyObject *
 633 unicode_internal_encode(PyObject *self,
 634                         PyObject *args)
 635 {
 636     PyObject *obj;
 637     const char *errors = NULL;
 638     const char *data;
 639     Py_ssize_t size;
 640
 641     if (!PyArg_ParseTuple(args, "O|z:unicode_internal_encode",
 642                           &obj, &errors))
 643         return NULL;
 644
 645     if (PyUnicode_Check(obj)) {
 646         data = PyUnicode_AS_DATA(obj);
 647         size = PyUnicode_GET_DATA_SIZE(obj);
 648         return codec_tuple(PyString_FromStringAndSize(data, size),
 649                            PyUnicode_GET_SIZE(obj));
 650     }
 651     else {
 652         if (PyObject_AsReadBuffer(obj, (const void **)&data, &size))
 653             return NULL;
 654         return codec_tuple(PyString_FromStringAndSize(data, size),
 655                            size);
 656     }
 657 }
 658
 659 static PyObject *
 660 utf_7_encode(PyObject *self,
 661             PyObject *args)
 662 {
 663     PyObject *str, *v;
 664     const char *errors = NULL;
 665
 666     if (!PyArg_ParseTuple(args, "O|z:utf_7_encode",
 667                           &str, &errors))
 668         return NULL;
 669
 670     str = PyUnicode_FromObject(str);
 671     if (str == NULL)
 672         return NULL;
 673     v = codec_tuple(PyUnicode_EncodeUTF7(PyUnicode_AS_UNICODE(str),
 674                                          PyUnicode_GET_SIZE(str),
 675                                          0,
 676                                          0,
 677                                          errors),
 678                     PyUnicode_GET_SIZE(str));
 679     Py_DECREF(str);
 680     return v;
 681 }
 682
 683 static PyObject *
 684 utf_8_encode(PyObject *self,
 685             PyObject *args)
 686 {
 687     PyObject *str, *v;
 688     const char *errors = NULL;
 689
 690     if (!PyArg_ParseTuple(args, "O|z:utf_8_encode",
 691                           &str, &errors))
 692         return NULL;
 693
 694     str = PyUnicode_FromObject(str);
 695     if (str == NULL)
 696         return NULL;
 697     v = codec_tuple(PyUnicode_EncodeUTF8(PyUnicode_AS_UNICODE(str),
 698                                          PyUnicode_GET_SIZE(str),
 699                                          errors),
 700                     PyUnicode_GET_SIZE(str));
 701     Py_DECREF(str);
 702     return v;
 703 }
 704
 705 /* This version provides access to the byteorder parameter of the
 706    builtin UTF-16 codecs as optional third argument. It defaults to 0
 707    which means: use the native byte order and prepend the data with a
 708    BOM mark.
 709
 710 */
 711
 712 static PyObject *
 713 utf_16_encode(PyObject *self,
 714             PyObject *args)
 715 {
 716     PyObject *str, *v;
 717     const char *errors = NULL;
 718     int byteorder = 0;
 719
 720     if (!PyArg_ParseTuple(args, "O|zi:utf_16_encode",
 721                           &str, &errors, &byteorder))
 722         return NULL;
 723
 724     str = PyUnicode_FromObject(str);
 725     if (str == NULL)
 726         return NULL;
 727     v = codec_tuple(PyUnicode_EncodeUTF16(PyUnicode_AS_UNICODE(str),
 728                                           PyUnicode_GET_SIZE(str),
 729                                           errors,
 730                                           byteorder),
 731                     PyUnicode_GET_SIZE(str));
 732     Py_DECREF(str);
 733     return v;
 734 }
 735
 736 static PyObject *
 737 utf_16_le_encode(PyObject *self,
 738                  PyObject *args)
 739 {
 740     PyObject *str, *v;
 741     const char *errors = NULL;
 742
 743     if (!PyArg_ParseTuple(args, "O|z:utf_16_le_encode",
 744                           &str, &errors))
 745         return NULL;
 746
 747     str = PyUnicode_FromObject(str);
 748     if (str == NULL)
 749         return NULL;
 750     v = codec_tuple(PyUnicode_EncodeUTF16(PyUnicode_AS_UNICODE(str),
 751                                              PyUnicode_GET_SIZE(str),
 752                                              errors,
 753                                              -1),
 754                        PyUnicode_GET_SIZE(str));
 755     Py_DECREF(str);
 756     return v;
 757 }
 758
 759 static PyObject *
 760 utf_16_be_encode(PyObject *self,
 761                  PyObject *args)
 762 {
 763     PyObject *str, *v;
 764     const char *errors = NULL;
 765
 766     if (!PyArg_ParseTuple(args, "O|z:utf_16_be_encode",
 767                           &str, &errors))
 768         return NULL;
 769
 770     str = PyUnicode_FromObject(str);
 771     if (str == NULL)
 772         return NULL;
 773     v = codec_tuple(PyUnicode_EncodeUTF16(PyUnicode_AS_UNICODE(str),
 774                                           PyUnicode_GET_SIZE(str),
 775                                           errors,
 776                                           +1),
 777                     PyUnicode_GET_SIZE(str));
 778     Py_DECREF(str);
 779     return v;
 780 }
 781
 782 /* This version provides access to the byteorder parameter of the
 783    builtin UTF-32 codecs as optional third argument. It defaults to 0
 784    which means: use the native byte order and prepend the data with a
 785    BOM mark.
 786
 787 */
 788
 789 static PyObject *
 790 utf_32_encode(PyObject *self,
 791             PyObject *args)
 792 {
 793     PyObject *str, *v;
 794     const char *errors = NULL;
 795     int byteorder = 0;
 796
 797     if (!PyArg_ParseTuple(args, "O|zi:utf_32_encode",
 798                           &str, &errors, &byteorder))
 799         return NULL;
 800
 801     str = PyUnicode_FromObject(str);
 802     if (str == NULL)
 803         return NULL;
 804     v = codec_tuple(PyUnicode_EncodeUTF32(PyUnicode_AS_UNICODE(str),
 805                                           PyUnicode_GET_SIZE(str),
 806                                           errors,
 807                                           byteorder),
 808                     PyUnicode_GET_SIZE(str));
 809     Py_DECREF(str);
 810     return v;
 811 }
 812
 813 static PyObject *
 814 utf_32_le_encode(PyObject *self,
 815                  PyObject *args)
 816 {
 817     PyObject *str, *v;
 818     const char *errors = NULL;
 819
 820     if (!PyArg_ParseTuple(args, "O|z:utf_32_le_encode",
 821                           &str, &errors))
 822         return NULL;
 823
 824     str = PyUnicode_FromObject(str);
 825     if (str == NULL)
 826         return NULL;
 827     v = codec_tuple(PyUnicode_EncodeUTF32(PyUnicode_AS_UNICODE(str),
 828                                              PyUnicode_GET_SIZE(str),
 829                                              errors,
 830                                              -1),
 831                        PyUnicode_GET_SIZE(str));
 832     Py_DECREF(str);
 833     return v;
 834 }
 835
 836 static PyObject *
 837 utf_32_be_encode(PyObject *self,
 838                  PyObject *args)
 839 {
 840     PyObject *str, *v;
 841     const char *errors = NULL;
 842
 843     if (!PyArg_ParseTuple(args, "O|z:utf_32_be_encode",
 844                           &str, &errors))
 845         return NULL;
 846
 847     str = PyUnicode_FromObject(str);
 848     if (str == NULL)
 849         return NULL;
 850     v = codec_tuple(PyUnicode_EncodeUTF32(PyUnicode_AS_UNICODE(str),
 851                                           PyUnicode_GET_SIZE(str),
 852                                           errors,
 853                                           +1),
 854                     PyUnicode_GET_SIZE(str));
 855     Py_DECREF(str);
 856     return v;
 857 }
 858
 859 static PyObject *
 860 unicode_escape_encode(PyObject *self,
 861                      PyObject *args)
 862 {
 863     PyObject *str, *v;
 864     const char *errors = NULL;
 865
 866     if (!PyArg_ParseTuple(args, "O|z:unicode_escape_encode",
 867                           &str, &errors))
 868         return NULL;
 869
 870     str = PyUnicode_FromObject(str);
 871     if (str == NULL)
 872         return NULL;
 873     v = codec_tuple(PyUnicode_EncodeUnicodeEscape(PyUnicode_AS_UNICODE(str),
 874                                                   PyUnicode_GET_SIZE(str)),
 875                     PyUnicode_GET_SIZE(str));
 876     Py_DECREF(str);
 877     return v;
 878 }
 879
 880 static PyObject *
 881 raw_unicode_escape_encode(PyObject *self,
 882                         PyObject *args)
 883 {
 884     PyObject *str, *v;
 885     const char *errors = NULL;
 886
 887     if (!PyArg_ParseTuple(args, "O|z:raw_unicode_escape_encode",
 888                           &str, &errors))
 889         return NULL;
 890
 891     str = PyUnicode_FromObject(str);
 892     if (str == NULL)
 893         return NULL;
 894     v = codec_tuple(PyUnicode_EncodeRawUnicodeEscape(
 895                                PyUnicode_AS_UNICODE(str),
 896                                PyUnicode_GET_SIZE(str)),
 897                     PyUnicode_GET_SIZE(str));
 898     Py_DECREF(str);
 899     return v;
 900 }
 901
 902 static PyObject *
 903 latin_1_encode(PyObject *self,
 904                PyObject *args)
 905 {
 906     PyObject *str, *v;
 907     const char *errors = NULL;
 908
 909     if (!PyArg_ParseTuple(args, "O|z:latin_1_encode",
 910                           &str, &errors))
 911         return NULL;
 912
 913     str = PyUnicode_FromObject(str);
 914     if (str == NULL)
 915         return NULL;
 916     v = codec_tuple(PyUnicode_EncodeLatin1(
 917                                PyUnicode_AS_UNICODE(str),
 918                                PyUnicode_GET_SIZE(str),
 919                                errors),
 920                     PyUnicode_GET_SIZE(str));
 921     Py_DECREF(str);
 922     return v;
 923 }
 924
 925 static PyObject *
 926 ascii_encode(PyObject *self,
 927              PyObject *args)
 928 {
 929     PyObject *str, *v;
 930     const char *errors = NULL;
 931
 932     if (!PyArg_ParseTuple(args, "O|z:ascii_encode",
 933                           &str, &errors))
 934         return NULL;
 935
 936     str = PyUnicode_FromObject(str);
 937     if (str == NULL)
 938         return NULL;
 939     v = codec_tuple(PyUnicode_EncodeASCII(
 940                                PyUnicode_AS_UNICODE(str),
 941                                PyUnicode_GET_SIZE(str),
 942                                errors),
 943                     PyUnicode_GET_SIZE(str));
 944     Py_DECREF(str);
 945     return v;
 946 }
 947
 948 static PyObject *
 949 charmap_encode(PyObject *self,
 950              PyObject *args)
 951 {
 952     PyObject *str, *v;
 953     const char *errors = NULL;
 954     PyObject *mapping = NULL;
 955
 956     if (!PyArg_ParseTuple(args, "O|zO:charmap_encode",
 957                           &str, &errors, &mapping))
 958         return NULL;
 959     if (mapping == Py_None)
 960         mapping = NULL;
 961
 962     str = PyUnicode_FromObject(str);
 963     if (str == NULL)
 964         return NULL;
 965     v = codec_tuple(PyUnicode_EncodeCharmap(
 966                                PyUnicode_AS_UNICODE(str),
 967                                PyUnicode_GET_SIZE(str),
 968                                mapping,
 969                                errors),
 970                     PyUnicode_GET_SIZE(str));
 971     Py_DECREF(str);
 972     return v;
 973 }
 974
 975 static PyObject*
 976 charmap_build(PyObject *self, PyObject *args)
 977 {
 978     PyObject *map;
 979     if (!PyArg_ParseTuple(args, "U:charmap_build", &map))
 980         return NULL;
 981     return PyUnicode_BuildEncodingMap(map);
 982 }
 983
 984 #if defined(MS_WINDOWS) && defined(HAVE_USABLE_WCHAR_T)
 985
 986 static PyObject *
 987 mbcs_encode(PyObject *self,
 988             PyObject *args)
 989 {
 990     PyObject *str, *v;
 991     const char *errors = NULL;
 992
 993     if (!PyArg_ParseTuple(args, "O|z:mbcs_encode",
 994                           &str, &errors))
 995         return NULL;
 996
 997     str = PyUnicode_FromObject(str);
 998     if (str == NULL)
 999         return NULL;
1000     v = codec_tuple(PyUnicode_EncodeMBCS(
1001                                PyUnicode_AS_UNICODE(str),
1002                                PyUnicode_GET_SIZE(str),
1003                                errors),
1004                     PyUnicode_GET_SIZE(str));
1005     Py_DECREF(str);
1006     return v;
1007 }
1008
1009 #endif /* MS_WINDOWS */
1010 #endif /* Py_USING_UNICODE */
1011
1012 /* --- Error handler registry --------------------------------------------- */
1013
1014 PyDoc_STRVAR(register_error__doc__,
1015 "register_error(errors, handler)\n\
1016 \n\
1017 Register the specified error handler under the name\n\
1018 errors. handler must be a callable object, that\n\
1019 will be called with an exception instance containing\n\
1020 information about the location of the encoding/decoding\n\
1021 error and must return a (replacement, new position) tuple.");
1022
1023 static PyObject *register_error(PyObject *self, PyObject *args)
1024 {
1025     const char *name;
1026     PyObject *handler;
1027
1028     if (!PyArg_ParseTuple(args, "sO:register_error",
1029                           &name, &handler))
1030         return NULL;
1031     if (PyCodec_RegisterError(name, handler))
1032         return NULL;
1033     Py_RETURN_NONE;
1034 }
1035
1036 PyDoc_STRVAR(lookup_error__doc__,
1037 "lookup_error(errors) -> handler\n\
1038 \n\
1039 Return the error handler for the specified error handling name\n\
1040 or raise a LookupError, if no handler exists under this name.");
1041
1042 static PyObject *lookup_error(PyObject *self, PyObject *args)
1043 {
1044     const char *name;
1045
1046     if (!PyArg_ParseTuple(args, "s:lookup_error",
1047                           &name))
1048         return NULL;
1049     return PyCodec_LookupError(name);
1050 }
1051
1052 /* --- Module API --------------------------------------------------------- */
1053
1054 static PyMethodDef _codecs_functions[] = {
1055     {"register",                codec_register,                 METH_O,
1056         register__doc__},
1057     {"lookup",                  codec_lookup,                   METH_VARARGS,
1058         lookup__doc__},
1059     {"encode",                  codec_encode,                   METH_VARARGS,
1060         encode__doc__},
1061     {"decode",                  codec_decode,                   METH_VARARGS,
1062         decode__doc__},
1063     {"escape_encode",           escape_encode,                  METH_VARARGS},
1064     {"escape_decode",           escape_decode,                  METH_VARARGS},
1065 #ifdef Py_USING_UNICODE
1066     {"utf_8_encode",            utf_8_encode,                   METH_VARARGS},
1067     {"utf_8_decode",            utf_8_decode,                   METH_VARARGS},
1068     {"utf_7_encode",            utf_7_encode,                   METH_VARARGS},
1069     {"utf_7_decode",            utf_7_decode,                   METH_VARARGS},
1070     {"utf_16_encode",           utf_16_encode,                  METH_VARARGS},
1071     {"utf_16_le_encode",        utf_16_le_encode,               METH_VARARGS},
1072     {"utf_16_be_encode",        utf_16_be_encode,               METH_VARARGS},
1073     {"utf_16_decode",           utf_16_decode,                  METH_VARARGS},
1074     {"utf_16_le_decode",        utf_16_le_decode,               METH_VARARGS},
1075     {"utf_16_be_decode",        utf_16_be_decode,               METH_VARARGS},
1076     {"utf_16_ex_decode",        utf_16_ex_decode,               METH_VARARGS},
1077     {"utf_32_encode",           utf_32_encode,                  METH_VARARGS},
1078     {"utf_32_le_encode",        utf_32_le_encode,               METH_VARARGS},
1079     {"utf_32_be_encode",        utf_32_be_encode,               METH_VARARGS},
1080     {"utf_32_decode",           utf_32_decode,                  METH_VARARGS},
1081     {"utf_32_le_decode",        utf_32_le_decode,               METH_VARARGS},
1082     {"utf_32_be_decode",        utf_32_be_decode,               METH_VARARGS},
1083     {"utf_32_ex_decode",        utf_32_ex_decode,               METH_VARARGS},
1084     {"unicode_escape_encode",   unicode_escape_encode,          METH_VARARGS},
1085     {"unicode_escape_decode",   unicode_escape_decode,          METH_VARARGS},
1086     {"unicode_internal_encode", unicode_internal_encode,        METH_VARARGS},
1087     {"unicode_internal_decode", unicode_internal_decode,        METH_VARARGS},
1088     {"raw_unicode_escape_encode", raw_unicode_escape_encode,    METH_VARARGS},
1089     {"raw_unicode_escape_decode", raw_unicode_escape_decode,    METH_VARARGS},
1090     {"latin_1_encode",          latin_1_encode,                 METH_VARARGS},
1091     {"latin_1_decode",          latin_1_decode,                 METH_VARARGS},
1092     {"ascii_encode",            ascii_encode,                   METH_VARARGS},
1093     {"ascii_decode",            ascii_decode,                   METH_VARARGS},
1094     {"charmap_encode",          charmap_encode,                 METH_VARARGS},
1095     {"charmap_decode",          charmap_decode,                 METH_VARARGS},
1096     {"charmap_build",           charmap_build,                  METH_VARARGS},
1097     {"readbuffer_encode",       readbuffer_encode,              METH_VARARGS},
1098     {"charbuffer_encode",       charbuffer_encode,              METH_VARARGS},
1099 #if defined(MS_WINDOWS) && defined(HAVE_USABLE_WCHAR_T)
1100     {"mbcs_encode",             mbcs_encode,                    METH_VARARGS},
1101     {"mbcs_decode",             mbcs_decode,                    METH_VARARGS},
1102 #endif
1103 #endif /* Py_USING_UNICODE */
1104     {"register_error",          register_error,                 METH_VARARGS,
1105         register_error__doc__},
1106     {"lookup_error",            lookup_error,                   METH_VARARGS,
1107         lookup_error__doc__},
1108     {NULL, NULL}                /* sentinel */
1109 };
1110
1111 PyMODINIT_FUNC
1112 init_codecs(void)
1113 {
1114     Py_InitModule("_codecs", _codecs_functions);
1115 }