?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Added updates with respect to recent changes to TimedRotatingFileHandler.
[python.git] / Modules / cjkcodecs / multibytecodec.c
blobbbd4c1a04c8d7bedaef0f98e246bc8f0b3d3a4b4
1 /*
2 * multibytecodec.c: Common Multibyte Codec Implementation
3 *
4 * Written by Hye-Shik Chang <perky@FreeBSD.org>
5 */
6
7 #define PY_SSIZE_T_CLEAN
8 #include "Python.h"
9 #include "structmember.h"
10 #include "multibytecodec.h"
11
12 typedef struct {
13 const Py_UNICODE *inbuf, *inbuf_top, *inbuf_end;
14 unsigned char *outbuf, *outbuf_end;
15 PyObject *excobj, *outobj;
16 } MultibyteEncodeBuffer;
17
18 typedef struct {
19 const unsigned char *inbuf, *inbuf_top, *inbuf_end;
20 Py_UNICODE *outbuf, *outbuf_end;
21 PyObject *excobj, *outobj;
22 } MultibyteDecodeBuffer;
23
24 PyDoc_STRVAR(MultibyteCodec_Encode__doc__,
25 "I.encode(unicode[, errors]) -> (string, length consumed)\n\
26 \n\
27 Return an encoded string version of `unicode'. errors may be given to\n\
28 set a different error handling scheme. Default is 'strict' meaning that\n\
29 encoding errors raise a UnicodeEncodeError. Other possible values are\n\
30 'ignore', 'replace' and 'xmlcharrefreplace' as well as any other name\n\
31 registered with codecs.register_error that can handle UnicodeEncodeErrors.");
32
33 PyDoc_STRVAR(MultibyteCodec_Decode__doc__,
34 "I.decode(string[, errors]) -> (unicodeobject, length consumed)\n\
35 \n\
36 Decodes `string' using I, an MultibyteCodec instance. errors may be given\n\
37 to set a different error handling scheme. Default is 'strict' meaning\n\
38 that encoding errors raise a UnicodeDecodeError. Other possible values\n\
39 are 'ignore' and 'replace' as well as any other name registerd with\n\
40 codecs.register_error that is able to handle UnicodeDecodeErrors.");
41
42 static char *codeckwarglist[] = {"input", "errors", NULL};
43 static char *incnewkwarglist[] = {"errors", NULL};
44 static char *incrementalkwarglist[] = {"input", "final", NULL};
45 static char *streamkwarglist[] = {"stream", "errors", NULL};
46
47 static PyObject *multibytecodec_encode(MultibyteCodec *,
48 MultibyteCodec_State *, const Py_UNICODE **, Py_ssize_t,
49 PyObject *, int);
50
51 #define MBENC_RESET MBENC_MAX<<1 /* reset after an encoding session */
52
53 static PyObject *
54 make_tuple(PyObject *object, Py_ssize_t len)
55 {
56 PyObject *v, *w;
57
58 if (object == NULL)
59 return NULL;
60
61 v = PyTuple_New(2);
62 if (v == NULL) {
63 Py_DECREF(object);
64 return NULL;
65 }
66 PyTuple_SET_ITEM(v, 0, object);
67
68 w = PyInt_FromSsize_t(len);
69 if (w == NULL) {
70 Py_DECREF(v);
71 return NULL;
72 }
73 PyTuple_SET_ITEM(v, 1, w);
74
75 return v;
76 }
77
78 static PyObject *
79 internal_error_callback(const char *errors)
80 {
81 if (errors == NULL || strcmp(errors, "strict") == 0)
82 return ERROR_STRICT;
83 else if (strcmp(errors, "ignore") == 0)
84 return ERROR_IGNORE;
85 else if (strcmp(errors, "replace") == 0)
86 return ERROR_REPLACE;
87 else
88 return PyString_FromString(errors);
89 }
90
91 static PyObject *
92 call_error_callback(PyObject *errors, PyObject *exc)
93 {
94 PyObject *args, *cb, *r;
95
96 assert(PyString_Check(errors));
97 cb = PyCodec_LookupError(PyString_AS_STRING(errors));
98 if (cb == NULL)
99 return NULL;
100
101 args = PyTuple_New(1);
102 if (args == NULL) {
103 Py_DECREF(cb);
104 return NULL;
105 }
106
107 PyTuple_SET_ITEM(args, 0, exc);
108 Py_INCREF(exc);
109
110 r = PyObject_CallObject(cb, args);
111 Py_DECREF(args);
112 Py_DECREF(cb);
113 return r;
114 }
115
116 static PyObject *
117 codecctx_errors_get(MultibyteStatefulCodecContext *self)
118 {
119 const char *errors;
120
121 if (self->errors == ERROR_STRICT)
122 errors = "strict";
123 else if (self->errors == ERROR_IGNORE)
124 errors = "ignore";
125 else if (self->errors == ERROR_REPLACE)
126 errors = "replace";
127 else {
128 Py_INCREF(self->errors);
129 return self->errors;
130 }
131
132 return PyString_FromString(errors);
133 }
134
135 static int
136 codecctx_errors_set(MultibyteStatefulCodecContext *self, PyObject *value,
137 void *closure)
138 {
139 PyObject *cb;
140
141 if (!PyString_Check(value)) {
142 PyErr_SetString(PyExc_TypeError, "errors must be a string");
143 return -1;
144 }
145
146 cb = internal_error_callback(PyString_AS_STRING(value));
147 if (cb == NULL)
148 return -1;
149
150 ERROR_DECREF(self->errors);
151 self->errors = cb;
152 return 0;
153 }
154
155 /* This getset handlers list is used by all the stateful codec objects */
156 static PyGetSetDef codecctx_getsets[] = {
157 {"errors", (getter)codecctx_errors_get,
158 (setter)codecctx_errors_set,
159 PyDoc_STR("how to treat errors")},
160 {NULL,}
161 };
162
163 static int
164 expand_encodebuffer(MultibyteEncodeBuffer *buf, Py_ssize_t esize)
165 {
166 Py_ssize_t orgpos, orgsize;
167
168 orgpos = (Py_ssize_t)((char *)buf->outbuf -
169 PyString_AS_STRING(buf->outobj));
170 orgsize = PyString_GET_SIZE(buf->outobj);
171 if (_PyString_Resize(&buf->outobj, orgsize + (
172 esize < (orgsize >> 1) ? (orgsize >> 1) | 1 : esize)) == -1)
173 return -1;
174
175 buf->outbuf = (unsigned char *)PyString_AS_STRING(buf->outobj) +orgpos;
176 buf->outbuf_end = (unsigned char *)PyString_AS_STRING(buf->outobj)
177 + PyString_GET_SIZE(buf->outobj);
178
179 return 0;
180 }
181 #define REQUIRE_ENCODEBUFFER(buf, s) { \
182 if ((s) < 1 || (buf)->outbuf + (s) > (buf)->outbuf_end) \
183 if (expand_encodebuffer(buf, s) == -1) \
184 goto errorexit; \
185 }
186
187 static int
188 expand_decodebuffer(MultibyteDecodeBuffer *buf, Py_ssize_t esize)
189 {
190 Py_ssize_t orgpos, orgsize;
191
192 orgpos = (Py_ssize_t)(buf->outbuf - PyUnicode_AS_UNICODE(buf->outobj));
193 orgsize = PyUnicode_GET_SIZE(buf->outobj);
194 if (PyUnicode_Resize(&buf->outobj, orgsize + (
195 esize < (orgsize >> 1) ? (orgsize >> 1) | 1 : esize)) == -1)
196 return -1;
197
198 buf->outbuf = PyUnicode_AS_UNICODE(buf->outobj) + orgpos;
199 buf->outbuf_end = PyUnicode_AS_UNICODE(buf->outobj)
200 + PyUnicode_GET_SIZE(buf->outobj);
201
202 return 0;
203 }
204 #define REQUIRE_DECODEBUFFER(buf, s) { \
205 if ((s) < 1 || (buf)->outbuf + (s) > (buf)->outbuf_end) \
206 if (expand_decodebuffer(buf, s) == -1) \
207 goto errorexit; \
208 }
209
210
211 /**
212 * MultibyteCodec object
213 */
214
215 static int
216 multibytecodec_encerror(MultibyteCodec *codec,
217 MultibyteCodec_State *state,
218 MultibyteEncodeBuffer *buf,
219 PyObject *errors, Py_ssize_t e)
220 {
221 PyObject *retobj = NULL, *retstr = NULL, *tobj;
222 Py_ssize_t retstrsize, newpos;
223 Py_ssize_t esize, start, end;
224 const char *reason;
225
226 if (e > 0) {
227 reason = "illegal multibyte sequence";
228 esize = e;
229 }
230 else {
231 switch (e) {
232 case MBERR_TOOSMALL:
233 REQUIRE_ENCODEBUFFER(buf, -1);
234 return 0; /* retry it */
235 case MBERR_TOOFEW:
236 reason = "incomplete multibyte sequence";
237 esize = (Py_ssize_t)(buf->inbuf_end - buf->inbuf);
238 break;
239 case MBERR_INTERNAL:
240 PyErr_SetString(PyExc_RuntimeError,
241 "internal codec error");
242 return -1;
243 default:
244 PyErr_SetString(PyExc_RuntimeError,
245 "unknown runtime error");
246 return -1;
247 }
248 }
249
250 if (errors == ERROR_REPLACE) {
251 const Py_UNICODE replchar = '?', *inbuf = &replchar;
252 Py_ssize_t r;
253
254 for (;;) {
255 Py_ssize_t outleft;
256
257 outleft = (Py_ssize_t)(buf->outbuf_end - buf->outbuf);
258 r = codec->encode(state, codec->config, &inbuf, 1,
259 &buf->outbuf, outleft, 0);
260 if (r == MBERR_TOOSMALL) {
261 REQUIRE_ENCODEBUFFER(buf, -1);
262 continue;
263 }
264 else
265 break;
266 }
267
268 if (r != 0) {
269 REQUIRE_ENCODEBUFFER(buf, 1);
270 *buf->outbuf++ = '?';
271 }
272 }
273 if (errors == ERROR_IGNORE || errors == ERROR_REPLACE) {
274 buf->inbuf += esize;
275 return 0;
276 }
277
278 start = (Py_ssize_t)(buf->inbuf - buf->inbuf_top);
279 end = start + esize;
280
281 /* use cached exception object if available */
282 if (buf->excobj == NULL) {
283 buf->excobj = PyUnicodeEncodeError_Create(codec->encoding,
284 buf->inbuf_top,
285 buf->inbuf_end - buf->inbuf_top,
286 start, end, reason);
287 if (buf->excobj == NULL)
288 goto errorexit;
289 }
290 else
291 if (PyUnicodeEncodeError_SetStart(buf->excobj, start) != 0 ||
292 PyUnicodeEncodeError_SetEnd(buf->excobj, end) != 0 ||
293 PyUnicodeEncodeError_SetReason(buf->excobj, reason) != 0)
294 goto errorexit;
295
296 if (errors == ERROR_STRICT) {
297 PyCodec_StrictErrors(buf->excobj);
298 goto errorexit;
299 }
300
301 retobj = call_error_callback(errors, buf->excobj);
302 if (retobj == NULL)
303 goto errorexit;
304
305 if (!PyTuple_Check(retobj) || PyTuple_GET_SIZE(retobj) != 2 ||
306 !PyUnicode_Check((tobj = PyTuple_GET_ITEM(retobj, 0))) ||
307 !(PyInt_Check(PyTuple_GET_ITEM(retobj, 1)) ||
308 PyLong_Check(PyTuple_GET_ITEM(retobj, 1)))) {
309 PyErr_SetString(PyExc_TypeError,
310 "encoding error handler must return "
311 "(unicode, int) tuple");
312 goto errorexit;
313 }
314
315 {
316 const Py_UNICODE *uraw = PyUnicode_AS_UNICODE(tobj);
317
318 retstr = multibytecodec_encode(codec, state, &uraw,
319 PyUnicode_GET_SIZE(tobj), ERROR_STRICT,
320 MBENC_FLUSH);
321 if (retstr == NULL)
322 goto errorexit;
323 }
324
325 retstrsize = PyString_GET_SIZE(retstr);
326 REQUIRE_ENCODEBUFFER(buf, retstrsize);
327
328 memcpy(buf->outbuf, PyString_AS_STRING(retstr), retstrsize);
329 buf->outbuf += retstrsize;
330
331 newpos = PyInt_AsSsize_t(PyTuple_GET_ITEM(retobj, 1));
332 if (newpos < 0 && !PyErr_Occurred())
333 newpos += (Py_ssize_t)(buf->inbuf_end - buf->inbuf_top);
334 if (newpos < 0 || buf->inbuf_top + newpos > buf->inbuf_end) {
335 PyErr_Clear();
336 PyErr_Format(PyExc_IndexError,
337 "position %zd from error handler out of bounds",
338 newpos);
339 goto errorexit;
340 }
341 buf->inbuf = buf->inbuf_top + newpos;
342
343 Py_DECREF(retobj);
344 Py_DECREF(retstr);
345 return 0;
346
347 errorexit:
348 Py_XDECREF(retobj);
349 Py_XDECREF(retstr);
350 return -1;
351 }
352
353 static int
354 multibytecodec_decerror(MultibyteCodec *codec,
355 MultibyteCodec_State *state,
356 MultibyteDecodeBuffer *buf,
357 PyObject *errors, Py_ssize_t e)
358 {
359 PyObject *retobj = NULL, *retuni = NULL;
360 Py_ssize_t retunisize, newpos;
361 const char *reason;
362 Py_ssize_t esize, start, end;
363
364 if (e > 0) {
365 reason = "illegal multibyte sequence";
366 esize = e;
367 }
368 else {
369 switch (e) {
370 case MBERR_TOOSMALL:
371 REQUIRE_DECODEBUFFER(buf, -1);
372 return 0; /* retry it */
373 case MBERR_TOOFEW:
374 reason = "incomplete multibyte sequence";
375 esize = (Py_ssize_t)(buf->inbuf_end - buf->inbuf);
376 break;
377 case MBERR_INTERNAL:
378 PyErr_SetString(PyExc_RuntimeError,
379 "internal codec error");
380 return -1;
381 default:
382 PyErr_SetString(PyExc_RuntimeError,
383 "unknown runtime error");
384 return -1;
385 }
386 }
387
388 if (errors == ERROR_REPLACE) {
389 REQUIRE_DECODEBUFFER(buf, 1);
390 *buf->outbuf++ = Py_UNICODE_REPLACEMENT_CHARACTER;
391 }
392 if (errors == ERROR_IGNORE || errors == ERROR_REPLACE) {
393 buf->inbuf += esize;
394 return 0;
395 }
396
397 start = (Py_ssize_t)(buf->inbuf - buf->inbuf_top);
398 end = start + esize;
399
400 /* use cached exception object if available */
401 if (buf->excobj == NULL) {
402 buf->excobj = PyUnicodeDecodeError_Create(codec->encoding,
403 (const char *)buf->inbuf_top,
404 (Py_ssize_t)(buf->inbuf_end - buf->inbuf_top),
405 start, end, reason);
406 if (buf->excobj == NULL)
407 goto errorexit;
408 }
409 else
410 if (PyUnicodeDecodeError_SetStart(buf->excobj, start) ||
411 PyUnicodeDecodeError_SetEnd(buf->excobj, end) ||
412 PyUnicodeDecodeError_SetReason(buf->excobj, reason))
413 goto errorexit;
414
415 if (errors == ERROR_STRICT) {
416 PyCodec_StrictErrors(buf->excobj);
417 goto errorexit;
418 }
419
420 retobj = call_error_callback(errors, buf->excobj);
421 if (retobj == NULL)
422 goto errorexit;
423
424 if (!PyTuple_Check(retobj) || PyTuple_GET_SIZE(retobj) != 2 ||
425 !PyUnicode_Check((retuni = PyTuple_GET_ITEM(retobj, 0))) ||
426 !(PyInt_Check(PyTuple_GET_ITEM(retobj, 1)) ||
427 PyLong_Check(PyTuple_GET_ITEM(retobj, 1)))) {
428 PyErr_SetString(PyExc_TypeError,
429 "decoding error handler must return "
430 "(unicode, int) tuple");
431 goto errorexit;
432 }
433
434 retunisize = PyUnicode_GET_SIZE(retuni);
435 if (retunisize > 0) {
436 REQUIRE_DECODEBUFFER(buf, retunisize);
437 memcpy((char *)buf->outbuf, PyUnicode_AS_DATA(retuni),
438 retunisize * Py_UNICODE_SIZE);
439 buf->outbuf += retunisize;
440 }
441
442 newpos = PyInt_AsSsize_t(PyTuple_GET_ITEM(retobj, 1));
443 if (newpos < 0 && !PyErr_Occurred())
444 newpos += (Py_ssize_t)(buf->inbuf_end - buf->inbuf_top);
445 if (newpos < 0 || buf->inbuf_top + newpos > buf->inbuf_end) {
446 PyErr_Clear();
447 PyErr_Format(PyExc_IndexError,
448 "position %zd from error handler out of bounds",
449 newpos);
450 goto errorexit;
451 }
452 buf->inbuf = buf->inbuf_top + newpos;
453 Py_DECREF(retobj);
454 return 0;
455
456 errorexit:
457 Py_XDECREF(retobj);
458 return -1;
459 }
460
461 static PyObject *
462 multibytecodec_encode(MultibyteCodec *codec,
463 MultibyteCodec_State *state,
464 const Py_UNICODE **data, Py_ssize_t datalen,
465 PyObject *errors, int flags)
466 {
467 MultibyteEncodeBuffer buf;
468 Py_ssize_t finalsize, r = 0;
469
470 if (datalen == 0)
471 return PyString_FromString("");
472
473 buf.excobj = NULL;
474 buf.inbuf = buf.inbuf_top = *data;
475 buf.inbuf_end = buf.inbuf_top + datalen;
476 buf.outobj = PyString_FromStringAndSize(NULL, datalen * 2 + 16);
477 if (buf.outobj == NULL)
478 goto errorexit;
479 buf.outbuf = (unsigned char *)PyString_AS_STRING(buf.outobj);
480 buf.outbuf_end = buf.outbuf + PyString_GET_SIZE(buf.outobj);
481
482 while (buf.inbuf < buf.inbuf_end) {
483 Py_ssize_t inleft, outleft;
484
485 /* we don't reuse inleft and outleft here.
486 * error callbacks can relocate the cursor anywhere on buffer*/
487 inleft = (Py_ssize_t)(buf.inbuf_end - buf.inbuf);
488 outleft = (Py_ssize_t)(buf.outbuf_end - buf.outbuf);
489 r = codec->encode(state, codec->config, &buf.inbuf, inleft,
490 &buf.outbuf, outleft, flags);
491 *data = buf.inbuf;
492 if ((r == 0) || (r == MBERR_TOOFEW && !(flags & MBENC_FLUSH)))
493 break;
494 else if (multibytecodec_encerror(codec, state, &buf, errors,r))
495 goto errorexit;
496 else if (r == MBERR_TOOFEW)
497 break;
498 }
499
500 if (codec->encreset != NULL)
501 for (;;) {
502 Py_ssize_t outleft;
503
504 outleft = (Py_ssize_t)(buf.outbuf_end - buf.outbuf);
505 r = codec->encreset(state, codec->config, &buf.outbuf,
506 outleft);
507 if (r == 0)
508 break;
509 else if (multibytecodec_encerror(codec, state,
510 &buf, errors, r))
511 goto errorexit;
512 }
513
514 finalsize = (Py_ssize_t)((char *)buf.outbuf -
515 PyString_AS_STRING(buf.outobj));
516
517 if (finalsize != PyString_GET_SIZE(buf.outobj))
518 if (_PyString_Resize(&buf.outobj, finalsize) == -1)
519 goto errorexit;
520
521 Py_XDECREF(buf.excobj);
522 return buf.outobj;
523
524 errorexit:
525 Py_XDECREF(buf.excobj);
526 Py_XDECREF(buf.outobj);
527 return NULL;
528 }
529
530 static PyObject *
531 MultibyteCodec_Encode(MultibyteCodecObject *self,
532 PyObject *args, PyObject *kwargs)
533 {
534 MultibyteCodec_State state;
535 Py_UNICODE *data;
536 PyObject *errorcb, *r, *arg, *ucvt;
537 const char *errors = NULL;
538 Py_ssize_t datalen;
539
540 if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|z:encode",
541 codeckwarglist, &arg, &errors))
542 return NULL;
543
544 if (PyUnicode_Check(arg))
545 ucvt = NULL;
546 else {
547 arg = ucvt = PyObject_Unicode(arg);
548 if (arg == NULL)
549 return NULL;
550 else if (!PyUnicode_Check(arg)) {
551 PyErr_SetString(PyExc_TypeError,
552 "couldn't convert the object to unicode.");
553 Py_DECREF(ucvt);
554 return NULL;
555 }
556 }
557
558 data = PyUnicode_AS_UNICODE(arg);
559 datalen = PyUnicode_GET_SIZE(arg);
560
561 errorcb = internal_error_callback(errors);
562 if (errorcb == NULL) {
563 Py_XDECREF(ucvt);
564 return NULL;
565 }
566
567 if (self->codec->encinit != NULL &&
568 self->codec->encinit(&state, self->codec->config) != 0)
569 goto errorexit;
570 r = multibytecodec_encode(self->codec, &state,
571 (const Py_UNICODE **)&data, datalen, errorcb,
572 MBENC_FLUSH | MBENC_RESET);
573 if (r == NULL)
574 goto errorexit;
575
576 ERROR_DECREF(errorcb);
577 Py_XDECREF(ucvt);
578 return make_tuple(r, datalen);
579
580 errorexit:
581 ERROR_DECREF(errorcb);
582 Py_XDECREF(ucvt);
583 return NULL;
584 }
585
586 static PyObject *
587 MultibyteCodec_Decode(MultibyteCodecObject *self,
588 PyObject *args, PyObject *kwargs)
589 {
590 MultibyteCodec_State state;
591 MultibyteDecodeBuffer buf;
592 PyObject *errorcb;
593 const char *data, *errors = NULL;
594 Py_ssize_t datalen, finalsize;
595
596 if (!PyArg_ParseTupleAndKeywords(args, kwargs, "s#|z:decode",
597 codeckwarglist, &data, &datalen, &errors))
598 return NULL;
599
600 errorcb = internal_error_callback(errors);
601 if (errorcb == NULL)
602 return NULL;
603
604 if (datalen == 0) {
605 ERROR_DECREF(errorcb);
606 return make_tuple(PyUnicode_FromUnicode(NULL, 0), 0);
607 }
608
609 buf.excobj = NULL;
610 buf.inbuf = buf.inbuf_top = (unsigned char *)data;
611 buf.inbuf_end = buf.inbuf_top + datalen;
612 buf.outobj = PyUnicode_FromUnicode(NULL, datalen);
613 if (buf.outobj == NULL)
614 goto errorexit;
615 buf.outbuf = PyUnicode_AS_UNICODE(buf.outobj);
616 buf.outbuf_end = buf.outbuf + PyUnicode_GET_SIZE(buf.outobj);
617
618 if (self->codec->decinit != NULL &&
619 self->codec->decinit(&state, self->codec->config) != 0)
620 goto errorexit;
621
622 while (buf.inbuf < buf.inbuf_end) {
623 Py_ssize_t inleft, outleft, r;
624
625 inleft = (Py_ssize_t)(buf.inbuf_end - buf.inbuf);
626 outleft = (Py_ssize_t)(buf.outbuf_end - buf.outbuf);
627
628 r = self->codec->decode(&state, self->codec->config,
629 &buf.inbuf, inleft, &buf.outbuf, outleft);
630 if (r == 0)
631 break;
632 else if (multibytecodec_decerror(self->codec, &state,
633 &buf, errorcb, r))
634 goto errorexit;
635 }
636
637 finalsize = (Py_ssize_t)(buf.outbuf -
638 PyUnicode_AS_UNICODE(buf.outobj));
639
640 if (finalsize != PyUnicode_GET_SIZE(buf.outobj))
641 if (PyUnicode_Resize(&buf.outobj, finalsize) == -1)
642 goto errorexit;
643
644 Py_XDECREF(buf.excobj);
645 ERROR_DECREF(errorcb);
646 return make_tuple(buf.outobj, datalen);
647
648 errorexit:
649 ERROR_DECREF(errorcb);
650 Py_XDECREF(buf.excobj);
651 Py_XDECREF(buf.outobj);
652
653 return NULL;
654 }
655
656 static struct PyMethodDef multibytecodec_methods[] = {
657 {"encode", (PyCFunction)MultibyteCodec_Encode,
658 METH_VARARGS | METH_KEYWORDS,
659 MultibyteCodec_Encode__doc__},
660 {"decode", (PyCFunction)MultibyteCodec_Decode,
661 METH_VARARGS | METH_KEYWORDS,
662 MultibyteCodec_Decode__doc__},
663 {NULL, NULL},
664 };
665
666 static void
667 multibytecodec_dealloc(MultibyteCodecObject *self)
668 {
669 PyObject_Del(self);
670 }
671
672 static PyTypeObject MultibyteCodec_Type = {
673 PyVarObject_HEAD_INIT(NULL, 0)
674 "MultibyteCodec", /* tp_name */
675 sizeof(MultibyteCodecObject), /* tp_basicsize */
676 0, /* tp_itemsize */
677 /* methods */
678 (destructor)multibytecodec_dealloc, /* tp_dealloc */
679 0, /* tp_print */
680 0, /* tp_getattr */
681 0, /* tp_setattr */
682 0, /* tp_compare */
683 0, /* tp_repr */
684 0, /* tp_as_number */
685 0, /* tp_as_sequence */
686 0, /* tp_as_mapping */
687 0, /* tp_hash */
688 0, /* tp_call */
689 0, /* tp_str */
690 PyObject_GenericGetAttr, /* tp_getattro */
691 0, /* tp_setattro */
692 0, /* tp_as_buffer */
693 Py_TPFLAGS_DEFAULT, /* tp_flags */
694 0, /* tp_doc */
695 0, /* tp_traverse */
696 0, /* tp_clear */
697 0, /* tp_richcompare */
698 0, /* tp_weaklistoffset */
699 0, /* tp_iter */
700 0, /* tp_iterext */
701 multibytecodec_methods, /* tp_methods */
702 };
703
704
705 /**
706 * Utility functions for stateful codec mechanism
707 */
708
709 #define STATEFUL_DCTX(o) ((MultibyteStatefulDecoderContext *)(o))
710 #define STATEFUL_ECTX(o) ((MultibyteStatefulEncoderContext *)(o))
711
712 static PyObject *
713 encoder_encode_stateful(MultibyteStatefulEncoderContext *ctx,
714 PyObject *unistr, int final)
715 {
716 PyObject *ucvt, *r = NULL;
717 Py_UNICODE *inbuf, *inbuf_end, *inbuf_tmp = NULL;
718 Py_ssize_t datalen, origpending;
719
720 if (PyUnicode_Check(unistr))
721 ucvt = NULL;
722 else {
723 unistr = ucvt = PyObject_Unicode(unistr);
724 if (unistr == NULL)
725 return NULL;
726 else if (!PyUnicode_Check(unistr)) {
727 PyErr_SetString(PyExc_TypeError,
728 "couldn't convert the object to unicode.");
729 Py_DECREF(ucvt);
730 return NULL;
731 }
732 }
733
734 datalen = PyUnicode_GET_SIZE(unistr);
735 origpending = ctx->pendingsize;
736
737 if (origpending > 0) {
738 inbuf_tmp = PyMem_New(Py_UNICODE, datalen + ctx->pendingsize);
739 if (inbuf_tmp == NULL)
740 goto errorexit;
741 memcpy(inbuf_tmp, ctx->pending,
742 Py_UNICODE_SIZE * ctx->pendingsize);
743 memcpy(inbuf_tmp + ctx->pendingsize,
744 PyUnicode_AS_UNICODE(unistr),
745 Py_UNICODE_SIZE * datalen);
746 datalen += ctx->pendingsize;
747 ctx->pendingsize = 0;
748 inbuf = inbuf_tmp;
749 }
750 else
751 inbuf = (Py_UNICODE *)PyUnicode_AS_UNICODE(unistr);
752
753 inbuf_end = inbuf + datalen;
754
755 r = multibytecodec_encode(ctx->codec, &ctx->state,
756 (const Py_UNICODE **)&inbuf,
757 datalen, ctx->errors, final ? MBENC_FLUSH : 0);
758 if (r == NULL) {
759 /* recover the original pending buffer */
760 if (origpending > 0)
761 memcpy(ctx->pending, inbuf_tmp,
762 Py_UNICODE_SIZE * origpending);
763 ctx->pendingsize = origpending;
764 goto errorexit;
765 }
766
767 if (inbuf < inbuf_end) {
768 ctx->pendingsize = (Py_ssize_t)(inbuf_end - inbuf);
769 if (ctx->pendingsize > MAXENCPENDING) {
770 /* normal codecs can't reach here */
771 ctx->pendingsize = 0;
772 PyErr_SetString(PyExc_UnicodeError,
773 "pending buffer overflow");
774 goto errorexit;
775 }
776 memcpy(ctx->pending, inbuf,
777 ctx->pendingsize * Py_UNICODE_SIZE);
778 }
779
780 if (inbuf_tmp != NULL)
781 PyMem_Del(inbuf_tmp);
782 Py_XDECREF(ucvt);
783 return r;
784
785 errorexit:
786 if (inbuf_tmp != NULL)
787 PyMem_Del(inbuf_tmp);
788 Py_XDECREF(r);
789 Py_XDECREF(ucvt);
790 return NULL;
791 }
792
793 static int
794 decoder_append_pending(MultibyteStatefulDecoderContext *ctx,
795 MultibyteDecodeBuffer *buf)
796 {
797 Py_ssize_t npendings;
798
799 npendings = (Py_ssize_t)(buf->inbuf_end - buf->inbuf);
800 if (npendings + ctx->pendingsize > MAXDECPENDING) {
801 PyErr_SetString(PyExc_UnicodeError, "pending buffer overflow");
802 return -1;
803 }
804 memcpy(ctx->pending + ctx->pendingsize, buf->inbuf, npendings);
805 ctx->pendingsize += npendings;
806 return 0;
807 }
808
809 static int
810 decoder_prepare_buffer(MultibyteDecodeBuffer *buf, const char *data,
811 Py_ssize_t size)
812 {
813 buf->inbuf = buf->inbuf_top = (const unsigned char *)data;
814 buf->inbuf_end = buf->inbuf_top + size;
815 if (buf->outobj == NULL) { /* only if outobj is not allocated yet */
816 buf->outobj = PyUnicode_FromUnicode(NULL, size);
817 if (buf->outobj == NULL)
818 return -1;
819 buf->outbuf = PyUnicode_AS_UNICODE(buf->outobj);
820 buf->outbuf_end = buf->outbuf +
821 PyUnicode_GET_SIZE(buf->outobj);
822 }
823
824 return 0;
825 }
826
827 static int
828 decoder_feed_buffer(MultibyteStatefulDecoderContext *ctx,
829 MultibyteDecodeBuffer *buf)
830 {
831 while (buf->inbuf < buf->inbuf_end) {
832 Py_ssize_t inleft, outleft;
833 Py_ssize_t r;
834
835 inleft = (Py_ssize_t)(buf->inbuf_end - buf->inbuf);
836 outleft = (Py_ssize_t)(buf->outbuf_end - buf->outbuf);
837
838 r = ctx->codec->decode(&ctx->state, ctx->codec->config,
839 &buf->inbuf, inleft, &buf->outbuf, outleft);
840 if (r == 0 || r == MBERR_TOOFEW)
841 break;
842 else if (multibytecodec_decerror(ctx->codec, &ctx->state,
843 buf, ctx->errors, r))
844 return -1;
845 }
846 return 0;
847 }
848
849
850 /**
851 * MultibyteIncrementalEncoder object
852 */
853
854 static PyObject *
855 mbiencoder_encode(MultibyteIncrementalEncoderObject *self,
856 PyObject *args, PyObject *kwargs)
857 {
858 PyObject *data;
859 int final = 0;
860
861 if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|i:encode",
862 incrementalkwarglist, &data, &final))
863 return NULL;
864
865 return encoder_encode_stateful(STATEFUL_ECTX(self), data, final);
866 }
867
868 static PyObject *
869 mbiencoder_reset(MultibyteIncrementalEncoderObject *self)
870 {
871 if (self->codec->decreset != NULL &&
872 self->codec->decreset(&self->state, self->codec->config) != 0)
873 return NULL;
874 self->pendingsize = 0;
875
876 Py_RETURN_NONE;
877 }
878
879 static struct PyMethodDef mbiencoder_methods[] = {
880 {"encode", (PyCFunction)mbiencoder_encode,
881 METH_VARARGS | METH_KEYWORDS, NULL},
882 {"reset", (PyCFunction)mbiencoder_reset,
883 METH_NOARGS, NULL},
884 {NULL, NULL},
885 };
886
887 static PyObject *
888 mbiencoder_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
889 {
890 MultibyteIncrementalEncoderObject *self;
891 PyObject *codec = NULL;
892 char *errors = NULL;
893
894 if (!PyArg_ParseTupleAndKeywords(args, kwds, "|s:IncrementalEncoder",
895 incnewkwarglist, &errors))
896 return NULL;
897
898 self = (MultibyteIncrementalEncoderObject *)type->tp_alloc(type, 0);
899 if (self == NULL)
900 return NULL;
901
902 codec = PyObject_GetAttrString((PyObject *)type, "codec");
903 if (codec == NULL)
904 goto errorexit;
905 if (!MultibyteCodec_Check(codec)) {
906 PyErr_SetString(PyExc_TypeError, "codec is unexpected type");
907 goto errorexit;
908 }
909
910 self->codec = ((MultibyteCodecObject *)codec)->codec;
911 self->pendingsize = 0;
912 self->errors = internal_error_callback(errors);
913 if (self->errors == NULL)
914 goto errorexit;
915 if (self->codec->encinit != NULL &&
916 self->codec->encinit(&self->state, self->codec->config) != 0)
917 goto errorexit;
918
919 Py_DECREF(codec);
920 return (PyObject *)self;
921
922 errorexit:
923 Py_XDECREF(self);
924 Py_XDECREF(codec);
925 return NULL;
926 }
927
928 static int
929 mbiencoder_init(PyObject *self, PyObject *args, PyObject *kwds)
930 {
931 return 0;
932 }
933
934 static int
935 mbiencoder_traverse(MultibyteIncrementalEncoderObject *self,
936 visitproc visit, void *arg)
937 {
938 if (ERROR_ISCUSTOM(self->errors))
939 Py_VISIT(self->errors);
940 return 0;
941 }
942
943 static void
944 mbiencoder_dealloc(MultibyteIncrementalEncoderObject *self)
945 {
946 PyObject_GC_UnTrack(self);
947 ERROR_DECREF(self->errors);
948 Py_TYPE(self)->tp_free(self);
949 }
950
951 static PyTypeObject MultibyteIncrementalEncoder_Type = {
952 PyVarObject_HEAD_INIT(NULL, 0)
953 "MultibyteIncrementalEncoder", /* tp_name */
954 sizeof(MultibyteIncrementalEncoderObject), /* tp_basicsize */
955 0, /* tp_itemsize */
956 /* methods */
957 (destructor)mbiencoder_dealloc, /* tp_dealloc */
958 0, /* tp_print */
959 0, /* tp_getattr */
960 0, /* tp_setattr */
961 0, /* tp_compare */
962 0, /* tp_repr */
963 0, /* tp_as_number */
964 0, /* tp_as_sequence */
965 0, /* tp_as_mapping */
966 0, /* tp_hash */
967 0, /* tp_call */
968 0, /* tp_str */
969 PyObject_GenericGetAttr, /* tp_getattro */
970 0, /* tp_setattro */
971 0, /* tp_as_buffer */
972 Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC
973 | Py_TPFLAGS_BASETYPE, /* tp_flags */
974 0, /* tp_doc */
975 (traverseproc)mbiencoder_traverse, /* tp_traverse */
976 0, /* tp_clear */
977 0, /* tp_richcompare */
978 0, /* tp_weaklistoffset */
979 0, /* tp_iter */
980 0, /* tp_iterext */
981 mbiencoder_methods, /* tp_methods */
982 0, /* tp_members */
983 codecctx_getsets, /* tp_getset */
984 0, /* tp_base */
985 0, /* tp_dict */
986 0, /* tp_descr_get */
987 0, /* tp_descr_set */
988 0, /* tp_dictoffset */
989 mbiencoder_init, /* tp_init */
990 0, /* tp_alloc */
991 mbiencoder_new, /* tp_new */
992 };
993
994
995 /**
996 * MultibyteIncrementalDecoder object
997 */
998
999 static PyObject *
1000 mbidecoder_decode(MultibyteIncrementalDecoderObject *self,
1001 PyObject *args, PyObject *kwargs)
1002 {
1003 MultibyteDecodeBuffer buf;
1004 char *data, *wdata;
1005 Py_ssize_t wsize, finalsize = 0, size, origpending;
1006 int final = 0;
1007
1008 if (!PyArg_ParseTupleAndKeywords(args, kwargs, "t#|i:decode",
1009 incrementalkwarglist, &data, &size, &final))
1010 return NULL;
1011
1012 buf.outobj = buf.excobj = NULL;
1013 origpending = self->pendingsize;
1014
1015 if (self->pendingsize == 0) {
1016 wsize = size;
1017 wdata = data;
1018 }
1019 else {
1020 wsize = size + self->pendingsize;
1021 wdata = PyMem_Malloc(wsize);
1022 if (wdata == NULL)
1023 goto errorexit;
1024 memcpy(wdata, self->pending, self->pendingsize);
1025 memcpy(wdata + self->pendingsize, data, size);
1026 self->pendingsize = 0;
1027 }
1028
1029 if (decoder_prepare_buffer(&buf, wdata, wsize) != 0)
1030 goto errorexit;
1031
1032 if (decoder_feed_buffer(STATEFUL_DCTX(self), &buf))
1033 goto errorexit;
1034
1035 if (final && buf.inbuf < buf.inbuf_end) {
1036 if (multibytecodec_decerror(self->codec, &self->state,
1037 &buf, self->errors, MBERR_TOOFEW)) {
1038 /* recover the original pending buffer */
1039 memcpy(self->pending, wdata, origpending);
1040 self->pendingsize = origpending;
1041 goto errorexit;
1042 }
1043 }
1044
1045 if (buf.inbuf < buf.inbuf_end) { /* pending sequence still exists */
1046 if (decoder_append_pending(STATEFUL_DCTX(self), &buf) != 0)
1047 goto errorexit;
1048 }
1049
1050 finalsize = (Py_ssize_t)(buf.outbuf - PyUnicode_AS_UNICODE(buf.outobj));
1051 if (finalsize != PyUnicode_GET_SIZE(buf.outobj))
1052 if (PyUnicode_Resize(&buf.outobj, finalsize) == -1)
1053 goto errorexit;
1054
1055 if (wdata != data)
1056 PyMem_Del(wdata);
1057 Py_XDECREF(buf.excobj);
1058 return buf.outobj;
1059
1060 errorexit:
1061 if (wdata != NULL && wdata != data)
1062 PyMem_Del(wdata);
1063 Py_XDECREF(buf.excobj);
1064 Py_XDECREF(buf.outobj);
1065 return NULL;
1066 }
1067
1068 static PyObject *
1069 mbidecoder_reset(MultibyteIncrementalDecoderObject *self)
1070 {
1071 if (self->codec->decreset != NULL &&
1072 self->codec->decreset(&self->state, self->codec->config) != 0)
1073 return NULL;
1074 self->pendingsize = 0;
1075
1076 Py_RETURN_NONE;
1077 }
1078
1079 static struct PyMethodDef mbidecoder_methods[] = {
1080 {"decode", (PyCFunction)mbidecoder_decode,
1081 METH_VARARGS | METH_KEYWORDS, NULL},
1082 {"reset", (PyCFunction)mbidecoder_reset,
1083 METH_NOARGS, NULL},
1084 {NULL, NULL},
1085 };
1086
1087 static PyObject *
1088 mbidecoder_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
1089 {
1090 MultibyteIncrementalDecoderObject *self;
1091 PyObject *codec = NULL;
1092 char *errors = NULL;
1093
1094 if (!PyArg_ParseTupleAndKeywords(args, kwds, "|s:IncrementalDecoder",
1095 incnewkwarglist, &errors))
1096 return NULL;
1097
1098 self = (MultibyteIncrementalDecoderObject *)type->tp_alloc(type, 0);
1099 if (self == NULL)
1100 return NULL;
1101
1102 codec = PyObject_GetAttrString((PyObject *)type, "codec");
1103 if (codec == NULL)
1104 goto errorexit;
1105 if (!MultibyteCodec_Check(codec)) {
1106 PyErr_SetString(PyExc_TypeError, "codec is unexpected type");
1107 goto errorexit;
1108 }
1109
1110 self->codec = ((MultibyteCodecObject *)codec)->codec;
1111 self->pendingsize = 0;
1112 self->errors = internal_error_callback(errors);
1113 if (self->errors == NULL)
1114 goto errorexit;
1115 if (self->codec->decinit != NULL &&
1116 self->codec->decinit(&self->state, self->codec->config) != 0)
1117 goto errorexit;
1118
1119 Py_DECREF(codec);
1120 return (PyObject *)self;
1121
1122 errorexit:
1123 Py_XDECREF(self);
1124 Py_XDECREF(codec);
1125 return NULL;
1126 }
1127
1128 static int
1129 mbidecoder_init(PyObject *self, PyObject *args, PyObject *kwds)
1130 {
1131 return 0;
1132 }
1133
1134 static int
1135 mbidecoder_traverse(MultibyteIncrementalDecoderObject *self,
1136 visitproc visit, void *arg)
1137 {
1138 if (ERROR_ISCUSTOM(self->errors))
1139 Py_VISIT(self->errors);
1140 return 0;
1141 }
1142
1143 static void
1144 mbidecoder_dealloc(MultibyteIncrementalDecoderObject *self)
1145 {
1146 PyObject_GC_UnTrack(self);
1147 ERROR_DECREF(self->errors);
1148 Py_TYPE(self)->tp_free(self);
1149 }
1150
1151 static PyTypeObject MultibyteIncrementalDecoder_Type = {
1152 PyVarObject_HEAD_INIT(NULL, 0)
1153 "MultibyteIncrementalDecoder", /* tp_name */
1154 sizeof(MultibyteIncrementalDecoderObject), /* tp_basicsize */
1155 0, /* tp_itemsize */
1156 /* methods */
1157 (destructor)mbidecoder_dealloc, /* tp_dealloc */
1158 0, /* tp_print */
1159 0, /* tp_getattr */
1160 0, /* tp_setattr */
1161 0, /* tp_compare */
1162 0, /* tp_repr */
1163 0, /* tp_as_number */
1164 0, /* tp_as_sequence */
1165 0, /* tp_as_mapping */
1166 0, /* tp_hash */
1167 0, /* tp_call */
1168 0, /* tp_str */
1169 PyObject_GenericGetAttr, /* tp_getattro */
1170 0, /* tp_setattro */
1171 0, /* tp_as_buffer */
1172 Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC
1173 | Py_TPFLAGS_BASETYPE, /* tp_flags */
1174 0, /* tp_doc */
1175 (traverseproc)mbidecoder_traverse, /* tp_traverse */
1176 0, /* tp_clear */
1177 0, /* tp_richcompare */
1178 0, /* tp_weaklistoffset */
1179 0, /* tp_iter */
1180 0, /* tp_iterext */
1181 mbidecoder_methods, /* tp_methods */
1182 0, /* tp_members */
1183 codecctx_getsets, /* tp_getset */
1184 0, /* tp_base */
1185 0, /* tp_dict */
1186 0, /* tp_descr_get */
1187 0, /* tp_descr_set */
1188 0, /* tp_dictoffset */
1189 mbidecoder_init, /* tp_init */
1190 0, /* tp_alloc */
1191 mbidecoder_new, /* tp_new */
1192 };
1193
1194
1195 /**
1196 * MultibyteStreamReader object
1197 */
1198
1199 static PyObject *
1200 mbstreamreader_iread(MultibyteStreamReaderObject *self,
1201 const char *method, Py_ssize_t sizehint)
1202 {
1203 MultibyteDecodeBuffer buf;
1204 PyObject *cres;
1205 Py_ssize_t rsize, finalsize = 0;
1206
1207 if (sizehint == 0)
1208 return PyUnicode_FromUnicode(NULL, 0);
1209
1210 buf.outobj = buf.excobj = NULL;
1211 cres = NULL;
1212
1213 for (;;) {
1214 int endoffile;
1215
1216 if (sizehint < 0)
1217 cres = PyObject_CallMethod(self->stream,
1218 (char *)method, NULL);
1219 else
1220 cres = PyObject_CallMethod(self->stream,
1221 (char *)method, "i", sizehint);
1222 if (cres == NULL)
1223 goto errorexit;
1224
1225 if (!PyString_Check(cres)) {
1226 PyErr_SetString(PyExc_TypeError,
1227 "stream function returned a "
1228 "non-string object");
1229 goto errorexit;
1230 }
1231
1232 endoffile = (PyString_GET_SIZE(cres) == 0);
1233
1234 if (self->pendingsize > 0) {
1235 PyObject *ctr;
1236 char *ctrdata;
1237
1238 rsize = PyString_GET_SIZE(cres) + self->pendingsize;
1239 ctr = PyString_FromStringAndSize(NULL, rsize);
1240 if (ctr == NULL)
1241 goto errorexit;
1242 ctrdata = PyString_AS_STRING(ctr);
1243 memcpy(ctrdata, self->pending, self->pendingsize);
1244 memcpy(ctrdata + self->pendingsize,
1245 PyString_AS_STRING(cres),
1246 PyString_GET_SIZE(cres));
1247 Py_DECREF(cres);
1248 cres = ctr;
1249 self->pendingsize = 0;
1250 }
1251
1252 rsize = PyString_GET_SIZE(cres);
1253 if (decoder_prepare_buffer(&buf, PyString_AS_STRING(cres),
1254 rsize) != 0)
1255 goto errorexit;
1256
1257 if (rsize > 0 && decoder_feed_buffer(
1258 (MultibyteStatefulDecoderContext *)self, &buf))
1259 goto errorexit;
1260
1261 if (endoffile || sizehint < 0) {
1262 if (buf.inbuf < buf.inbuf_end &&
1263 multibytecodec_decerror(self->codec, &self->state,
1264 &buf, self->errors, MBERR_TOOFEW))
1265 goto errorexit;
1266 }
1267
1268 if (buf.inbuf < buf.inbuf_end) { /* pending sequence exists */
1269 if (decoder_append_pending(STATEFUL_DCTX(self),
1270 &buf) != 0)
1271 goto errorexit;
1272 }
1273
1274 finalsize = (Py_ssize_t)(buf.outbuf -
1275 PyUnicode_AS_UNICODE(buf.outobj));
1276 Py_DECREF(cres);
1277 cres = NULL;
1278
1279 if (sizehint < 0 || finalsize != 0 || rsize == 0)
1280 break;
1281
1282 sizehint = 1; /* read 1 more byte and retry */
1283 }
1284
1285 if (finalsize != PyUnicode_GET_SIZE(buf.outobj))
1286 if (PyUnicode_Resize(&buf.outobj, finalsize) == -1)
1287 goto errorexit;
1288
1289 Py_XDECREF(cres);
1290 Py_XDECREF(buf.excobj);
1291 return buf.outobj;
1292
1293 errorexit:
1294 Py_XDECREF(cres);
1295 Py_XDECREF(buf.excobj);
1296 Py_XDECREF(buf.outobj);
1297 return NULL;
1298 }
1299
1300 static PyObject *
1301 mbstreamreader_read(MultibyteStreamReaderObject *self, PyObject *args)
1302 {
1303 PyObject *sizeobj = NULL;
1304 Py_ssize_t size;
1305
1306 if (!PyArg_UnpackTuple(args, "read", 0, 1, &sizeobj))
1307 return NULL;
1308
1309 if (sizeobj == Py_None || sizeobj == NULL)
1310 size = -1;
1311 else if (PyInt_Check(sizeobj))
1312 size = PyInt_AsSsize_t(sizeobj);
1313 else {
1314 PyErr_SetString(PyExc_TypeError, "arg 1 must be an integer");
1315 return NULL;
1316 }
1317
1318 return mbstreamreader_iread(self, "read", size);
1319 }
1320
1321 static PyObject *
1322 mbstreamreader_readline(MultibyteStreamReaderObject *self, PyObject *args)
1323 {
1324 PyObject *sizeobj = NULL;
1325 Py_ssize_t size;
1326
1327 if (!PyArg_UnpackTuple(args, "readline", 0, 1, &sizeobj))
1328 return NULL;
1329
1330 if (sizeobj == Py_None || sizeobj == NULL)
1331 size = -1;
1332 else if (PyInt_Check(sizeobj))
1333 size = PyInt_AsSsize_t(sizeobj);
1334 else {
1335 PyErr_SetString(PyExc_TypeError, "arg 1 must be an integer");
1336 return NULL;
1337 }
1338
1339 return mbstreamreader_iread(self, "readline", size);
1340 }
1341
1342 static PyObject *
1343 mbstreamreader_readlines(MultibyteStreamReaderObject *self, PyObject *args)
1344 {
1345 PyObject *sizehintobj = NULL, *r, *sr;
1346 Py_ssize_t sizehint;
1347
1348 if (!PyArg_UnpackTuple(args, "readlines", 0, 1, &sizehintobj))
1349 return NULL;
1350
1351 if (sizehintobj == Py_None || sizehintobj == NULL)
1352 sizehint = -1;
1353 else if (PyInt_Check(sizehintobj))
1354 sizehint = PyInt_AsSsize_t(sizehintobj);
1355 else {
1356 PyErr_SetString(PyExc_TypeError, "arg 1 must be an integer");
1357 return NULL;
1358 }
1359
1360 r = mbstreamreader_iread(self, "read", sizehint);
1361 if (r == NULL)
1362 return NULL;
1363
1364 sr = PyUnicode_Splitlines(r, 1);
1365 Py_DECREF(r);
1366 return sr;
1367 }
1368
1369 static PyObject *
1370 mbstreamreader_reset(MultibyteStreamReaderObject *self)
1371 {
1372 if (self->codec->decreset != NULL &&
1373 self->codec->decreset(&self->state, self->codec->config) != 0)
1374 return NULL;
1375 self->pendingsize = 0;
1376
1377 Py_RETURN_NONE;
1378 }
1379
1380 static struct PyMethodDef mbstreamreader_methods[] = {
1381 {"read", (PyCFunction)mbstreamreader_read,
1382 METH_VARARGS, NULL},
1383 {"readline", (PyCFunction)mbstreamreader_readline,
1384 METH_VARARGS, NULL},
1385 {"readlines", (PyCFunction)mbstreamreader_readlines,
1386 METH_VARARGS, NULL},
1387 {"reset", (PyCFunction)mbstreamreader_reset,
1388 METH_NOARGS, NULL},
1389 {NULL, NULL},
1390 };
1391
1392 static PyMemberDef mbstreamreader_members[] = {
1393 {"stream", T_OBJECT,
1394 offsetof(MultibyteStreamReaderObject, stream),
1395 READONLY, NULL},
1396 {NULL,}
1397 };
1398
1399 static PyObject *
1400 mbstreamreader_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
1401 {
1402 MultibyteStreamReaderObject *self;
1403 PyObject *stream, *codec = NULL;
1404 char *errors = NULL;
1405
1406 if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|s:StreamReader",
1407 streamkwarglist, &stream, &errors))
1408 return NULL;
1409
1410 self = (MultibyteStreamReaderObject *)type->tp_alloc(type, 0);
1411 if (self == NULL)
1412 return NULL;
1413
1414 codec = PyObject_GetAttrString((PyObject *)type, "codec");
1415 if (codec == NULL)
1416 goto errorexit;
1417 if (!MultibyteCodec_Check(codec)) {
1418 PyErr_SetString(PyExc_TypeError, "codec is unexpected type");
1419 goto errorexit;
1420 }
1421
1422 self->codec = ((MultibyteCodecObject *)codec)->codec;
1423 self->stream = stream;
1424 Py_INCREF(stream);
1425 self->pendingsize = 0;
1426 self->errors = internal_error_callback(errors);
1427 if (self->errors == NULL)
1428 goto errorexit;
1429 if (self->codec->decinit != NULL &&
1430 self->codec->decinit(&self->state, self->codec->config) != 0)
1431 goto errorexit;
1432
1433 Py_DECREF(codec);
1434 return (PyObject *)self;
1435
1436 errorexit:
1437 Py_XDECREF(self);
1438 Py_XDECREF(codec);
1439 return NULL;
1440 }
1441
1442 static int
1443 mbstreamreader_init(PyObject *self, PyObject *args, PyObject *kwds)
1444 {
1445 return 0;
1446 }
1447
1448 static int
1449 mbstreamreader_traverse(MultibyteStreamReaderObject *self,
1450 visitproc visit, void *arg)
1451 {
1452 if (ERROR_ISCUSTOM(self->errors))
1453 Py_VISIT(self->errors);
1454 Py_VISIT(self->stream);
1455 return 0;
1456 }
1457
1458 static void
1459 mbstreamreader_dealloc(MultibyteStreamReaderObject *self)
1460 {
1461 PyObject_GC_UnTrack(self);
1462 ERROR_DECREF(self->errors);
1463 Py_DECREF(self->stream);
1464 Py_TYPE(self)->tp_free(self);
1465 }
1466
1467 static PyTypeObject MultibyteStreamReader_Type = {
1468 PyVarObject_HEAD_INIT(NULL, 0)
1469 "MultibyteStreamReader", /* tp_name */
1470 sizeof(MultibyteStreamReaderObject), /* tp_basicsize */
1471 0, /* tp_itemsize */
1472 /* methods */
1473 (destructor)mbstreamreader_dealloc, /* tp_dealloc */
1474 0, /* tp_print */
1475 0, /* tp_getattr */
1476 0, /* tp_setattr */
1477 0, /* tp_compare */
1478 0, /* tp_repr */
1479 0, /* tp_as_number */
1480 0, /* tp_as_sequence */
1481 0, /* tp_as_mapping */
1482 0, /* tp_hash */
1483 0, /* tp_call */
1484 0, /* tp_str */
1485 PyObject_GenericGetAttr, /* tp_getattro */
1486 0, /* tp_setattro */
1487 0, /* tp_as_buffer */
1488 Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC
1489 | Py_TPFLAGS_BASETYPE, /* tp_flags */
1490 0, /* tp_doc */
1491 (traverseproc)mbstreamreader_traverse, /* tp_traverse */
1492 0, /* tp_clear */
1493 0, /* tp_richcompare */
1494 0, /* tp_weaklistoffset */
1495 0, /* tp_iter */
1496 0, /* tp_iterext */
1497 mbstreamreader_methods, /* tp_methods */
1498 mbstreamreader_members, /* tp_members */
1499 codecctx_getsets, /* tp_getset */
1500 0, /* tp_base */
1501 0, /* tp_dict */
1502 0, /* tp_descr_get */
1503 0, /* tp_descr_set */
1504 0, /* tp_dictoffset */
1505 mbstreamreader_init, /* tp_init */
1506 0, /* tp_alloc */
1507 mbstreamreader_new, /* tp_new */
1508 };
1509
1510
1511 /**
1512 * MultibyteStreamWriter object
1513 */
1514
1515 static int
1516 mbstreamwriter_iwrite(MultibyteStreamWriterObject *self,
1517 PyObject *unistr)
1518 {
1519 PyObject *str, *wr;
1520
1521 str = encoder_encode_stateful(STATEFUL_ECTX(self), unistr, 0);
1522 if (str == NULL)
1523 return -1;
1524
1525 wr = PyObject_CallMethod(self->stream, "write", "O", str);
1526 Py_DECREF(str);
1527 if (wr == NULL)
1528 return -1;
1529
1530 Py_DECREF(wr);
1531 return 0;
1532 }
1533
1534 static PyObject *
1535 mbstreamwriter_write(MultibyteStreamWriterObject *self, PyObject *strobj)
1536 {
1537 if (mbstreamwriter_iwrite(self, strobj))
1538 return NULL;
1539 else
1540 Py_RETURN_NONE;
1541 }
1542
1543 static PyObject *
1544 mbstreamwriter_writelines(MultibyteStreamWriterObject *self, PyObject *lines)
1545 {
1546 PyObject *strobj;
1547 int i, r;
1548
1549 if (!PySequence_Check(lines)) {
1550 PyErr_SetString(PyExc_TypeError,
1551 "arg must be a sequence object");
1552 return NULL;
1553 }
1554
1555 for (i = 0; i < PySequence_Length(lines); i++) {
1556 /* length can be changed even within this loop */
1557 strobj = PySequence_GetItem(lines, i);
1558 if (strobj == NULL)
1559 return NULL;
1560
1561 r = mbstreamwriter_iwrite(self, strobj);
1562 Py_DECREF(strobj);
1563 if (r == -1)
1564 return NULL;
1565 }
1566
1567 Py_RETURN_NONE;
1568 }
1569
1570 static PyObject *
1571 mbstreamwriter_reset(MultibyteStreamWriterObject *self)
1572 {
1573 const Py_UNICODE *pending;
1574 PyObject *pwrt;
1575
1576 pending = self->pending;
1577 pwrt = multibytecodec_encode(self->codec, &self->state,
1578 &pending, self->pendingsize, self->errors,
1579 MBENC_FLUSH | MBENC_RESET);
1580 /* some pending buffer can be truncated when UnicodeEncodeError is
1581 * raised on 'strict' mode. but, 'reset' method is designed to
1582 * reset the pending buffer or states so failed string sequence
1583 * ought to be missed */
1584 self->pendingsize = 0;
1585 if (pwrt == NULL)
1586 return NULL;
1587
1588 if (PyString_Size(pwrt) > 0) {
1589 PyObject *wr;
1590 wr = PyObject_CallMethod(self->stream, "write", "O", pwrt);
1591 if (wr == NULL) {
1592 Py_DECREF(pwrt);
1593 return NULL;
1594 }
1595 }
1596 Py_DECREF(pwrt);
1597
1598 Py_RETURN_NONE;
1599 }
1600
1601 static PyObject *
1602 mbstreamwriter_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
1603 {
1604 MultibyteStreamWriterObject *self;
1605 PyObject *stream, *codec = NULL;
1606 char *errors = NULL;
1607
1608 if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|s:StreamWriter",
1609 streamkwarglist, &stream, &errors))
1610 return NULL;
1611
1612 self = (MultibyteStreamWriterObject *)type->tp_alloc(type, 0);
1613 if (self == NULL)
1614 return NULL;
1615
1616 codec = PyObject_GetAttrString((PyObject *)type, "codec");
1617 if (codec == NULL)
1618 goto errorexit;
1619 if (!MultibyteCodec_Check(codec)) {
1620 PyErr_SetString(PyExc_TypeError, "codec is unexpected type");
1621 goto errorexit;
1622 }
1623
1624 self->codec = ((MultibyteCodecObject *)codec)->codec;
1625 self->stream = stream;
1626 Py_INCREF(stream);
1627 self->pendingsize = 0;
1628 self->errors = internal_error_callback(errors);
1629 if (self->errors == NULL)
1630 goto errorexit;
1631 if (self->codec->encinit != NULL &&
1632 self->codec->encinit(&self->state, self->codec->config) != 0)
1633 goto errorexit;
1634
1635 Py_DECREF(codec);
1636 return (PyObject *)self;
1637
1638 errorexit:
1639 Py_XDECREF(self);
1640 Py_XDECREF(codec);
1641 return NULL;
1642 }
1643
1644 static int
1645 mbstreamwriter_init(PyObject *self, PyObject *args, PyObject *kwds)
1646 {
1647 return 0;
1648 }
1649
1650 static int
1651 mbstreamwriter_traverse(MultibyteStreamWriterObject *self,
1652 visitproc visit, void *arg)
1653 {
1654 if (ERROR_ISCUSTOM(self->errors))
1655 Py_VISIT(self->errors);
1656 Py_VISIT(self->stream);
1657 return 0;
1658 }
1659
1660 static void
1661 mbstreamwriter_dealloc(MultibyteStreamWriterObject *self)
1662 {
1663 PyObject_GC_UnTrack(self);
1664 ERROR_DECREF(self->errors);
1665 Py_DECREF(self->stream);
1666 Py_TYPE(self)->tp_free(self);
1667 }
1668
1669 static struct PyMethodDef mbstreamwriter_methods[] = {
1670 {"write", (PyCFunction)mbstreamwriter_write,
1671 METH_O, NULL},
1672 {"writelines", (PyCFunction)mbstreamwriter_writelines,
1673 METH_O, NULL},
1674 {"reset", (PyCFunction)mbstreamwriter_reset,
1675 METH_NOARGS, NULL},
1676 {NULL, NULL},
1677 };
1678
1679 static PyMemberDef mbstreamwriter_members[] = {
1680 {"stream", T_OBJECT,
1681 offsetof(MultibyteStreamWriterObject, stream),
1682 READONLY, NULL},
1683 {NULL,}
1684 };
1685
1686 static PyTypeObject MultibyteStreamWriter_Type = {
1687 PyVarObject_HEAD_INIT(NULL, 0)
1688 "MultibyteStreamWriter", /* tp_name */
1689 sizeof(MultibyteStreamWriterObject), /* tp_basicsize */
1690 0, /* tp_itemsize */
1691 /* methods */
1692 (destructor)mbstreamwriter_dealloc, /* tp_dealloc */
1693 0, /* tp_print */
1694 0, /* tp_getattr */
1695 0, /* tp_setattr */
1696 0, /* tp_compare */
1697 0, /* tp_repr */
1698 0, /* tp_as_number */
1699 0, /* tp_as_sequence */
1700 0, /* tp_as_mapping */
1701 0, /* tp_hash */
1702 0, /* tp_call */
1703 0, /* tp_str */
1704 PyObject_GenericGetAttr, /* tp_getattro */
1705 0, /* tp_setattro */
1706 0, /* tp_as_buffer */
1707 Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC
1708 | Py_TPFLAGS_BASETYPE, /* tp_flags */
1709 0, /* tp_doc */
1710 (traverseproc)mbstreamwriter_traverse, /* tp_traverse */
1711 0, /* tp_clear */
1712 0, /* tp_richcompare */
1713 0, /* tp_weaklistoffset */
1714 0, /* tp_iter */
1715 0, /* tp_iterext */
1716 mbstreamwriter_methods, /* tp_methods */
1717 mbstreamwriter_members, /* tp_members */
1718 codecctx_getsets, /* tp_getset */
1719 0, /* tp_base */
1720 0, /* tp_dict */
1721 0, /* tp_descr_get */
1722 0, /* tp_descr_set */
1723 0, /* tp_dictoffset */
1724 mbstreamwriter_init, /* tp_init */
1725 0, /* tp_alloc */
1726 mbstreamwriter_new, /* tp_new */
1727 };
1728
1729
1730 /**
1731 * Exposed factory function
1732 */
1733
1734 static PyObject *
1735 __create_codec(PyObject *ignore, PyObject *arg)
1736 {
1737 MultibyteCodecObject *self;
1738 MultibyteCodec *codec;
1739
1740 if (!PyCObject_Check(arg)) {
1741 PyErr_SetString(PyExc_ValueError, "argument type invalid");
1742 return NULL;
1743 }
1744
1745 codec = PyCObject_AsVoidPtr(arg);
1746 if (codec->codecinit != NULL && codec->codecinit(codec->config) != 0)
1747 return NULL;
1748
1749 self = PyObject_New(MultibyteCodecObject, &MultibyteCodec_Type);
1750 if (self == NULL)
1751 return NULL;
1752 self->codec = codec;
1753
1754 return (PyObject *)self;
1755 }
1756
1757 static struct PyMethodDef __methods[] = {
1758 {"__create_codec", (PyCFunction)__create_codec, METH_O},
1759 {NULL, NULL},
1760 };
1761
1762 PyMODINIT_FUNC
1763 init_multibytecodec(void)
1764 {
1765 int i;
1766 PyObject *m;
1767 PyTypeObject *typelist[] = {
1768 &MultibyteIncrementalEncoder_Type,
1769 &MultibyteIncrementalDecoder_Type,
1770 &MultibyteStreamReader_Type,
1771 &MultibyteStreamWriter_Type,
1772 NULL
1773 };
1774
1775 if (PyType_Ready(&MultibyteCodec_Type) < 0)
1776 return;
1777
1778 m = Py_InitModule("_multibytecodec", __methods);
1779 if (m == NULL)
1780 return;
1781
1782 for (i = 0; typelist[i] != NULL; i++) {
1783 if (PyType_Ready(typelist[i]) < 0)
1784 return;
1785 Py_INCREF(typelist[i]);
1786 PyModule_AddObject(m, typelist[i]->tp_name,
1787 (PyObject *)typelist[i]);
1788 }
1789
1790 if (PyErr_Occurred())
1791 Py_FatalError("can't initialize the _multibytecodec module");
1792 }
python git mirror