3 Unicode implementation based on original code by Fredrik Lundh,
4 modified by Marc-Andre Lemburg <mal@lemburg.com> according to the
5 Unicode Integration Proposal (see file Misc/unicode.txt).
7 Major speed upgrades to the method implementations at the Reykjavik
8 NeedForSpeed sprint, by Fredrik Lundh and Andrew Dalke.
10 Copyright (c) Corporation for National Research Initiatives.
12 --------------------------------------------------------------------
13 The original string type implementation is:
15 Copyright (c) 1999 by Secret Labs AB
16 Copyright (c) 1999 by Fredrik Lundh
18 By obtaining, using, and/or copying this software and/or its
19 associated documentation, you agree that you have read, understood,
20 and will comply with the following terms and conditions:
22 Permission to use, copy, modify, and distribute this software and its
23 associated documentation for any purpose and without fee is hereby
24 granted, provided that the above copyright notice appears in all
25 copies, and that both that copyright notice and this permission notice
26 appear in supporting documentation, and that the name of Secret Labs
27 AB or the author not be used in advertising or publicity pertaining to
28 distribution of the software without specific, written prior
31 SECRET LABS AB AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO
32 THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND
33 FITNESS. IN NO EVENT SHALL SECRET LABS AB OR THE AUTHOR BE LIABLE FOR
34 ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
35 WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
36 ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT
37 OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
38 --------------------------------------------------------------------
42 #define PY_SSIZE_T_CLEAN
45 #include "unicodeobject.h"
52 /* Limit for the Unicode object free list */
54 #define PyUnicode_MAXFREELIST 1024
56 /* Limit for the Unicode object free list stay alive optimization.
58 The implementation will keep allocated Unicode memory intact for
59 all objects on the free list having a size less than this
60 limit. This reduces malloc() overhead for small Unicode objects.
62 At worst this will result in PyUnicode_MAXFREELIST *
63 (sizeof(PyUnicodeObject) + KEEPALIVE_SIZE_LIMIT +
64 malloc()-overhead) bytes of unused garbage.
66 Setting the limit to 0 effectively turns the feature off.
68 Note: This is an experimental feature ! If you get core dumps when
69 using Unicode objects, turn this feature off.
73 #define KEEPALIVE_SIZE_LIMIT 9
75 /* Endianness switches; defaults to little endian */
77 #ifdef WORDS_BIGENDIAN
78 # define BYTEORDER_IS_BIG_ENDIAN
80 # define BYTEORDER_IS_LITTLE_ENDIAN
83 /* --- Globals ------------------------------------------------------------
85 The globals are initialized by the _PyUnicode_Init() API and should
86 not be used before calling that API.
95 /* Free list for Unicode objects */
96 static PyUnicodeObject
*free_list
;
99 /* The empty Unicode object is shared to improve performance. */
100 static PyUnicodeObject
*unicode_empty
;
102 /* Single character Unicode strings in the Latin-1 range are being
104 static PyUnicodeObject
*unicode_latin1
[256];
106 /* Default encoding to use and assume when NULL is passed as encoding
107 parameter; it is initialized by _PyUnicode_Init().
109 Always use the PyUnicode_SetDefaultEncoding() and
110 PyUnicode_GetDefaultEncoding() APIs to access this global.
113 static char unicode_default_encoding
[100];
115 /* Fast detection of the most frequent whitespace characters */
116 const unsigned char _Py_ascii_whitespace
[] = {
117 0, 0, 0, 0, 0, 0, 0, 0,
118 /* case 0x0009: * HORIZONTAL TABULATION */
119 /* case 0x000A: * LINE FEED */
120 /* case 0x000B: * VERTICAL TABULATION */
121 /* case 0x000C: * FORM FEED */
122 /* case 0x000D: * CARRIAGE RETURN */
123 0, 1, 1, 1, 1, 1, 0, 0,
124 0, 0, 0, 0, 0, 0, 0, 0,
125 /* case 0x001C: * FILE SEPARATOR */
126 /* case 0x001D: * GROUP SEPARATOR */
127 /* case 0x001E: * RECORD SEPARATOR */
128 /* case 0x001F: * UNIT SEPARATOR */
129 0, 0, 0, 0, 1, 1, 1, 1,
130 /* case 0x0020: * SPACE */
131 1, 0, 0, 0, 0, 0, 0, 0,
132 0, 0, 0, 0, 0, 0, 0, 0,
133 0, 0, 0, 0, 0, 0, 0, 0,
134 0, 0, 0, 0, 0, 0, 0, 0,
136 0, 0, 0, 0, 0, 0, 0, 0,
137 0, 0, 0, 0, 0, 0, 0, 0,
138 0, 0, 0, 0, 0, 0, 0, 0,
139 0, 0, 0, 0, 0, 0, 0, 0,
140 0, 0, 0, 0, 0, 0, 0, 0,
141 0, 0, 0, 0, 0, 0, 0, 0,
142 0, 0, 0, 0, 0, 0, 0, 0,
143 0, 0, 0, 0, 0, 0, 0, 0
146 /* Same for linebreaks */
147 static unsigned char ascii_linebreak
[] = {
148 0, 0, 0, 0, 0, 0, 0, 0,
149 /* 0x000A, * LINE FEED */
150 /* 0x000D, * CARRIAGE RETURN */
151 0, 0, 1, 0, 0, 1, 0, 0,
152 0, 0, 0, 0, 0, 0, 0, 0,
153 /* 0x001C, * FILE SEPARATOR */
154 /* 0x001D, * GROUP SEPARATOR */
155 /* 0x001E, * RECORD SEPARATOR */
156 0, 0, 0, 0, 1, 1, 1, 0,
157 0, 0, 0, 0, 0, 0, 0, 0,
158 0, 0, 0, 0, 0, 0, 0, 0,
159 0, 0, 0, 0, 0, 0, 0, 0,
160 0, 0, 0, 0, 0, 0, 0, 0,
162 0, 0, 0, 0, 0, 0, 0, 0,
163 0, 0, 0, 0, 0, 0, 0, 0,
164 0, 0, 0, 0, 0, 0, 0, 0,
165 0, 0, 0, 0, 0, 0, 0, 0,
166 0, 0, 0, 0, 0, 0, 0, 0,
167 0, 0, 0, 0, 0, 0, 0, 0,
168 0, 0, 0, 0, 0, 0, 0, 0,
169 0, 0, 0, 0, 0, 0, 0, 0
174 PyUnicode_GetMax(void)
176 #ifdef Py_UNICODE_WIDE
179 /* This is actually an illegal character, so it should
180 not be passed to unichr. */
185 /* --- Bloom Filters ----------------------------------------------------- */
187 /* stuff to implement simple "bloom filters" for Unicode characters.
188 to keep things simple, we use a single bitmask, using the least 5
189 bits from each unicode characters as the bit index. */
191 /* the linebreak mask is set up by Unicode_Init below */
193 #define BLOOM_MASK unsigned long
195 static BLOOM_MASK bloom_linebreak
;
197 #define BLOOM(mask, ch) ((mask & (1 << ((ch) & 0x1F))))
199 #define BLOOM_LINEBREAK(ch) \
200 ((ch) < 128U ? ascii_linebreak[(ch)] : \
201 (BLOOM(bloom_linebreak, (ch)) && Py_UNICODE_ISLINEBREAK(ch)))
203 Py_LOCAL_INLINE(BLOOM_MASK
) make_bloom_mask(Py_UNICODE
* ptr
, Py_ssize_t len
)
205 /* calculate simple bloom-style bitmask for a given unicode string */
211 for (i
= 0; i
< len
; i
++)
212 mask
|= (1 << (ptr
[i
] & 0x1F));
217 Py_LOCAL_INLINE(int) unicode_member(Py_UNICODE chr
, Py_UNICODE
* set
, Py_ssize_t setlen
)
221 for (i
= 0; i
< setlen
; i
++)
228 #define BLOOM_MEMBER(mask, chr, set, setlen)\
229 BLOOM(mask, chr) && unicode_member(chr, set, setlen)
231 /* --- Unicode Object ----------------------------------------------------- */
234 int unicode_resize(register PyUnicodeObject
*unicode
,
239 /* Shortcut if there's nothing much to do. */
240 if (unicode
->length
== length
)
243 /* Resizing shared object (unicode_empty or single character
244 objects) in-place is not allowed. Use PyUnicode_Resize()
247 if (unicode
== unicode_empty
||
248 (unicode
->length
== 1 &&
249 unicode
->str
[0] < 256U &&
250 unicode_latin1
[unicode
->str
[0]] == unicode
)) {
251 PyErr_SetString(PyExc_SystemError
,
252 "can't resize shared unicode objects");
256 /* We allocate one more byte to make sure the string is Ux0000 terminated.
257 The overallocation is also used by fastsearch, which assumes that it's
258 safe to look at str[length] (without making any assumptions about what
261 oldstr
= unicode
->str
;
262 unicode
->str
= PyObject_REALLOC(unicode
->str
,
263 sizeof(Py_UNICODE
) * (length
+ 1));
265 unicode
->str
= (Py_UNICODE
*)oldstr
;
269 unicode
->str
[length
] = 0;
270 unicode
->length
= length
;
273 /* Reset the object caches */
274 if (unicode
->defenc
) {
275 Py_DECREF(unicode
->defenc
);
276 unicode
->defenc
= NULL
;
283 /* We allocate one more byte to make sure the string is
284 Ux0000 terminated -- XXX is this needed ?
286 XXX This allocator could further be enhanced by assuring that the
287 free list never reduces its size below 1.
292 PyUnicodeObject
*_PyUnicode_New(Py_ssize_t length
)
294 register PyUnicodeObject
*unicode
;
296 /* Optimization for empty strings */
297 if (length
== 0 && unicode_empty
!= NULL
) {
298 Py_INCREF(unicode_empty
);
299 return unicode_empty
;
302 /* Ensure we won't overflow the size. */
303 if (length
> ((PY_SSIZE_T_MAX
/ sizeof(Py_UNICODE
)) - 1)) {
304 return (PyUnicodeObject
*)PyErr_NoMemory();
307 /* Unicode freelist & memory allocation */
310 free_list
= *(PyUnicodeObject
**)unicode
;
313 /* Keep-Alive optimization: we only upsize the buffer,
314 never downsize it. */
315 if ((unicode
->length
< length
) &&
316 unicode_resize(unicode
, length
) < 0) {
317 PyObject_DEL(unicode
->str
);
322 size_t new_size
= sizeof(Py_UNICODE
) * ((size_t)length
+ 1);
323 unicode
->str
= (Py_UNICODE
*) PyObject_MALLOC(new_size
);
325 PyObject_INIT(unicode
, &PyUnicode_Type
);
329 unicode
= PyObject_New(PyUnicodeObject
, &PyUnicode_Type
);
332 new_size
= sizeof(Py_UNICODE
) * ((size_t)length
+ 1);
333 unicode
->str
= (Py_UNICODE
*) PyObject_MALLOC(new_size
);
340 /* Initialize the first element to guard against cases where
341 * the caller fails before initializing str -- unicode_resize()
342 * reads str[0], and the Keep-Alive optimization can keep memory
343 * allocated for str alive across a call to unicode_dealloc(unicode).
344 * We don't want unicode_resize to read uninitialized memory in
348 unicode
->str
[length
] = 0;
349 unicode
->length
= length
;
351 unicode
->defenc
= NULL
;
355 /* XXX UNREF/NEWREF interface should be more symmetrical */
357 _Py_ForgetReference((PyObject
*)unicode
);
358 PyObject_Del(unicode
);
363 void unicode_dealloc(register PyUnicodeObject
*unicode
)
365 if (PyUnicode_CheckExact(unicode
) &&
366 numfree
< PyUnicode_MAXFREELIST
) {
367 /* Keep-Alive optimization */
368 if (unicode
->length
>= KEEPALIVE_SIZE_LIMIT
) {
369 PyObject_DEL(unicode
->str
);
373 if (unicode
->defenc
) {
374 Py_DECREF(unicode
->defenc
);
375 unicode
->defenc
= NULL
;
377 /* Add to free list */
378 *(PyUnicodeObject
**)unicode
= free_list
;
383 PyObject_DEL(unicode
->str
);
384 Py_XDECREF(unicode
->defenc
);
385 Py_TYPE(unicode
)->tp_free((PyObject
*)unicode
);
390 int _PyUnicode_Resize(PyUnicodeObject
**unicode
, Py_ssize_t length
)
392 register PyUnicodeObject
*v
;
394 /* Argument checks */
395 if (unicode
== NULL
) {
396 PyErr_BadInternalCall();
400 if (v
== NULL
|| !PyUnicode_Check(v
) || Py_REFCNT(v
) != 1 || length
< 0) {
401 PyErr_BadInternalCall();
405 /* Resizing unicode_empty and single character objects is not
406 possible since these are being shared. We simply return a fresh
407 copy with the same Unicode content. */
408 if (v
->length
!= length
&&
409 (v
== unicode_empty
|| v
->length
== 1)) {
410 PyUnicodeObject
*w
= _PyUnicode_New(length
);
413 Py_UNICODE_COPY(w
->str
, v
->str
,
414 length
< v
->length
? length
: v
->length
);
420 /* Note that we don't have to modify *unicode for unshared Unicode
421 objects, since we can modify them in-place. */
422 return unicode_resize(v
, length
);
425 int PyUnicode_Resize(PyObject
**unicode
, Py_ssize_t length
)
427 return _PyUnicode_Resize((PyUnicodeObject
**)unicode
, length
);
430 PyObject
*PyUnicode_FromUnicode(const Py_UNICODE
*u
,
433 PyUnicodeObject
*unicode
;
435 /* If the Unicode data is known at construction time, we can apply
436 some optimizations which share commonly used objects. */
439 /* Optimization for empty strings */
440 if (size
== 0 && unicode_empty
!= NULL
) {
441 Py_INCREF(unicode_empty
);
442 return (PyObject
*)unicode_empty
;
445 /* Single character Unicode objects in the Latin-1 range are
446 shared when using this constructor */
447 if (size
== 1 && *u
< 256) {
448 unicode
= unicode_latin1
[*u
];
450 unicode
= _PyUnicode_New(1);
453 unicode
->str
[0] = *u
;
454 unicode_latin1
[*u
] = unicode
;
457 return (PyObject
*)unicode
;
461 unicode
= _PyUnicode_New(size
);
465 /* Copy the Unicode data into the new object */
467 Py_UNICODE_COPY(unicode
->str
, u
, size
);
469 return (PyObject
*)unicode
;
472 PyObject
*PyUnicode_FromStringAndSize(const char *u
, Py_ssize_t size
)
474 PyUnicodeObject
*unicode
;
477 PyErr_SetString(PyExc_SystemError
,
478 "Negative size passed to PyUnicode_FromStringAndSize");
482 /* If the Unicode data is known at construction time, we can apply
483 some optimizations which share commonly used objects.
484 Also, this means the input must be UTF-8, so fall back to the
485 UTF-8 decoder at the end. */
488 /* Optimization for empty strings */
489 if (size
== 0 && unicode_empty
!= NULL
) {
490 Py_INCREF(unicode_empty
);
491 return (PyObject
*)unicode_empty
;
494 /* Single characters are shared when using this constructor.
495 Restrict to ASCII, since the input must be UTF-8. */
496 if (size
== 1 && Py_CHARMASK(*u
) < 128) {
497 unicode
= unicode_latin1
[Py_CHARMASK(*u
)];
499 unicode
= _PyUnicode_New(1);
502 unicode
->str
[0] = Py_CHARMASK(*u
);
503 unicode_latin1
[Py_CHARMASK(*u
)] = unicode
;
506 return (PyObject
*)unicode
;
509 return PyUnicode_DecodeUTF8(u
, size
, NULL
);
512 unicode
= _PyUnicode_New(size
);
516 return (PyObject
*)unicode
;
519 PyObject
*PyUnicode_FromString(const char *u
)
521 size_t size
= strlen(u
);
522 if (size
> PY_SSIZE_T_MAX
) {
523 PyErr_SetString(PyExc_OverflowError
, "input too long");
527 return PyUnicode_FromStringAndSize(u
, size
);
532 PyObject
*PyUnicode_FromWideChar(register const wchar_t *w
,
535 PyUnicodeObject
*unicode
;
538 PyErr_BadInternalCall();
542 unicode
= _PyUnicode_New(size
);
546 /* Copy the wchar_t data into the new object */
547 #ifdef HAVE_USABLE_WCHAR_T
548 memcpy(unicode
->str
, w
, size
* sizeof(wchar_t));
551 register Py_UNICODE
*u
;
552 register Py_ssize_t i
;
553 u
= PyUnicode_AS_UNICODE(unicode
);
554 for (i
= size
; i
> 0; i
--)
559 return (PyObject
*)unicode
;
563 makefmt(char *fmt
, int longflag
, int size_tflag
, int zeropad
, int width
, int precision
, char c
)
569 fmt
+= sprintf(fmt
, "%d", width
);
572 fmt
+= sprintf(fmt
, ".%d", precision
);
575 else if (size_tflag
) {
576 char *f
= PY_FORMAT_SIZE_T
;
584 #define appendstring(string) {for (copy = string;*copy;) *s++ = *copy++;}
587 PyUnicode_FromFormatV(const char *format
, va_list vargs
)
590 Py_ssize_t callcount
= 0;
591 PyObject
**callresults
= NULL
;
592 PyObject
**callresult
= NULL
;
600 /* used by sprintf */
602 /* use abuffer instead of buffer, if we need more space
603 * (which can happen if there's a format specifier with width). */
604 char *abuffer
= NULL
;
606 Py_ssize_t abuffersize
= 0;
607 char fmt
[60]; /* should be enough for %0width.precisionld */
610 #ifdef VA_LIST_IS_ARRAY
611 Py_MEMCPY(count
, vargs
, sizeof(va_list));
614 __va_copy(count
, vargs
);
619 /* step 1: count the number of %S/%R format specifications
620 * (we call PyObject_Str()/PyObject_Repr() for these objects
621 * once during step 3 and put the result in an array) */
622 for (f
= format
; *f
; f
++) {
623 if (*f
== '%' && (*(f
+1)=='S' || *(f
+1)=='R'))
626 /* step 2: allocate memory for the results of
627 * PyObject_Str()/PyObject_Repr() calls */
629 callresults
= PyObject_Malloc(sizeof(PyObject
*)*callcount
);
634 callresult
= callresults
;
636 /* step 3: figure out how large a buffer we need */
637 for (f
= format
; *f
; f
++) {
641 while (isdigit((unsigned)*f
))
642 width
= (width
*10) + *f
++ - '0';
643 while (*++f
&& *f
!= '%' && !isalpha((unsigned)*f
))
646 /* skip the 'l' or 'z' in {%ld, %zd, %lu, %zu} since
647 * they don't affect the amount of space we reserve.
649 if ((*f
== 'l' || *f
== 'z') &&
650 (f
[1] == 'd' || f
[1] == 'u'))
655 (void)va_arg(count
, int);
656 /* fall through... */
660 case 'd': case 'u': case 'i': case 'x':
661 (void) va_arg(count
, int);
662 /* 20 bytes is enough to hold a 64-bit
663 integer. Decimal takes the most space.
664 This isn't enough for octal.
665 If a width is specified we need more
666 (which we allocate later). */
670 if (abuffersize
< width
)
677 s
= va_arg(count
, unsigned char*);
681 } else if (*s
< 0xc0) {
684 } else if (*s
< 0xc0) {
688 } else if (*s
< 0xe0) {
694 #ifdef Py_UNICODE_WIDE
709 PyObject
*obj
= va_arg(count
, PyObject
*);
710 assert(obj
&& PyUnicode_Check(obj
));
711 n
+= PyUnicode_GET_SIZE(obj
);
716 PyObject
*obj
= va_arg(count
, PyObject
*);
717 const char *str
= va_arg(count
, const char *);
719 assert(!obj
|| PyUnicode_Check(obj
));
721 n
+= PyUnicode_GET_SIZE(obj
);
728 PyObject
*obj
= va_arg(count
, PyObject
*);
731 str
= PyObject_Str(obj
);
734 n
+= PyUnicode_GET_SIZE(str
);
735 /* Remember the str and switch to the next slot */
741 PyObject
*obj
= va_arg(count
, PyObject
*);
744 repr
= PyObject_Repr(obj
);
747 n
+= PyUnicode_GET_SIZE(repr
);
748 /* Remember the repr and switch to the next slot */
749 *callresult
++ = repr
;
753 (void) va_arg(count
, int);
754 /* maximum 64-bit pointer representation:
756 * so 19 characters is enough.
757 * XXX I count 18 -- what's the extra for?
762 /* if we stumble upon an unknown
763 formatting code, copy the rest of
764 the format string to the output
765 string. (we cannot just skip the
766 code, since there's no way to know
767 what's in the argument list) */
775 if (abuffersize
> 20) {
776 abuffer
= PyObject_Malloc(abuffersize
);
781 realbuffer
= abuffer
;
785 /* step 4: fill the buffer */
786 /* Since we've analyzed how much space we need for the worst case,
787 we don't have to resize the string.
788 There can be no errors beyond this point. */
789 string
= PyUnicode_FromUnicode(NULL
, n
);
793 s
= PyUnicode_AS_UNICODE(string
);
794 callresult
= callresults
;
796 for (f
= format
; *f
; f
++) {
801 zeropad
= (*f
== '0');
802 /* parse the width.precision part */
804 while (isdigit((unsigned)*f
))
805 width
= (width
*10) + *f
++ - '0';
809 while (isdigit((unsigned)*f
))
810 precision
= (precision
*10) + *f
++ - '0';
812 /* handle the long flag, but only for %ld and %lu.
813 others can be added when necessary. */
814 if (*f
== 'l' && (f
[1] == 'd' || f
[1] == 'u')) {
818 /* handle the size_t flag. */
819 if (*f
== 'z' && (f
[1] == 'd' || f
[1] == 'u')) {
826 *s
++ = va_arg(vargs
, int);
829 makefmt(fmt
, longflag
, size_tflag
, zeropad
, width
, precision
, 'd');
831 sprintf(realbuffer
, fmt
, va_arg(vargs
, long));
833 sprintf(realbuffer
, fmt
, va_arg(vargs
, Py_ssize_t
));
835 sprintf(realbuffer
, fmt
, va_arg(vargs
, int));
836 appendstring(realbuffer
);
839 makefmt(fmt
, longflag
, size_tflag
, zeropad
, width
, precision
, 'u');
841 sprintf(realbuffer
, fmt
, va_arg(vargs
, unsigned long));
843 sprintf(realbuffer
, fmt
, va_arg(vargs
, size_t));
845 sprintf(realbuffer
, fmt
, va_arg(vargs
, unsigned int));
846 appendstring(realbuffer
);
849 makefmt(fmt
, 0, 0, zeropad
, width
, precision
, 'i');
850 sprintf(realbuffer
, fmt
, va_arg(vargs
, int));
851 appendstring(realbuffer
);
854 makefmt(fmt
, 0, 0, zeropad
, width
, precision
, 'x');
855 sprintf(realbuffer
, fmt
, va_arg(vargs
, int));
856 appendstring(realbuffer
);
860 /* Parameter must be UTF-8 encoded.
861 In case of encoding errors, use
862 the replacement character. */
864 p
= va_arg(vargs
, char*);
865 u
= PyUnicode_DecodeUTF8(p
, strlen(p
),
869 Py_UNICODE_COPY(s
, PyUnicode_AS_UNICODE(u
),
870 PyUnicode_GET_SIZE(u
));
871 s
+= PyUnicode_GET_SIZE(u
);
877 PyObject
*obj
= va_arg(vargs
, PyObject
*);
878 Py_ssize_t size
= PyUnicode_GET_SIZE(obj
);
879 Py_UNICODE_COPY(s
, PyUnicode_AS_UNICODE(obj
), size
);
885 PyObject
*obj
= va_arg(vargs
, PyObject
*);
886 const char *str
= va_arg(vargs
, const char *);
888 Py_ssize_t size
= PyUnicode_GET_SIZE(obj
);
889 Py_UNICODE_COPY(s
, PyUnicode_AS_UNICODE(obj
), size
);
902 /* unused, since we already have the result */
903 (void) va_arg(vargs
, PyObject
*);
904 ucopy
= PyUnicode_AS_UNICODE(*callresult
);
905 usize
= PyUnicode_GET_SIZE(*callresult
);
906 for (upos
= 0; upos
<usize
;)
907 *s
++ = ucopy
[upos
++];
908 /* We're done with the unicode()/repr() => forget it */
909 Py_DECREF(*callresult
);
910 /* switch to next unicode()/repr() result */
915 sprintf(buffer
, "%p", va_arg(vargs
, void*));
916 /* %p is ill-defined: ensure leading 0x. */
917 if (buffer
[1] == 'X')
919 else if (buffer
[1] != 'x') {
920 memmove(buffer
+2, buffer
, strlen(buffer
)+1);
924 appendstring(buffer
);
939 PyObject_Free(callresults
);
941 PyObject_Free(abuffer
);
942 PyUnicode_Resize(&string
, s
- PyUnicode_AS_UNICODE(string
));
946 PyObject
**callresult2
= callresults
;
947 while (callresult2
< callresult
) {
948 Py_DECREF(*callresult2
);
951 PyObject_Free(callresults
);
954 PyObject_Free(abuffer
);
961 PyUnicode_FromFormat(const char *format
, ...)
966 #ifdef HAVE_STDARG_PROTOTYPES
967 va_start(vargs
, format
);
971 ret
= PyUnicode_FromFormatV(format
, vargs
);
976 Py_ssize_t
PyUnicode_AsWideChar(PyUnicodeObject
*unicode
,
980 if (unicode
== NULL
) {
981 PyErr_BadInternalCall();
985 /* If possible, try to copy the 0-termination as well */
986 if (size
> PyUnicode_GET_SIZE(unicode
))
987 size
= PyUnicode_GET_SIZE(unicode
) + 1;
989 #ifdef HAVE_USABLE_WCHAR_T
990 memcpy(w
, unicode
->str
, size
* sizeof(wchar_t));
993 register Py_UNICODE
*u
;
994 register Py_ssize_t i
;
995 u
= PyUnicode_AS_UNICODE(unicode
);
996 for (i
= size
; i
> 0; i
--)
1001 if (size
> PyUnicode_GET_SIZE(unicode
))
1002 return PyUnicode_GET_SIZE(unicode
);
1009 PyObject
*PyUnicode_FromOrdinal(int ordinal
)
1013 #ifdef Py_UNICODE_WIDE
1014 if (ordinal
< 0 || ordinal
> 0x10ffff) {
1015 PyErr_SetString(PyExc_ValueError
,
1016 "unichr() arg not in range(0x110000) "
1017 "(wide Python build)");
1021 if (ordinal
< 0 || ordinal
> 0xffff) {
1022 PyErr_SetString(PyExc_ValueError
,
1023 "unichr() arg not in range(0x10000) "
1024 "(narrow Python build)");
1029 s
[0] = (Py_UNICODE
)ordinal
;
1030 return PyUnicode_FromUnicode(s
, 1);
1033 PyObject
*PyUnicode_FromObject(register PyObject
*obj
)
1035 /* XXX Perhaps we should make this API an alias of
1036 PyObject_Unicode() instead ?! */
1037 if (PyUnicode_CheckExact(obj
)) {
1041 if (PyUnicode_Check(obj
)) {
1042 /* For a Unicode subtype that's not a Unicode object,
1043 return a true Unicode object with the same data. */
1044 return PyUnicode_FromUnicode(PyUnicode_AS_UNICODE(obj
),
1045 PyUnicode_GET_SIZE(obj
));
1047 return PyUnicode_FromEncodedObject(obj
, NULL
, "strict");
1050 PyObject
*PyUnicode_FromEncodedObject(register PyObject
*obj
,
1051 const char *encoding
,
1054 const char *s
= NULL
;
1059 PyErr_BadInternalCall();
1064 /* For b/w compatibility we also accept Unicode objects provided
1065 that no encodings is given and then redirect to
1066 PyObject_Unicode() which then applies the additional logic for
1069 NOTE: This API should really only be used for object which
1070 represent *encoded* Unicode !
1073 if (PyUnicode_Check(obj
)) {
1075 PyErr_SetString(PyExc_TypeError
,
1076 "decoding Unicode is not supported");
1079 return PyObject_Unicode(obj
);
1082 if (PyUnicode_Check(obj
)) {
1083 PyErr_SetString(PyExc_TypeError
,
1084 "decoding Unicode is not supported");
1090 if (PyString_Check(obj
)) {
1091 s
= PyString_AS_STRING(obj
);
1092 len
= PyString_GET_SIZE(obj
);
1094 else if (PyByteArray_Check(obj
)) {
1095 /* Python 2.x specific */
1096 PyErr_Format(PyExc_TypeError
,
1097 "decoding bytearray is not supported");
1100 else if (PyObject_AsCharBuffer(obj
, &s
, &len
)) {
1101 /* Overwrite the error message with something more useful in
1102 case of a TypeError. */
1103 if (PyErr_ExceptionMatches(PyExc_TypeError
))
1104 PyErr_Format(PyExc_TypeError
,
1105 "coercing to Unicode: need string or buffer, "
1107 Py_TYPE(obj
)->tp_name
);
1111 /* Convert to Unicode */
1113 Py_INCREF(unicode_empty
);
1114 v
= (PyObject
*)unicode_empty
;
1117 v
= PyUnicode_Decode(s
, len
, encoding
, errors
);
1125 PyObject
*PyUnicode_Decode(const char *s
,
1127 const char *encoding
,
1130 PyObject
*buffer
= NULL
, *unicode
;
1132 if (encoding
== NULL
)
1133 encoding
= PyUnicode_GetDefaultEncoding();
1135 /* Shortcuts for common default encodings */
1136 if (strcmp(encoding
, "utf-8") == 0)
1137 return PyUnicode_DecodeUTF8(s
, size
, errors
);
1138 else if (strcmp(encoding
, "latin-1") == 0)
1139 return PyUnicode_DecodeLatin1(s
, size
, errors
);
1140 #if defined(MS_WINDOWS) && defined(HAVE_USABLE_WCHAR_T)
1141 else if (strcmp(encoding
, "mbcs") == 0)
1142 return PyUnicode_DecodeMBCS(s
, size
, errors
);
1144 else if (strcmp(encoding
, "ascii") == 0)
1145 return PyUnicode_DecodeASCII(s
, size
, errors
);
1147 /* Decode via the codec registry */
1148 buffer
= PyBuffer_FromMemory((void *)s
, size
);
1151 unicode
= PyCodec_Decode(buffer
, encoding
, errors
);
1152 if (unicode
== NULL
)
1154 if (!PyUnicode_Check(unicode
)) {
1155 PyErr_Format(PyExc_TypeError
,
1156 "decoder did not return an unicode object (type=%.400s)",
1157 Py_TYPE(unicode
)->tp_name
);
1169 PyObject
*PyUnicode_AsDecodedObject(PyObject
*unicode
,
1170 const char *encoding
,
1175 if (!PyUnicode_Check(unicode
)) {
1176 PyErr_BadArgument();
1180 if (encoding
== NULL
)
1181 encoding
= PyUnicode_GetDefaultEncoding();
1183 /* Decode via the codec registry */
1184 v
= PyCodec_Decode(unicode
, encoding
, errors
);
1193 PyObject
*PyUnicode_Encode(const Py_UNICODE
*s
,
1195 const char *encoding
,
1198 PyObject
*v
, *unicode
;
1200 unicode
= PyUnicode_FromUnicode(s
, size
);
1201 if (unicode
== NULL
)
1203 v
= PyUnicode_AsEncodedString(unicode
, encoding
, errors
);
1208 PyObject
*PyUnicode_AsEncodedObject(PyObject
*unicode
,
1209 const char *encoding
,
1214 if (!PyUnicode_Check(unicode
)) {
1215 PyErr_BadArgument();
1219 if (encoding
== NULL
)
1220 encoding
= PyUnicode_GetDefaultEncoding();
1222 /* Encode via the codec registry */
1223 v
= PyCodec_Encode(unicode
, encoding
, errors
);
1232 PyObject
*PyUnicode_AsEncodedString(PyObject
*unicode
,
1233 const char *encoding
,
1238 if (!PyUnicode_Check(unicode
)) {
1239 PyErr_BadArgument();
1243 if (encoding
== NULL
)
1244 encoding
= PyUnicode_GetDefaultEncoding();
1246 /* Shortcuts for common default encodings */
1247 if (errors
== NULL
) {
1248 if (strcmp(encoding
, "utf-8") == 0)
1249 return PyUnicode_AsUTF8String(unicode
);
1250 else if (strcmp(encoding
, "latin-1") == 0)
1251 return PyUnicode_AsLatin1String(unicode
);
1252 #if defined(MS_WINDOWS) && defined(HAVE_USABLE_WCHAR_T)
1253 else if (strcmp(encoding
, "mbcs") == 0)
1254 return PyUnicode_AsMBCSString(unicode
);
1256 else if (strcmp(encoding
, "ascii") == 0)
1257 return PyUnicode_AsASCIIString(unicode
);
1260 /* Encode via the codec registry */
1261 v
= PyCodec_Encode(unicode
, encoding
, errors
);
1264 if (!PyString_Check(v
)) {
1265 PyErr_Format(PyExc_TypeError
,
1266 "encoder did not return a string object (type=%.400s)",
1267 Py_TYPE(v
)->tp_name
);
1277 PyObject
*_PyUnicode_AsDefaultEncodedString(PyObject
*unicode
,
1280 PyObject
*v
= ((PyUnicodeObject
*)unicode
)->defenc
;
1284 v
= PyUnicode_AsEncodedString(unicode
, NULL
, errors
);
1285 if (v
&& errors
== NULL
)
1286 ((PyUnicodeObject
*)unicode
)->defenc
= v
;
1290 Py_UNICODE
*PyUnicode_AsUnicode(PyObject
*unicode
)
1292 if (!PyUnicode_Check(unicode
)) {
1293 PyErr_BadArgument();
1296 return PyUnicode_AS_UNICODE(unicode
);
1302 Py_ssize_t
PyUnicode_GetSize(PyObject
*unicode
)
1304 if (!PyUnicode_Check(unicode
)) {
1305 PyErr_BadArgument();
1308 return PyUnicode_GET_SIZE(unicode
);
1314 const char *PyUnicode_GetDefaultEncoding(void)
1316 return unicode_default_encoding
;
1319 int PyUnicode_SetDefaultEncoding(const char *encoding
)
1323 /* Make sure the encoding is valid. As side effect, this also
1324 loads the encoding into the codec registry cache. */
1325 v
= _PyCodec_Lookup(encoding
);
1329 strncpy(unicode_default_encoding
,
1331 sizeof(unicode_default_encoding
));
1338 /* error handling callback helper:
1339 build arguments, call the callback and check the arguments,
1340 if no exception occurred, copy the replacement to the output
1341 and adjust various state variables.
1342 return 0 on success, -1 on error
1346 int unicode_decode_call_errorhandler(const char *errors
, PyObject
**errorHandler
,
1347 const char *encoding
, const char *reason
,
1348 const char *input
, Py_ssize_t insize
, Py_ssize_t
*startinpos
,
1349 Py_ssize_t
*endinpos
, PyObject
**exceptionObject
, const char **inptr
,
1350 PyUnicodeObject
**output
, Py_ssize_t
*outpos
, Py_UNICODE
**outptr
)
1352 static char *argparse
= "O!n;decoding error handler must return (unicode, int) tuple";
1354 PyObject
*restuple
= NULL
;
1355 PyObject
*repunicode
= NULL
;
1356 Py_ssize_t outsize
= PyUnicode_GET_SIZE(*output
);
1357 Py_ssize_t requiredsize
;
1363 if (*errorHandler
== NULL
) {
1364 *errorHandler
= PyCodec_LookupError(errors
);
1365 if (*errorHandler
== NULL
)
1369 if (*exceptionObject
== NULL
) {
1370 *exceptionObject
= PyUnicodeDecodeError_Create(
1371 encoding
, input
, insize
, *startinpos
, *endinpos
, reason
);
1372 if (*exceptionObject
== NULL
)
1376 if (PyUnicodeDecodeError_SetStart(*exceptionObject
, *startinpos
))
1378 if (PyUnicodeDecodeError_SetEnd(*exceptionObject
, *endinpos
))
1380 if (PyUnicodeDecodeError_SetReason(*exceptionObject
, reason
))
1384 restuple
= PyObject_CallFunctionObjArgs(*errorHandler
, *exceptionObject
, NULL
);
1385 if (restuple
== NULL
)
1387 if (!PyTuple_Check(restuple
)) {
1388 PyErr_Format(PyExc_TypeError
, &argparse
[4]);
1391 if (!PyArg_ParseTuple(restuple
, argparse
, &PyUnicode_Type
, &repunicode
, &newpos
))
1394 newpos
= insize
+newpos
;
1395 if (newpos
<0 || newpos
>insize
) {
1396 PyErr_Format(PyExc_IndexError
, "position %zd from error handler out of bounds", newpos
);
1400 /* need more space? (at least enough for what we
1401 have+the replacement+the rest of the string (starting
1402 at the new input position), so we won't have to check space
1403 when there are no errors in the rest of the string) */
1404 repptr
= PyUnicode_AS_UNICODE(repunicode
);
1405 repsize
= PyUnicode_GET_SIZE(repunicode
);
1406 requiredsize
= *outpos
+ repsize
+ insize
-newpos
;
1407 if (requiredsize
> outsize
) {
1408 if (requiredsize
<2*outsize
)
1409 requiredsize
= 2*outsize
;
1410 if (_PyUnicode_Resize(output
, requiredsize
) < 0)
1412 *outptr
= PyUnicode_AS_UNICODE(*output
) + *outpos
;
1415 *inptr
= input
+ newpos
;
1416 Py_UNICODE_COPY(*outptr
, repptr
, repsize
);
1423 Py_XDECREF(restuple
);
1427 /* --- UTF-7 Codec -------------------------------------------------------- */
1429 /* see RFC2152 for details */
1432 char utf7_special
[128] = {
1433 /* indicate whether a UTF-7 character is special i.e. cannot be directly
1437 2 - whitespace (optional)
1438 3 - RFC2152 Set O (optional) */
1439 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 1, 1, 2, 1, 1,
1440 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1441 2, 3, 3, 3, 3, 3, 3, 0, 0, 0, 3, 1, 0, 0, 0, 1,
1442 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 3, 3, 0,
1443 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1444 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 3, 3, 3,
1445 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1446 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 3, 1, 1,
1450 /* Note: The comparison (c) <= 0 is a trick to work-around gcc
1451 warnings about the comparison always being false; since
1452 utf7_special[0] is 1, we can safely make that one comparison
1455 #define SPECIAL(c, encodeO, encodeWS) \
1456 ((c) > 127 || (c) <= 0 || utf7_special[(c)] == 1 || \
1457 (encodeWS && (utf7_special[(c)] == 2)) || \
1458 (encodeO && (utf7_special[(c)] == 3)))
1461 ("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"[(n) & 0x3f])
1462 #define B64CHAR(c) \
1463 (isalnum(c) || (c) == '+' || (c) == '/')
1465 ((c) == '+' ? 62 : (c) == '/' ? 63 : (c) >= 'a' ? \
1466 (c) - 71 : (c) >= 'A' ? (c) - 65 : (c) + 4 )
1468 #define ENCODE(out, ch, bits) \
1469 while (bits >= 6) { \
1470 *out++ = B64(ch >> (bits-6)); \
1474 #define DECODE(out, ch, bits, surrogate) \
1475 while (bits >= 16) { \
1476 Py_UNICODE outCh = (Py_UNICODE) ((ch >> (bits-16)) & 0xffff); \
1479 /* We have already generated an error for the high surrogate \
1480 so let's not bother seeing if the low surrogate is correct or not */ \
1482 } else if (0xDC00 <= outCh && outCh <= 0xDFFF) { \
1483 /* This is a surrogate pair. Unfortunately we can't represent \
1484 it in a 16-bit character */ \
1486 errmsg = "code pairs are not supported"; \
1493 PyObject
*PyUnicode_DecodeUTF7(const char *s
,
1497 return PyUnicode_DecodeUTF7Stateful(s
, size
, errors
, NULL
);
1500 PyObject
*PyUnicode_DecodeUTF7Stateful(const char *s
,
1503 Py_ssize_t
*consumed
)
1505 const char *starts
= s
;
1506 Py_ssize_t startinpos
;
1507 Py_ssize_t endinpos
;
1510 PyUnicodeObject
*unicode
;
1512 const char *errmsg
= "";
1514 unsigned int bitsleft
= 0;
1515 unsigned long charsleft
= 0;
1517 PyObject
*errorHandler
= NULL
;
1518 PyObject
*exc
= NULL
;
1520 unicode
= _PyUnicode_New(size
);
1526 return (PyObject
*)unicode
;
1535 ch
= (unsigned char) *s
;
1538 if ((ch
== '-') || !B64CHAR(ch
)) {
1542 /* p, charsleft, bitsleft, surrogate = */ DECODE(p
, charsleft
, bitsleft
, surrogate
);
1543 if (bitsleft
>= 6) {
1544 /* The shift sequence has a partial character in it. If
1545 bitsleft < 6 then we could just classify it as padding
1546 but that is not the case here */
1548 errmsg
= "partial character in shift sequence";
1551 /* According to RFC2152 the remaining bits should be zero. We
1552 choose to signal an error/insert a replacement character
1553 here so indicate the potential of a misencoded character. */
1555 /* On x86, a << b == a << (b%32) so make sure that bitsleft != 0 */
1556 if (bitsleft
&& charsleft
<< (sizeof(charsleft
) * 8 - bitsleft
)) {
1557 errmsg
= "non-zero padding bits in shift sequence";
1562 if ((s
< e
) && (*(s
) == '-')) {
1566 } else if (SPECIAL(ch
,0,0)) {
1567 errmsg
= "unexpected special character";
1573 charsleft
= (charsleft
<< 6) | UB64(ch
);
1576 /* p, charsleft, bitsleft, surrogate = */ DECODE(p
, charsleft
, bitsleft
, surrogate
);
1579 else if ( ch
== '+' ) {
1580 startinpos
= s
-starts
;
1582 if (s
< e
&& *s
== '-') {
1591 else if (SPECIAL(ch
,0,0)) {
1592 startinpos
= s
-starts
;
1593 errmsg
= "unexpected special character";
1603 outpos
= p
-PyUnicode_AS_UNICODE(unicode
);
1604 endinpos
= s
-starts
;
1605 if (unicode_decode_call_errorhandler(
1606 errors
, &errorHandler
,
1608 starts
, size
, &startinpos
, &endinpos
, &exc
, &s
,
1609 &unicode
, &outpos
, &p
))
1613 if (inShift
&& !consumed
) {
1614 outpos
= p
-PyUnicode_AS_UNICODE(unicode
);
1616 if (unicode_decode_call_errorhandler(
1617 errors
, &errorHandler
,
1618 "utf7", "unterminated shift sequence",
1619 starts
, size
, &startinpos
, &endinpos
, &exc
, &s
,
1620 &unicode
, &outpos
, &p
))
1627 *consumed
= startinpos
;
1629 *consumed
= s
-starts
;
1632 if (_PyUnicode_Resize(&unicode
, p
- PyUnicode_AS_UNICODE(unicode
)) < 0)
1635 Py_XDECREF(errorHandler
);
1637 return (PyObject
*)unicode
;
1640 Py_XDECREF(errorHandler
);
1647 PyObject
*PyUnicode_EncodeUTF7(const Py_UNICODE
*s
,
1650 int encodeWhiteSpace
,
1654 /* It might be possible to tighten this worst case */
1655 Py_ssize_t cbAllocated
= 5 * size
;
1658 unsigned int bitsleft
= 0;
1659 unsigned long charsleft
= 0;
1663 if (cbAllocated
/ 5 != size
)
1664 return PyErr_NoMemory();
1667 return PyString_FromStringAndSize(NULL
, 0);
1669 v
= PyString_FromStringAndSize(NULL
, cbAllocated
);
1673 start
= out
= PyString_AS_STRING(v
);
1674 for (;i
< size
; ++i
) {
1675 Py_UNICODE ch
= s
[i
];
1681 } else if (SPECIAL(ch
, encodeSetO
, encodeWhiteSpace
)) {
1685 /* out, charsleft, bitsleft = */ ENCODE(out
, charsleft
, bitsleft
);
1686 inShift
= bitsleft
> 0;
1691 if (!SPECIAL(ch
, encodeSetO
, encodeWhiteSpace
)) {
1692 *out
++ = B64(charsleft
<< (6-bitsleft
));
1695 /* Characters not in the BASE64 set implicitly unshift the sequence
1696 so no '-' is required, except if the character is itself a '-' */
1697 if (B64CHAR(ch
) || ch
== '-') {
1704 charsleft
= (charsleft
<< 16) | ch
;
1705 /* out, charsleft, bitsleft = */ ENCODE(out
, charsleft
, bitsleft
);
1707 /* If the next character is special then we dont' need to terminate
1708 the shift sequence. If the next character is not a BASE64 character
1709 or '-' then the shift sequence will be terminated implicitly and we
1710 don't have to insert a '-'. */
1712 if (bitsleft
== 0) {
1714 Py_UNICODE ch2
= s
[i
+1];
1716 if (SPECIAL(ch2
, encodeSetO
, encodeWhiteSpace
)) {
1718 } else if (B64CHAR(ch2
) || ch2
== '-') {
1735 *out
++= B64(charsleft
<< (6-bitsleft
) );
1739 _PyString_Resize(&v
, out
- start
);
1750 /* --- UTF-8 Codec -------------------------------------------------------- */
1753 char utf8_code_length
[256] = {
1754 /* Map UTF-8 encoded prefix byte to sequence length. zero means
1755 illegal prefix. see RFC 2279 for details */
1756 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1757 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1758 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1759 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1760 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1761 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1762 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1763 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1764 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1765 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1766 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1767 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1768 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
1769 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
1770 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
1771 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 0, 0
1774 PyObject
*PyUnicode_DecodeUTF8(const char *s
,
1778 return PyUnicode_DecodeUTF8Stateful(s
, size
, errors
, NULL
);
1781 PyObject
*PyUnicode_DecodeUTF8Stateful(const char *s
,
1784 Py_ssize_t
*consumed
)
1786 const char *starts
= s
;
1788 Py_ssize_t startinpos
;
1789 Py_ssize_t endinpos
;
1792 PyUnicodeObject
*unicode
;
1794 const char *errmsg
= "";
1795 PyObject
*errorHandler
= NULL
;
1796 PyObject
*exc
= NULL
;
1798 /* Note: size will always be longer than the resulting Unicode
1800 unicode
= _PyUnicode_New(size
);
1806 return (PyObject
*)unicode
;
1809 /* Unpack UTF-8 encoded data */
1814 Py_UCS4 ch
= (unsigned char)*s
;
1817 *p
++ = (Py_UNICODE
)ch
;
1822 n
= utf8_code_length
[ch
];
1828 errmsg
= "unexpected end of data";
1829 startinpos
= s
-starts
;
1838 errmsg
= "unexpected code byte";
1839 startinpos
= s
-starts
;
1840 endinpos
= startinpos
+1;
1844 errmsg
= "internal error";
1845 startinpos
= s
-starts
;
1846 endinpos
= startinpos
+1;
1850 if ((s
[1] & 0xc0) != 0x80) {
1851 errmsg
= "invalid data";
1852 startinpos
= s
-starts
;
1853 endinpos
= startinpos
+2;
1856 ch
= ((s
[0] & 0x1f) << 6) + (s
[1] & 0x3f);
1858 startinpos
= s
-starts
;
1859 endinpos
= startinpos
+2;
1860 errmsg
= "illegal encoding";
1864 *p
++ = (Py_UNICODE
)ch
;
1868 if ((s
[1] & 0xc0) != 0x80 ||
1869 (s
[2] & 0xc0) != 0x80) {
1870 errmsg
= "invalid data";
1871 startinpos
= s
-starts
;
1872 endinpos
= startinpos
+3;
1875 ch
= ((s
[0] & 0x0f) << 12) + ((s
[1] & 0x3f) << 6) + (s
[2] & 0x3f);
1877 /* Note: UTF-8 encodings of surrogates are considered
1878 legal UTF-8 sequences;
1880 XXX For wide builds (UCS-4) we should probably try
1881 to recombine the surrogates into a single code
1884 errmsg
= "illegal encoding";
1885 startinpos
= s
-starts
;
1886 endinpos
= startinpos
+3;
1890 *p
++ = (Py_UNICODE
)ch
;
1894 if ((s
[1] & 0xc0) != 0x80 ||
1895 (s
[2] & 0xc0) != 0x80 ||
1896 (s
[3] & 0xc0) != 0x80) {
1897 errmsg
= "invalid data";
1898 startinpos
= s
-starts
;
1899 endinpos
= startinpos
+4;
1902 ch
= ((s
[0] & 0x7) << 18) + ((s
[1] & 0x3f) << 12) +
1903 ((s
[2] & 0x3f) << 6) + (s
[3] & 0x3f);
1904 /* validate and convert to UTF-16 */
1905 if ((ch
< 0x10000) /* minimum value allowed for 4
1907 || (ch
> 0x10ffff)) /* maximum value allowed for
1910 errmsg
= "illegal encoding";
1911 startinpos
= s
-starts
;
1912 endinpos
= startinpos
+4;
1915 #ifdef Py_UNICODE_WIDE
1916 *p
++ = (Py_UNICODE
)ch
;
1918 /* compute and append the two surrogates: */
1920 /* translate from 10000..10FFFF to 0..FFFF */
1923 /* high surrogate = top 10 bits added to D800 */
1924 *p
++ = (Py_UNICODE
)(0xD800 + (ch
>> 10));
1926 /* low surrogate = bottom 10 bits added to DC00 */
1927 *p
++ = (Py_UNICODE
)(0xDC00 + (ch
& 0x03FF));
1932 /* Other sizes are only needed for UCS-4 */
1933 errmsg
= "unsupported Unicode code range";
1934 startinpos
= s
-starts
;
1935 endinpos
= startinpos
+n
;
1942 outpos
= p
-PyUnicode_AS_UNICODE(unicode
);
1943 if (unicode_decode_call_errorhandler(
1944 errors
, &errorHandler
,
1946 starts
, size
, &startinpos
, &endinpos
, &exc
, &s
,
1947 &unicode
, &outpos
, &p
))
1951 *consumed
= s
-starts
;
1954 if (_PyUnicode_Resize(&unicode
, p
- unicode
->str
) < 0)
1957 Py_XDECREF(errorHandler
);
1959 return (PyObject
*)unicode
;
1962 Py_XDECREF(errorHandler
);
1968 /* Allocation strategy: if the string is short, convert into a stack buffer
1969 and allocate exactly as much space needed at the end. Else allocate the
1970 maximum possible needed (4 result bytes per Unicode character), and return
1971 the excess memory at the end.
1974 PyUnicode_EncodeUTF8(const Py_UNICODE
*s
,
1978 #define MAX_SHORT_UNICHARS 300 /* largest size we'll do on the stack */
1980 Py_ssize_t i
; /* index into s of next input byte */
1981 PyObject
*v
; /* result string object */
1982 char *p
; /* next free byte in output buffer */
1983 Py_ssize_t nallocated
; /* number of result bytes allocated */
1984 Py_ssize_t nneeded
; /* number of result bytes needed */
1985 char stackbuf
[MAX_SHORT_UNICHARS
* 4];
1990 if (size
<= MAX_SHORT_UNICHARS
) {
1991 /* Write into the stack buffer; nallocated can't overflow.
1992 * At the end, we'll allocate exactly as much heap space as it
1993 * turns out we need.
1995 nallocated
= Py_SAFE_DOWNCAST(sizeof(stackbuf
), size_t, int);
1996 v
= NULL
; /* will allocate after we're done */
2000 /* Overallocate on the heap, and give the excess back at the end. */
2001 nallocated
= size
* 4;
2002 if (nallocated
/ 4 != size
) /* overflow! */
2003 return PyErr_NoMemory();
2004 v
= PyString_FromStringAndSize(NULL
, nallocated
);
2007 p
= PyString_AS_STRING(v
);
2010 for (i
= 0; i
< size
;) {
2011 Py_UCS4 ch
= s
[i
++];
2017 else if (ch
< 0x0800) {
2018 /* Encode Latin-1 */
2019 *p
++ = (char)(0xc0 | (ch
>> 6));
2020 *p
++ = (char)(0x80 | (ch
& 0x3f));
2023 /* Encode UCS2 Unicode ordinals */
2025 /* Special case: check for high surrogate */
2026 if (0xD800 <= ch
&& ch
<= 0xDBFF && i
!= size
) {
2028 /* Check for low surrogate and combine the two to
2029 form a UCS4 value */
2030 if (0xDC00 <= ch2
&& ch2
<= 0xDFFF) {
2031 ch
= ((ch
- 0xD800) << 10 | (ch2
- 0xDC00)) + 0x10000;
2035 /* Fall through: handles isolated high surrogates */
2037 *p
++ = (char)(0xe0 | (ch
>> 12));
2038 *p
++ = (char)(0x80 | ((ch
>> 6) & 0x3f));
2039 *p
++ = (char)(0x80 | (ch
& 0x3f));
2043 /* Encode UCS4 Unicode ordinals */
2044 *p
++ = (char)(0xf0 | (ch
>> 18));
2045 *p
++ = (char)(0x80 | ((ch
>> 12) & 0x3f));
2046 *p
++ = (char)(0x80 | ((ch
>> 6) & 0x3f));
2047 *p
++ = (char)(0x80 | (ch
& 0x3f));
2052 /* This was stack allocated. */
2053 nneeded
= p
- stackbuf
;
2054 assert(nneeded
<= nallocated
);
2055 v
= PyString_FromStringAndSize(stackbuf
, nneeded
);
2058 /* Cut back to size actually needed. */
2059 nneeded
= p
- PyString_AS_STRING(v
);
2060 assert(nneeded
<= nallocated
);
2061 _PyString_Resize(&v
, nneeded
);
2065 #undef MAX_SHORT_UNICHARS
2068 PyObject
*PyUnicode_AsUTF8String(PyObject
*unicode
)
2070 if (!PyUnicode_Check(unicode
)) {
2071 PyErr_BadArgument();
2074 return PyUnicode_EncodeUTF8(PyUnicode_AS_UNICODE(unicode
),
2075 PyUnicode_GET_SIZE(unicode
),
2079 /* --- UTF-32 Codec ------------------------------------------------------- */
2082 PyUnicode_DecodeUTF32(const char *s
,
2087 return PyUnicode_DecodeUTF32Stateful(s
, size
, errors
, byteorder
, NULL
);
2091 PyUnicode_DecodeUTF32Stateful(const char *s
,
2095 Py_ssize_t
*consumed
)
2097 const char *starts
= s
;
2098 Py_ssize_t startinpos
;
2099 Py_ssize_t endinpos
;
2101 PyUnicodeObject
*unicode
;
2103 #ifndef Py_UNICODE_WIDE
2106 const int pairs
= 0;
2108 const unsigned char *q
, *e
;
2109 int bo
= 0; /* assume native ordering by default */
2110 const char *errmsg
= "";
2111 /* Offsets from q for retrieving bytes in the right order. */
2112 #ifdef BYTEORDER_IS_LITTLE_ENDIAN
2113 int iorder
[] = {0, 1, 2, 3};
2115 int iorder
[] = {3, 2, 1, 0};
2117 PyObject
*errorHandler
= NULL
;
2118 PyObject
*exc
= NULL
;
2119 /* On narrow builds we split characters outside the BMP into two
2120 codepoints => count how much extra space we need. */
2121 #ifndef Py_UNICODE_WIDE
2122 for (i
= pairs
= 0; i
< size
/4; i
++)
2123 if (((Py_UCS4
*)s
)[i
] >= 0x10000)
2127 /* This might be one to much, because of a BOM */
2128 unicode
= _PyUnicode_New((size
+3)/4+pairs
);
2132 return (PyObject
*)unicode
;
2134 /* Unpack UTF-32 encoded data */
2136 q
= (unsigned char *)s
;
2142 /* Check for BOM marks (U+FEFF) in the input and adjust current
2143 byte order setting accordingly. In native mode, the leading BOM
2144 mark is skipped, in all other modes, it is copied to the output
2145 stream as-is (giving a ZWNBSP character). */
2148 const Py_UCS4 bom
= (q
[iorder
[3]] << 24) | (q
[iorder
[2]] << 16) |
2149 (q
[iorder
[1]] << 8) | q
[iorder
[0]];
2150 #ifdef BYTEORDER_IS_LITTLE_ENDIAN
2151 if (bom
== 0x0000FEFF) {
2155 else if (bom
== 0xFFFE0000) {
2160 if (bom
== 0x0000FEFF) {
2164 else if (bom
== 0xFFFE0000) {
2189 /* remaining bytes at the end? (size should be divisible by 4) */
2193 errmsg
= "truncated data";
2194 startinpos
= ((const char *)q
)-starts
;
2195 endinpos
= ((const char *)e
)-starts
;
2197 /* The remaining input chars are ignored if the callback
2198 chooses to skip the input */
2200 ch
= (q
[iorder
[3]] << 24) | (q
[iorder
[2]] << 16) |
2201 (q
[iorder
[1]] << 8) | q
[iorder
[0]];
2205 errmsg
= "codepoint not in range(0x110000)";
2206 startinpos
= ((const char *)q
)-starts
;
2207 endinpos
= startinpos
+4;
2210 #ifndef Py_UNICODE_WIDE
2213 *p
++ = 0xD800 | ((ch
-0x10000) >> 10);
2214 *p
++ = 0xDC00 | ((ch
-0x10000) & 0x3FF);
2222 outpos
= p
-PyUnicode_AS_UNICODE(unicode
);
2223 if (unicode_decode_call_errorhandler(
2224 errors
, &errorHandler
,
2226 starts
, size
, &startinpos
, &endinpos
, &exc
, &s
,
2227 &unicode
, &outpos
, &p
))
2235 *consumed
= (const char *)q
-starts
;
2238 if (_PyUnicode_Resize(&unicode
, p
- unicode
->str
) < 0)
2241 Py_XDECREF(errorHandler
);
2243 return (PyObject
*)unicode
;
2247 Py_XDECREF(errorHandler
);
2253 PyUnicode_EncodeUTF32(const Py_UNICODE
*s
,
2260 Py_ssize_t nsize
, bytesize
;
2261 #ifndef Py_UNICODE_WIDE
2262 Py_ssize_t i
, pairs
;
2264 const int pairs
= 0;
2266 /* Offsets from p for storing byte pairs in the right order. */
2267 #ifdef BYTEORDER_IS_LITTLE_ENDIAN
2268 int iorder
[] = {0, 1, 2, 3};
2270 int iorder
[] = {3, 2, 1, 0};
2273 #define STORECHAR(CH) \
2275 p[iorder[3]] = ((CH) >> 24) & 0xff; \
2276 p[iorder[2]] = ((CH) >> 16) & 0xff; \
2277 p[iorder[1]] = ((CH) >> 8) & 0xff; \
2278 p[iorder[0]] = (CH) & 0xff; \
2282 /* In narrow builds we can output surrogate pairs as one codepoint,
2283 so we need less space. */
2284 #ifndef Py_UNICODE_WIDE
2285 for (i
= pairs
= 0; i
< size
-1; i
++)
2286 if (0xD800 <= s
[i
] && s
[i
] <= 0xDBFF &&
2287 0xDC00 <= s
[i
+1] && s
[i
+1] <= 0xDFFF)
2290 nsize
= (size
- pairs
+ (byteorder
== 0));
2291 bytesize
= nsize
* 4;
2292 if (bytesize
/ 4 != nsize
)
2293 return PyErr_NoMemory();
2294 v
= PyString_FromStringAndSize(NULL
, bytesize
);
2298 p
= (unsigned char *)PyString_AS_STRING(v
);
2304 if (byteorder
== -1) {
2311 else if (byteorder
== 1) {
2319 while (size
-- > 0) {
2321 #ifndef Py_UNICODE_WIDE
2322 if (0xD800 <= ch
&& ch
<= 0xDBFF && size
> 0) {
2324 if (0xDC00 <= ch2
&& ch2
<= 0xDFFF) {
2325 ch
= (((ch
& 0x3FF)<<10) | (ch2
& 0x3FF)) + 0x10000;
2337 PyObject
*PyUnicode_AsUTF32String(PyObject
*unicode
)
2339 if (!PyUnicode_Check(unicode
)) {
2340 PyErr_BadArgument();
2343 return PyUnicode_EncodeUTF32(PyUnicode_AS_UNICODE(unicode
),
2344 PyUnicode_GET_SIZE(unicode
),
2349 /* --- UTF-16 Codec ------------------------------------------------------- */
2352 PyUnicode_DecodeUTF16(const char *s
,
2357 return PyUnicode_DecodeUTF16Stateful(s
, size
, errors
, byteorder
, NULL
);
2361 PyUnicode_DecodeUTF16Stateful(const char *s
,
2365 Py_ssize_t
*consumed
)
2367 const char *starts
= s
;
2368 Py_ssize_t startinpos
;
2369 Py_ssize_t endinpos
;
2371 PyUnicodeObject
*unicode
;
2373 const unsigned char *q
, *e
;
2374 int bo
= 0; /* assume native ordering by default */
2375 const char *errmsg
= "";
2376 /* Offsets from q for retrieving byte pairs in the right order. */
2377 #ifdef BYTEORDER_IS_LITTLE_ENDIAN
2378 int ihi
= 1, ilo
= 0;
2380 int ihi
= 0, ilo
= 1;
2382 PyObject
*errorHandler
= NULL
;
2383 PyObject
*exc
= NULL
;
2385 /* Note: size will always be longer than the resulting Unicode
2387 unicode
= _PyUnicode_New(size
);
2391 return (PyObject
*)unicode
;
2393 /* Unpack UTF-16 encoded data */
2395 q
= (unsigned char *)s
;
2401 /* Check for BOM marks (U+FEFF) in the input and adjust current
2402 byte order setting accordingly. In native mode, the leading BOM
2403 mark is skipped, in all other modes, it is copied to the output
2404 stream as-is (giving a ZWNBSP character). */
2407 const Py_UNICODE bom
= (q
[ihi
] << 8) | q
[ilo
];
2408 #ifdef BYTEORDER_IS_LITTLE_ENDIAN
2409 if (bom
== 0xFEFF) {
2413 else if (bom
== 0xFFFE) {
2418 if (bom
== 0xFEFF) {
2422 else if (bom
== 0xFFFE) {
2443 /* remaining bytes at the end? (size should be even) */
2447 errmsg
= "truncated data";
2448 startinpos
= ((const char *)q
)-starts
;
2449 endinpos
= ((const char *)e
)-starts
;
2451 /* The remaining input chars are ignored if the callback
2452 chooses to skip the input */
2454 ch
= (q
[ihi
] << 8) | q
[ilo
];
2458 if (ch
< 0xD800 || ch
> 0xDFFF) {
2463 /* UTF-16 code pair: */
2465 errmsg
= "unexpected end of data";
2466 startinpos
= (((const char *)q
)-2)-starts
;
2467 endinpos
= ((const char *)e
)-starts
;
2470 if (0xD800 <= ch
&& ch
<= 0xDBFF) {
2471 Py_UNICODE ch2
= (q
[ihi
] << 8) | q
[ilo
];
2473 if (0xDC00 <= ch2
&& ch2
<= 0xDFFF) {
2474 #ifndef Py_UNICODE_WIDE
2478 *p
++ = (((ch
& 0x3FF)<<10) | (ch2
& 0x3FF)) + 0x10000;
2483 errmsg
= "illegal UTF-16 surrogate";
2484 startinpos
= (((const char *)q
)-4)-starts
;
2485 endinpos
= startinpos
+2;
2490 errmsg
= "illegal encoding";
2491 startinpos
= (((const char *)q
)-2)-starts
;
2492 endinpos
= startinpos
+2;
2493 /* Fall through to report the error */
2496 outpos
= p
-PyUnicode_AS_UNICODE(unicode
);
2497 if (unicode_decode_call_errorhandler(
2498 errors
, &errorHandler
,
2500 starts
, size
, &startinpos
, &endinpos
, &exc
, (const char **)&q
,
2501 &unicode
, &outpos
, &p
))
2509 *consumed
= (const char *)q
-starts
;
2512 if (_PyUnicode_Resize(&unicode
, p
- unicode
->str
) < 0)
2515 Py_XDECREF(errorHandler
);
2517 return (PyObject
*)unicode
;
2521 Py_XDECREF(errorHandler
);
2527 PyUnicode_EncodeUTF16(const Py_UNICODE
*s
,
2534 Py_ssize_t nsize
, bytesize
;
2535 #ifdef Py_UNICODE_WIDE
2536 Py_ssize_t i
, pairs
;
2538 const int pairs
= 0;
2540 /* Offsets from p for storing byte pairs in the right order. */
2541 #ifdef BYTEORDER_IS_LITTLE_ENDIAN
2542 int ihi
= 1, ilo
= 0;
2544 int ihi
= 0, ilo
= 1;
2547 #define STORECHAR(CH) \
2549 p[ihi] = ((CH) >> 8) & 0xff; \
2550 p[ilo] = (CH) & 0xff; \
2554 #ifdef Py_UNICODE_WIDE
2555 for (i
= pairs
= 0; i
< size
; i
++)
2556 if (s
[i
] >= 0x10000)
2559 /* 2 * (size + pairs + (byteorder == 0)) */
2560 if (size
> PY_SSIZE_T_MAX
||
2561 size
> PY_SSIZE_T_MAX
- pairs
- (byteorder
== 0))
2562 return PyErr_NoMemory();
2563 nsize
= size
+ pairs
+ (byteorder
== 0);
2564 bytesize
= nsize
* 2;
2565 if (bytesize
/ 2 != nsize
)
2566 return PyErr_NoMemory();
2567 v
= PyString_FromStringAndSize(NULL
, bytesize
);
2571 p
= (unsigned char *)PyString_AS_STRING(v
);
2577 if (byteorder
== -1) {
2582 else if (byteorder
== 1) {
2588 while (size
-- > 0) {
2589 Py_UNICODE ch
= *s
++;
2591 #ifdef Py_UNICODE_WIDE
2592 if (ch
>= 0x10000) {
2593 ch2
= 0xDC00 | ((ch
-0x10000) & 0x3FF);
2594 ch
= 0xD800 | ((ch
-0x10000) >> 10);
2605 PyObject
*PyUnicode_AsUTF16String(PyObject
*unicode
)
2607 if (!PyUnicode_Check(unicode
)) {
2608 PyErr_BadArgument();
2611 return PyUnicode_EncodeUTF16(PyUnicode_AS_UNICODE(unicode
),
2612 PyUnicode_GET_SIZE(unicode
),
2617 /* --- Unicode Escape Codec ----------------------------------------------- */
2619 static _PyUnicode_Name_CAPI
*ucnhash_CAPI
= NULL
;
2621 PyObject
*PyUnicode_DecodeUnicodeEscape(const char *s
,
2625 const char *starts
= s
;
2626 Py_ssize_t startinpos
;
2627 Py_ssize_t endinpos
;
2634 Py_UCS4 chr
= 0xffffffff; /* in case 'getcode' messes up */
2635 PyObject
*errorHandler
= NULL
;
2636 PyObject
*exc
= NULL
;
2638 /* Escaped strings will always be longer than the resulting
2639 Unicode string, so we start with size here and then reduce the
2640 length after conversion to the true value.
2641 (but if the error callback returns a long replacement string
2642 we'll have to allocate more space) */
2643 v
= _PyUnicode_New(size
);
2647 return (PyObject
*)v
;
2649 p
= PyUnicode_AS_UNICODE(v
);
2657 /* Non-escape characters are interpreted as Unicode ordinals */
2659 *p
++ = (unsigned char) *s
++;
2663 startinpos
= s
-starts
;
2668 c
= '\0'; /* Invalid after \ */
2673 case '\\': *p
++ = '\\'; break;
2674 case '\'': *p
++ = '\''; break;
2675 case '\"': *p
++ = '\"'; break;
2676 case 'b': *p
++ = '\b'; break;
2677 case 'f': *p
++ = '\014'; break; /* FF */
2678 case 't': *p
++ = '\t'; break;
2679 case 'n': *p
++ = '\n'; break;
2680 case 'r': *p
++ = '\r'; break;
2681 case 'v': *p
++ = '\013'; break; /* VT */
2682 case 'a': *p
++ = '\007'; break; /* BEL, not classic C */
2684 /* \OOO (octal) escapes */
2685 case '0': case '1': case '2': case '3':
2686 case '4': case '5': case '6': case '7':
2688 if (s
< end
&& '0' <= *s
&& *s
<= '7') {
2689 x
= (x
<<3) + *s
++ - '0';
2690 if (s
< end
&& '0' <= *s
&& *s
<= '7')
2691 x
= (x
<<3) + *s
++ - '0';
2700 message
= "truncated \\xXX escape";
2706 message
= "truncated \\uXXXX escape";
2712 message
= "truncated \\UXXXXXXXX escape";
2715 outpos
= p
-PyUnicode_AS_UNICODE(v
);
2718 if (unicode_decode_call_errorhandler(
2719 errors
, &errorHandler
,
2720 "unicodeescape", "end of string in escape sequence",
2721 starts
, size
, &startinpos
, &endinpos
, &exc
, &s
,
2726 for (i
= 0; i
< digits
; ++i
) {
2727 c
= (unsigned char) s
[i
];
2729 endinpos
= (s
+i
+1)-starts
;
2730 if (unicode_decode_call_errorhandler(
2731 errors
, &errorHandler
,
2732 "unicodeescape", message
,
2733 starts
, size
, &startinpos
, &endinpos
, &exc
, &s
,
2738 chr
= (chr
<<4) & ~0xF;
2739 if (c
>= '0' && c
<= '9')
2741 else if (c
>= 'a' && c
<= 'f')
2742 chr
+= 10 + c
- 'a';
2744 chr
+= 10 + c
- 'A';
2747 if (chr
== 0xffffffff && PyErr_Occurred())
2748 /* _decoding_error will have already written into the
2752 /* when we get here, chr is a 32-bit unicode character */
2754 /* UCS-2 character */
2755 *p
++ = (Py_UNICODE
) chr
;
2756 else if (chr
<= 0x10ffff) {
2757 /* UCS-4 character. Either store directly, or as
2759 #ifdef Py_UNICODE_WIDE
2763 *p
++ = 0xD800 + (Py_UNICODE
) (chr
>> 10);
2764 *p
++ = 0xDC00 + (Py_UNICODE
) (chr
& 0x03FF);
2767 endinpos
= s
-starts
;
2768 outpos
= p
-PyUnicode_AS_UNICODE(v
);
2769 if (unicode_decode_call_errorhandler(
2770 errors
, &errorHandler
,
2771 "unicodeescape", "illegal Unicode character",
2772 starts
, size
, &startinpos
, &endinpos
, &exc
, &s
,
2780 message
= "malformed \\N character escape";
2781 if (ucnhash_CAPI
== NULL
) {
2782 /* load the unicode data module */
2784 m
= PyImport_ImportModuleNoBlock("unicodedata");
2787 api
= PyObject_GetAttrString(m
, "ucnhash_CAPI");
2791 ucnhash_CAPI
= (_PyUnicode_Name_CAPI
*)PyCObject_AsVoidPtr(api
);
2793 if (ucnhash_CAPI
== NULL
)
2797 const char *start
= s
+1;
2798 /* look for the closing brace */
2799 while (*s
!= '}' && s
< end
)
2801 if (s
> start
&& s
< end
&& *s
== '}') {
2802 /* found a name. look it up in the unicode database */
2803 message
= "unknown Unicode character name";
2805 if (ucnhash_CAPI
->getcode(NULL
, start
, (int)(s
-start
-1), &chr
))
2809 endinpos
= s
-starts
;
2810 outpos
= p
-PyUnicode_AS_UNICODE(v
);
2811 if (unicode_decode_call_errorhandler(
2812 errors
, &errorHandler
,
2813 "unicodeescape", message
,
2814 starts
, size
, &startinpos
, &endinpos
, &exc
, &s
,
2821 message
= "\\ at end of string";
2823 endinpos
= s
-starts
;
2824 outpos
= p
-PyUnicode_AS_UNICODE(v
);
2825 if (unicode_decode_call_errorhandler(
2826 errors
, &errorHandler
,
2827 "unicodeescape", message
,
2828 starts
, size
, &startinpos
, &endinpos
, &exc
, &s
,
2834 *p
++ = (unsigned char)s
[-1];
2841 if (_PyUnicode_Resize(&v
, p
- PyUnicode_AS_UNICODE(v
)) < 0)
2843 Py_XDECREF(errorHandler
);
2845 return (PyObject
*)v
;
2850 "\\N escapes not supported (can't load unicodedata module)"
2853 Py_XDECREF(errorHandler
);
2859 Py_XDECREF(errorHandler
);
2864 /* Return a Unicode-Escape string version of the Unicode object.
2866 If quotes is true, the string is enclosed in u"" or u'' quotes as
2871 Py_LOCAL_INLINE(const Py_UNICODE
*) findchar(const Py_UNICODE
*s
,
2875 /* like wcschr, but doesn't stop at NULL characters */
2877 while (size
-- > 0) {
2887 PyObject
*unicodeescape_string(const Py_UNICODE
*s
,
2894 static const char *hexdigit
= "0123456789abcdef";
2895 #ifdef Py_UNICODE_WIDE
2896 const Py_ssize_t expandsize
= 10;
2898 const Py_ssize_t expandsize
= 6;
2901 /* XXX(nnorwitz): rather than over-allocating, it would be
2902 better to choose a different scheme. Perhaps scan the
2903 first N-chars of the string and allocate based on that size.
2905 /* Initial allocation is based on the longest-possible unichr
2908 In wide (UTF-32) builds '\U00xxxxxx' is 10 chars per source
2909 unichr, so in this case it's the longest unichr escape. In
2910 narrow (UTF-16) builds this is five chars per source unichr
2911 since there are two unichrs in the surrogate pair, so in narrow
2912 (UTF-16) builds it's not the longest unichr escape.
2914 In wide or narrow builds '\uxxxx' is 6 chars per source unichr,
2915 so in the narrow (UTF-16) build case it's the longest unichr
2919 if (size
> (PY_SSIZE_T_MAX
- 2 - 1) / expandsize
)
2920 return PyErr_NoMemory();
2922 repr
= PyString_FromStringAndSize(NULL
,
2929 p
= PyString_AS_STRING(repr
);
2933 *p
++ = (findchar(s
, size
, '\'') &&
2934 !findchar(s
, size
, '"')) ? '"' : '\'';
2936 while (size
-- > 0) {
2937 Py_UNICODE ch
= *s
++;
2939 /* Escape quotes and backslashes */
2941 ch
== (Py_UNICODE
) PyString_AS_STRING(repr
)[1]) || ch
== '\\') {
2947 #ifdef Py_UNICODE_WIDE
2948 /* Map 21-bit characters to '\U00xxxxxx' */
2949 else if (ch
>= 0x10000) {
2952 *p
++ = hexdigit
[(ch
>> 28) & 0x0000000F];
2953 *p
++ = hexdigit
[(ch
>> 24) & 0x0000000F];
2954 *p
++ = hexdigit
[(ch
>> 20) & 0x0000000F];
2955 *p
++ = hexdigit
[(ch
>> 16) & 0x0000000F];
2956 *p
++ = hexdigit
[(ch
>> 12) & 0x0000000F];
2957 *p
++ = hexdigit
[(ch
>> 8) & 0x0000000F];
2958 *p
++ = hexdigit
[(ch
>> 4) & 0x0000000F];
2959 *p
++ = hexdigit
[ch
& 0x0000000F];
2963 /* Map UTF-16 surrogate pairs to '\U00xxxxxx' */
2964 else if (ch
>= 0xD800 && ch
< 0xDC00) {
2970 if (ch2
>= 0xDC00 && ch2
<= 0xDFFF) {
2971 ucs
= (((ch
& 0x03FF) << 10) | (ch2
& 0x03FF)) + 0x00010000;
2974 *p
++ = hexdigit
[(ucs
>> 28) & 0x0000000F];
2975 *p
++ = hexdigit
[(ucs
>> 24) & 0x0000000F];
2976 *p
++ = hexdigit
[(ucs
>> 20) & 0x0000000F];
2977 *p
++ = hexdigit
[(ucs
>> 16) & 0x0000000F];
2978 *p
++ = hexdigit
[(ucs
>> 12) & 0x0000000F];
2979 *p
++ = hexdigit
[(ucs
>> 8) & 0x0000000F];
2980 *p
++ = hexdigit
[(ucs
>> 4) & 0x0000000F];
2981 *p
++ = hexdigit
[ucs
& 0x0000000F];
2984 /* Fall through: isolated surrogates are copied as-is */
2990 /* Map 16-bit characters to '\uxxxx' */
2994 *p
++ = hexdigit
[(ch
>> 12) & 0x000F];
2995 *p
++ = hexdigit
[(ch
>> 8) & 0x000F];
2996 *p
++ = hexdigit
[(ch
>> 4) & 0x000F];
2997 *p
++ = hexdigit
[ch
& 0x000F];
3000 /* Map special whitespace to '\t', \n', '\r' */
3001 else if (ch
== '\t') {
3005 else if (ch
== '\n') {
3009 else if (ch
== '\r') {
3014 /* Map non-printable US ASCII to '\xhh' */
3015 else if (ch
< ' ' || ch
>= 0x7F) {
3018 *p
++ = hexdigit
[(ch
>> 4) & 0x000F];
3019 *p
++ = hexdigit
[ch
& 0x000F];
3022 /* Copy everything else as-is */
3027 *p
++ = PyString_AS_STRING(repr
)[1];
3030 _PyString_Resize(&repr
, p
- PyString_AS_STRING(repr
));
3034 PyObject
*PyUnicode_EncodeUnicodeEscape(const Py_UNICODE
*s
,
3037 return unicodeescape_string(s
, size
, 0);
3040 PyObject
*PyUnicode_AsUnicodeEscapeString(PyObject
*unicode
)
3042 if (!PyUnicode_Check(unicode
)) {
3043 PyErr_BadArgument();
3046 return PyUnicode_EncodeUnicodeEscape(PyUnicode_AS_UNICODE(unicode
),
3047 PyUnicode_GET_SIZE(unicode
));
3050 /* --- Raw Unicode Escape Codec ------------------------------------------- */
3052 PyObject
*PyUnicode_DecodeRawUnicodeEscape(const char *s
,
3056 const char *starts
= s
;
3057 Py_ssize_t startinpos
;
3058 Py_ssize_t endinpos
;
3064 PyObject
*errorHandler
= NULL
;
3065 PyObject
*exc
= NULL
;
3067 /* Escaped strings will always be longer than the resulting
3068 Unicode string, so we start with size here and then reduce the
3069 length after conversion to the true value. (But decoding error
3070 handler might have to resize the string) */
3071 v
= _PyUnicode_New(size
);
3075 return (PyObject
*)v
;
3076 p
= PyUnicode_AS_UNICODE(v
);
3084 /* Non-escape characters are interpreted as Unicode ordinals */
3086 *p
++ = (unsigned char)*s
++;
3089 startinpos
= s
-starts
;
3091 /* \u-escapes are only interpreted iff the number of leading
3092 backslashes if odd */
3097 *p
++ = (unsigned char)*s
++;
3099 if (((s
- bs
) & 1) == 0 ||
3101 (*s
!= 'u' && *s
!= 'U')) {
3105 count
= *s
=='u' ? 4 : 8;
3108 /* \uXXXX with 4 hex digits, \Uxxxxxxxx with 8 */
3109 outpos
= p
-PyUnicode_AS_UNICODE(v
);
3110 for (x
= 0, i
= 0; i
< count
; ++i
, ++s
) {
3111 c
= (unsigned char)*s
;
3113 endinpos
= s
-starts
;
3114 if (unicode_decode_call_errorhandler(
3115 errors
, &errorHandler
,
3116 "rawunicodeescape", "truncated \\uXXXX",
3117 starts
, size
, &startinpos
, &endinpos
, &exc
, &s
,
3123 if (c
>= '0' && c
<= '9')
3125 else if (c
>= 'a' && c
<= 'f')
3131 /* UCS-2 character */
3132 *p
++ = (Py_UNICODE
) x
;
3133 else if (x
<= 0x10ffff) {
3134 /* UCS-4 character. Either store directly, or as
3136 #ifdef Py_UNICODE_WIDE
3137 *p
++ = (Py_UNICODE
) x
;
3140 *p
++ = 0xD800 + (Py_UNICODE
) (x
>> 10);
3141 *p
++ = 0xDC00 + (Py_UNICODE
) (x
& 0x03FF);
3144 endinpos
= s
-starts
;
3145 outpos
= p
-PyUnicode_AS_UNICODE(v
);
3146 if (unicode_decode_call_errorhandler(
3147 errors
, &errorHandler
,
3148 "rawunicodeescape", "\\Uxxxxxxxx out of range",
3149 starts
, size
, &startinpos
, &endinpos
, &exc
, &s
,
3156 if (_PyUnicode_Resize(&v
, p
- PyUnicode_AS_UNICODE(v
)) < 0)
3158 Py_XDECREF(errorHandler
);
3160 return (PyObject
*)v
;
3164 Py_XDECREF(errorHandler
);
3169 PyObject
*PyUnicode_EncodeRawUnicodeEscape(const Py_UNICODE
*s
,
3176 static const char *hexdigit
= "0123456789abcdef";
3177 #ifdef Py_UNICODE_WIDE
3178 const Py_ssize_t expandsize
= 10;
3180 const Py_ssize_t expandsize
= 6;
3183 if (size
> PY_SSIZE_T_MAX
/ expandsize
)
3184 return PyErr_NoMemory();
3186 repr
= PyString_FromStringAndSize(NULL
, expandsize
* size
);
3192 p
= q
= PyString_AS_STRING(repr
);
3193 while (size
-- > 0) {
3194 Py_UNICODE ch
= *s
++;
3195 #ifdef Py_UNICODE_WIDE
3196 /* Map 32-bit characters to '\Uxxxxxxxx' */
3197 if (ch
>= 0x10000) {
3200 *p
++ = hexdigit
[(ch
>> 28) & 0xf];
3201 *p
++ = hexdigit
[(ch
>> 24) & 0xf];
3202 *p
++ = hexdigit
[(ch
>> 20) & 0xf];
3203 *p
++ = hexdigit
[(ch
>> 16) & 0xf];
3204 *p
++ = hexdigit
[(ch
>> 12) & 0xf];
3205 *p
++ = hexdigit
[(ch
>> 8) & 0xf];
3206 *p
++ = hexdigit
[(ch
>> 4) & 0xf];
3207 *p
++ = hexdigit
[ch
& 15];
3211 /* Map UTF-16 surrogate pairs to '\U00xxxxxx' */
3212 if (ch
>= 0xD800 && ch
< 0xDC00) {
3218 if (ch2
>= 0xDC00 && ch2
<= 0xDFFF) {
3219 ucs
= (((ch
& 0x03FF) << 10) | (ch2
& 0x03FF)) + 0x00010000;
3222 *p
++ = hexdigit
[(ucs
>> 28) & 0xf];
3223 *p
++ = hexdigit
[(ucs
>> 24) & 0xf];
3224 *p
++ = hexdigit
[(ucs
>> 20) & 0xf];
3225 *p
++ = hexdigit
[(ucs
>> 16) & 0xf];
3226 *p
++ = hexdigit
[(ucs
>> 12) & 0xf];
3227 *p
++ = hexdigit
[(ucs
>> 8) & 0xf];
3228 *p
++ = hexdigit
[(ucs
>> 4) & 0xf];
3229 *p
++ = hexdigit
[ucs
& 0xf];
3232 /* Fall through: isolated surrogates are copied as-is */
3237 /* Map 16-bit characters to '\uxxxx' */
3241 *p
++ = hexdigit
[(ch
>> 12) & 0xf];
3242 *p
++ = hexdigit
[(ch
>> 8) & 0xf];
3243 *p
++ = hexdigit
[(ch
>> 4) & 0xf];
3244 *p
++ = hexdigit
[ch
& 15];
3246 /* Copy everything else as-is */
3251 _PyString_Resize(&repr
, p
- q
);
3255 PyObject
*PyUnicode_AsRawUnicodeEscapeString(PyObject
*unicode
)
3257 if (!PyUnicode_Check(unicode
)) {
3258 PyErr_BadArgument();
3261 return PyUnicode_EncodeRawUnicodeEscape(PyUnicode_AS_UNICODE(unicode
),
3262 PyUnicode_GET_SIZE(unicode
));
3265 /* --- Unicode Internal Codec ------------------------------------------- */
3267 PyObject
*_PyUnicode_DecodeUnicodeInternal(const char *s
,
3271 const char *starts
= s
;
3272 Py_ssize_t startinpos
;
3273 Py_ssize_t endinpos
;
3279 PyObject
*errorHandler
= NULL
;
3280 PyObject
*exc
= NULL
;
3282 #ifdef Py_UNICODE_WIDE
3283 Py_UNICODE unimax
= PyUnicode_GetMax();
3286 /* XXX overflow detection missing */
3287 v
= _PyUnicode_New((size
+Py_UNICODE_SIZE
-1)/ Py_UNICODE_SIZE
);
3290 if (PyUnicode_GetSize((PyObject
*)v
) == 0)
3291 return (PyObject
*)v
;
3292 p
= PyUnicode_AS_UNICODE(v
);
3296 memcpy(p
, s
, sizeof(Py_UNICODE
));
3297 /* We have to sanity check the raw data, otherwise doom looms for
3298 some malformed UCS-4 data. */
3300 #ifdef Py_UNICODE_WIDE
3301 *p
> unimax
|| *p
< 0 ||
3303 end
-s
< Py_UNICODE_SIZE
3306 startinpos
= s
- starts
;
3307 if (end
-s
< Py_UNICODE_SIZE
) {
3308 endinpos
= end
-starts
;
3309 reason
= "truncated input";
3312 endinpos
= s
- starts
+ Py_UNICODE_SIZE
;
3313 reason
= "illegal code point (> 0x10FFFF)";
3315 outpos
= p
- PyUnicode_AS_UNICODE(v
);
3316 if (unicode_decode_call_errorhandler(
3317 errors
, &errorHandler
,
3318 "unicode_internal", reason
,
3319 starts
, size
, &startinpos
, &endinpos
, &exc
, &s
,
3326 s
+= Py_UNICODE_SIZE
;
3330 if (_PyUnicode_Resize(&v
, p
- PyUnicode_AS_UNICODE(v
)) < 0)
3332 Py_XDECREF(errorHandler
);
3334 return (PyObject
*)v
;
3338 Py_XDECREF(errorHandler
);
3343 /* --- Latin-1 Codec ------------------------------------------------------ */
3345 PyObject
*PyUnicode_DecodeLatin1(const char *s
,
3352 /* Latin-1 is equivalent to the first 256 ordinals in Unicode. */
3354 Py_UNICODE r
= *(unsigned char*)s
;
3355 return PyUnicode_FromUnicode(&r
, 1);
3358 v
= _PyUnicode_New(size
);
3362 return (PyObject
*)v
;
3363 p
= PyUnicode_AS_UNICODE(v
);
3365 *p
++ = (unsigned char)*s
++;
3366 return (PyObject
*)v
;
3373 /* create or adjust a UnicodeEncodeError */
3374 static void make_encode_exception(PyObject
**exceptionObject
,
3375 const char *encoding
,
3376 const Py_UNICODE
*unicode
, Py_ssize_t size
,
3377 Py_ssize_t startpos
, Py_ssize_t endpos
,
3380 if (*exceptionObject
== NULL
) {
3381 *exceptionObject
= PyUnicodeEncodeError_Create(
3382 encoding
, unicode
, size
, startpos
, endpos
, reason
);
3385 if (PyUnicodeEncodeError_SetStart(*exceptionObject
, startpos
))
3387 if (PyUnicodeEncodeError_SetEnd(*exceptionObject
, endpos
))
3389 if (PyUnicodeEncodeError_SetReason(*exceptionObject
, reason
))
3393 Py_DECREF(*exceptionObject
);
3394 *exceptionObject
= NULL
;
3398 /* raises a UnicodeEncodeError */
3399 static void raise_encode_exception(PyObject
**exceptionObject
,
3400 const char *encoding
,
3401 const Py_UNICODE
*unicode
, Py_ssize_t size
,
3402 Py_ssize_t startpos
, Py_ssize_t endpos
,
3405 make_encode_exception(exceptionObject
,
3406 encoding
, unicode
, size
, startpos
, endpos
, reason
);
3407 if (*exceptionObject
!= NULL
)
3408 PyCodec_StrictErrors(*exceptionObject
);
3411 /* error handling callback helper:
3412 build arguments, call the callback and check the arguments,
3413 put the result into newpos and return the replacement string, which
3414 has to be freed by the caller */
3415 static PyObject
*unicode_encode_call_errorhandler(const char *errors
,
3416 PyObject
**errorHandler
,
3417 const char *encoding
, const char *reason
,
3418 const Py_UNICODE
*unicode
, Py_ssize_t size
, PyObject
**exceptionObject
,
3419 Py_ssize_t startpos
, Py_ssize_t endpos
,
3422 static char *argparse
= "O!n;encoding error handler must return (unicode, int) tuple";
3425 PyObject
*resunicode
;
3427 if (*errorHandler
== NULL
) {
3428 *errorHandler
= PyCodec_LookupError(errors
);
3429 if (*errorHandler
== NULL
)
3433 make_encode_exception(exceptionObject
,
3434 encoding
, unicode
, size
, startpos
, endpos
, reason
);
3435 if (*exceptionObject
== NULL
)
3438 restuple
= PyObject_CallFunctionObjArgs(
3439 *errorHandler
, *exceptionObject
, NULL
);
3440 if (restuple
== NULL
)
3442 if (!PyTuple_Check(restuple
)) {
3443 PyErr_Format(PyExc_TypeError
, &argparse
[4]);
3444 Py_DECREF(restuple
);
3447 if (!PyArg_ParseTuple(restuple
, argparse
, &PyUnicode_Type
,
3448 &resunicode
, newpos
)) {
3449 Py_DECREF(restuple
);
3453 *newpos
= size
+*newpos
;
3454 if (*newpos
<0 || *newpos
>size
) {
3455 PyErr_Format(PyExc_IndexError
, "position %zd from error handler out of bounds", *newpos
);
3456 Py_DECREF(restuple
);
3459 Py_INCREF(resunicode
);
3460 Py_DECREF(restuple
);
3464 static PyObject
*unicode_encode_ucs1(const Py_UNICODE
*p
,
3471 /* pointers to the beginning and end+1 of input */
3472 const Py_UNICODE
*startp
= p
;
3473 const Py_UNICODE
*endp
= p
+ size
;
3474 /* pointer to the beginning of the unencodable characters */
3475 /* const Py_UNICODE *badp = NULL; */
3476 /* pointer into the output */
3478 /* current output position */
3479 Py_ssize_t respos
= 0;
3481 const char *encoding
= (limit
== 256) ? "latin-1" : "ascii";
3482 const char *reason
= (limit
== 256) ? "ordinal not in range(256)" : "ordinal not in range(128)";
3483 PyObject
*errorHandler
= NULL
;
3484 PyObject
*exc
= NULL
;
3485 /* the following variable is used for caching string comparisons
3486 * -1=not initialized, 0=unknown, 1=strict, 2=replace, 3=ignore, 4=xmlcharrefreplace */
3487 int known_errorHandler
= -1;
3489 /* allocate enough for a simple encoding without
3490 replacements, if we need more, we'll resize */
3491 res
= PyString_FromStringAndSize(NULL
, size
);
3496 str
= PyString_AS_STRING(res
);
3502 /* can we encode this? */
3504 /* no overflow check, because we know that the space is enough */
3509 Py_ssize_t unicodepos
= p
-startp
;
3510 Py_ssize_t requiredsize
;
3511 PyObject
*repunicode
;
3516 /* startpos for collecting unencodable chars */
3517 const Py_UNICODE
*collstart
= p
;
3518 const Py_UNICODE
*collend
= p
;
3519 /* find all unecodable characters */
3520 while ((collend
< endp
) && ((*collend
)>=limit
))
3522 /* cache callback name lookup (if not done yet, i.e. it's the first error) */
3523 if (known_errorHandler
==-1) {
3524 if ((errors
==NULL
) || (!strcmp(errors
, "strict")))
3525 known_errorHandler
= 1;
3526 else if (!strcmp(errors
, "replace"))
3527 known_errorHandler
= 2;
3528 else if (!strcmp(errors
, "ignore"))
3529 known_errorHandler
= 3;
3530 else if (!strcmp(errors
, "xmlcharrefreplace"))
3531 known_errorHandler
= 4;
3533 known_errorHandler
= 0;
3535 switch (known_errorHandler
) {
3536 case 1: /* strict */
3537 raise_encode_exception(&exc
, encoding
, startp
, size
, collstart
-startp
, collend
-startp
, reason
);
3539 case 2: /* replace */
3540 while (collstart
++<collend
)
3541 *str
++ = '?'; /* fall through */
3542 case 3: /* ignore */
3545 case 4: /* xmlcharrefreplace */
3546 respos
= str
-PyString_AS_STRING(res
);
3547 /* determine replacement size (temporarily (mis)uses p) */
3548 for (p
= collstart
, repsize
= 0; p
< collend
; ++p
) {
3557 #ifndef Py_UNICODE_WIDE
3563 else if (*p
<1000000)
3569 requiredsize
= respos
+repsize
+(endp
-collend
);
3570 if (requiredsize
> ressize
) {
3571 if (requiredsize
<2*ressize
)
3572 requiredsize
= 2*ressize
;
3573 if (_PyString_Resize(&res
, requiredsize
))
3575 str
= PyString_AS_STRING(res
) + respos
;
3576 ressize
= requiredsize
;
3578 /* generate replacement (temporarily (mis)uses p) */
3579 for (p
= collstart
; p
< collend
; ++p
) {
3580 str
+= sprintf(str
, "&#%d;", (int)*p
);
3585 repunicode
= unicode_encode_call_errorhandler(errors
, &errorHandler
,
3586 encoding
, reason
, startp
, size
, &exc
,
3587 collstart
-startp
, collend
-startp
, &newpos
);
3588 if (repunicode
== NULL
)
3590 /* need more space? (at least enough for what we
3591 have+the replacement+the rest of the string, so
3592 we won't have to check space for encodable characters) */
3593 respos
= str
-PyString_AS_STRING(res
);
3594 repsize
= PyUnicode_GET_SIZE(repunicode
);
3595 requiredsize
= respos
+repsize
+(endp
-collend
);
3596 if (requiredsize
> ressize
) {
3597 if (requiredsize
<2*ressize
)
3598 requiredsize
= 2*ressize
;
3599 if (_PyString_Resize(&res
, requiredsize
)) {
3600 Py_DECREF(repunicode
);
3603 str
= PyString_AS_STRING(res
) + respos
;
3604 ressize
= requiredsize
;
3606 /* check if there is anything unencodable in the replacement
3607 and copy it to the output */
3608 for (uni2
= PyUnicode_AS_UNICODE(repunicode
);repsize
-->0; ++uni2
, ++str
) {
3611 raise_encode_exception(&exc
, encoding
, startp
, size
,
3612 unicodepos
, unicodepos
+1, reason
);
3613 Py_DECREF(repunicode
);
3618 p
= startp
+ newpos
;
3619 Py_DECREF(repunicode
);
3623 /* Resize if we allocated to much */
3624 respos
= str
-PyString_AS_STRING(res
);
3626 /* If this falls res will be NULL */
3627 _PyString_Resize(&res
, respos
);
3628 Py_XDECREF(errorHandler
);
3634 Py_XDECREF(errorHandler
);
3639 PyObject
*PyUnicode_EncodeLatin1(const Py_UNICODE
*p
,
3643 return unicode_encode_ucs1(p
, size
, errors
, 256);
3646 PyObject
*PyUnicode_AsLatin1String(PyObject
*unicode
)
3648 if (!PyUnicode_Check(unicode
)) {
3649 PyErr_BadArgument();
3652 return PyUnicode_EncodeLatin1(PyUnicode_AS_UNICODE(unicode
),
3653 PyUnicode_GET_SIZE(unicode
),
3657 /* --- 7-bit ASCII Codec -------------------------------------------------- */
3659 PyObject
*PyUnicode_DecodeASCII(const char *s
,
3663 const char *starts
= s
;
3666 Py_ssize_t startinpos
;
3667 Py_ssize_t endinpos
;
3670 PyObject
*errorHandler
= NULL
;
3671 PyObject
*exc
= NULL
;
3673 /* ASCII is equivalent to the first 128 ordinals in Unicode. */
3674 if (size
== 1 && *(unsigned char*)s
< 128) {
3675 Py_UNICODE r
= *(unsigned char*)s
;
3676 return PyUnicode_FromUnicode(&r
, 1);
3679 v
= _PyUnicode_New(size
);
3683 return (PyObject
*)v
;
3684 p
= PyUnicode_AS_UNICODE(v
);
3687 register unsigned char c
= (unsigned char)*s
;
3693 startinpos
= s
-starts
;
3694 endinpos
= startinpos
+ 1;
3695 outpos
= p
- (Py_UNICODE
*)PyUnicode_AS_UNICODE(v
);
3696 if (unicode_decode_call_errorhandler(
3697 errors
, &errorHandler
,
3698 "ascii", "ordinal not in range(128)",
3699 starts
, size
, &startinpos
, &endinpos
, &exc
, &s
,
3704 if (p
- PyUnicode_AS_UNICODE(v
) < PyString_GET_SIZE(v
))
3705 if (_PyUnicode_Resize(&v
, p
- PyUnicode_AS_UNICODE(v
)) < 0)
3707 Py_XDECREF(errorHandler
);
3709 return (PyObject
*)v
;
3713 Py_XDECREF(errorHandler
);
3718 PyObject
*PyUnicode_EncodeASCII(const Py_UNICODE
*p
,
3722 return unicode_encode_ucs1(p
, size
, errors
, 128);
3725 PyObject
*PyUnicode_AsASCIIString(PyObject
*unicode
)
3727 if (!PyUnicode_Check(unicode
)) {
3728 PyErr_BadArgument();
3731 return PyUnicode_EncodeASCII(PyUnicode_AS_UNICODE(unicode
),
3732 PyUnicode_GET_SIZE(unicode
),
3736 #if defined(MS_WINDOWS) && defined(HAVE_USABLE_WCHAR_T)
3738 /* --- MBCS codecs for Windows -------------------------------------------- */
3740 #if SIZEOF_INT < SIZEOF_SSIZE_T
3744 /* XXX This code is limited to "true" double-byte encodings, as
3745 a) it assumes an incomplete character consists of a single byte, and
3746 b) IsDBCSLeadByte (probably) does not work for non-DBCS multi-byte
3747 encodings, see IsDBCSLeadByteEx documentation. */
3749 static int is_dbcs_lead_byte(const char *s
, int offset
)
3751 const char *curr
= s
+ offset
;
3753 if (IsDBCSLeadByte(*curr
)) {
3754 const char *prev
= CharPrev(s
, curr
);
3755 return (prev
== curr
) || !IsDBCSLeadByte(*prev
) || (curr
- prev
== 2);
3761 * Decode MBCS string into unicode object. If 'final' is set, converts
3762 * trailing lead-byte too. Returns consumed size if succeed, -1 otherwise.
3764 static int decode_mbcs(PyUnicodeObject
**v
,
3765 const char *s
, /* MBCS string */
3766 int size
, /* sizeof MBCS string */
3775 /* Skip trailing lead-byte unless 'final' is set */
3776 if (!final
&& size
>= 1 && is_dbcs_lead_byte(s
, size
- 1))
3779 /* First get the size of the result */
3781 usize
= MultiByteToWideChar(CP_ACP
, 0, s
, size
, NULL
, 0);
3783 PyErr_SetFromWindowsErrWithFilename(0, NULL
);
3789 /* Create unicode object */
3790 *v
= _PyUnicode_New(usize
);
3795 /* Extend unicode object */
3796 n
= PyUnicode_GET_SIZE(*v
);
3797 if (_PyUnicode_Resize(v
, n
+ usize
) < 0)
3801 /* Do the conversion */
3803 p
= PyUnicode_AS_UNICODE(*v
) + n
;
3804 if (0 == MultiByteToWideChar(CP_ACP
, 0, s
, size
, p
, usize
)) {
3805 PyErr_SetFromWindowsErrWithFilename(0, NULL
);
3813 PyObject
*PyUnicode_DecodeMBCSStateful(const char *s
,
3816 Py_ssize_t
*consumed
)
3818 PyUnicodeObject
*v
= NULL
;
3827 done
= decode_mbcs(&v
, s
, INT_MAX
, 0);
3830 done
= decode_mbcs(&v
, s
, (int)size
, !consumed
);
3841 if (size
> INT_MAX
) {
3848 return (PyObject
*)v
;
3851 PyObject
*PyUnicode_DecodeMBCS(const char *s
,
3855 return PyUnicode_DecodeMBCSStateful(s
, size
, errors
, NULL
);
3859 * Convert unicode into string object (MBCS).
3860 * Returns 0 if succeed, -1 otherwise.
3862 static int encode_mbcs(PyObject
**repr
,
3863 const Py_UNICODE
*p
, /* unicode */
3864 int size
) /* size of unicode */
3871 /* First get the size of the result */
3873 mbcssize
= WideCharToMultiByte(CP_ACP
, 0, p
, size
, NULL
, 0, NULL
, NULL
);
3874 if (mbcssize
== 0) {
3875 PyErr_SetFromWindowsErrWithFilename(0, NULL
);
3880 if (*repr
== NULL
) {
3881 /* Create string object */
3882 *repr
= PyString_FromStringAndSize(NULL
, mbcssize
);
3887 /* Extend string object */
3888 n
= PyString_Size(*repr
);
3889 if (_PyString_Resize(repr
, n
+ mbcssize
) < 0)
3893 /* Do the conversion */
3895 char *s
= PyString_AS_STRING(*repr
) + n
;
3896 if (0 == WideCharToMultiByte(CP_ACP
, 0, p
, size
, s
, mbcssize
, NULL
, NULL
)) {
3897 PyErr_SetFromWindowsErrWithFilename(0, NULL
);
3905 PyObject
*PyUnicode_EncodeMBCS(const Py_UNICODE
*p
,
3909 PyObject
*repr
= NULL
;
3915 ret
= encode_mbcs(&repr
, p
, INT_MAX
);
3918 ret
= encode_mbcs(&repr
, p
, (int)size
);
3926 if (size
> INT_MAX
) {
3936 PyObject
*PyUnicode_AsMBCSString(PyObject
*unicode
)
3938 if (!PyUnicode_Check(unicode
)) {
3939 PyErr_BadArgument();
3942 return PyUnicode_EncodeMBCS(PyUnicode_AS_UNICODE(unicode
),
3943 PyUnicode_GET_SIZE(unicode
),
3949 #endif /* MS_WINDOWS */
3951 /* --- Character Mapping Codec -------------------------------------------- */
3953 PyObject
*PyUnicode_DecodeCharmap(const char *s
,
3958 const char *starts
= s
;
3959 Py_ssize_t startinpos
;
3960 Py_ssize_t endinpos
;
3965 Py_ssize_t extrachars
= 0;
3966 PyObject
*errorHandler
= NULL
;
3967 PyObject
*exc
= NULL
;
3968 Py_UNICODE
*mapstring
= NULL
;
3969 Py_ssize_t maplen
= 0;
3971 /* Default to Latin-1 */
3972 if (mapping
== NULL
)
3973 return PyUnicode_DecodeLatin1(s
, size
, errors
);
3975 v
= _PyUnicode_New(size
);
3979 return (PyObject
*)v
;
3980 p
= PyUnicode_AS_UNICODE(v
);
3982 if (PyUnicode_CheckExact(mapping
)) {
3983 mapstring
= PyUnicode_AS_UNICODE(mapping
);
3984 maplen
= PyUnicode_GET_SIZE(mapping
);
3986 unsigned char ch
= *s
;
3987 Py_UNICODE x
= 0xfffe; /* illegal value */
3993 /* undefined mapping */
3994 outpos
= p
-PyUnicode_AS_UNICODE(v
);
3995 startinpos
= s
-starts
;
3996 endinpos
= startinpos
+1;
3997 if (unicode_decode_call_errorhandler(
3998 errors
, &errorHandler
,
3999 "charmap", "character maps to <undefined>",
4000 starts
, size
, &startinpos
, &endinpos
, &exc
, &s
,
4012 unsigned char ch
= *s
;
4015 /* Get mapping (char ordinal -> integer, Unicode char or None) */
4016 w
= PyInt_FromLong((long)ch
);
4019 x
= PyObject_GetItem(mapping
, w
);
4022 if (PyErr_ExceptionMatches(PyExc_LookupError
)) {
4023 /* No mapping found means: mapping is undefined. */
4032 if (PyInt_Check(x
)) {
4033 long value
= PyInt_AS_LONG(x
);
4034 if (value
< 0 || value
> 65535) {
4035 PyErr_SetString(PyExc_TypeError
,
4036 "character mapping must be in range(65536)");
4040 *p
++ = (Py_UNICODE
)value
;
4042 else if (x
== Py_None
) {
4043 /* undefined mapping */
4044 outpos
= p
-PyUnicode_AS_UNICODE(v
);
4045 startinpos
= s
-starts
;
4046 endinpos
= startinpos
+1;
4047 if (unicode_decode_call_errorhandler(
4048 errors
, &errorHandler
,
4049 "charmap", "character maps to <undefined>",
4050 starts
, size
, &startinpos
, &endinpos
, &exc
, &s
,
4058 else if (PyUnicode_Check(x
)) {
4059 Py_ssize_t targetsize
= PyUnicode_GET_SIZE(x
);
4061 if (targetsize
== 1)
4063 *p
++ = *PyUnicode_AS_UNICODE(x
);
4065 else if (targetsize
> 1) {
4067 if (targetsize
> extrachars
) {
4069 Py_ssize_t oldpos
= p
- PyUnicode_AS_UNICODE(v
);
4070 Py_ssize_t needed
= (targetsize
- extrachars
) + \
4072 extrachars
+= needed
;
4073 /* XXX overflow detection missing */
4074 if (_PyUnicode_Resize(&v
,
4075 PyUnicode_GET_SIZE(v
) + needed
) < 0) {
4079 p
= PyUnicode_AS_UNICODE(v
) + oldpos
;
4082 PyUnicode_AS_UNICODE(x
),
4085 extrachars
-= targetsize
;
4087 /* 1-0 mapping: skip the character */
4090 /* wrong return value */
4091 PyErr_SetString(PyExc_TypeError
,
4092 "character mapping must return integer, None or unicode");
4100 if (p
- PyUnicode_AS_UNICODE(v
) < PyUnicode_GET_SIZE(v
))
4101 if (_PyUnicode_Resize(&v
, p
- PyUnicode_AS_UNICODE(v
)) < 0)
4103 Py_XDECREF(errorHandler
);
4105 return (PyObject
*)v
;
4108 Py_XDECREF(errorHandler
);
4114 /* Charmap encoding: the lookup table */
4116 struct encoding_map
{
4118 unsigned char level1
[32];
4120 unsigned char level23
[1];
4124 encoding_map_size(PyObject
*obj
, PyObject
* args
)
4126 struct encoding_map
*map
= (struct encoding_map
*)obj
;
4127 return PyInt_FromLong(sizeof(*map
) - 1 + 16*map
->count2
+
4131 static PyMethodDef encoding_map_methods
[] = {
4132 {"size", encoding_map_size
, METH_NOARGS
,
4133 PyDoc_STR("Return the size (in bytes) of this object") },
4138 encoding_map_dealloc(PyObject
* o
)
4143 static PyTypeObject EncodingMapType
= {
4144 PyVarObject_HEAD_INIT(NULL
, 0)
4145 "EncodingMap", /*tp_name*/
4146 sizeof(struct encoding_map
), /*tp_basicsize*/
4149 encoding_map_dealloc
, /*tp_dealloc*/
4156 0, /*tp_as_sequence*/
4157 0, /*tp_as_mapping*/
4164 Py_TPFLAGS_DEFAULT
, /*tp_flags*/
4168 0, /*tp_richcompare*/
4169 0, /*tp_weaklistoffset*/
4172 encoding_map_methods
, /*tp_methods*/
4179 0, /*tp_dictoffset*/
4188 PyUnicode_BuildEncodingMap(PyObject
* string
)
4192 struct encoding_map
*mresult
;
4195 unsigned char level1
[32];
4196 unsigned char level2
[512];
4197 unsigned char *mlevel1
, *mlevel2
, *mlevel3
;
4198 int count2
= 0, count3
= 0;
4200 if (!PyUnicode_Check(string
) || PyUnicode_GetSize(string
) != 256) {
4201 PyErr_BadArgument();
4204 decode
= PyUnicode_AS_UNICODE(string
);
4205 memset(level1
, 0xFF, sizeof level1
);
4206 memset(level2
, 0xFF, sizeof level2
);
4208 /* If there isn't a one-to-one mapping of NULL to \0,
4209 or if there are non-BMP characters, we need to use
4210 a mapping dictionary. */
4213 for (i
= 1; i
< 256; i
++) {
4216 #ifdef Py_UNICODE_WIDE
4217 || decode
[i
] > 0xFFFF
4223 if (decode
[i
] == 0xFFFE)
4224 /* unmapped character */
4226 l1
= decode
[i
] >> 11;
4227 l2
= decode
[i
] >> 7;
4228 if (level1
[l1
] == 0xFF)
4229 level1
[l1
] = count2
++;
4230 if (level2
[l2
] == 0xFF)
4231 level2
[l2
] = count3
++;
4234 if (count2
>= 0xFF || count3
>= 0xFF)
4238 PyObject
*result
= PyDict_New();
4239 PyObject
*key
, *value
;
4242 for (i
= 0; i
< 256; i
++) {
4244 key
= PyInt_FromLong(decode
[i
]);
4245 value
= PyInt_FromLong(i
);
4248 if (PyDict_SetItem(result
, key
, value
) == -1)
4261 /* Create a three-level trie */
4262 result
= PyObject_MALLOC(sizeof(struct encoding_map
) +
4263 16*count2
+ 128*count3
- 1);
4265 return PyErr_NoMemory();
4266 PyObject_Init(result
, &EncodingMapType
);
4267 mresult
= (struct encoding_map
*)result
;
4268 mresult
->count2
= count2
;
4269 mresult
->count3
= count3
;
4270 mlevel1
= mresult
->level1
;
4271 mlevel2
= mresult
->level23
;
4272 mlevel3
= mresult
->level23
+ 16*count2
;
4273 memcpy(mlevel1
, level1
, 32);
4274 memset(mlevel2
, 0xFF, 16*count2
);
4275 memset(mlevel3
, 0, 128*count3
);
4277 for (i
= 1; i
< 256; i
++) {
4278 int o1
, o2
, o3
, i2
, i3
;
4279 if (decode
[i
] == 0xFFFE)
4280 /* unmapped character */
4283 o2
= (decode
[i
]>>7) & 0xF;
4284 i2
= 16*mlevel1
[o1
] + o2
;
4285 if (mlevel2
[i2
] == 0xFF)
4286 mlevel2
[i2
] = count3
++;
4287 o3
= decode
[i
] & 0x7F;
4288 i3
= 128*mlevel2
[i2
] + o3
;
4295 encoding_map_lookup(Py_UNICODE c
, PyObject
*mapping
)
4297 struct encoding_map
*map
= (struct encoding_map
*)mapping
;
4299 int l2
= (c
>>7) & 0xF;
4303 #ifdef Py_UNICODE_WIDE
4311 i
= map
->level1
[l1
];
4316 i
= map
->level23
[16*i
+l2
];
4321 i
= map
->level23
[16*map
->count2
+ 128*i
+ l3
];
4328 /* Lookup the character ch in the mapping. If the character
4329 can't be found, Py_None is returned (or NULL, if another
4331 static PyObject
*charmapencode_lookup(Py_UNICODE c
, PyObject
*mapping
)
4333 PyObject
*w
= PyInt_FromLong((long)c
);
4338 x
= PyObject_GetItem(mapping
, w
);
4341 if (PyErr_ExceptionMatches(PyExc_LookupError
)) {
4342 /* No mapping found means: mapping is undefined. */
4350 else if (x
== Py_None
)
4352 else if (PyInt_Check(x
)) {
4353 long value
= PyInt_AS_LONG(x
);
4354 if (value
< 0 || value
> 255) {
4355 PyErr_SetString(PyExc_TypeError
,
4356 "character mapping must be in range(256)");
4362 else if (PyString_Check(x
))
4365 /* wrong return value */
4366 PyErr_SetString(PyExc_TypeError
,
4367 "character mapping must return integer, None or str");
4374 charmapencode_resize(PyObject
**outobj
, Py_ssize_t
*outpos
, Py_ssize_t requiredsize
)
4376 Py_ssize_t outsize
= PyString_GET_SIZE(*outobj
);
4377 /* exponentially overallocate to minimize reallocations */
4378 if (requiredsize
< 2*outsize
)
4379 requiredsize
= 2*outsize
;
4380 if (_PyString_Resize(outobj
, requiredsize
)) {
4386 typedef enum charmapencode_result
{
4387 enc_SUCCESS
, enc_FAILED
, enc_EXCEPTION
4388 }charmapencode_result
;
4389 /* lookup the character, put the result in the output string and adjust
4390 various state variables. Reallocate the output string if not enough
4391 space is available. Return a new reference to the object that
4392 was put in the output buffer, or Py_None, if the mapping was undefined
4393 (in which case no character was written) or NULL, if a
4394 reallocation error occurred. The caller must decref the result */
4396 charmapencode_result
charmapencode_output(Py_UNICODE c
, PyObject
*mapping
,
4397 PyObject
**outobj
, Py_ssize_t
*outpos
)
4401 Py_ssize_t outsize
= PyString_GET_SIZE(*outobj
);
4403 if (Py_TYPE(mapping
) == &EncodingMapType
) {
4404 int res
= encoding_map_lookup(c
, mapping
);
4405 Py_ssize_t requiredsize
= *outpos
+1;
4408 if (outsize
<requiredsize
)
4409 if (!charmapencode_resize(outobj
, outpos
, requiredsize
))
4410 return enc_EXCEPTION
;
4411 outstart
= PyString_AS_STRING(*outobj
);
4412 outstart
[(*outpos
)++] = (char)res
;
4416 rep
= charmapencode_lookup(c
, mapping
);
4418 return enc_EXCEPTION
;
4419 else if (rep
==Py_None
) {
4423 if (PyInt_Check(rep
)) {
4424 Py_ssize_t requiredsize
= *outpos
+1;
4425 if (outsize
<requiredsize
)
4426 if (!charmapencode_resize(outobj
, outpos
, requiredsize
)) {
4428 return enc_EXCEPTION
;
4430 outstart
= PyString_AS_STRING(*outobj
);
4431 outstart
[(*outpos
)++] = (char)PyInt_AS_LONG(rep
);
4434 const char *repchars
= PyString_AS_STRING(rep
);
4435 Py_ssize_t repsize
= PyString_GET_SIZE(rep
);
4436 Py_ssize_t requiredsize
= *outpos
+repsize
;
4437 if (outsize
<requiredsize
)
4438 if (!charmapencode_resize(outobj
, outpos
, requiredsize
)) {
4440 return enc_EXCEPTION
;
4442 outstart
= PyString_AS_STRING(*outobj
);
4443 memcpy(outstart
+ *outpos
, repchars
, repsize
);
4451 /* handle an error in PyUnicode_EncodeCharmap
4452 Return 0 on success, -1 on error */
4454 int charmap_encoding_error(
4455 const Py_UNICODE
*p
, Py_ssize_t size
, Py_ssize_t
*inpos
, PyObject
*mapping
,
4456 PyObject
**exceptionObject
,
4457 int *known_errorHandler
, PyObject
**errorHandler
, const char *errors
,
4458 PyObject
**res
, Py_ssize_t
*respos
)
4460 PyObject
*repunicode
= NULL
; /* initialize to prevent gcc warning */
4464 /* startpos for collecting unencodable chars */
4465 Py_ssize_t collstartpos
= *inpos
;
4466 Py_ssize_t collendpos
= *inpos
+1;
4468 char *encoding
= "charmap";
4469 char *reason
= "character maps to <undefined>";
4470 charmapencode_result x
;
4472 /* find all unencodable characters */
4473 while (collendpos
< size
) {
4475 if (Py_TYPE(mapping
) == &EncodingMapType
) {
4476 int res
= encoding_map_lookup(p
[collendpos
], mapping
);
4483 rep
= charmapencode_lookup(p
[collendpos
], mapping
);
4486 else if (rep
!=Py_None
) {
4493 /* cache callback name lookup
4494 * (if not done yet, i.e. it's the first error) */
4495 if (*known_errorHandler
==-1) {
4496 if ((errors
==NULL
) || (!strcmp(errors
, "strict")))
4497 *known_errorHandler
= 1;
4498 else if (!strcmp(errors
, "replace"))
4499 *known_errorHandler
= 2;
4500 else if (!strcmp(errors
, "ignore"))
4501 *known_errorHandler
= 3;
4502 else if (!strcmp(errors
, "xmlcharrefreplace"))
4503 *known_errorHandler
= 4;
4505 *known_errorHandler
= 0;
4507 switch (*known_errorHandler
) {
4508 case 1: /* strict */
4509 raise_encode_exception(exceptionObject
, encoding
, p
, size
, collstartpos
, collendpos
, reason
);
4511 case 2: /* replace */
4512 for (collpos
= collstartpos
; collpos
<collendpos
; ++collpos
) {
4513 x
= charmapencode_output('?', mapping
, res
, respos
);
4514 if (x
==enc_EXCEPTION
) {
4517 else if (x
==enc_FAILED
) {
4518 raise_encode_exception(exceptionObject
, encoding
, p
, size
, collstartpos
, collendpos
, reason
);
4523 case 3: /* ignore */
4524 *inpos
= collendpos
;
4526 case 4: /* xmlcharrefreplace */
4527 /* generate replacement (temporarily (mis)uses p) */
4528 for (collpos
= collstartpos
; collpos
< collendpos
; ++collpos
) {
4529 char buffer
[2+29+1+1];
4531 sprintf(buffer
, "&#%d;", (int)p
[collpos
]);
4532 for (cp
= buffer
; *cp
; ++cp
) {
4533 x
= charmapencode_output(*cp
, mapping
, res
, respos
);
4534 if (x
==enc_EXCEPTION
)
4536 else if (x
==enc_FAILED
) {
4537 raise_encode_exception(exceptionObject
, encoding
, p
, size
, collstartpos
, collendpos
, reason
);
4542 *inpos
= collendpos
;
4545 repunicode
= unicode_encode_call_errorhandler(errors
, errorHandler
,
4546 encoding
, reason
, p
, size
, exceptionObject
,
4547 collstartpos
, collendpos
, &newpos
);
4548 if (repunicode
== NULL
)
4550 /* generate replacement */
4551 repsize
= PyUnicode_GET_SIZE(repunicode
);
4552 for (uni2
= PyUnicode_AS_UNICODE(repunicode
); repsize
-->0; ++uni2
) {
4553 x
= charmapencode_output(*uni2
, mapping
, res
, respos
);
4554 if (x
==enc_EXCEPTION
) {
4557 else if (x
==enc_FAILED
) {
4558 Py_DECREF(repunicode
);
4559 raise_encode_exception(exceptionObject
, encoding
, p
, size
, collstartpos
, collendpos
, reason
);
4564 Py_DECREF(repunicode
);
4569 PyObject
*PyUnicode_EncodeCharmap(const Py_UNICODE
*p
,
4575 PyObject
*res
= NULL
;
4576 /* current input position */
4577 Py_ssize_t inpos
= 0;
4578 /* current output position */
4579 Py_ssize_t respos
= 0;
4580 PyObject
*errorHandler
= NULL
;
4581 PyObject
*exc
= NULL
;
4582 /* the following variable is used for caching string comparisons
4583 * -1=not initialized, 0=unknown, 1=strict, 2=replace,
4584 * 3=ignore, 4=xmlcharrefreplace */
4585 int known_errorHandler
= -1;
4587 /* Default to Latin-1 */
4588 if (mapping
== NULL
)
4589 return PyUnicode_EncodeLatin1(p
, size
, errors
);
4591 /* allocate enough for a simple encoding without
4592 replacements, if we need more, we'll resize */
4593 res
= PyString_FromStringAndSize(NULL
, size
);
4599 while (inpos
<size
) {
4600 /* try to encode it */
4601 charmapencode_result x
= charmapencode_output(p
[inpos
], mapping
, &res
, &respos
);
4602 if (x
==enc_EXCEPTION
) /* error */
4604 if (x
==enc_FAILED
) { /* unencodable character */
4605 if (charmap_encoding_error(p
, size
, &inpos
, mapping
,
4607 &known_errorHandler
, &errorHandler
, errors
,
4613 /* done with this character => adjust input position */
4617 /* Resize if we allocated to much */
4618 if (respos
<PyString_GET_SIZE(res
)) {
4619 if (_PyString_Resize(&res
, respos
))
4623 Py_XDECREF(errorHandler
);
4629 Py_XDECREF(errorHandler
);
4633 PyObject
*PyUnicode_AsCharmapString(PyObject
*unicode
,
4636 if (!PyUnicode_Check(unicode
) || mapping
== NULL
) {
4637 PyErr_BadArgument();
4640 return PyUnicode_EncodeCharmap(PyUnicode_AS_UNICODE(unicode
),
4641 PyUnicode_GET_SIZE(unicode
),
4646 /* create or adjust a UnicodeTranslateError */
4647 static void make_translate_exception(PyObject
**exceptionObject
,
4648 const Py_UNICODE
*unicode
, Py_ssize_t size
,
4649 Py_ssize_t startpos
, Py_ssize_t endpos
,
4652 if (*exceptionObject
== NULL
) {
4653 *exceptionObject
= PyUnicodeTranslateError_Create(
4654 unicode
, size
, startpos
, endpos
, reason
);
4657 if (PyUnicodeTranslateError_SetStart(*exceptionObject
, startpos
))
4659 if (PyUnicodeTranslateError_SetEnd(*exceptionObject
, endpos
))
4661 if (PyUnicodeTranslateError_SetReason(*exceptionObject
, reason
))
4665 Py_DECREF(*exceptionObject
);
4666 *exceptionObject
= NULL
;
4670 /* raises a UnicodeTranslateError */
4671 static void raise_translate_exception(PyObject
**exceptionObject
,
4672 const Py_UNICODE
*unicode
, Py_ssize_t size
,
4673 Py_ssize_t startpos
, Py_ssize_t endpos
,
4676 make_translate_exception(exceptionObject
,
4677 unicode
, size
, startpos
, endpos
, reason
);
4678 if (*exceptionObject
!= NULL
)
4679 PyCodec_StrictErrors(*exceptionObject
);
4682 /* error handling callback helper:
4683 build arguments, call the callback and check the arguments,
4684 put the result into newpos and return the replacement string, which
4685 has to be freed by the caller */
4686 static PyObject
*unicode_translate_call_errorhandler(const char *errors
,
4687 PyObject
**errorHandler
,
4689 const Py_UNICODE
*unicode
, Py_ssize_t size
, PyObject
**exceptionObject
,
4690 Py_ssize_t startpos
, Py_ssize_t endpos
,
4693 static char *argparse
= "O!n;translating error handler must return (unicode, int) tuple";
4695 Py_ssize_t i_newpos
;
4697 PyObject
*resunicode
;
4699 if (*errorHandler
== NULL
) {
4700 *errorHandler
= PyCodec_LookupError(errors
);
4701 if (*errorHandler
== NULL
)
4705 make_translate_exception(exceptionObject
,
4706 unicode
, size
, startpos
, endpos
, reason
);
4707 if (*exceptionObject
== NULL
)
4710 restuple
= PyObject_CallFunctionObjArgs(
4711 *errorHandler
, *exceptionObject
, NULL
);
4712 if (restuple
== NULL
)
4714 if (!PyTuple_Check(restuple
)) {
4715 PyErr_Format(PyExc_TypeError
, &argparse
[4]);
4716 Py_DECREF(restuple
);
4719 if (!PyArg_ParseTuple(restuple
, argparse
, &PyUnicode_Type
,
4720 &resunicode
, &i_newpos
)) {
4721 Py_DECREF(restuple
);
4725 *newpos
= size
+i_newpos
;
4728 if (*newpos
<0 || *newpos
>size
) {
4729 PyErr_Format(PyExc_IndexError
, "position %zd from error handler out of bounds", *newpos
);
4730 Py_DECREF(restuple
);
4733 Py_INCREF(resunicode
);
4734 Py_DECREF(restuple
);
4738 /* Lookup the character ch in the mapping and put the result in result,
4739 which must be decrefed by the caller.
4740 Return 0 on success, -1 on error */
4742 int charmaptranslate_lookup(Py_UNICODE c
, PyObject
*mapping
, PyObject
**result
)
4744 PyObject
*w
= PyInt_FromLong((long)c
);
4749 x
= PyObject_GetItem(mapping
, w
);
4752 if (PyErr_ExceptionMatches(PyExc_LookupError
)) {
4753 /* No mapping found means: use 1:1 mapping. */
4760 else if (x
== Py_None
) {
4764 else if (PyInt_Check(x
)) {
4765 long value
= PyInt_AS_LONG(x
);
4766 long max
= PyUnicode_GetMax();
4767 if (value
< 0 || value
> max
) {
4768 PyErr_Format(PyExc_TypeError
,
4769 "character mapping must be in range(0x%lx)", max
+1);
4776 else if (PyUnicode_Check(x
)) {
4781 /* wrong return value */
4782 PyErr_SetString(PyExc_TypeError
,
4783 "character mapping must return integer, None or unicode");
4788 /* ensure that *outobj is at least requiredsize characters long,
4789 if not reallocate and adjust various state variables.
4790 Return 0 on success, -1 on error */
4792 int charmaptranslate_makespace(PyObject
**outobj
, Py_UNICODE
**outp
,
4793 Py_ssize_t requiredsize
)
4795 Py_ssize_t oldsize
= PyUnicode_GET_SIZE(*outobj
);
4796 if (requiredsize
> oldsize
) {
4797 /* remember old output position */
4798 Py_ssize_t outpos
= *outp
-PyUnicode_AS_UNICODE(*outobj
);
4799 /* exponentially overallocate to minimize reallocations */
4800 if (requiredsize
< 2 * oldsize
)
4801 requiredsize
= 2 * oldsize
;
4802 if (PyUnicode_Resize(outobj
, requiredsize
) < 0)
4804 *outp
= PyUnicode_AS_UNICODE(*outobj
) + outpos
;
4808 /* lookup the character, put the result in the output string and adjust
4809 various state variables. Return a new reference to the object that
4810 was put in the output buffer in *result, or Py_None, if the mapping was
4811 undefined (in which case no character was written).
4812 The called must decref result.
4813 Return 0 on success, -1 on error. */
4815 int charmaptranslate_output(const Py_UNICODE
*startinp
, const Py_UNICODE
*curinp
,
4816 Py_ssize_t insize
, PyObject
*mapping
, PyObject
**outobj
, Py_UNICODE
**outp
,
4819 if (charmaptranslate_lookup(*curinp
, mapping
, res
))
4822 /* not found => default to 1:1 mapping */
4823 *(*outp
)++ = *curinp
;
4825 else if (*res
==Py_None
)
4827 else if (PyInt_Check(*res
)) {
4828 /* no overflow check, because we know that the space is enough */
4829 *(*outp
)++ = (Py_UNICODE
)PyInt_AS_LONG(*res
);
4831 else if (PyUnicode_Check(*res
)) {
4832 Py_ssize_t repsize
= PyUnicode_GET_SIZE(*res
);
4834 /* no overflow check, because we know that the space is enough */
4835 *(*outp
)++ = *PyUnicode_AS_UNICODE(*res
);
4837 else if (repsize
!=0) {
4838 /* more than one character */
4839 Py_ssize_t requiredsize
= (*outp
-PyUnicode_AS_UNICODE(*outobj
)) +
4840 (insize
- (curinp
-startinp
)) +
4842 if (charmaptranslate_makespace(outobj
, outp
, requiredsize
))
4844 memcpy(*outp
, PyUnicode_AS_UNICODE(*res
), sizeof(Py_UNICODE
)*repsize
);
4853 PyObject
*PyUnicode_TranslateCharmap(const Py_UNICODE
*p
,
4859 PyObject
*res
= NULL
;
4860 /* pointers to the beginning and end+1 of input */
4861 const Py_UNICODE
*startp
= p
;
4862 const Py_UNICODE
*endp
= p
+ size
;
4863 /* pointer into the output */
4865 /* current output position */
4866 Py_ssize_t respos
= 0;
4867 char *reason
= "character maps to <undefined>";
4868 PyObject
*errorHandler
= NULL
;
4869 PyObject
*exc
= NULL
;
4870 /* the following variable is used for caching string comparisons
4871 * -1=not initialized, 0=unknown, 1=strict, 2=replace,
4872 * 3=ignore, 4=xmlcharrefreplace */
4873 int known_errorHandler
= -1;
4875 if (mapping
== NULL
) {
4876 PyErr_BadArgument();
4880 /* allocate enough for a simple 1:1 translation without
4881 replacements, if we need more, we'll resize */
4882 res
= PyUnicode_FromUnicode(NULL
, size
);
4887 str
= PyUnicode_AS_UNICODE(res
);
4890 /* try to encode it */
4892 if (charmaptranslate_output(startp
, p
, size
, mapping
, &res
, &str
, &x
)) {
4897 if (x
!=Py_None
) /* it worked => adjust input pointer */
4899 else { /* untranslatable character */
4900 PyObject
*repunicode
= NULL
; /* initialize to prevent gcc warning */
4904 /* startpos for collecting untranslatable chars */
4905 const Py_UNICODE
*collstart
= p
;
4906 const Py_UNICODE
*collend
= p
+1;
4907 const Py_UNICODE
*coll
;
4909 /* find all untranslatable characters */
4910 while (collend
< endp
) {
4911 if (charmaptranslate_lookup(*collend
, mapping
, &x
))
4918 /* cache callback name lookup
4919 * (if not done yet, i.e. it's the first error) */
4920 if (known_errorHandler
==-1) {
4921 if ((errors
==NULL
) || (!strcmp(errors
, "strict")))
4922 known_errorHandler
= 1;
4923 else if (!strcmp(errors
, "replace"))
4924 known_errorHandler
= 2;
4925 else if (!strcmp(errors
, "ignore"))
4926 known_errorHandler
= 3;
4927 else if (!strcmp(errors
, "xmlcharrefreplace"))
4928 known_errorHandler
= 4;
4930 known_errorHandler
= 0;
4932 switch (known_errorHandler
) {
4933 case 1: /* strict */
4934 raise_translate_exception(&exc
, startp
, size
, collstart
-startp
, collend
-startp
, reason
);
4936 case 2: /* replace */
4937 /* No need to check for space, this is a 1:1 replacement */
4938 for (coll
= collstart
; coll
<collend
; ++coll
)
4941 case 3: /* ignore */
4944 case 4: /* xmlcharrefreplace */
4945 /* generate replacement (temporarily (mis)uses p) */
4946 for (p
= collstart
; p
< collend
; ++p
) {
4947 char buffer
[2+29+1+1];
4949 sprintf(buffer
, "&#%d;", (int)*p
);
4950 if (charmaptranslate_makespace(&res
, &str
,
4951 (str
-PyUnicode_AS_UNICODE(res
))+strlen(buffer
)+(endp
-collend
)))
4953 for (cp
= buffer
; *cp
; ++cp
)
4959 repunicode
= unicode_translate_call_errorhandler(errors
, &errorHandler
,
4960 reason
, startp
, size
, &exc
,
4961 collstart
-startp
, collend
-startp
, &newpos
);
4962 if (repunicode
== NULL
)
4964 /* generate replacement */
4965 repsize
= PyUnicode_GET_SIZE(repunicode
);
4966 if (charmaptranslate_makespace(&res
, &str
,
4967 (str
-PyUnicode_AS_UNICODE(res
))+repsize
+(endp
-collend
))) {
4968 Py_DECREF(repunicode
);
4971 for (uni2
= PyUnicode_AS_UNICODE(repunicode
); repsize
-->0; ++uni2
)
4973 p
= startp
+ newpos
;
4974 Py_DECREF(repunicode
);
4978 /* Resize if we allocated to much */
4979 respos
= str
-PyUnicode_AS_UNICODE(res
);
4980 if (respos
<PyUnicode_GET_SIZE(res
)) {
4981 if (PyUnicode_Resize(&res
, respos
) < 0)
4985 Py_XDECREF(errorHandler
);
4991 Py_XDECREF(errorHandler
);
4995 PyObject
*PyUnicode_Translate(PyObject
*str
,
5001 str
= PyUnicode_FromObject(str
);
5004 result
= PyUnicode_TranslateCharmap(PyUnicode_AS_UNICODE(str
),
5005 PyUnicode_GET_SIZE(str
),
5016 /* --- Decimal Encoder ---------------------------------------------------- */
5018 int PyUnicode_EncodeDecimal(Py_UNICODE
*s
,
5023 Py_UNICODE
*p
, *end
;
5024 PyObject
*errorHandler
= NULL
;
5025 PyObject
*exc
= NULL
;
5026 const char *encoding
= "decimal";
5027 const char *reason
= "invalid decimal Unicode string";
5028 /* the following variable is used for caching string comparisons
5029 * -1=not initialized, 0=unknown, 1=strict, 2=replace, 3=ignore, 4=xmlcharrefreplace */
5030 int known_errorHandler
= -1;
5032 if (output
== NULL
) {
5033 PyErr_BadArgument();
5040 register Py_UNICODE ch
= *p
;
5042 PyObject
*repunicode
;
5046 Py_UNICODE
*collstart
;
5047 Py_UNICODE
*collend
;
5049 if (Py_UNICODE_ISSPACE(ch
)) {
5054 decimal
= Py_UNICODE_TODECIMAL(ch
);
5056 *output
++ = '0' + decimal
;
5060 if (0 < ch
&& ch
< 256) {
5061 *output
++ = (char)ch
;
5065 /* All other characters are considered unencodable */
5068 while (collend
< end
) {
5069 if ((0 < *collend
&& *collend
< 256) ||
5070 !Py_UNICODE_ISSPACE(*collend
) ||
5071 Py_UNICODE_TODECIMAL(*collend
))
5074 /* cache callback name lookup
5075 * (if not done yet, i.e. it's the first error) */
5076 if (known_errorHandler
==-1) {
5077 if ((errors
==NULL
) || (!strcmp(errors
, "strict")))
5078 known_errorHandler
= 1;
5079 else if (!strcmp(errors
, "replace"))
5080 known_errorHandler
= 2;
5081 else if (!strcmp(errors
, "ignore"))
5082 known_errorHandler
= 3;
5083 else if (!strcmp(errors
, "xmlcharrefreplace"))
5084 known_errorHandler
= 4;
5086 known_errorHandler
= 0;
5088 switch (known_errorHandler
) {
5089 case 1: /* strict */
5090 raise_encode_exception(&exc
, encoding
, s
, length
, collstart
-s
, collend
-s
, reason
);
5092 case 2: /* replace */
5093 for (p
= collstart
; p
< collend
; ++p
)
5096 case 3: /* ignore */
5099 case 4: /* xmlcharrefreplace */
5100 /* generate replacement (temporarily (mis)uses p) */
5101 for (p
= collstart
; p
< collend
; ++p
)
5102 output
+= sprintf(output
, "&#%d;", (int)*p
);
5106 repunicode
= unicode_encode_call_errorhandler(errors
, &errorHandler
,
5107 encoding
, reason
, s
, length
, &exc
,
5108 collstart
-s
, collend
-s
, &newpos
);
5109 if (repunicode
== NULL
)
5111 /* generate replacement */
5112 repsize
= PyUnicode_GET_SIZE(repunicode
);
5113 for (uni2
= PyUnicode_AS_UNICODE(repunicode
); repsize
-->0; ++uni2
) {
5114 Py_UNICODE ch
= *uni2
;
5115 if (Py_UNICODE_ISSPACE(ch
))
5118 decimal
= Py_UNICODE_TODECIMAL(ch
);
5120 *output
++ = '0' + decimal
;
5121 else if (0 < ch
&& ch
< 256)
5122 *output
++ = (char)ch
;
5124 Py_DECREF(repunicode
);
5125 raise_encode_exception(&exc
, encoding
,
5126 s
, length
, collstart
-s
, collend
-s
, reason
);
5132 Py_DECREF(repunicode
);
5135 /* 0-terminate the output string */
5138 Py_XDECREF(errorHandler
);
5143 Py_XDECREF(errorHandler
);
5147 /* --- Helpers ------------------------------------------------------------ */
5149 #include "stringlib/unicodedefs.h"
5151 #define FROM_UNICODE
5153 #include "stringlib/fastsearch.h"
5155 #include "stringlib/count.h"
5156 #include "stringlib/find.h"
5157 #include "stringlib/partition.h"
5159 /* helper macro to fixup start/end slice values */
5160 #define FIX_START_END(obj) \
5162 start += (obj)->length; \
5165 if (end > (obj)->length) \
5166 end = (obj)->length; \
5168 end += (obj)->length; \
5172 Py_ssize_t
PyUnicode_Count(PyObject
*str
,
5178 PyUnicodeObject
* str_obj
;
5179 PyUnicodeObject
* sub_obj
;
5181 str_obj
= (PyUnicodeObject
*) PyUnicode_FromObject(str
);
5184 sub_obj
= (PyUnicodeObject
*) PyUnicode_FromObject(substr
);
5190 FIX_START_END(str_obj
);
5192 result
= stringlib_count(
5193 str_obj
->str
+ start
, end
- start
, sub_obj
->str
, sub_obj
->length
5202 Py_ssize_t
PyUnicode_Find(PyObject
*str
,
5210 str
= PyUnicode_FromObject(str
);
5213 sub
= PyUnicode_FromObject(sub
);
5220 result
= stringlib_find_slice(
5221 PyUnicode_AS_UNICODE(str
), PyUnicode_GET_SIZE(str
),
5222 PyUnicode_AS_UNICODE(sub
), PyUnicode_GET_SIZE(sub
),
5226 result
= stringlib_rfind_slice(
5227 PyUnicode_AS_UNICODE(str
), PyUnicode_GET_SIZE(str
),
5228 PyUnicode_AS_UNICODE(sub
), PyUnicode_GET_SIZE(sub
),
5239 int tailmatch(PyUnicodeObject
*self
,
5240 PyUnicodeObject
*substring
,
5245 if (substring
->length
== 0)
5248 FIX_START_END(self
);
5250 end
-= substring
->length
;
5254 if (direction
> 0) {
5255 if (Py_UNICODE_MATCH(self
, end
, substring
))
5258 if (Py_UNICODE_MATCH(self
, start
, substring
))
5265 Py_ssize_t
PyUnicode_Tailmatch(PyObject
*str
,
5273 str
= PyUnicode_FromObject(str
);
5276 substr
= PyUnicode_FromObject(substr
);
5277 if (substr
== NULL
) {
5282 result
= tailmatch((PyUnicodeObject
*)str
,
5283 (PyUnicodeObject
*)substr
,
5284 start
, end
, direction
);
5290 /* Apply fixfct filter to the Unicode object self and return a
5291 reference to the modified object */
5294 PyObject
*fixup(PyUnicodeObject
*self
,
5295 int (*fixfct
)(PyUnicodeObject
*s
))
5300 u
= (PyUnicodeObject
*) PyUnicode_FromUnicode(NULL
, self
->length
);
5304 Py_UNICODE_COPY(u
->str
, self
->str
, self
->length
);
5306 if (!fixfct(u
) && PyUnicode_CheckExact(self
)) {
5307 /* fixfct should return TRUE if it modified the buffer. If
5308 FALSE, return a reference to the original buffer instead
5309 (to save space, not time) */
5312 return (PyObject
*) self
;
5314 return (PyObject
*) u
;
5318 int fixupper(PyUnicodeObject
*self
)
5320 Py_ssize_t len
= self
->length
;
5321 Py_UNICODE
*s
= self
->str
;
5325 register Py_UNICODE ch
;
5327 ch
= Py_UNICODE_TOUPPER(*s
);
5339 int fixlower(PyUnicodeObject
*self
)
5341 Py_ssize_t len
= self
->length
;
5342 Py_UNICODE
*s
= self
->str
;
5346 register Py_UNICODE ch
;
5348 ch
= Py_UNICODE_TOLOWER(*s
);
5360 int fixswapcase(PyUnicodeObject
*self
)
5362 Py_ssize_t len
= self
->length
;
5363 Py_UNICODE
*s
= self
->str
;
5367 if (Py_UNICODE_ISUPPER(*s
)) {
5368 *s
= Py_UNICODE_TOLOWER(*s
);
5370 } else if (Py_UNICODE_ISLOWER(*s
)) {
5371 *s
= Py_UNICODE_TOUPPER(*s
);
5381 int fixcapitalize(PyUnicodeObject
*self
)
5383 Py_ssize_t len
= self
->length
;
5384 Py_UNICODE
*s
= self
->str
;
5389 if (Py_UNICODE_ISLOWER(*s
)) {
5390 *s
= Py_UNICODE_TOUPPER(*s
);
5395 if (Py_UNICODE_ISUPPER(*s
)) {
5396 *s
= Py_UNICODE_TOLOWER(*s
);
5405 int fixtitle(PyUnicodeObject
*self
)
5407 register Py_UNICODE
*p
= PyUnicode_AS_UNICODE(self
);
5408 register Py_UNICODE
*e
;
5409 int previous_is_cased
;
5411 /* Shortcut for single character strings */
5412 if (PyUnicode_GET_SIZE(self
) == 1) {
5413 Py_UNICODE ch
= Py_UNICODE_TOTITLE(*p
);
5422 e
= p
+ PyUnicode_GET_SIZE(self
);
5423 previous_is_cased
= 0;
5424 for (; p
< e
; p
++) {
5425 register const Py_UNICODE ch
= *p
;
5427 if (previous_is_cased
)
5428 *p
= Py_UNICODE_TOLOWER(ch
);
5430 *p
= Py_UNICODE_TOTITLE(ch
);
5432 if (Py_UNICODE_ISLOWER(ch
) ||
5433 Py_UNICODE_ISUPPER(ch
) ||
5434 Py_UNICODE_ISTITLE(ch
))
5435 previous_is_cased
= 1;
5437 previous_is_cased
= 0;
5443 PyUnicode_Join(PyObject
*separator
, PyObject
*seq
)
5445 PyObject
*internal_separator
= NULL
;
5446 const Py_UNICODE blank
= ' ';
5447 const Py_UNICODE
*sep
= &blank
;
5448 Py_ssize_t seplen
= 1;
5449 PyUnicodeObject
*res
= NULL
; /* the result */
5450 Py_ssize_t res_alloc
= 100; /* # allocated bytes for string in res */
5451 Py_ssize_t res_used
; /* # used bytes */
5452 Py_UNICODE
*res_p
; /* pointer to free byte in res's string area */
5453 PyObject
*fseq
; /* PySequence_Fast(seq) */
5454 Py_ssize_t seqlen
; /* len(fseq) -- number of items in sequence */
5458 fseq
= PySequence_Fast(seq
, "");
5463 /* Grrrr. A codec may be invoked to convert str objects to
5464 * Unicode, and so it's possible to call back into Python code
5465 * during PyUnicode_FromObject(), and so it's possible for a sick
5466 * codec to change the size of fseq (if seq is a list). Therefore
5467 * we have to keep refetching the size -- can't assume seqlen
5470 seqlen
= PySequence_Fast_GET_SIZE(fseq
);
5471 /* If empty sequence, return u"". */
5473 res
= _PyUnicode_New(0); /* empty sequence; return u"" */
5476 /* If singleton sequence with an exact Unicode, return that. */
5478 item
= PySequence_Fast_GET_ITEM(fseq
, 0);
5479 if (PyUnicode_CheckExact(item
)) {
5481 res
= (PyUnicodeObject
*)item
;
5486 /* At least two items to join, or one that isn't exact Unicode. */
5488 /* Set up sep and seplen -- they're needed. */
5489 if (separator
== NULL
) {
5494 internal_separator
= PyUnicode_FromObject(separator
);
5495 if (internal_separator
== NULL
)
5497 sep
= PyUnicode_AS_UNICODE(internal_separator
);
5498 seplen
= PyUnicode_GET_SIZE(internal_separator
);
5499 /* In case PyUnicode_FromObject() mutated seq. */
5500 seqlen
= PySequence_Fast_GET_SIZE(fseq
);
5505 res
= _PyUnicode_New(res_alloc
);
5508 res_p
= PyUnicode_AS_UNICODE(res
);
5511 for (i
= 0; i
< seqlen
; ++i
) {
5513 Py_ssize_t new_res_used
;
5515 item
= PySequence_Fast_GET_ITEM(fseq
, i
);
5516 /* Convert item to Unicode. */
5517 if (! PyUnicode_Check(item
) && ! PyString_Check(item
)) {
5518 PyErr_Format(PyExc_TypeError
,
5519 "sequence item %zd: expected string or Unicode,"
5521 i
, Py_TYPE(item
)->tp_name
);
5524 item
= PyUnicode_FromObject(item
);
5527 /* We own a reference to item from here on. */
5529 /* In case PyUnicode_FromObject() mutated seq. */
5530 seqlen
= PySequence_Fast_GET_SIZE(fseq
);
5532 /* Make sure we have enough space for the separator and the item. */
5533 itemlen
= PyUnicode_GET_SIZE(item
);
5534 new_res_used
= res_used
+ itemlen
;
5535 if (new_res_used
< 0)
5537 if (i
< seqlen
- 1) {
5538 new_res_used
+= seplen
;
5539 if (new_res_used
< 0)
5542 if (new_res_used
> res_alloc
) {
5543 /* double allocated size until it's big enough */
5545 res_alloc
+= res_alloc
;
5548 } while (new_res_used
> res_alloc
);
5549 if (_PyUnicode_Resize(&res
, res_alloc
) < 0) {
5553 res_p
= PyUnicode_AS_UNICODE(res
) + res_used
;
5556 /* Copy item, and maybe the separator. */
5557 Py_UNICODE_COPY(res_p
, PyUnicode_AS_UNICODE(item
), itemlen
);
5559 if (i
< seqlen
- 1) {
5560 Py_UNICODE_COPY(res_p
, sep
, seplen
);
5564 res_used
= new_res_used
;
5567 /* Shrink res to match the used area; this probably can't fail,
5568 * but it's cheap to check.
5570 if (_PyUnicode_Resize(&res
, res_used
) < 0)
5574 Py_XDECREF(internal_separator
);
5576 return (PyObject
*)res
;
5579 PyErr_SetString(PyExc_OverflowError
,
5580 "join() result is too long for a Python string");
5585 Py_XDECREF(internal_separator
);
5592 PyUnicodeObject
*pad(PyUnicodeObject
*self
,
5604 if (left
== 0 && right
== 0 && PyUnicode_CheckExact(self
)) {
5609 if (left
> PY_SSIZE_T_MAX
- self
->length
||
5610 right
> PY_SSIZE_T_MAX
- (left
+ self
->length
)) {
5611 PyErr_SetString(PyExc_OverflowError
, "padded string is too long");
5614 u
= _PyUnicode_New(left
+ self
->length
+ right
);
5617 Py_UNICODE_FILL(u
->str
, fill
, left
);
5618 Py_UNICODE_COPY(u
->str
+ left
, self
->str
, self
->length
);
5620 Py_UNICODE_FILL(u
->str
+ left
+ self
->length
, fill
, right
);
5626 #define SPLIT_APPEND(data, left, right) \
5627 str = PyUnicode_FromUnicode((data) + (left), (right) - (left)); \
5630 if (PyList_Append(list, str)) { \
5638 PyObject
*split_whitespace(PyUnicodeObject
*self
,
5640 Py_ssize_t maxcount
)
5642 register Py_ssize_t i
;
5643 register Py_ssize_t j
;
5644 Py_ssize_t len
= self
->length
;
5646 register const Py_UNICODE
*buf
= self
->str
;
5648 for (i
= j
= 0; i
< len
; ) {
5650 while (i
< len
&& Py_UNICODE_ISSPACE(buf
[i
]))
5653 while (i
< len
&& !Py_UNICODE_ISSPACE(buf
[i
]))
5656 if (maxcount
-- <= 0)
5658 SPLIT_APPEND(buf
, j
, i
);
5659 while (i
< len
&& Py_UNICODE_ISSPACE(buf
[i
]))
5665 SPLIT_APPEND(buf
, j
, len
);
5674 PyObject
*PyUnicode_Splitlines(PyObject
*string
,
5677 register Py_ssize_t i
;
5678 register Py_ssize_t j
;
5684 string
= PyUnicode_FromObject(string
);
5687 data
= PyUnicode_AS_UNICODE(string
);
5688 len
= PyUnicode_GET_SIZE(string
);
5690 list
= PyList_New(0);
5694 for (i
= j
= 0; i
< len
; ) {
5697 /* Find a line and append it */
5698 while (i
< len
&& !BLOOM_LINEBREAK(data
[i
]))
5701 /* Skip the line break reading CRLF as one line break */
5704 if (data
[i
] == '\r' && i
+ 1 < len
&&
5712 SPLIT_APPEND(data
, j
, eol
);
5716 SPLIT_APPEND(data
, j
, len
);
5729 PyObject
*split_char(PyUnicodeObject
*self
,
5732 Py_ssize_t maxcount
)
5734 register Py_ssize_t i
;
5735 register Py_ssize_t j
;
5736 Py_ssize_t len
= self
->length
;
5738 register const Py_UNICODE
*buf
= self
->str
;
5740 for (i
= j
= 0; i
< len
; ) {
5742 if (maxcount
-- <= 0)
5744 SPLIT_APPEND(buf
, j
, i
);
5750 SPLIT_APPEND(buf
, j
, len
);
5760 PyObject
*split_substring(PyUnicodeObject
*self
,
5762 PyUnicodeObject
*substring
,
5763 Py_ssize_t maxcount
)
5765 register Py_ssize_t i
;
5766 register Py_ssize_t j
;
5767 Py_ssize_t len
= self
->length
;
5768 Py_ssize_t sublen
= substring
->length
;
5771 for (i
= j
= 0; i
<= len
- sublen
; ) {
5772 if (Py_UNICODE_MATCH(self
, i
, substring
)) {
5773 if (maxcount
-- <= 0)
5775 SPLIT_APPEND(self
->str
, j
, i
);
5781 SPLIT_APPEND(self
->str
, j
, len
);
5791 PyObject
*rsplit_whitespace(PyUnicodeObject
*self
,
5793 Py_ssize_t maxcount
)
5795 register Py_ssize_t i
;
5796 register Py_ssize_t j
;
5797 Py_ssize_t len
= self
->length
;
5799 register const Py_UNICODE
*buf
= self
->str
;
5801 for (i
= j
= len
- 1; i
>= 0; ) {
5803 while (i
>= 0 && Py_UNICODE_ISSPACE(buf
[i
]))
5806 while (i
>= 0 && !Py_UNICODE_ISSPACE(buf
[i
]))
5809 if (maxcount
-- <= 0)
5811 SPLIT_APPEND(buf
, i
+ 1, j
+ 1);
5812 while (i
>= 0 && Py_UNICODE_ISSPACE(buf
[i
]))
5818 SPLIT_APPEND(buf
, 0, j
+ 1);
5820 if (PyList_Reverse(list
) < 0)
5830 PyObject
*rsplit_char(PyUnicodeObject
*self
,
5833 Py_ssize_t maxcount
)
5835 register Py_ssize_t i
;
5836 register Py_ssize_t j
;
5837 Py_ssize_t len
= self
->length
;
5839 register const Py_UNICODE
*buf
= self
->str
;
5841 for (i
= j
= len
- 1; i
>= 0; ) {
5843 if (maxcount
-- <= 0)
5845 SPLIT_APPEND(buf
, i
+ 1, j
+ 1);
5851 SPLIT_APPEND(buf
, 0, j
+ 1);
5853 if (PyList_Reverse(list
) < 0)
5863 PyObject
*rsplit_substring(PyUnicodeObject
*self
,
5865 PyUnicodeObject
*substring
,
5866 Py_ssize_t maxcount
)
5868 register Py_ssize_t i
;
5869 register Py_ssize_t j
;
5870 Py_ssize_t len
= self
->length
;
5871 Py_ssize_t sublen
= substring
->length
;
5874 for (i
= len
- sublen
, j
= len
; i
>= 0; ) {
5875 if (Py_UNICODE_MATCH(self
, i
, substring
)) {
5876 if (maxcount
-- <= 0)
5878 SPLIT_APPEND(self
->str
, i
+ sublen
, j
);
5885 SPLIT_APPEND(self
->str
, 0, j
);
5887 if (PyList_Reverse(list
) < 0)
5899 PyObject
*split(PyUnicodeObject
*self
,
5900 PyUnicodeObject
*substring
,
5901 Py_ssize_t maxcount
)
5906 maxcount
= PY_SSIZE_T_MAX
;
5908 list
= PyList_New(0);
5912 if (substring
== NULL
)
5913 return split_whitespace(self
,list
,maxcount
);
5915 else if (substring
->length
== 1)
5916 return split_char(self
,list
,substring
->str
[0],maxcount
);
5918 else if (substring
->length
== 0) {
5920 PyErr_SetString(PyExc_ValueError
, "empty separator");
5924 return split_substring(self
,list
,substring
,maxcount
);
5928 PyObject
*rsplit(PyUnicodeObject
*self
,
5929 PyUnicodeObject
*substring
,
5930 Py_ssize_t maxcount
)
5935 maxcount
= PY_SSIZE_T_MAX
;
5937 list
= PyList_New(0);
5941 if (substring
== NULL
)
5942 return rsplit_whitespace(self
,list
,maxcount
);
5944 else if (substring
->length
== 1)
5945 return rsplit_char(self
,list
,substring
->str
[0],maxcount
);
5947 else if (substring
->length
== 0) {
5949 PyErr_SetString(PyExc_ValueError
, "empty separator");
5953 return rsplit_substring(self
,list
,substring
,maxcount
);
5957 PyObject
*replace(PyUnicodeObject
*self
,
5958 PyUnicodeObject
*str1
,
5959 PyUnicodeObject
*str2
,
5960 Py_ssize_t maxcount
)
5965 maxcount
= PY_SSIZE_T_MAX
;
5967 if (str1
->length
== str2
->length
) {
5970 if (str1
->length
== 1) {
5971 /* replace characters */
5973 if (!findchar(self
->str
, self
->length
, str1
->str
[0]))
5975 u
= (PyUnicodeObject
*) PyUnicode_FromUnicode(NULL
, self
->length
);
5978 Py_UNICODE_COPY(u
->str
, self
->str
, self
->length
);
5981 for (i
= 0; i
< u
->length
; i
++)
5982 if (u
->str
[i
] == u1
) {
5989 self
->str
, self
->length
, str1
->str
, str1
->length
, FAST_SEARCH
5993 u
= (PyUnicodeObject
*) PyUnicode_FromUnicode(NULL
, self
->length
);
5996 Py_UNICODE_COPY(u
->str
, self
->str
, self
->length
);
5997 while (i
<= self
->length
- str1
->length
)
5998 if (Py_UNICODE_MATCH(self
, i
, str1
)) {
6001 Py_UNICODE_COPY(u
->str
+i
, str2
->str
, str2
->length
);
6008 Py_ssize_t n
, i
, j
, e
;
6009 Py_ssize_t product
, new_size
, delta
;
6012 /* replace strings */
6013 n
= stringlib_count(self
->str
, self
->length
, str1
->str
, str1
->length
);
6018 /* new_size = self->length + n * (str2->length - str1->length)); */
6019 delta
= (str2
->length
- str1
->length
);
6021 new_size
= self
->length
;
6023 product
= n
* (str2
->length
- str1
->length
);
6024 if ((product
/ (str2
->length
- str1
->length
)) != n
) {
6025 PyErr_SetString(PyExc_OverflowError
,
6026 "replace string is too long");
6029 new_size
= self
->length
+ product
;
6031 PyErr_SetString(PyExc_OverflowError
,
6032 "replace string is too long");
6036 u
= _PyUnicode_New(new_size
);
6041 e
= self
->length
- str1
->length
;
6042 if (str1
->length
> 0) {
6044 /* look for next match */
6047 if (Py_UNICODE_MATCH(self
, j
, str1
))
6054 /* copy unchanged part [i:j] */
6055 Py_UNICODE_COPY(p
, self
->str
+i
, j
-i
);
6058 /* copy substitution string */
6059 if (str2
->length
> 0) {
6060 Py_UNICODE_COPY(p
, str2
->str
, str2
->length
);
6063 i
= j
+ str1
->length
;
6065 if (i
< self
->length
)
6066 /* copy tail [i:] */
6067 Py_UNICODE_COPY(p
, self
->str
+i
, self
->length
-i
);
6071 Py_UNICODE_COPY(p
, str2
->str
, str2
->length
);
6075 *p
++ = self
->str
[i
++];
6077 Py_UNICODE_COPY(p
, self
->str
+i
, self
->length
-i
);
6080 return (PyObject
*) u
;
6083 /* nothing to replace; return original string (when possible) */
6084 if (PyUnicode_CheckExact(self
)) {
6086 return (PyObject
*) self
;
6088 return PyUnicode_FromUnicode(self
->str
, self
->length
);
6091 /* --- Unicode Object Methods --------------------------------------------- */
6093 PyDoc_STRVAR(title__doc__
,
6094 "S.title() -> unicode\n\
6096 Return a titlecased version of S, i.e. words start with title case\n\
6097 characters, all remaining cased characters have lower case.");
6100 unicode_title(PyUnicodeObject
*self
)
6102 return fixup(self
, fixtitle
);
6105 PyDoc_STRVAR(capitalize__doc__
,
6106 "S.capitalize() -> unicode\n\
6108 Return a capitalized version of S, i.e. make the first character\n\
6112 unicode_capitalize(PyUnicodeObject
*self
)
6114 return fixup(self
, fixcapitalize
);
6118 PyDoc_STRVAR(capwords__doc__
,
6119 "S.capwords() -> unicode\n\
6121 Apply .capitalize() to all words in S and return the result with\n\
6122 normalized whitespace (all whitespace strings are replaced by ' ').");
6125 unicode_capwords(PyUnicodeObject
*self
)
6131 /* Split into words */
6132 list
= split(self
, NULL
, -1);
6136 /* Capitalize each word */
6137 for (i
= 0; i
< PyList_GET_SIZE(list
); i
++) {
6138 item
= fixup((PyUnicodeObject
*)PyList_GET_ITEM(list
, i
),
6142 Py_DECREF(PyList_GET_ITEM(list
, i
));
6143 PyList_SET_ITEM(list
, i
, item
);
6146 /* Join the words to form a new string */
6147 item
= PyUnicode_Join(NULL
, list
);
6151 return (PyObject
*)item
;
6155 /* Argument converter. Coerces to a single unicode character */
6158 convert_uc(PyObject
*obj
, void *addr
)
6160 Py_UNICODE
*fillcharloc
= (Py_UNICODE
*)addr
;
6164 uniobj
= PyUnicode_FromObject(obj
);
6165 if (uniobj
== NULL
) {
6166 PyErr_SetString(PyExc_TypeError
,
6167 "The fill character cannot be converted to Unicode");
6170 if (PyUnicode_GET_SIZE(uniobj
) != 1) {
6171 PyErr_SetString(PyExc_TypeError
,
6172 "The fill character must be exactly one character long");
6176 unistr
= PyUnicode_AS_UNICODE(uniobj
);
6177 *fillcharloc
= unistr
[0];
6182 PyDoc_STRVAR(center__doc__
,
6183 "S.center(width[, fillchar]) -> unicode\n\
6185 Return S centered in a Unicode string of length width. Padding is\n\
6186 done using the specified fill character (default is a space)");
6189 unicode_center(PyUnicodeObject
*self
, PyObject
*args
)
6191 Py_ssize_t marg
, left
;
6193 Py_UNICODE fillchar
= ' ';
6195 if (!PyArg_ParseTuple(args
, "n|O&:center", &width
, convert_uc
, &fillchar
))
6198 if (self
->length
>= width
&& PyUnicode_CheckExact(self
)) {
6200 return (PyObject
*) self
;
6203 marg
= width
- self
->length
;
6204 left
= marg
/ 2 + (marg
& width
& 1);
6206 return (PyObject
*) pad(self
, left
, marg
- left
, fillchar
);
6211 /* This code should go into some future Unicode collation support
6212 module. The basic comparison should compare ordinals on a naive
6213 basis (this is what Java does and thus JPython too). */
6215 /* speedy UTF-16 code point order comparison */
6217 /* http://www-4.ibm.com/software/developer/library/utf16.html?dwzone=unicode */
6219 static short utf16Fixup
[32] =
6221 0, 0, 0, 0, 0, 0, 0, 0,
6222 0, 0, 0, 0, 0, 0, 0, 0,
6223 0, 0, 0, 0, 0, 0, 0, 0,
6224 0, 0, 0, 0x2000, -0x800, -0x800, -0x800, -0x800
6228 unicode_compare(PyUnicodeObject
*str1
, PyUnicodeObject
*str2
)
6230 Py_ssize_t len1
, len2
;
6232 Py_UNICODE
*s1
= str1
->str
;
6233 Py_UNICODE
*s2
= str2
->str
;
6235 len1
= str1
->length
;
6236 len2
= str2
->length
;
6238 while (len1
> 0 && len2
> 0) {
6244 if (c1
> (1<<11) * 26)
6245 c1
+= utf16Fixup
[c1
>>11];
6246 if (c2
> (1<<11) * 26)
6247 c2
+= utf16Fixup
[c2
>>11];
6248 /* now c1 and c2 are in UTF-32-compatible order */
6251 return (c1
< c2
) ? -1 : 1;
6256 return (len1
< len2
) ? -1 : (len1
!= len2
);
6262 unicode_compare(PyUnicodeObject
*str1
, PyUnicodeObject
*str2
)
6264 register Py_ssize_t len1
, len2
;
6266 Py_UNICODE
*s1
= str1
->str
;
6267 Py_UNICODE
*s2
= str2
->str
;
6269 len1
= str1
->length
;
6270 len2
= str2
->length
;
6272 while (len1
> 0 && len2
> 0) {
6279 return (c1
< c2
) ? -1 : 1;
6284 return (len1
< len2
) ? -1 : (len1
!= len2
);
6289 int PyUnicode_Compare(PyObject
*left
,
6292 PyUnicodeObject
*u
= NULL
, *v
= NULL
;
6295 /* Coerce the two arguments */
6296 u
= (PyUnicodeObject
*)PyUnicode_FromObject(left
);
6299 v
= (PyUnicodeObject
*)PyUnicode_FromObject(right
);
6303 /* Shortcut for empty or interned objects */
6310 result
= unicode_compare(u
, v
);
6322 PyObject
*PyUnicode_RichCompare(PyObject
*left
,
6328 result
= PyUnicode_Compare(left
, right
);
6329 if (result
== -1 && PyErr_Occurred())
6332 /* Convert the return value to a Boolean */
6335 result
= (result
== 0);
6338 result
= (result
!= 0);
6341 result
= (result
<= 0);
6344 result
= (result
>= 0);
6347 result
= (result
== -1);
6350 result
= (result
== 1);
6353 return PyBool_FromLong(result
);
6359 Type errors mean that PyUnicode_FromObject() could not convert
6360 one of the arguments (usually the right hand side) to Unicode,
6361 ie. we can't handle the comparison request. However, it is
6362 possible that the other object knows a comparison method, which
6363 is why we return Py_NotImplemented to give the other object a
6367 if (PyErr_ExceptionMatches(PyExc_TypeError
)) {
6369 Py_INCREF(Py_NotImplemented
);
6370 return Py_NotImplemented
;
6372 if (op
!= Py_EQ
&& op
!= Py_NE
)
6375 /* Equality comparison.
6377 This is a special case: we silence any PyExc_UnicodeDecodeError
6378 and instead turn it into a PyErr_UnicodeWarning.
6381 if (!PyErr_ExceptionMatches(PyExc_UnicodeDecodeError
))
6384 if (PyErr_Warn(PyExc_UnicodeWarning
,
6386 "Unicode equal comparison "
6387 "failed to convert both arguments to Unicode - "
6388 "interpreting them as being unequal" :
6389 "Unicode unequal comparison "
6390 "failed to convert both arguments to Unicode - "
6391 "interpreting them as being unequal"
6394 result
= (op
== Py_NE
);
6395 return PyBool_FromLong(result
);
6398 int PyUnicode_Contains(PyObject
*container
,
6401 PyObject
*str
, *sub
;
6404 /* Coerce the two arguments */
6405 sub
= PyUnicode_FromObject(element
);
6407 PyErr_SetString(PyExc_TypeError
,
6408 "'in <string>' requires string as left operand");
6412 str
= PyUnicode_FromObject(container
);
6418 result
= stringlib_contains_obj(str
, sub
);
6426 /* Concat to string or Unicode object giving a new Unicode object. */
6428 PyObject
*PyUnicode_Concat(PyObject
*left
,
6431 PyUnicodeObject
*u
= NULL
, *v
= NULL
, *w
;
6433 /* Coerce the two arguments */
6434 u
= (PyUnicodeObject
*)PyUnicode_FromObject(left
);
6437 v
= (PyUnicodeObject
*)PyUnicode_FromObject(right
);
6442 if (v
== unicode_empty
) {
6444 return (PyObject
*)u
;
6446 if (u
== unicode_empty
) {
6448 return (PyObject
*)v
;
6451 /* Concat the two Unicode strings */
6452 w
= _PyUnicode_New(u
->length
+ v
->length
);
6455 Py_UNICODE_COPY(w
->str
, u
->str
, u
->length
);
6456 Py_UNICODE_COPY(w
->str
+ u
->length
, v
->str
, v
->length
);
6460 return (PyObject
*)w
;
6468 PyDoc_STRVAR(count__doc__
,
6469 "S.count(sub[, start[, end]]) -> int\n\
6471 Return the number of non-overlapping occurrences of substring sub in\n\
6472 Unicode string S[start:end]. Optional arguments start and end are\n\
6473 interpreted as in slice notation.");
6476 unicode_count(PyUnicodeObject
*self
, PyObject
*args
)
6478 PyUnicodeObject
*substring
;
6479 Py_ssize_t start
= 0;
6480 Py_ssize_t end
= PY_SSIZE_T_MAX
;
6483 if (!PyArg_ParseTuple(args
, "O|O&O&:count", &substring
,
6484 _PyEval_SliceIndex
, &start
, _PyEval_SliceIndex
, &end
))
6487 substring
= (PyUnicodeObject
*)PyUnicode_FromObject(
6488 (PyObject
*)substring
);
6489 if (substring
== NULL
)
6492 FIX_START_END(self
);
6494 result
= PyInt_FromSsize_t(
6495 stringlib_count(self
->str
+ start
, end
- start
,
6496 substring
->str
, substring
->length
)
6499 Py_DECREF(substring
);
6504 PyDoc_STRVAR(encode__doc__
,
6505 "S.encode([encoding[,errors]]) -> string or unicode\n\
6507 Encodes S using the codec registered for encoding. encoding defaults\n\
6508 to the default encoding. errors may be given to set a different error\n\
6509 handling scheme. Default is 'strict' meaning that encoding errors raise\n\
6510 a UnicodeEncodeError. Other possible values are 'ignore', 'replace' and\n\
6511 'xmlcharrefreplace' as well as any other name registered with\n\
6512 codecs.register_error that can handle UnicodeEncodeErrors.");
6515 unicode_encode(PyUnicodeObject
*self
, PyObject
*args
)
6517 char *encoding
= NULL
;
6518 char *errors
= NULL
;
6521 if (!PyArg_ParseTuple(args
, "|ss:encode", &encoding
, &errors
))
6523 v
= PyUnicode_AsEncodedObject((PyObject
*)self
, encoding
, errors
);
6526 if (!PyString_Check(v
) && !PyUnicode_Check(v
)) {
6527 PyErr_Format(PyExc_TypeError
,
6528 "encoder did not return a string/unicode object "
6530 Py_TYPE(v
)->tp_name
);
6540 PyDoc_STRVAR(decode__doc__
,
6541 "S.decode([encoding[,errors]]) -> string or unicode\n\
6543 Decodes S using the codec registered for encoding. encoding defaults\n\
6544 to the default encoding. errors may be given to set a different error\n\
6545 handling scheme. Default is 'strict' meaning that encoding errors raise\n\
6546 a UnicodeDecodeError. Other possible values are 'ignore' and 'replace'\n\
6547 as well as any other name registerd with codecs.register_error that is\n\
6548 able to handle UnicodeDecodeErrors.");
6551 unicode_decode(PyUnicodeObject
*self
, PyObject
*args
)
6553 char *encoding
= NULL
;
6554 char *errors
= NULL
;
6557 if (!PyArg_ParseTuple(args
, "|ss:decode", &encoding
, &errors
))
6559 v
= PyUnicode_AsDecodedObject((PyObject
*)self
, encoding
, errors
);
6562 if (!PyString_Check(v
) && !PyUnicode_Check(v
)) {
6563 PyErr_Format(PyExc_TypeError
,
6564 "decoder did not return a string/unicode object "
6566 Py_TYPE(v
)->tp_name
);
6576 PyDoc_STRVAR(expandtabs__doc__
,
6577 "S.expandtabs([tabsize]) -> unicode\n\
6579 Return a copy of S where all tab characters are expanded using spaces.\n\
6580 If tabsize is not given, a tab size of 8 characters is assumed.");
6583 unicode_expandtabs(PyUnicodeObject
*self
, PyObject
*args
)
6589 Py_ssize_t i
, j
, incr
;
6593 if (!PyArg_ParseTuple(args
, "|i:expandtabs", &tabsize
))
6596 /* First pass: determine size of output string */
6597 i
= 0; /* chars up to and including most recent \n or \r */
6598 j
= 0; /* chars since most recent \n or \r (use in tab calculations) */
6599 e
= self
->str
+ self
->length
; /* end of input */
6600 for (p
= self
->str
; p
< e
; p
++)
6603 incr
= tabsize
- (j
% tabsize
); /* cannot overflow */
6604 if (j
> PY_SSIZE_T_MAX
- incr
)
6610 if (j
> PY_SSIZE_T_MAX
- 1)
6613 if (*p
== '\n' || *p
== '\r') {
6614 if (i
> PY_SSIZE_T_MAX
- j
)
6621 if (i
> PY_SSIZE_T_MAX
- j
)
6624 /* Second pass: create output string and fill it */
6625 u
= _PyUnicode_New(i
+ j
);
6629 j
= 0; /* same as in first pass */
6630 q
= u
->str
; /* next output char */
6631 qe
= u
->str
+ u
->length
; /* end of output */
6633 for (p
= self
->str
; p
< e
; p
++)
6636 i
= tabsize
- (j
% tabsize
);
6650 if (*p
== '\n' || *p
== '\r')
6654 return (PyObject
*) u
;
6659 PyErr_SetString(PyExc_OverflowError
, "new string is too long");
6663 PyDoc_STRVAR(find__doc__
,
6664 "S.find(sub [,start [,end]]) -> int\n\
6666 Return the lowest index in S where substring sub is found,\n\
6667 such that sub is contained within s[start:end]. Optional\n\
6668 arguments start and end are interpreted as in slice notation.\n\
6670 Return -1 on failure.");
6673 unicode_find(PyUnicodeObject
*self
, PyObject
*args
)
6675 PyObject
*substring
;
6680 if (!_ParseTupleFinds(args
, &substring
, &start
, &end
))
6683 result
= stringlib_find_slice(
6684 PyUnicode_AS_UNICODE(self
), PyUnicode_GET_SIZE(self
),
6685 PyUnicode_AS_UNICODE(substring
), PyUnicode_GET_SIZE(substring
),
6689 Py_DECREF(substring
);
6691 return PyInt_FromSsize_t(result
);
6695 unicode_getitem(PyUnicodeObject
*self
, Py_ssize_t index
)
6697 if (index
< 0 || index
>= self
->length
) {
6698 PyErr_SetString(PyExc_IndexError
, "string index out of range");
6702 return (PyObject
*) PyUnicode_FromUnicode(&self
->str
[index
], 1);
6706 unicode_hash(PyUnicodeObject
*self
)
6708 /* Since Unicode objects compare equal to their ASCII string
6709 counterparts, they should use the individual character values
6710 as basis for their hash value. This is needed to assure that
6711 strings and Unicode objects behave in the same way as
6714 register Py_ssize_t len
;
6715 register Py_UNICODE
*p
;
6718 if (self
->hash
!= -1)
6720 len
= PyUnicode_GET_SIZE(self
);
6721 p
= PyUnicode_AS_UNICODE(self
);
6724 x
= (1000003*x
) ^ *p
++;
6725 x
^= PyUnicode_GET_SIZE(self
);
6732 PyDoc_STRVAR(index__doc__
,
6733 "S.index(sub [,start [,end]]) -> int\n\
6735 Like S.find() but raise ValueError when the substring is not found.");
6738 unicode_index(PyUnicodeObject
*self
, PyObject
*args
)
6741 PyObject
*substring
;
6745 if (!_ParseTupleFinds(args
, &substring
, &start
, &end
))
6748 result
= stringlib_find_slice(
6749 PyUnicode_AS_UNICODE(self
), PyUnicode_GET_SIZE(self
),
6750 PyUnicode_AS_UNICODE(substring
), PyUnicode_GET_SIZE(substring
),
6754 Py_DECREF(substring
);
6757 PyErr_SetString(PyExc_ValueError
, "substring not found");
6761 return PyInt_FromSsize_t(result
);
6764 PyDoc_STRVAR(islower__doc__
,
6765 "S.islower() -> bool\n\
6767 Return True if all cased characters in S are lowercase and there is\n\
6768 at least one cased character in S, False otherwise.");
6771 unicode_islower(PyUnicodeObject
*self
)
6773 register const Py_UNICODE
*p
= PyUnicode_AS_UNICODE(self
);
6774 register const Py_UNICODE
*e
;
6777 /* Shortcut for single character strings */
6778 if (PyUnicode_GET_SIZE(self
) == 1)
6779 return PyBool_FromLong(Py_UNICODE_ISLOWER(*p
));
6781 /* Special case for empty strings */
6782 if (PyUnicode_GET_SIZE(self
) == 0)
6783 return PyBool_FromLong(0);
6785 e
= p
+ PyUnicode_GET_SIZE(self
);
6787 for (; p
< e
; p
++) {
6788 register const Py_UNICODE ch
= *p
;
6790 if (Py_UNICODE_ISUPPER(ch
) || Py_UNICODE_ISTITLE(ch
))
6791 return PyBool_FromLong(0);
6792 else if (!cased
&& Py_UNICODE_ISLOWER(ch
))
6795 return PyBool_FromLong(cased
);
6798 PyDoc_STRVAR(isupper__doc__
,
6799 "S.isupper() -> bool\n\
6801 Return True if all cased characters in S are uppercase and there is\n\
6802 at least one cased character in S, False otherwise.");
6805 unicode_isupper(PyUnicodeObject
*self
)
6807 register const Py_UNICODE
*p
= PyUnicode_AS_UNICODE(self
);
6808 register const Py_UNICODE
*e
;
6811 /* Shortcut for single character strings */
6812 if (PyUnicode_GET_SIZE(self
) == 1)
6813 return PyBool_FromLong(Py_UNICODE_ISUPPER(*p
) != 0);
6815 /* Special case for empty strings */
6816 if (PyUnicode_GET_SIZE(self
) == 0)
6817 return PyBool_FromLong(0);
6819 e
= p
+ PyUnicode_GET_SIZE(self
);
6821 for (; p
< e
; p
++) {
6822 register const Py_UNICODE ch
= *p
;
6824 if (Py_UNICODE_ISLOWER(ch
) || Py_UNICODE_ISTITLE(ch
))
6825 return PyBool_FromLong(0);
6826 else if (!cased
&& Py_UNICODE_ISUPPER(ch
))
6829 return PyBool_FromLong(cased
);
6832 PyDoc_STRVAR(istitle__doc__
,
6833 "S.istitle() -> bool\n\
6835 Return True if S is a titlecased string and there is at least one\n\
6836 character in S, i.e. upper- and titlecase characters may only\n\
6837 follow uncased characters and lowercase characters only cased ones.\n\
6838 Return False otherwise.");
6841 unicode_istitle(PyUnicodeObject
*self
)
6843 register const Py_UNICODE
*p
= PyUnicode_AS_UNICODE(self
);
6844 register const Py_UNICODE
*e
;
6845 int cased
, previous_is_cased
;
6847 /* Shortcut for single character strings */
6848 if (PyUnicode_GET_SIZE(self
) == 1)
6849 return PyBool_FromLong((Py_UNICODE_ISTITLE(*p
) != 0) ||
6850 (Py_UNICODE_ISUPPER(*p
) != 0));
6852 /* Special case for empty strings */
6853 if (PyUnicode_GET_SIZE(self
) == 0)
6854 return PyBool_FromLong(0);
6856 e
= p
+ PyUnicode_GET_SIZE(self
);
6858 previous_is_cased
= 0;
6859 for (; p
< e
; p
++) {
6860 register const Py_UNICODE ch
= *p
;
6862 if (Py_UNICODE_ISUPPER(ch
) || Py_UNICODE_ISTITLE(ch
)) {
6863 if (previous_is_cased
)
6864 return PyBool_FromLong(0);
6865 previous_is_cased
= 1;
6868 else if (Py_UNICODE_ISLOWER(ch
)) {
6869 if (!previous_is_cased
)
6870 return PyBool_FromLong(0);
6871 previous_is_cased
= 1;
6875 previous_is_cased
= 0;
6877 return PyBool_FromLong(cased
);
6880 PyDoc_STRVAR(isspace__doc__
,
6881 "S.isspace() -> bool\n\
6883 Return True if all characters in S are whitespace\n\
6884 and there is at least one character in S, False otherwise.");
6887 unicode_isspace(PyUnicodeObject
*self
)
6889 register const Py_UNICODE
*p
= PyUnicode_AS_UNICODE(self
);
6890 register const Py_UNICODE
*e
;
6892 /* Shortcut for single character strings */
6893 if (PyUnicode_GET_SIZE(self
) == 1 &&
6894 Py_UNICODE_ISSPACE(*p
))
6895 return PyBool_FromLong(1);
6897 /* Special case for empty strings */
6898 if (PyUnicode_GET_SIZE(self
) == 0)
6899 return PyBool_FromLong(0);
6901 e
= p
+ PyUnicode_GET_SIZE(self
);
6902 for (; p
< e
; p
++) {
6903 if (!Py_UNICODE_ISSPACE(*p
))
6904 return PyBool_FromLong(0);
6906 return PyBool_FromLong(1);
6909 PyDoc_STRVAR(isalpha__doc__
,
6910 "S.isalpha() -> bool\n\
6912 Return True if all characters in S are alphabetic\n\
6913 and there is at least one character in S, False otherwise.");
6916 unicode_isalpha(PyUnicodeObject
*self
)
6918 register const Py_UNICODE
*p
= PyUnicode_AS_UNICODE(self
);
6919 register const Py_UNICODE
*e
;
6921 /* Shortcut for single character strings */
6922 if (PyUnicode_GET_SIZE(self
) == 1 &&
6923 Py_UNICODE_ISALPHA(*p
))
6924 return PyBool_FromLong(1);
6926 /* Special case for empty strings */
6927 if (PyUnicode_GET_SIZE(self
) == 0)
6928 return PyBool_FromLong(0);
6930 e
= p
+ PyUnicode_GET_SIZE(self
);
6931 for (; p
< e
; p
++) {
6932 if (!Py_UNICODE_ISALPHA(*p
))
6933 return PyBool_FromLong(0);
6935 return PyBool_FromLong(1);
6938 PyDoc_STRVAR(isalnum__doc__
,
6939 "S.isalnum() -> bool\n\
6941 Return True if all characters in S are alphanumeric\n\
6942 and there is at least one character in S, False otherwise.");
6945 unicode_isalnum(PyUnicodeObject
*self
)
6947 register const Py_UNICODE
*p
= PyUnicode_AS_UNICODE(self
);
6948 register const Py_UNICODE
*e
;
6950 /* Shortcut for single character strings */
6951 if (PyUnicode_GET_SIZE(self
) == 1 &&
6952 Py_UNICODE_ISALNUM(*p
))
6953 return PyBool_FromLong(1);
6955 /* Special case for empty strings */
6956 if (PyUnicode_GET_SIZE(self
) == 0)
6957 return PyBool_FromLong(0);
6959 e
= p
+ PyUnicode_GET_SIZE(self
);
6960 for (; p
< e
; p
++) {
6961 if (!Py_UNICODE_ISALNUM(*p
))
6962 return PyBool_FromLong(0);
6964 return PyBool_FromLong(1);
6967 PyDoc_STRVAR(isdecimal__doc__
,
6968 "S.isdecimal() -> bool\n\
6970 Return True if there are only decimal characters in S,\n\
6974 unicode_isdecimal(PyUnicodeObject
*self
)
6976 register const Py_UNICODE
*p
= PyUnicode_AS_UNICODE(self
);
6977 register const Py_UNICODE
*e
;
6979 /* Shortcut for single character strings */
6980 if (PyUnicode_GET_SIZE(self
) == 1 &&
6981 Py_UNICODE_ISDECIMAL(*p
))
6982 return PyBool_FromLong(1);
6984 /* Special case for empty strings */
6985 if (PyUnicode_GET_SIZE(self
) == 0)
6986 return PyBool_FromLong(0);
6988 e
= p
+ PyUnicode_GET_SIZE(self
);
6989 for (; p
< e
; p
++) {
6990 if (!Py_UNICODE_ISDECIMAL(*p
))
6991 return PyBool_FromLong(0);
6993 return PyBool_FromLong(1);
6996 PyDoc_STRVAR(isdigit__doc__
,
6997 "S.isdigit() -> bool\n\
6999 Return True if all characters in S are digits\n\
7000 and there is at least one character in S, False otherwise.");
7003 unicode_isdigit(PyUnicodeObject
*self
)
7005 register const Py_UNICODE
*p
= PyUnicode_AS_UNICODE(self
);
7006 register const Py_UNICODE
*e
;
7008 /* Shortcut for single character strings */
7009 if (PyUnicode_GET_SIZE(self
) == 1 &&
7010 Py_UNICODE_ISDIGIT(*p
))
7011 return PyBool_FromLong(1);
7013 /* Special case for empty strings */
7014 if (PyUnicode_GET_SIZE(self
) == 0)
7015 return PyBool_FromLong(0);
7017 e
= p
+ PyUnicode_GET_SIZE(self
);
7018 for (; p
< e
; p
++) {
7019 if (!Py_UNICODE_ISDIGIT(*p
))
7020 return PyBool_FromLong(0);
7022 return PyBool_FromLong(1);
7025 PyDoc_STRVAR(isnumeric__doc__
,
7026 "S.isnumeric() -> bool\n\
7028 Return True if there are only numeric characters in S,\n\
7032 unicode_isnumeric(PyUnicodeObject
*self
)
7034 register const Py_UNICODE
*p
= PyUnicode_AS_UNICODE(self
);
7035 register const Py_UNICODE
*e
;
7037 /* Shortcut for single character strings */
7038 if (PyUnicode_GET_SIZE(self
) == 1 &&
7039 Py_UNICODE_ISNUMERIC(*p
))
7040 return PyBool_FromLong(1);
7042 /* Special case for empty strings */
7043 if (PyUnicode_GET_SIZE(self
) == 0)
7044 return PyBool_FromLong(0);
7046 e
= p
+ PyUnicode_GET_SIZE(self
);
7047 for (; p
< e
; p
++) {
7048 if (!Py_UNICODE_ISNUMERIC(*p
))
7049 return PyBool_FromLong(0);
7051 return PyBool_FromLong(1);
7054 PyDoc_STRVAR(join__doc__
,
7055 "S.join(sequence) -> unicode\n\
7057 Return a string which is the concatenation of the strings in the\n\
7058 sequence. The separator between elements is S.");
7061 unicode_join(PyObject
*self
, PyObject
*data
)
7063 return PyUnicode_Join(self
, data
);
7067 unicode_length(PyUnicodeObject
*self
)
7069 return self
->length
;
7072 PyDoc_STRVAR(ljust__doc__
,
7073 "S.ljust(width[, fillchar]) -> int\n\
7075 Return S left-justified in a Unicode string of length width. Padding is\n\
7076 done using the specified fill character (default is a space).");
7079 unicode_ljust(PyUnicodeObject
*self
, PyObject
*args
)
7082 Py_UNICODE fillchar
= ' ';
7084 if (!PyArg_ParseTuple(args
, "n|O&:ljust", &width
, convert_uc
, &fillchar
))
7087 if (self
->length
>= width
&& PyUnicode_CheckExact(self
)) {
7089 return (PyObject
*) self
;
7092 return (PyObject
*) pad(self
, 0, width
- self
->length
, fillchar
);
7095 PyDoc_STRVAR(lower__doc__
,
7096 "S.lower() -> unicode\n\
7098 Return a copy of the string S converted to lowercase.");
7101 unicode_lower(PyUnicodeObject
*self
)
7103 return fixup(self
, fixlower
);
7107 #define RIGHTSTRIP 1
7110 /* Arrays indexed by above */
7111 static const char *stripformat
[] = {"|O:lstrip", "|O:rstrip", "|O:strip"};
7113 #define STRIPNAME(i) (stripformat[i]+3)
7115 /* externally visible for str.strip(unicode) */
7117 _PyUnicode_XStrip(PyUnicodeObject
*self
, int striptype
, PyObject
*sepobj
)
7119 Py_UNICODE
*s
= PyUnicode_AS_UNICODE(self
);
7120 Py_ssize_t len
= PyUnicode_GET_SIZE(self
);
7121 Py_UNICODE
*sep
= PyUnicode_AS_UNICODE(sepobj
);
7122 Py_ssize_t seplen
= PyUnicode_GET_SIZE(sepobj
);
7125 BLOOM_MASK sepmask
= make_bloom_mask(sep
, seplen
);
7128 if (striptype
!= RIGHTSTRIP
) {
7129 while (i
< len
&& BLOOM_MEMBER(sepmask
, s
[i
], sep
, seplen
)) {
7135 if (striptype
!= LEFTSTRIP
) {
7138 } while (j
>= i
&& BLOOM_MEMBER(sepmask
, s
[j
], sep
, seplen
));
7142 if (i
== 0 && j
== len
&& PyUnicode_CheckExact(self
)) {
7144 return (PyObject
*)self
;
7147 return PyUnicode_FromUnicode(s
+i
, j
-i
);
7152 do_strip(PyUnicodeObject
*self
, int striptype
)
7154 Py_UNICODE
*s
= PyUnicode_AS_UNICODE(self
);
7155 Py_ssize_t len
= PyUnicode_GET_SIZE(self
), i
, j
;
7158 if (striptype
!= RIGHTSTRIP
) {
7159 while (i
< len
&& Py_UNICODE_ISSPACE(s
[i
])) {
7165 if (striptype
!= LEFTSTRIP
) {
7168 } while (j
>= i
&& Py_UNICODE_ISSPACE(s
[j
]));
7172 if (i
== 0 && j
== len
&& PyUnicode_CheckExact(self
)) {
7174 return (PyObject
*)self
;
7177 return PyUnicode_FromUnicode(s
+i
, j
-i
);
7182 do_argstrip(PyUnicodeObject
*self
, int striptype
, PyObject
*args
)
7184 PyObject
*sep
= NULL
;
7186 if (!PyArg_ParseTuple(args
, (char *)stripformat
[striptype
], &sep
))
7189 if (sep
!= NULL
&& sep
!= Py_None
) {
7190 if (PyUnicode_Check(sep
))
7191 return _PyUnicode_XStrip(self
, striptype
, sep
);
7192 else if (PyString_Check(sep
)) {
7194 sep
= PyUnicode_FromObject(sep
);
7197 res
= _PyUnicode_XStrip(self
, striptype
, sep
);
7202 PyErr_Format(PyExc_TypeError
,
7203 "%s arg must be None, unicode or str",
7204 STRIPNAME(striptype
));
7209 return do_strip(self
, striptype
);
7213 PyDoc_STRVAR(strip__doc__
,
7214 "S.strip([chars]) -> unicode\n\
7216 Return a copy of the string S with leading and trailing\n\
7217 whitespace removed.\n\
7218 If chars is given and not None, remove characters in chars instead.\n\
7219 If chars is a str, it will be converted to unicode before stripping");
7222 unicode_strip(PyUnicodeObject
*self
, PyObject
*args
)
7224 if (PyTuple_GET_SIZE(args
) == 0)
7225 return do_strip(self
, BOTHSTRIP
); /* Common case */
7227 return do_argstrip(self
, BOTHSTRIP
, args
);
7231 PyDoc_STRVAR(lstrip__doc__
,
7232 "S.lstrip([chars]) -> unicode\n\
7234 Return a copy of the string S with leading whitespace removed.\n\
7235 If chars is given and not None, remove characters in chars instead.\n\
7236 If chars is a str, it will be converted to unicode before stripping");
7239 unicode_lstrip(PyUnicodeObject
*self
, PyObject
*args
)
7241 if (PyTuple_GET_SIZE(args
) == 0)
7242 return do_strip(self
, LEFTSTRIP
); /* Common case */
7244 return do_argstrip(self
, LEFTSTRIP
, args
);
7248 PyDoc_STRVAR(rstrip__doc__
,
7249 "S.rstrip([chars]) -> unicode\n\
7251 Return a copy of the string S with trailing whitespace removed.\n\
7252 If chars is given and not None, remove characters in chars instead.\n\
7253 If chars is a str, it will be converted to unicode before stripping");
7256 unicode_rstrip(PyUnicodeObject
*self
, PyObject
*args
)
7258 if (PyTuple_GET_SIZE(args
) == 0)
7259 return do_strip(self
, RIGHTSTRIP
); /* Common case */
7261 return do_argstrip(self
, RIGHTSTRIP
, args
);
7266 unicode_repeat(PyUnicodeObject
*str
, Py_ssize_t len
)
7276 if (len
== 1 && PyUnicode_CheckExact(str
)) {
7277 /* no repeat, return original string */
7279 return (PyObject
*) str
;
7282 /* ensure # of chars needed doesn't overflow int and # of bytes
7283 * needed doesn't overflow size_t
7285 nchars
= len
* str
->length
;
7286 if (len
&& nchars
/ len
!= str
->length
) {
7287 PyErr_SetString(PyExc_OverflowError
,
7288 "repeated string is too long");
7291 nbytes
= (nchars
+ 1) * sizeof(Py_UNICODE
);
7292 if (nbytes
/ sizeof(Py_UNICODE
) != (size_t)(nchars
+ 1)) {
7293 PyErr_SetString(PyExc_OverflowError
,
7294 "repeated string is too long");
7297 u
= _PyUnicode_New(nchars
);
7303 if (str
->length
== 1 && len
> 0) {
7304 Py_UNICODE_FILL(p
, str
->str
[0], len
);
7306 Py_ssize_t done
= 0; /* number of characters copied this far */
7307 if (done
< nchars
) {
7308 Py_UNICODE_COPY(p
, str
->str
, str
->length
);
7311 while (done
< nchars
) {
7312 Py_ssize_t n
= (done
<= nchars
-done
) ? done
: nchars
-done
;
7313 Py_UNICODE_COPY(p
+done
, p
, n
);
7318 return (PyObject
*) u
;
7321 PyObject
*PyUnicode_Replace(PyObject
*obj
,
7324 Py_ssize_t maxcount
)
7331 self
= PyUnicode_FromObject(obj
);
7334 str1
= PyUnicode_FromObject(subobj
);
7339 str2
= PyUnicode_FromObject(replobj
);
7345 result
= replace((PyUnicodeObject
*)self
,
7346 (PyUnicodeObject
*)str1
,
7347 (PyUnicodeObject
*)str2
,
7355 PyDoc_STRVAR(replace__doc__
,
7356 "S.replace (old, new[, count]) -> unicode\n\
7358 Return a copy of S with all occurrences of substring\n\
7359 old replaced by new. If the optional argument count is\n\
7360 given, only the first count occurrences are replaced.");
7363 unicode_replace(PyUnicodeObject
*self
, PyObject
*args
)
7365 PyUnicodeObject
*str1
;
7366 PyUnicodeObject
*str2
;
7367 Py_ssize_t maxcount
= -1;
7370 if (!PyArg_ParseTuple(args
, "OO|n:replace", &str1
, &str2
, &maxcount
))
7372 str1
= (PyUnicodeObject
*)PyUnicode_FromObject((PyObject
*)str1
);
7375 str2
= (PyUnicodeObject
*)PyUnicode_FromObject((PyObject
*)str2
);
7381 result
= replace(self
, str1
, str2
, maxcount
);
7389 PyObject
*unicode_repr(PyObject
*unicode
)
7391 return unicodeescape_string(PyUnicode_AS_UNICODE(unicode
),
7392 PyUnicode_GET_SIZE(unicode
),
7396 PyDoc_STRVAR(rfind__doc__
,
7397 "S.rfind(sub [,start [,end]]) -> int\n\
7399 Return the highest index in S where substring sub is found,\n\
7400 such that sub is contained within s[start:end]. Optional\n\
7401 arguments start and end are interpreted as in slice notation.\n\
7403 Return -1 on failure.");
7406 unicode_rfind(PyUnicodeObject
*self
, PyObject
*args
)
7408 PyObject
*substring
;
7413 if (!_ParseTupleFinds(args
, &substring
, &start
, &end
))
7416 result
= stringlib_rfind_slice(
7417 PyUnicode_AS_UNICODE(self
), PyUnicode_GET_SIZE(self
),
7418 PyUnicode_AS_UNICODE(substring
), PyUnicode_GET_SIZE(substring
),
7422 Py_DECREF(substring
);
7424 return PyInt_FromSsize_t(result
);
7427 PyDoc_STRVAR(rindex__doc__
,
7428 "S.rindex(sub [,start [,end]]) -> int\n\
7430 Like S.rfind() but raise ValueError when the substring is not found.");
7433 unicode_rindex(PyUnicodeObject
*self
, PyObject
*args
)
7435 PyObject
*substring
;
7440 if (!_ParseTupleFinds(args
, &substring
, &start
, &end
))
7443 result
= stringlib_rfind_slice(
7444 PyUnicode_AS_UNICODE(self
), PyUnicode_GET_SIZE(self
),
7445 PyUnicode_AS_UNICODE(substring
), PyUnicode_GET_SIZE(substring
),
7449 Py_DECREF(substring
);
7452 PyErr_SetString(PyExc_ValueError
, "substring not found");
7455 return PyInt_FromSsize_t(result
);
7458 PyDoc_STRVAR(rjust__doc__
,
7459 "S.rjust(width[, fillchar]) -> unicode\n\
7461 Return S right-justified in a Unicode string of length width. Padding is\n\
7462 done using the specified fill character (default is a space).");
7465 unicode_rjust(PyUnicodeObject
*self
, PyObject
*args
)
7468 Py_UNICODE fillchar
= ' ';
7470 if (!PyArg_ParseTuple(args
, "n|O&:rjust", &width
, convert_uc
, &fillchar
))
7473 if (self
->length
>= width
&& PyUnicode_CheckExact(self
)) {
7475 return (PyObject
*) self
;
7478 return (PyObject
*) pad(self
, width
- self
->length
, 0, fillchar
);
7482 unicode_slice(PyUnicodeObject
*self
, Py_ssize_t start
, Py_ssize_t end
)
7484 /* standard clamping */
7489 if (end
> self
->length
)
7491 if (start
== 0 && end
== self
->length
&& PyUnicode_CheckExact(self
)) {
7492 /* full slice, return original string */
7494 return (PyObject
*) self
;
7499 return (PyObject
*) PyUnicode_FromUnicode(self
->str
+ start
,
7503 PyObject
*PyUnicode_Split(PyObject
*s
,
7505 Py_ssize_t maxsplit
)
7509 s
= PyUnicode_FromObject(s
);
7513 sep
= PyUnicode_FromObject(sep
);
7520 result
= split((PyUnicodeObject
*)s
, (PyUnicodeObject
*)sep
, maxsplit
);
7527 PyDoc_STRVAR(split__doc__
,
7528 "S.split([sep [,maxsplit]]) -> list of strings\n\
7530 Return a list of the words in S, using sep as the\n\
7531 delimiter string. If maxsplit is given, at most maxsplit\n\
7532 splits are done. If sep is not specified or is None, any\n\
7533 whitespace string is a separator and empty strings are\n\
7534 removed from the result.");
7537 unicode_split(PyUnicodeObject
*self
, PyObject
*args
)
7539 PyObject
*substring
= Py_None
;
7540 Py_ssize_t maxcount
= -1;
7542 if (!PyArg_ParseTuple(args
, "|On:split", &substring
, &maxcount
))
7545 if (substring
== Py_None
)
7546 return split(self
, NULL
, maxcount
);
7547 else if (PyUnicode_Check(substring
))
7548 return split(self
, (PyUnicodeObject
*)substring
, maxcount
);
7550 return PyUnicode_Split((PyObject
*)self
, substring
, maxcount
);
7554 PyUnicode_Partition(PyObject
*str_in
, PyObject
*sep_in
)
7560 str_obj
= PyUnicode_FromObject(str_in
);
7563 sep_obj
= PyUnicode_FromObject(sep_in
);
7569 out
= stringlib_partition(
7570 str_obj
, PyUnicode_AS_UNICODE(str_obj
), PyUnicode_GET_SIZE(str_obj
),
7571 sep_obj
, PyUnicode_AS_UNICODE(sep_obj
), PyUnicode_GET_SIZE(sep_obj
)
7582 PyUnicode_RPartition(PyObject
*str_in
, PyObject
*sep_in
)
7588 str_obj
= PyUnicode_FromObject(str_in
);
7591 sep_obj
= PyUnicode_FromObject(sep_in
);
7597 out
= stringlib_rpartition(
7598 str_obj
, PyUnicode_AS_UNICODE(str_obj
), PyUnicode_GET_SIZE(str_obj
),
7599 sep_obj
, PyUnicode_AS_UNICODE(sep_obj
), PyUnicode_GET_SIZE(sep_obj
)
7608 PyDoc_STRVAR(partition__doc__
,
7609 "S.partition(sep) -> (head, sep, tail)\n\
7611 Search for the separator sep in S, and return the part before it,\n\
7612 the separator itself, and the part after it. If the separator is not\n\
7613 found, return S and two empty strings.");
7616 unicode_partition(PyUnicodeObject
*self
, PyObject
*separator
)
7618 return PyUnicode_Partition((PyObject
*)self
, separator
);
7621 PyDoc_STRVAR(rpartition__doc__
,
7622 "S.rpartition(sep) -> (tail, sep, head)\n\
7624 Search for the separator sep in S, starting at the end of S, and return\n\
7625 the part before it, the separator itself, and the part after it. If the\n\
7626 separator is not found, return two empty strings and S.");
7629 unicode_rpartition(PyUnicodeObject
*self
, PyObject
*separator
)
7631 return PyUnicode_RPartition((PyObject
*)self
, separator
);
7634 PyObject
*PyUnicode_RSplit(PyObject
*s
,
7636 Py_ssize_t maxsplit
)
7640 s
= PyUnicode_FromObject(s
);
7644 sep
= PyUnicode_FromObject(sep
);
7651 result
= rsplit((PyUnicodeObject
*)s
, (PyUnicodeObject
*)sep
, maxsplit
);
7658 PyDoc_STRVAR(rsplit__doc__
,
7659 "S.rsplit([sep [,maxsplit]]) -> list of strings\n\
7661 Return a list of the words in S, using sep as the\n\
7662 delimiter string, starting at the end of the string and\n\
7663 working to the front. If maxsplit is given, at most maxsplit\n\
7664 splits are done. If sep is not specified, any whitespace string\n\
7668 unicode_rsplit(PyUnicodeObject
*self
, PyObject
*args
)
7670 PyObject
*substring
= Py_None
;
7671 Py_ssize_t maxcount
= -1;
7673 if (!PyArg_ParseTuple(args
, "|On:rsplit", &substring
, &maxcount
))
7676 if (substring
== Py_None
)
7677 return rsplit(self
, NULL
, maxcount
);
7678 else if (PyUnicode_Check(substring
))
7679 return rsplit(self
, (PyUnicodeObject
*)substring
, maxcount
);
7681 return PyUnicode_RSplit((PyObject
*)self
, substring
, maxcount
);
7684 PyDoc_STRVAR(splitlines__doc__
,
7685 "S.splitlines([keepends]) -> list of strings\n\
7687 Return a list of the lines in S, breaking at line boundaries.\n\
7688 Line breaks are not included in the resulting list unless keepends\n\
7689 is given and true.");
7692 unicode_splitlines(PyUnicodeObject
*self
, PyObject
*args
)
7696 if (!PyArg_ParseTuple(args
, "|i:splitlines", &keepends
))
7699 return PyUnicode_Splitlines((PyObject
*)self
, keepends
);
7703 PyObject
*unicode_str(PyUnicodeObject
*self
)
7705 return PyUnicode_AsEncodedString((PyObject
*)self
, NULL
, NULL
);
7708 PyDoc_STRVAR(swapcase__doc__
,
7709 "S.swapcase() -> unicode\n\
7711 Return a copy of S with uppercase characters converted to lowercase\n\
7715 unicode_swapcase(PyUnicodeObject
*self
)
7717 return fixup(self
, fixswapcase
);
7720 PyDoc_STRVAR(translate__doc__
,
7721 "S.translate(table) -> unicode\n\
7723 Return a copy of the string S, where all characters have been mapped\n\
7724 through the given translation table, which must be a mapping of\n\
7725 Unicode ordinals to Unicode ordinals, Unicode strings or None.\n\
7726 Unmapped characters are left untouched. Characters mapped to None\n\
7730 unicode_translate(PyUnicodeObject
*self
, PyObject
*table
)
7732 return PyUnicode_TranslateCharmap(self
->str
,
7738 PyDoc_STRVAR(upper__doc__
,
7739 "S.upper() -> unicode\n\
7741 Return a copy of S converted to uppercase.");
7744 unicode_upper(PyUnicodeObject
*self
)
7746 return fixup(self
, fixupper
);
7749 PyDoc_STRVAR(zfill__doc__
,
7750 "S.zfill(width) -> unicode\n\
7752 Pad a numeric string S with zeros on the left, to fill a field\n\
7753 of the specified width. The string S is never truncated.");
7756 unicode_zfill(PyUnicodeObject
*self
, PyObject
*args
)
7762 if (!PyArg_ParseTuple(args
, "n:zfill", &width
))
7765 if (self
->length
>= width
) {
7766 if (PyUnicode_CheckExact(self
)) {
7768 return (PyObject
*) self
;
7771 return PyUnicode_FromUnicode(
7772 PyUnicode_AS_UNICODE(self
),
7773 PyUnicode_GET_SIZE(self
)
7777 fill
= width
- self
->length
;
7779 u
= pad(self
, fill
, 0, '0');
7784 if (u
->str
[fill
] == '+' || u
->str
[fill
] == '-') {
7785 /* move sign to beginning of string */
7786 u
->str
[0] = u
->str
[fill
];
7790 return (PyObject
*) u
;
7795 free_listsize(PyUnicodeObject
*self
)
7797 return PyInt_FromLong(numfree
);
7801 PyDoc_STRVAR(startswith__doc__
,
7802 "S.startswith(prefix[, start[, end]]) -> bool\n\
7804 Return True if S starts with the specified prefix, False otherwise.\n\
7805 With optional start, test S beginning at that position.\n\
7806 With optional end, stop comparing S at that position.\n\
7807 prefix can also be a tuple of strings to try.");
7810 unicode_startswith(PyUnicodeObject
*self
,
7814 PyUnicodeObject
*substring
;
7815 Py_ssize_t start
= 0;
7816 Py_ssize_t end
= PY_SSIZE_T_MAX
;
7819 if (!PyArg_ParseTuple(args
, "O|O&O&:startswith", &subobj
,
7820 _PyEval_SliceIndex
, &start
, _PyEval_SliceIndex
, &end
))
7822 if (PyTuple_Check(subobj
)) {
7824 for (i
= 0; i
< PyTuple_GET_SIZE(subobj
); i
++) {
7825 substring
= (PyUnicodeObject
*)PyUnicode_FromObject(
7826 PyTuple_GET_ITEM(subobj
, i
));
7827 if (substring
== NULL
)
7829 result
= tailmatch(self
, substring
, start
, end
, -1);
7830 Py_DECREF(substring
);
7835 /* nothing matched */
7838 substring
= (PyUnicodeObject
*)PyUnicode_FromObject(subobj
);
7839 if (substring
== NULL
)
7841 result
= tailmatch(self
, substring
, start
, end
, -1);
7842 Py_DECREF(substring
);
7843 return PyBool_FromLong(result
);
7847 PyDoc_STRVAR(endswith__doc__
,
7848 "S.endswith(suffix[, start[, end]]) -> bool\n\
7850 Return True if S ends with the specified suffix, False otherwise.\n\
7851 With optional start, test S beginning at that position.\n\
7852 With optional end, stop comparing S at that position.\n\
7853 suffix can also be a tuple of strings to try.");
7856 unicode_endswith(PyUnicodeObject
*self
,
7860 PyUnicodeObject
*substring
;
7861 Py_ssize_t start
= 0;
7862 Py_ssize_t end
= PY_SSIZE_T_MAX
;
7865 if (!PyArg_ParseTuple(args
, "O|O&O&:endswith", &subobj
,
7866 _PyEval_SliceIndex
, &start
, _PyEval_SliceIndex
, &end
))
7868 if (PyTuple_Check(subobj
)) {
7870 for (i
= 0; i
< PyTuple_GET_SIZE(subobj
); i
++) {
7871 substring
= (PyUnicodeObject
*)PyUnicode_FromObject(
7872 PyTuple_GET_ITEM(subobj
, i
));
7873 if (substring
== NULL
)
7875 result
= tailmatch(self
, substring
, start
, end
, +1);
7876 Py_DECREF(substring
);
7883 substring
= (PyUnicodeObject
*)PyUnicode_FromObject(subobj
);
7884 if (substring
== NULL
)
7887 result
= tailmatch(self
, substring
, start
, end
, +1);
7888 Py_DECREF(substring
);
7889 return PyBool_FromLong(result
);
7893 /* Implements do_string_format, which is unicode because of stringlib */
7894 #include "stringlib/string_format.h"
7896 PyDoc_STRVAR(format__doc__
,
7897 "S.format(*args, **kwargs) -> unicode\n\
7902 unicode__format__(PyObject
*self
, PyObject
*args
)
7904 PyObject
*format_spec
;
7905 PyObject
*result
= NULL
;
7906 PyObject
*tmp
= NULL
;
7908 /* If 2.x, convert format_spec to the same type as value */
7909 /* This is to allow things like u''.format('') */
7910 if (!PyArg_ParseTuple(args
, "O:__format__", &format_spec
))
7912 if (!(PyBytes_Check(format_spec
) || PyUnicode_Check(format_spec
))) {
7913 PyErr_Format(PyExc_TypeError
, "__format__ arg must be str "
7914 "or unicode, not %s", Py_TYPE(format_spec
)->tp_name
);
7917 tmp
= PyObject_Unicode(format_spec
);
7922 result
= _PyUnicode_FormatAdvanced(self
,
7923 PyUnicode_AS_UNICODE(format_spec
),
7924 PyUnicode_GET_SIZE(format_spec
));
7930 PyDoc_STRVAR(p_format__doc__
,
7931 "S.__format__(format_spec) -> unicode\n\
7936 unicode__sizeof__(PyUnicodeObject
*v
)
7938 return PyInt_FromSsize_t(sizeof(PyUnicodeObject
) +
7939 sizeof(Py_UNICODE
) * (v
->length
+ 1));
7942 PyDoc_STRVAR(sizeof__doc__
,
7943 "S.__sizeof__() -> size of S in memory, in bytes\n\
7948 unicode_getnewargs(PyUnicodeObject
*v
)
7950 return Py_BuildValue("(u#)", v
->str
, v
->length
);
7954 static PyMethodDef unicode_methods
[] = {
7956 /* Order is according to common usage: often used methods should
7957 appear first, since lookup is done sequentially. */
7959 {"encode", (PyCFunction
) unicode_encode
, METH_VARARGS
, encode__doc__
},
7960 {"replace", (PyCFunction
) unicode_replace
, METH_VARARGS
, replace__doc__
},
7961 {"split", (PyCFunction
) unicode_split
, METH_VARARGS
, split__doc__
},
7962 {"rsplit", (PyCFunction
) unicode_rsplit
, METH_VARARGS
, rsplit__doc__
},
7963 {"join", (PyCFunction
) unicode_join
, METH_O
, join__doc__
},
7964 {"capitalize", (PyCFunction
) unicode_capitalize
, METH_NOARGS
, capitalize__doc__
},
7965 {"title", (PyCFunction
) unicode_title
, METH_NOARGS
, title__doc__
},
7966 {"center", (PyCFunction
) unicode_center
, METH_VARARGS
, center__doc__
},
7967 {"count", (PyCFunction
) unicode_count
, METH_VARARGS
, count__doc__
},
7968 {"expandtabs", (PyCFunction
) unicode_expandtabs
, METH_VARARGS
, expandtabs__doc__
},
7969 {"find", (PyCFunction
) unicode_find
, METH_VARARGS
, find__doc__
},
7970 {"partition", (PyCFunction
) unicode_partition
, METH_O
, partition__doc__
},
7971 {"index", (PyCFunction
) unicode_index
, METH_VARARGS
, index__doc__
},
7972 {"ljust", (PyCFunction
) unicode_ljust
, METH_VARARGS
, ljust__doc__
},
7973 {"lower", (PyCFunction
) unicode_lower
, METH_NOARGS
, lower__doc__
},
7974 {"lstrip", (PyCFunction
) unicode_lstrip
, METH_VARARGS
, lstrip__doc__
},
7975 {"decode", (PyCFunction
) unicode_decode
, METH_VARARGS
, decode__doc__
},
7976 /* {"maketrans", (PyCFunction) unicode_maketrans, METH_VARARGS, maketrans__doc__}, */
7977 {"rfind", (PyCFunction
) unicode_rfind
, METH_VARARGS
, rfind__doc__
},
7978 {"rindex", (PyCFunction
) unicode_rindex
, METH_VARARGS
, rindex__doc__
},
7979 {"rjust", (PyCFunction
) unicode_rjust
, METH_VARARGS
, rjust__doc__
},
7980 {"rstrip", (PyCFunction
) unicode_rstrip
, METH_VARARGS
, rstrip__doc__
},
7981 {"rpartition", (PyCFunction
) unicode_rpartition
, METH_O
, rpartition__doc__
},
7982 {"splitlines", (PyCFunction
) unicode_splitlines
, METH_VARARGS
, splitlines__doc__
},
7983 {"strip", (PyCFunction
) unicode_strip
, METH_VARARGS
, strip__doc__
},
7984 {"swapcase", (PyCFunction
) unicode_swapcase
, METH_NOARGS
, swapcase__doc__
},
7985 {"translate", (PyCFunction
) unicode_translate
, METH_O
, translate__doc__
},
7986 {"upper", (PyCFunction
) unicode_upper
, METH_NOARGS
, upper__doc__
},
7987 {"startswith", (PyCFunction
) unicode_startswith
, METH_VARARGS
, startswith__doc__
},
7988 {"endswith", (PyCFunction
) unicode_endswith
, METH_VARARGS
, endswith__doc__
},
7989 {"islower", (PyCFunction
) unicode_islower
, METH_NOARGS
, islower__doc__
},
7990 {"isupper", (PyCFunction
) unicode_isupper
, METH_NOARGS
, isupper__doc__
},
7991 {"istitle", (PyCFunction
) unicode_istitle
, METH_NOARGS
, istitle__doc__
},
7992 {"isspace", (PyCFunction
) unicode_isspace
, METH_NOARGS
, isspace__doc__
},
7993 {"isdecimal", (PyCFunction
) unicode_isdecimal
, METH_NOARGS
, isdecimal__doc__
},
7994 {"isdigit", (PyCFunction
) unicode_isdigit
, METH_NOARGS
, isdigit__doc__
},
7995 {"isnumeric", (PyCFunction
) unicode_isnumeric
, METH_NOARGS
, isnumeric__doc__
},
7996 {"isalpha", (PyCFunction
) unicode_isalpha
, METH_NOARGS
, isalpha__doc__
},
7997 {"isalnum", (PyCFunction
) unicode_isalnum
, METH_NOARGS
, isalnum__doc__
},
7998 {"zfill", (PyCFunction
) unicode_zfill
, METH_VARARGS
, zfill__doc__
},
7999 {"format", (PyCFunction
) do_string_format
, METH_VARARGS
| METH_KEYWORDS
, format__doc__
},
8000 {"__format__", (PyCFunction
) unicode__format__
, METH_VARARGS
, p_format__doc__
},
8001 {"_formatter_field_name_split", (PyCFunction
) formatter_field_name_split
, METH_NOARGS
},
8002 {"_formatter_parser", (PyCFunction
) formatter_parser
, METH_NOARGS
},
8003 {"__sizeof__", (PyCFunction
) unicode__sizeof__
, METH_NOARGS
, sizeof__doc__
},
8005 {"capwords", (PyCFunction
) unicode_capwords
, METH_NOARGS
, capwords__doc__
},
8009 /* This one is just used for debugging the implementation. */
8010 {"freelistsize", (PyCFunction
) free_listsize
, METH_NOARGS
},
8013 {"__getnewargs__", (PyCFunction
)unicode_getnewargs
, METH_NOARGS
},
8018 unicode_mod(PyObject
*v
, PyObject
*w
)
8020 if (!PyUnicode_Check(v
)) {
8021 Py_INCREF(Py_NotImplemented
);
8022 return Py_NotImplemented
;
8024 return PyUnicode_Format(v
, w
);
8027 static PyNumberMethods unicode_as_number
= {
8032 unicode_mod
, /*nb_remainder*/
8035 static PySequenceMethods unicode_as_sequence
= {
8036 (lenfunc
) unicode_length
, /* sq_length */
8037 PyUnicode_Concat
, /* sq_concat */
8038 (ssizeargfunc
) unicode_repeat
, /* sq_repeat */
8039 (ssizeargfunc
) unicode_getitem
, /* sq_item */
8040 (ssizessizeargfunc
) unicode_slice
, /* sq_slice */
8041 0, /* sq_ass_item */
8042 0, /* sq_ass_slice */
8043 PyUnicode_Contains
, /* sq_contains */
8047 unicode_subscript(PyUnicodeObject
* self
, PyObject
* item
)
8049 if (PyIndex_Check(item
)) {
8050 Py_ssize_t i
= PyNumber_AsSsize_t(item
, PyExc_IndexError
);
8051 if (i
== -1 && PyErr_Occurred())
8054 i
+= PyUnicode_GET_SIZE(self
);
8055 return unicode_getitem(self
, i
);
8056 } else if (PySlice_Check(item
)) {
8057 Py_ssize_t start
, stop
, step
, slicelength
, cur
, i
;
8058 Py_UNICODE
* source_buf
;
8059 Py_UNICODE
* result_buf
;
8062 if (PySlice_GetIndicesEx((PySliceObject
*)item
, PyUnicode_GET_SIZE(self
),
8063 &start
, &stop
, &step
, &slicelength
) < 0) {
8067 if (slicelength
<= 0) {
8068 return PyUnicode_FromUnicode(NULL
, 0);
8069 } else if (start
== 0 && step
== 1 && slicelength
== self
->length
&&
8070 PyUnicode_CheckExact(self
)) {
8072 return (PyObject
*)self
;
8073 } else if (step
== 1) {
8074 return PyUnicode_FromUnicode(self
->str
+ start
, slicelength
);
8076 source_buf
= PyUnicode_AS_UNICODE((PyObject
*)self
);
8077 result_buf
= (Py_UNICODE
*)PyObject_MALLOC(slicelength
*
8078 sizeof(Py_UNICODE
));
8080 if (result_buf
== NULL
)
8081 return PyErr_NoMemory();
8083 for (cur
= start
, i
= 0; i
< slicelength
; cur
+= step
, i
++) {
8084 result_buf
[i
] = source_buf
[cur
];
8087 result
= PyUnicode_FromUnicode(result_buf
, slicelength
);
8088 PyObject_FREE(result_buf
);
8092 PyErr_SetString(PyExc_TypeError
, "string indices must be integers");
8097 static PyMappingMethods unicode_as_mapping
= {
8098 (lenfunc
)unicode_length
, /* mp_length */
8099 (binaryfunc
)unicode_subscript
, /* mp_subscript */
8100 (objobjargproc
)0, /* mp_ass_subscript */
8104 unicode_buffer_getreadbuf(PyUnicodeObject
*self
,
8109 PyErr_SetString(PyExc_SystemError
,
8110 "accessing non-existent unicode segment");
8113 *ptr
= (void *) self
->str
;
8114 return PyUnicode_GET_DATA_SIZE(self
);
8118 unicode_buffer_getwritebuf(PyUnicodeObject
*self
, Py_ssize_t index
,
8121 PyErr_SetString(PyExc_TypeError
,
8122 "cannot use unicode as modifiable buffer");
8127 unicode_buffer_getsegcount(PyUnicodeObject
*self
,
8131 *lenp
= PyUnicode_GET_DATA_SIZE(self
);
8136 unicode_buffer_getcharbuf(PyUnicodeObject
*self
,
8143 PyErr_SetString(PyExc_SystemError
,
8144 "accessing non-existent unicode segment");
8147 str
= _PyUnicode_AsDefaultEncodedString((PyObject
*)self
, NULL
);
8150 *ptr
= (void *) PyString_AS_STRING(str
);
8151 return PyString_GET_SIZE(str
);
8154 /* Helpers for PyUnicode_Format() */
8157 getnextarg(PyObject
*args
, Py_ssize_t arglen
, Py_ssize_t
*p_argidx
)
8159 Py_ssize_t argidx
= *p_argidx
;
8160 if (argidx
< arglen
) {
8165 return PyTuple_GetItem(args
, argidx
);
8167 PyErr_SetString(PyExc_TypeError
,
8168 "not enough arguments for format string");
8172 #define F_LJUST (1<<0)
8173 #define F_SIGN (1<<1)
8174 #define F_BLANK (1<<2)
8175 #define F_ALT (1<<3)
8176 #define F_ZERO (1<<4)
8179 strtounicode(Py_UNICODE
*buffer
, const char *charbuffer
)
8181 register Py_ssize_t i
;
8182 Py_ssize_t len
= strlen(charbuffer
);
8183 for (i
= len
- 1; i
>= 0; i
--)
8184 buffer
[i
] = (Py_UNICODE
) charbuffer
[i
];
8190 doubletounicode(Py_UNICODE
*buffer
, size_t len
, const char *format
, double x
)
8194 PyOS_ascii_formatd((char *)buffer
, len
, format
, x
);
8195 result
= strtounicode(buffer
, (char *)buffer
);
8196 return Py_SAFE_DOWNCAST(result
, Py_ssize_t
, int);
8200 longtounicode(Py_UNICODE
*buffer
, size_t len
, const char *format
, long x
)
8204 PyOS_snprintf((char *)buffer
, len
, format
, x
);
8205 result
= strtounicode(buffer
, (char *)buffer
);
8206 return Py_SAFE_DOWNCAST(result
, Py_ssize_t
, int);
8209 /* XXX To save some code duplication, formatfloat/long/int could have been
8210 shared with stringobject.c, converting from 8-bit to Unicode after the
8211 formatting is done. */
8214 formatfloat(Py_UNICODE
*buf
,
8221 /* fmt = '%#.' + `prec` + `type`
8222 worst case length = 3 + 10 (len of INT_MAX) + 1 = 14 (use 20)*/
8226 x
= PyFloat_AsDouble(v
);
8227 if (x
== -1.0 && PyErr_Occurred())
8231 if (type
== 'f' && (fabs(x
) / 1e25
) >= 1e25
)
8233 /* Worst case length calc to ensure no buffer overrun:
8237 buf = '-' + [0-9]*prec + '.' + 'e+' + (longest exp
8238 for any double rep.)
8239 len = 1 + prec + 1 + 2 + 5 = 9 + prec
8242 buf = '-' + [0-9]*x + '.' + [0-9]*prec (with x < 50)
8243 len = 1 + 50 + 1 + prec = 52 + prec
8245 If prec=0 the effective precision is 1 (the leading digit is
8246 always given), therefore increase the length by one.
8249 if (((type
== 'g' || type
== 'G') &&
8250 buflen
<= (size_t)10 + (size_t)prec
) ||
8251 (type
== 'f' && buflen
<= (size_t)53 + (size_t)prec
)) {
8252 PyErr_SetString(PyExc_OverflowError
,
8253 "formatted float is too long (precision too large?)");
8256 PyOS_snprintf(fmt
, sizeof(fmt
), "%%%s.%d%c",
8257 (flags
&F_ALT
) ? "#" : "",
8259 return doubletounicode(buf
, buflen
, fmt
, x
);
8263 formatlong(PyObject
*val
, int flags
, int prec
, int type
)
8267 PyObject
*str
; /* temporary string object. */
8268 PyUnicodeObject
*result
;
8270 str
= _PyString_FormatLong(val
, flags
, prec
, type
, &buf
, &len
);
8273 result
= _PyUnicode_New(len
);
8278 for (i
= 0; i
< len
; i
++)
8279 result
->str
[i
] = buf
[i
];
8280 result
->str
[len
] = 0;
8282 return (PyObject
*)result
;
8286 formatint(Py_UNICODE
*buf
,
8293 /* fmt = '%#.' + `prec` + 'l' + `type`
8294 * worst case length = 3 + 19 (worst len of INT_MAX on 64-bit machine)
8298 char fmt
[64]; /* plenty big enough! */
8302 x
= PyInt_AsLong(v
);
8303 if (x
== -1 && PyErr_Occurred())
8305 if (x
< 0 && type
== 'u') {
8308 if (x
< 0 && (type
== 'x' || type
== 'X' || type
== 'o'))
8315 /* buf = '+'/'-'/'' + '0'/'0x'/'' + '[0-9]'*max(prec, len(x in octal))
8316 * worst case buf = '-0x' + [0-9]*prec, where prec >= 11
8318 if (buflen
<= 14 || buflen
<= (size_t)3 + (size_t)prec
) {
8319 PyErr_SetString(PyExc_OverflowError
,
8320 "formatted integer is too long (precision too large?)");
8324 if ((flags
& F_ALT
) &&
8325 (type
== 'x' || type
== 'X')) {
8326 /* When converting under %#x or %#X, there are a number
8327 * of issues that cause pain:
8328 * - when 0 is being converted, the C standard leaves off
8329 * the '0x' or '0X', which is inconsistent with other
8330 * %#x/%#X conversions and inconsistent with Python's
8332 * - there are platforms that violate the standard and
8333 * convert 0 with the '0x' or '0X'
8334 * (Metrowerks, Compaq Tru64)
8335 * - there are platforms that give '0x' when converting
8336 * under %#X, but convert 0 in accordance with the
8337 * standard (OS/2 EMX)
8339 * We can achieve the desired consistency by inserting our
8340 * own '0x' or '0X' prefix, and substituting %x/%X in place
8343 * Note that this is the same approach as used in
8344 * formatint() in stringobject.c
8346 PyOS_snprintf(fmt
, sizeof(fmt
), "%s0%c%%.%dl%c",
8347 sign
, type
, prec
, type
);
8350 PyOS_snprintf(fmt
, sizeof(fmt
), "%s%%%s.%dl%c",
8351 sign
, (flags
&F_ALT
) ? "#" : "",
8355 return longtounicode(buf
, buflen
, fmt
, -x
);
8357 return longtounicode(buf
, buflen
, fmt
, x
);
8361 formatchar(Py_UNICODE
*buf
,
8365 /* presume that the buffer is at least 2 characters long */
8366 if (PyUnicode_Check(v
)) {
8367 if (PyUnicode_GET_SIZE(v
) != 1)
8369 buf
[0] = PyUnicode_AS_UNICODE(v
)[0];
8372 else if (PyString_Check(v
)) {
8373 if (PyString_GET_SIZE(v
) != 1)
8375 buf
[0] = (Py_UNICODE
)PyString_AS_STRING(v
)[0];
8379 /* Integer input truncated to a character */
8381 x
= PyInt_AsLong(v
);
8382 if (x
== -1 && PyErr_Occurred())
8384 #ifdef Py_UNICODE_WIDE
8385 if (x
< 0 || x
> 0x10ffff) {
8386 PyErr_SetString(PyExc_OverflowError
,
8387 "%c arg not in range(0x110000) "
8388 "(wide Python build)");
8392 if (x
< 0 || x
> 0xffff) {
8393 PyErr_SetString(PyExc_OverflowError
,
8394 "%c arg not in range(0x10000) "
8395 "(narrow Python build)");
8399 buf
[0] = (Py_UNICODE
) x
;
8405 PyErr_SetString(PyExc_TypeError
,
8406 "%c requires int or char");
8410 /* fmt%(v1,v2,...) is roughly equivalent to sprintf(fmt, v1, v2, ...)
8412 FORMATBUFLEN is the length of the buffer in which the floats, ints, &
8413 chars are formatted. XXX This is a magic number. Each formatting
8414 routine does bounds checking to ensure no overflow, but a better
8415 solution may be to malloc a buffer of appropriate size for each
8416 format. For now, the current solution is sufficient.
8418 #define FORMATBUFLEN (size_t)120
8420 PyObject
*PyUnicode_Format(PyObject
*format
,
8423 Py_UNICODE
*fmt
, *res
;
8424 Py_ssize_t fmtcnt
, rescnt
, reslen
, arglen
, argidx
;
8426 PyUnicodeObject
*result
= NULL
;
8427 PyObject
*dict
= NULL
;
8430 if (format
== NULL
|| args
== NULL
) {
8431 PyErr_BadInternalCall();
8434 uformat
= PyUnicode_FromObject(format
);
8435 if (uformat
== NULL
)
8437 fmt
= PyUnicode_AS_UNICODE(uformat
);
8438 fmtcnt
= PyUnicode_GET_SIZE(uformat
);
8440 reslen
= rescnt
= fmtcnt
+ 100;
8441 result
= _PyUnicode_New(reslen
);
8444 res
= PyUnicode_AS_UNICODE(result
);
8446 if (PyTuple_Check(args
)) {
8447 arglen
= PyTuple_Size(args
);
8454 if (Py_TYPE(args
)->tp_as_mapping
&& !PyTuple_Check(args
) &&
8455 !PyObject_TypeCheck(args
, &PyBaseString_Type
))
8458 while (--fmtcnt
>= 0) {
8461 rescnt
= fmtcnt
+ 100;
8463 if (_PyUnicode_Resize(&result
, reslen
) < 0)
8465 res
= PyUnicode_AS_UNICODE(result
) + reslen
- rescnt
;
8471 /* Got a format specifier */
8473 Py_ssize_t width
= -1;
8475 Py_UNICODE c
= '\0';
8479 PyObject
*temp
= NULL
;
8483 Py_UNICODE formatbuf
[FORMATBUFLEN
]; /* For format{float,int,char}() */
8487 Py_UNICODE
*keystart
;
8493 PyErr_SetString(PyExc_TypeError
,
8494 "format requires a mapping");
8500 /* Skip over balanced parentheses */
8501 while (pcount
> 0 && --fmtcnt
>= 0) {
8504 else if (*fmt
== '(')
8508 keylen
= fmt
- keystart
- 1;
8509 if (fmtcnt
< 0 || pcount
> 0) {
8510 PyErr_SetString(PyExc_ValueError
,
8511 "incomplete format key");
8515 /* keys are converted to strings using UTF-8 and
8516 then looked up since Python uses strings to hold
8517 variables names etc. in its namespaces and we
8518 wouldn't want to break common idioms. */
8519 key
= PyUnicode_EncodeUTF8(keystart
,
8523 key
= PyUnicode_FromUnicode(keystart
, keylen
);
8531 args
= PyObject_GetItem(dict
, key
);
8540 while (--fmtcnt
>= 0) {
8541 switch (c
= *fmt
++) {
8542 case '-': flags
|= F_LJUST
; continue;
8543 case '+': flags
|= F_SIGN
; continue;
8544 case ' ': flags
|= F_BLANK
; continue;
8545 case '#': flags
|= F_ALT
; continue;
8546 case '0': flags
|= F_ZERO
; continue;
8551 v
= getnextarg(args
, arglen
, &argidx
);
8554 if (!PyInt_Check(v
)) {
8555 PyErr_SetString(PyExc_TypeError
,
8559 width
= PyInt_AsLong(v
);
8567 else if (c
>= '0' && c
<= '9') {
8569 while (--fmtcnt
>= 0) {
8571 if (c
< '0' || c
> '9')
8573 if ((width
*10) / 10 != width
) {
8574 PyErr_SetString(PyExc_ValueError
,
8578 width
= width
*10 + (c
- '0');
8586 v
= getnextarg(args
, arglen
, &argidx
);
8589 if (!PyInt_Check(v
)) {
8590 PyErr_SetString(PyExc_TypeError
,
8594 prec
= PyInt_AsLong(v
);
8600 else if (c
>= '0' && c
<= '9') {
8602 while (--fmtcnt
>= 0) {
8603 c
= Py_CHARMASK(*fmt
++);
8604 if (c
< '0' || c
> '9')
8606 if ((prec
*10) / 10 != prec
) {
8607 PyErr_SetString(PyExc_ValueError
,
8611 prec
= prec
*10 + (c
- '0');
8616 if (c
== 'h' || c
== 'l' || c
== 'L') {
8622 PyErr_SetString(PyExc_ValueError
,
8623 "incomplete format");
8627 v
= getnextarg(args
, arglen
, &argidx
);
8637 /* presume that buffer length is at least 1 */
8644 if (PyUnicode_Check(v
) && c
== 's') {
8651 temp
= PyObject_Unicode(v
);
8653 temp
= PyObject_Repr(v
);
8656 if (PyUnicode_Check(temp
))
8657 /* nothing to do */;
8658 else if (PyString_Check(temp
)) {
8659 /* convert to string to Unicode */
8660 unicode
= PyUnicode_Decode(PyString_AS_STRING(temp
),
8661 PyString_GET_SIZE(temp
),
8671 PyErr_SetString(PyExc_TypeError
,
8672 "%s argument has non-string str()");
8676 pbuf
= PyUnicode_AS_UNICODE(temp
);
8677 len
= PyUnicode_GET_SIZE(temp
);
8678 if (prec
>= 0 && len
> prec
)
8691 if (PyNumber_Check(v
)) {
8692 PyObject
*iobj
=NULL
;
8694 if (PyInt_Check(v
) || (PyLong_Check(v
))) {
8699 iobj
= PyNumber_Int(v
);
8700 if (iobj
==NULL
) iobj
= PyNumber_Long(v
);
8703 if (PyInt_Check(iobj
)) {
8706 len
= formatint(pbuf
, sizeof(formatbuf
)/sizeof(Py_UNICODE
),
8707 flags
, prec
, c
, iobj
);
8713 else if (PyLong_Check(iobj
)) {
8715 temp
= formatlong(iobj
, flags
, prec
, c
);
8719 pbuf
= PyUnicode_AS_UNICODE(temp
);
8720 len
= PyUnicode_GET_SIZE(temp
);
8729 PyErr_Format(PyExc_TypeError
,
8730 "%%%c format: a number is required, "
8731 "not %.200s", (char)c
, Py_TYPE(v
)->tp_name
);
8747 len
= formatfloat(pbuf
, sizeof(formatbuf
)/sizeof(Py_UNICODE
),
8758 len
= formatchar(pbuf
, sizeof(formatbuf
)/sizeof(Py_UNICODE
), v
);
8764 PyErr_Format(PyExc_ValueError
,
8765 "unsupported format character '%c' (0x%x) "
8767 (31<=c
&& c
<=126) ? (char)c
: '?',
8769 (Py_ssize_t
)(fmt
- 1 -
8770 PyUnicode_AS_UNICODE(uformat
)));
8774 if (*pbuf
== '-' || *pbuf
== '+') {
8778 else if (flags
& F_SIGN
)
8780 else if (flags
& F_BLANK
)
8787 if (rescnt
- (sign
!= 0) < width
) {
8789 rescnt
= width
+ fmtcnt
+ 100;
8796 if (_PyUnicode_Resize(&result
, reslen
) < 0) {
8800 res
= PyUnicode_AS_UNICODE(result
)
8810 if ((flags
& F_ALT
) && (c
== 'x' || c
== 'X')) {
8811 assert(pbuf
[0] == '0');
8812 assert(pbuf
[1] == c
);
8823 if (width
> len
&& !(flags
& F_LJUST
)) {
8827 } while (--width
> len
);
8832 if ((flags
& F_ALT
) && (c
== 'x' || c
== 'X')) {
8833 assert(pbuf
[0] == '0');
8834 assert(pbuf
[1] == c
);
8839 Py_UNICODE_COPY(res
, pbuf
, len
);
8842 while (--width
>= len
) {
8846 if (dict
&& (argidx
< arglen
) && c
!= '%') {
8847 PyErr_SetString(PyExc_TypeError
,
8848 "not all arguments converted during string formatting");
8855 if (argidx
< arglen
&& !dict
) {
8856 PyErr_SetString(PyExc_TypeError
,
8857 "not all arguments converted during string formatting");
8861 if (_PyUnicode_Resize(&result
, reslen
- rescnt
) < 0)
8867 return (PyObject
*)result
;
8878 static PyBufferProcs unicode_as_buffer
= {
8879 (readbufferproc
) unicode_buffer_getreadbuf
,
8880 (writebufferproc
) unicode_buffer_getwritebuf
,
8881 (segcountproc
) unicode_buffer_getsegcount
,
8882 (charbufferproc
) unicode_buffer_getcharbuf
,
8886 unicode_subtype_new(PyTypeObject
*type
, PyObject
*args
, PyObject
*kwds
);
8889 unicode_new(PyTypeObject
*type
, PyObject
*args
, PyObject
*kwds
)
8892 static char *kwlist
[] = {"string", "encoding", "errors", 0};
8893 char *encoding
= NULL
;
8894 char *errors
= NULL
;
8896 if (type
!= &PyUnicode_Type
)
8897 return unicode_subtype_new(type
, args
, kwds
);
8898 if (!PyArg_ParseTupleAndKeywords(args
, kwds
, "|Oss:unicode",
8899 kwlist
, &x
, &encoding
, &errors
))
8902 return (PyObject
*)_PyUnicode_New(0);
8903 if (encoding
== NULL
&& errors
== NULL
)
8904 return PyObject_Unicode(x
);
8906 return PyUnicode_FromEncodedObject(x
, encoding
, errors
);
8910 unicode_subtype_new(PyTypeObject
*type
, PyObject
*args
, PyObject
*kwds
)
8912 PyUnicodeObject
*tmp
, *pnew
;
8915 assert(PyType_IsSubtype(type
, &PyUnicode_Type
));
8916 tmp
= (PyUnicodeObject
*)unicode_new(&PyUnicode_Type
, args
, kwds
);
8919 assert(PyUnicode_Check(tmp
));
8920 pnew
= (PyUnicodeObject
*) type
->tp_alloc(type
, n
= tmp
->length
);
8925 pnew
->str
= (Py_UNICODE
*) PyObject_MALLOC(sizeof(Py_UNICODE
) * (n
+1));
8926 if (pnew
->str
== NULL
) {
8927 _Py_ForgetReference((PyObject
*)pnew
);
8930 return PyErr_NoMemory();
8932 Py_UNICODE_COPY(pnew
->str
, tmp
->str
, n
+1);
8934 pnew
->hash
= tmp
->hash
;
8936 return (PyObject
*)pnew
;
8939 PyDoc_STRVAR(unicode_doc
,
8940 "unicode(string [, encoding[, errors]]) -> object\n\
8942 Create a new Unicode object from the given encoded string.\n\
8943 encoding defaults to the current default string encoding.\n\
8944 errors can be 'strict', 'replace' or 'ignore' and defaults to 'strict'.");
8946 PyTypeObject PyUnicode_Type
= {
8947 PyVarObject_HEAD_INIT(&PyType_Type
, 0)
8948 "unicode", /* tp_name */
8949 sizeof(PyUnicodeObject
), /* tp_size */
8950 0, /* tp_itemsize */
8952 (destructor
)unicode_dealloc
, /* tp_dealloc */
8957 unicode_repr
, /* tp_repr */
8958 &unicode_as_number
, /* tp_as_number */
8959 &unicode_as_sequence
, /* tp_as_sequence */
8960 &unicode_as_mapping
, /* tp_as_mapping */
8961 (hashfunc
) unicode_hash
, /* tp_hash*/
8963 (reprfunc
) unicode_str
, /* tp_str */
8964 PyObject_GenericGetAttr
, /* tp_getattro */
8965 0, /* tp_setattro */
8966 &unicode_as_buffer
, /* tp_as_buffer */
8967 Py_TPFLAGS_DEFAULT
| Py_TPFLAGS_CHECKTYPES
|
8968 Py_TPFLAGS_BASETYPE
| Py_TPFLAGS_UNICODE_SUBCLASS
, /* tp_flags */
8969 unicode_doc
, /* tp_doc */
8970 0, /* tp_traverse */
8972 PyUnicode_RichCompare
, /* tp_richcompare */
8973 0, /* tp_weaklistoffset */
8975 0, /* tp_iternext */
8976 unicode_methods
, /* tp_methods */
8979 &PyBaseString_Type
, /* tp_base */
8981 0, /* tp_descr_get */
8982 0, /* tp_descr_set */
8983 0, /* tp_dictoffset */
8986 unicode_new
, /* tp_new */
8987 PyObject_Del
, /* tp_free */
8990 /* Initialize the Unicode implementation */
8992 void _PyUnicode_Init(void)
8996 /* XXX - move this array to unicodectype.c ? */
8997 Py_UNICODE linebreak
[] = {
8998 0x000A, /* LINE FEED */
8999 0x000D, /* CARRIAGE RETURN */
9000 0x001C, /* FILE SEPARATOR */
9001 0x001D, /* GROUP SEPARATOR */
9002 0x001E, /* RECORD SEPARATOR */
9003 0x0085, /* NEXT LINE */
9004 0x2028, /* LINE SEPARATOR */
9005 0x2029, /* PARAGRAPH SEPARATOR */
9008 /* Init the implementation */
9011 unicode_empty
= _PyUnicode_New(0);
9015 strcpy(unicode_default_encoding
, "ascii");
9016 for (i
= 0; i
< 256; i
++)
9017 unicode_latin1
[i
] = NULL
;
9018 if (PyType_Ready(&PyUnicode_Type
) < 0)
9019 Py_FatalError("Can't initialize 'unicode'");
9021 /* initialize the linebreak bloom filter */
9022 bloom_linebreak
= make_bloom_mask(
9023 linebreak
, sizeof(linebreak
) / sizeof(linebreak
[0])
9026 PyType_Ready(&EncodingMapType
);
9029 /* Finalize the Unicode implementation */
9032 PyUnicode_ClearFreeList(void)
9034 int freelist_size
= numfree
;
9037 for (u
= free_list
; u
!= NULL
;) {
9038 PyUnicodeObject
*v
= u
;
9039 u
= *(PyUnicodeObject
**)u
;
9041 PyObject_DEL(v
->str
);
9042 Py_XDECREF(v
->defenc
);
9047 assert(numfree
== 0);
9048 return freelist_size
;
9052 _PyUnicode_Fini(void)
9056 Py_XDECREF(unicode_empty
);
9057 unicode_empty
= NULL
;
9059 for (i
= 0; i
< 256; i
++) {
9060 if (unicode_latin1
[i
]) {
9061 Py_DECREF(unicode_latin1
[i
]);
9062 unicode_latin1
[i
] = NULL
;
9065 (void)PyUnicode_ClearFreeList();
9076 indent-tabs-mode: nil