Include/unicodeobject.h

   1 #ifndef Py_UNICODEOBJECT_H
   2 #define Py_UNICODEOBJECT_H
   3
   4 #include <stdarg.h>
   5
   6 /*
   7
   8 Unicode implementation based on original code by Fredrik Lundh,
   9 modified by Marc-Andre Lemburg (mal@lemburg.com) according to the
  10 Unicode Integration Proposal (see file Misc/unicode.txt).
  11
  12 Copyright (c) Corporation for National Research Initiatives.
  13
  14
  15  Original header:
  16  --------------------------------------------------------------------
  17
  18  * Yet another Unicode string type for Python.  This type supports the
  19  * 16-bit Basic Multilingual Plane (BMP) only.
  20  *
  21  * Written by Fredrik Lundh, January 1999.
  22  *
  23  * Copyright (c) 1999 by Secret Labs AB.
  24  * Copyright (c) 1999 by Fredrik Lundh.
  25  *
  26  * fredrik@pythonware.com
  27  * http://www.pythonware.com
  28  *
  29  * --------------------------------------------------------------------
  30  * This Unicode String Type is
  31  *
  32  * Copyright (c) 1999 by Secret Labs AB
  33  * Copyright (c) 1999 by Fredrik Lundh
  34  *
  35  * By obtaining, using, and/or copying this software and/or its
  36  * associated documentation, you agree that you have read, understood,
  37  * and will comply with the following terms and conditions:
  38  *
  39  * Permission to use, copy, modify, and distribute this software and its
  40  * associated documentation for any purpose and without fee is hereby
  41  * granted, provided that the above copyright notice appears in all
  42  * copies, and that both that copyright notice and this permission notice
  43  * appear in supporting documentation, and that the name of Secret Labs
  44  * AB or the author not be used in advertising or publicity pertaining to
  45  * distribution of the software without specific, written prior
  46  * permission.
  47  *
  48  * SECRET LABS AB AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO
  49  * THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND
  50  * FITNESS.  IN NO EVENT SHALL SECRET LABS AB OR THE AUTHOR BE LIABLE FOR
  51  * ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
  52  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
  53  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT
  54  * OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
  55  * -------------------------------------------------------------------- */
  56
  57 #include <ctype.h>
  58
  59 /* === Internal API ======================================================= */
  60
  61 /* --- Internal Unicode Format -------------------------------------------- */
  62
  63 #ifndef Py_USING_UNICODE
  64
  65 #define PyUnicode_Check(op)                 0
  66 #define PyUnicode_CheckExact(op)            0
  67
  68 #else
  69
  70 /* FIXME: MvL's new implementation assumes that Py_UNICODE_SIZE is
  71    properly set, but the default rules below doesn't set it.  I'll
  72    sort this out some other day -- fredrik@pythonware.com */
  73
  74 #ifndef Py_UNICODE_SIZE
  75 #error Must define Py_UNICODE_SIZE
  76 #endif
  77
  78 /* Setting Py_UNICODE_WIDE enables UCS-4 storage.  Otherwise, Unicode
  79    strings are stored as UCS-2 (with limited support for UTF-16) */
  80
  81 #if Py_UNICODE_SIZE >= 4
  82 #define Py_UNICODE_WIDE
  83 #endif
  84
  85 /* Set these flags if the platform has "wchar.h", "wctype.h" and the
  86    wchar_t type is a 16-bit unsigned type */
  87 /* #define HAVE_WCHAR_H */
  88 /* #define HAVE_USABLE_WCHAR_T */
  89
  90 /* Defaults for various platforms */
  91 #ifndef PY_UNICODE_TYPE
  92
  93 /* Windows has a usable wchar_t type (unless we're using UCS-4) */
  94 # if defined(MS_WIN32) && Py_UNICODE_SIZE == 2
  95 #  define HAVE_USABLE_WCHAR_T
  96 #  define PY_UNICODE_TYPE wchar_t
  97 # endif
  98
  99 # if defined(Py_UNICODE_WIDE)
 100 #  define PY_UNICODE_TYPE Py_UCS4
 101 # endif
 102
 103 #endif
 104
 105 /* If the compiler provides a wchar_t type we try to support it
 106    through the interface functions PyUnicode_FromWideChar() and
 107    PyUnicode_AsWideChar(). */
 108
 109 #ifdef HAVE_USABLE_WCHAR_T
 110 # ifndef HAVE_WCHAR_H
 111 #  define HAVE_WCHAR_H
 112 # endif
 113 #endif
 114
 115 #ifdef HAVE_WCHAR_H
 116 /* Work around a cosmetic bug in BSDI 4.x wchar.h; thanks to Thomas Wouters */
 117 # ifdef _HAVE_BSDI
 118 #  include <time.h>
 119 # endif
 120 #  include <wchar.h>
 121 #endif
 122
 123 /*
 124  * Use this typedef when you need to represent a UTF-16 surrogate pair
 125  * as single unsigned integer.
 126  */
 127 #if SIZEOF_INT >= 4
 128 typedef unsigned int Py_UCS4;
 129 #elif SIZEOF_LONG >= 4
 130 typedef unsigned long Py_UCS4;
 131 #endif
 132
 133 /* Py_UNICODE is the native Unicode storage format (code unit) used by
 134    Python and represents a single Unicode element in the Unicode
 135    type. */
 136
 137 typedef PY_UNICODE_TYPE Py_UNICODE;
 138
 139 /* --- UCS-2/UCS-4 Name Mangling ------------------------------------------ */
 140
 141 /* Unicode API names are mangled to assure that UCS-2 and UCS-4 builds
 142    produce different external names and thus cause import errors in
 143    case Python interpreters and extensions with mixed compiled in
 144    Unicode width assumptions are combined. */
 145
 146 #ifndef Py_UNICODE_WIDE
 147
 148 # define PyUnicode_AsASCIIString PyUnicodeUCS2_AsASCIIString
 149 # define PyUnicode_AsCharmapString PyUnicodeUCS2_AsCharmapString
 150 # define PyUnicode_AsEncodedObject PyUnicodeUCS2_AsEncodedObject
 151 # define PyUnicode_AsEncodedString PyUnicodeUCS2_AsEncodedString
 152 # define PyUnicode_AsLatin1String PyUnicodeUCS2_AsLatin1String
 153 # define PyUnicode_AsRawUnicodeEscapeString PyUnicodeUCS2_AsRawUnicodeEscapeString
 154 # define PyUnicode_AsUTF32String PyUnicodeUCS2_AsUTF32String
 155 # define PyUnicode_AsUTF16String PyUnicodeUCS2_AsUTF16String
 156 # define PyUnicode_AsUTF8String PyUnicodeUCS2_AsUTF8String
 157 # define PyUnicode_AsUnicode PyUnicodeUCS2_AsUnicode
 158 # define PyUnicode_AsUnicodeEscapeString PyUnicodeUCS2_AsUnicodeEscapeString
 159 # define PyUnicode_AsWideChar PyUnicodeUCS2_AsWideChar
 160 # define PyUnicode_ClearFreeList PyUnicodeUCS2_ClearFreelist
 161 # define PyUnicode_Compare PyUnicodeUCS2_Compare
 162 # define PyUnicode_Concat PyUnicodeUCS2_Concat
 163 # define PyUnicode_Contains PyUnicodeUCS2_Contains
 164 # define PyUnicode_Count PyUnicodeUCS2_Count
 165 # define PyUnicode_Decode PyUnicodeUCS2_Decode
 166 # define PyUnicode_DecodeASCII PyUnicodeUCS2_DecodeASCII
 167 # define PyUnicode_DecodeCharmap PyUnicodeUCS2_DecodeCharmap
 168 # define PyUnicode_DecodeLatin1 PyUnicodeUCS2_DecodeLatin1
 169 # define PyUnicode_DecodeRawUnicodeEscape PyUnicodeUCS2_DecodeRawUnicodeEscape
 170 # define PyUnicode_DecodeUTF32 PyUnicodeUCS2_DecodeUTF32
 171 # define PyUnicode_DecodeUTF32Stateful PyUnicodeUCS2_DecodeUTF32Stateful
 172 # define PyUnicode_DecodeUTF16 PyUnicodeUCS2_DecodeUTF16
 173 # define PyUnicode_DecodeUTF16Stateful PyUnicodeUCS2_DecodeUTF16Stateful
 174 # define PyUnicode_DecodeUTF8 PyUnicodeUCS2_DecodeUTF8
 175 # define PyUnicode_DecodeUTF8Stateful PyUnicodeUCS2_DecodeUTF8Stateful
 176 # define PyUnicode_DecodeUnicodeEscape PyUnicodeUCS2_DecodeUnicodeEscape
 177 # define PyUnicode_Encode PyUnicodeUCS2_Encode
 178 # define PyUnicode_EncodeASCII PyUnicodeUCS2_EncodeASCII
 179 # define PyUnicode_EncodeCharmap PyUnicodeUCS2_EncodeCharmap
 180 # define PyUnicode_EncodeDecimal PyUnicodeUCS2_EncodeDecimal
 181 # define PyUnicode_EncodeLatin1 PyUnicodeUCS2_EncodeLatin1
 182 # define PyUnicode_EncodeRawUnicodeEscape PyUnicodeUCS2_EncodeRawUnicodeEscape
 183 # define PyUnicode_EncodeUTF32 PyUnicodeUCS2_EncodeUTF32
 184 # define PyUnicode_EncodeUTF16 PyUnicodeUCS2_EncodeUTF16
 185 # define PyUnicode_EncodeUTF8 PyUnicodeUCS2_EncodeUTF8
 186 # define PyUnicode_EncodeUnicodeEscape PyUnicodeUCS2_EncodeUnicodeEscape
 187 # define PyUnicode_Find PyUnicodeUCS2_Find
 188 # define PyUnicode_Format PyUnicodeUCS2_Format
 189 # define PyUnicode_FromEncodedObject PyUnicodeUCS2_FromEncodedObject
 190 # define PyUnicode_FromFormat PyUnicodeUCS2_FromFormat
 191 # define PyUnicode_FromFormatV PyUnicodeUCS2_FromFormatV
 192 # define PyUnicode_FromObject PyUnicodeUCS2_FromObject
 193 # define PyUnicode_FromOrdinal PyUnicodeUCS2_FromOrdinal
 194 # define PyUnicode_FromString PyUnicodeUCS2_FromString
 195 # define PyUnicode_FromStringAndSize PyUnicodeUCS2_FromStringAndSize
 196 # define PyUnicode_FromUnicode PyUnicodeUCS2_FromUnicode
 197 # define PyUnicode_FromWideChar PyUnicodeUCS2_FromWideChar
 198 # define PyUnicode_GetDefaultEncoding PyUnicodeUCS2_GetDefaultEncoding
 199 # define PyUnicode_GetMax PyUnicodeUCS2_GetMax
 200 # define PyUnicode_GetSize PyUnicodeUCS2_GetSize
 201 # define PyUnicode_Join PyUnicodeUCS2_Join
 202 # define PyUnicode_Partition PyUnicodeUCS2_Partition
 203 # define PyUnicode_RPartition PyUnicodeUCS2_RPartition
 204 # define PyUnicode_RSplit PyUnicodeUCS2_RSplit
 205 # define PyUnicode_Replace PyUnicodeUCS2_Replace
 206 # define PyUnicode_Resize PyUnicodeUCS2_Resize
 207 # define PyUnicode_RichCompare PyUnicodeUCS2_RichCompare
 208 # define PyUnicode_SetDefaultEncoding PyUnicodeUCS2_SetDefaultEncoding
 209 # define PyUnicode_Split PyUnicodeUCS2_Split
 210 # define PyUnicode_Splitlines PyUnicodeUCS2_Splitlines
 211 # define PyUnicode_Tailmatch PyUnicodeUCS2_Tailmatch
 212 # define PyUnicode_Translate PyUnicodeUCS2_Translate
 213 # define PyUnicode_TranslateCharmap PyUnicodeUCS2_TranslateCharmap
 214 # define _PyUnicode_AsDefaultEncodedString _PyUnicodeUCS2_AsDefaultEncodedString
 215 # define _PyUnicode_Fini _PyUnicodeUCS2_Fini
 216 # define _PyUnicode_Init _PyUnicodeUCS2_Init
 217 # define _PyUnicode_IsAlpha _PyUnicodeUCS2_IsAlpha
 218 # define _PyUnicode_IsDecimalDigit _PyUnicodeUCS2_IsDecimalDigit
 219 # define _PyUnicode_IsDigit _PyUnicodeUCS2_IsDigit
 220 # define _PyUnicode_IsLinebreak _PyUnicodeUCS2_IsLinebreak
 221 # define _PyUnicode_IsLowercase _PyUnicodeUCS2_IsLowercase
 222 # define _PyUnicode_IsNumeric _PyUnicodeUCS2_IsNumeric
 223 # define _PyUnicode_IsTitlecase _PyUnicodeUCS2_IsTitlecase
 224 # define _PyUnicode_IsUppercase _PyUnicodeUCS2_IsUppercase
 225 # define _PyUnicode_IsWhitespace _PyUnicodeUCS2_IsWhitespace
 226 # define _PyUnicode_ToDecimalDigit _PyUnicodeUCS2_ToDecimalDigit
 227 # define _PyUnicode_ToDigit _PyUnicodeUCS2_ToDigit
 228 # define _PyUnicode_ToLowercase _PyUnicodeUCS2_ToLowercase
 229 # define _PyUnicode_ToNumeric _PyUnicodeUCS2_ToNumeric
 230 # define _PyUnicode_ToTitlecase _PyUnicodeUCS2_ToTitlecase
 231 # define _PyUnicode_ToUppercase _PyUnicodeUCS2_ToUppercase
 232
 233 #else
 234
 235 # define PyUnicode_AsASCIIString PyUnicodeUCS4_AsASCIIString
 236 # define PyUnicode_AsCharmapString PyUnicodeUCS4_AsCharmapString
 237 # define PyUnicode_AsEncodedObject PyUnicodeUCS4_AsEncodedObject
 238 # define PyUnicode_AsEncodedString PyUnicodeUCS4_AsEncodedString
 239 # define PyUnicode_AsLatin1String PyUnicodeUCS4_AsLatin1String
 240 # define PyUnicode_AsRawUnicodeEscapeString PyUnicodeUCS4_AsRawUnicodeEscapeString
 241 # define PyUnicode_AsUTF32String PyUnicodeUCS4_AsUTF32String
 242 # define PyUnicode_AsUTF16String PyUnicodeUCS4_AsUTF16String
 243 # define PyUnicode_AsUTF8String PyUnicodeUCS4_AsUTF8String
 244 # define PyUnicode_AsUnicode PyUnicodeUCS4_AsUnicode
 245 # define PyUnicode_AsUnicodeEscapeString PyUnicodeUCS4_AsUnicodeEscapeString
 246 # define PyUnicode_AsWideChar PyUnicodeUCS4_AsWideChar
 247 # define PyUnicode_ClearFreeList PyUnicodeUCS4_ClearFreelist
 248 # define PyUnicode_Compare PyUnicodeUCS4_Compare
 249 # define PyUnicode_Concat PyUnicodeUCS4_Concat
 250 # define PyUnicode_Contains PyUnicodeUCS4_Contains
 251 # define PyUnicode_Count PyUnicodeUCS4_Count
 252 # define PyUnicode_Decode PyUnicodeUCS4_Decode
 253 # define PyUnicode_DecodeASCII PyUnicodeUCS4_DecodeASCII
 254 # define PyUnicode_DecodeCharmap PyUnicodeUCS4_DecodeCharmap
 255 # define PyUnicode_DecodeLatin1 PyUnicodeUCS4_DecodeLatin1
 256 # define PyUnicode_DecodeRawUnicodeEscape PyUnicodeUCS4_DecodeRawUnicodeEscape
 257 # define PyUnicode_DecodeUTF32 PyUnicodeUCS4_DecodeUTF32
 258 # define PyUnicode_DecodeUTF32Stateful PyUnicodeUCS4_DecodeUTF32Stateful
 259 # define PyUnicode_DecodeUTF16 PyUnicodeUCS4_DecodeUTF16
 260 # define PyUnicode_DecodeUTF16Stateful PyUnicodeUCS4_DecodeUTF16Stateful
 261 # define PyUnicode_DecodeUTF8 PyUnicodeUCS4_DecodeUTF8
 262 # define PyUnicode_DecodeUTF8Stateful PyUnicodeUCS4_DecodeUTF8Stateful
 263 # define PyUnicode_DecodeUnicodeEscape PyUnicodeUCS4_DecodeUnicodeEscape
 264 # define PyUnicode_Encode PyUnicodeUCS4_Encode
 265 # define PyUnicode_EncodeASCII PyUnicodeUCS4_EncodeASCII
 266 # define PyUnicode_EncodeCharmap PyUnicodeUCS4_EncodeCharmap
 267 # define PyUnicode_EncodeDecimal PyUnicodeUCS4_EncodeDecimal
 268 # define PyUnicode_EncodeLatin1 PyUnicodeUCS4_EncodeLatin1
 269 # define PyUnicode_EncodeRawUnicodeEscape PyUnicodeUCS4_EncodeRawUnicodeEscape
 270 # define PyUnicode_EncodeUTF32 PyUnicodeUCS4_EncodeUTF32
 271 # define PyUnicode_EncodeUTF16 PyUnicodeUCS4_EncodeUTF16
 272 # define PyUnicode_EncodeUTF8 PyUnicodeUCS4_EncodeUTF8
 273 # define PyUnicode_EncodeUnicodeEscape PyUnicodeUCS4_EncodeUnicodeEscape
 274 # define PyUnicode_Find PyUnicodeUCS4_Find
 275 # define PyUnicode_Format PyUnicodeUCS4_Format
 276 # define PyUnicode_FromEncodedObject PyUnicodeUCS4_FromEncodedObject
 277 # define PyUnicode_FromFormat PyUnicodeUCS4_FromFormat
 278 # define PyUnicode_FromFormatV PyUnicodeUCS4_FromFormatV
 279 # define PyUnicode_FromObject PyUnicodeUCS4_FromObject
 280 # define PyUnicode_FromOrdinal PyUnicodeUCS4_FromOrdinal
 281 # define PyUnicode_FromString PyUnicodeUCS4_FromString
 282 # define PyUnicode_FromStringAndSize PyUnicodeUCS4_FromStringAndSize
 283 # define PyUnicode_FromUnicode PyUnicodeUCS4_FromUnicode
 284 # define PyUnicode_FromWideChar PyUnicodeUCS4_FromWideChar
 285 # define PyUnicode_GetDefaultEncoding PyUnicodeUCS4_GetDefaultEncoding
 286 # define PyUnicode_GetMax PyUnicodeUCS4_GetMax
 287 # define PyUnicode_GetSize PyUnicodeUCS4_GetSize
 288 # define PyUnicode_Join PyUnicodeUCS4_Join
 289 # define PyUnicode_Partition PyUnicodeUCS4_Partition
 290 # define PyUnicode_RPartition PyUnicodeUCS4_RPartition
 291 # define PyUnicode_RSplit PyUnicodeUCS4_RSplit
 292 # define PyUnicode_Replace PyUnicodeUCS4_Replace
 293 # define PyUnicode_Resize PyUnicodeUCS4_Resize
 294 # define PyUnicode_RichCompare PyUnicodeUCS4_RichCompare
 295 # define PyUnicode_SetDefaultEncoding PyUnicodeUCS4_SetDefaultEncoding
 296 # define PyUnicode_Split PyUnicodeUCS4_Split
 297 # define PyUnicode_Splitlines PyUnicodeUCS4_Splitlines
 298 # define PyUnicode_Tailmatch PyUnicodeUCS4_Tailmatch
 299 # define PyUnicode_Translate PyUnicodeUCS4_Translate
 300 # define PyUnicode_TranslateCharmap PyUnicodeUCS4_TranslateCharmap
 301 # define _PyUnicode_AsDefaultEncodedString _PyUnicodeUCS4_AsDefaultEncodedString
 302 # define _PyUnicode_Fini _PyUnicodeUCS4_Fini
 303 # define _PyUnicode_Init _PyUnicodeUCS4_Init
 304 # define _PyUnicode_IsAlpha _PyUnicodeUCS4_IsAlpha
 305 # define _PyUnicode_IsDecimalDigit _PyUnicodeUCS4_IsDecimalDigit
 306 # define _PyUnicode_IsDigit _PyUnicodeUCS4_IsDigit
 307 # define _PyUnicode_IsLinebreak _PyUnicodeUCS4_IsLinebreak
 308 # define _PyUnicode_IsLowercase _PyUnicodeUCS4_IsLowercase
 309 # define _PyUnicode_IsNumeric _PyUnicodeUCS4_IsNumeric
 310 # define _PyUnicode_IsTitlecase _PyUnicodeUCS4_IsTitlecase
 311 # define _PyUnicode_IsUppercase _PyUnicodeUCS4_IsUppercase
 312 # define _PyUnicode_IsWhitespace _PyUnicodeUCS4_IsWhitespace
 313 # define _PyUnicode_ToDecimalDigit _PyUnicodeUCS4_ToDecimalDigit
 314 # define _PyUnicode_ToDigit _PyUnicodeUCS4_ToDigit
 315 # define _PyUnicode_ToLowercase _PyUnicodeUCS4_ToLowercase
 316 # define _PyUnicode_ToNumeric _PyUnicodeUCS4_ToNumeric
 317 # define _PyUnicode_ToTitlecase _PyUnicodeUCS4_ToTitlecase
 318 # define _PyUnicode_ToUppercase _PyUnicodeUCS4_ToUppercase
 319
 320
 321 #endif
 322
 323 /* --- Internal Unicode Operations ---------------------------------------- */
 324
 325 /* If you want Python to use the compiler's wctype.h functions instead
 326    of the ones supplied with Python, define WANT_WCTYPE_FUNCTIONS or
 327    configure Python using --with-wctype-functions.  This reduces the
 328    interpreter's code size. */
 329
 330 #if defined(HAVE_USABLE_WCHAR_T) && defined(WANT_WCTYPE_FUNCTIONS)
 331
 332 #include <wctype.h>
 333
 334 #define Py_UNICODE_ISSPACE(ch) iswspace(ch)
 335
 336 #define Py_UNICODE_ISLOWER(ch) iswlower(ch)
 337 #define Py_UNICODE_ISUPPER(ch) iswupper(ch)
 338 #define Py_UNICODE_ISTITLE(ch) _PyUnicode_IsTitlecase(ch)
 339 #define Py_UNICODE_ISLINEBREAK(ch) _PyUnicode_IsLinebreak(ch)
 340
 341 #define Py_UNICODE_TOLOWER(ch) towlower(ch)
 342 #define Py_UNICODE_TOUPPER(ch) towupper(ch)
 343 #define Py_UNICODE_TOTITLE(ch) _PyUnicode_ToTitlecase(ch)
 344
 345 #define Py_UNICODE_ISDECIMAL(ch) _PyUnicode_IsDecimalDigit(ch)
 346 #define Py_UNICODE_ISDIGIT(ch) _PyUnicode_IsDigit(ch)
 347 #define Py_UNICODE_ISNUMERIC(ch) _PyUnicode_IsNumeric(ch)
 348
 349 #define Py_UNICODE_TODECIMAL(ch) _PyUnicode_ToDecimalDigit(ch)
 350 #define Py_UNICODE_TODIGIT(ch) _PyUnicode_ToDigit(ch)
 351 #define Py_UNICODE_TONUMERIC(ch) _PyUnicode_ToNumeric(ch)
 352
 353 #define Py_UNICODE_ISALPHA(ch) iswalpha(ch)
 354
 355 #else
 356
 357 /* Since splitting on whitespace is an important use case, and
 358    whitespace in most situations is solely ASCII whitespace, we
 359    optimize for the common case by using a quick look-up table
 360    _Py_ascii_whitespace (see below) with an inlined check.
 361
 362  */
 363 #define Py_UNICODE_ISSPACE(ch) \
 364         ((ch) < 128U ? _Py_ascii_whitespace[(ch)] : _PyUnicode_IsWhitespace(ch))
 365
 366 #define Py_UNICODE_ISLOWER(ch) _PyUnicode_IsLowercase(ch)
 367 #define Py_UNICODE_ISUPPER(ch) _PyUnicode_IsUppercase(ch)
 368 #define Py_UNICODE_ISTITLE(ch) _PyUnicode_IsTitlecase(ch)
 369 #define Py_UNICODE_ISLINEBREAK(ch) _PyUnicode_IsLinebreak(ch)
 370
 371 #define Py_UNICODE_TOLOWER(ch) _PyUnicode_ToLowercase(ch)
 372 #define Py_UNICODE_TOUPPER(ch) _PyUnicode_ToUppercase(ch)
 373 #define Py_UNICODE_TOTITLE(ch) _PyUnicode_ToTitlecase(ch)
 374
 375 #define Py_UNICODE_ISDECIMAL(ch) _PyUnicode_IsDecimalDigit(ch)
 376 #define Py_UNICODE_ISDIGIT(ch) _PyUnicode_IsDigit(ch)
 377 #define Py_UNICODE_ISNUMERIC(ch) _PyUnicode_IsNumeric(ch)
 378
 379 #define Py_UNICODE_TODECIMAL(ch) _PyUnicode_ToDecimalDigit(ch)
 380 #define Py_UNICODE_TODIGIT(ch) _PyUnicode_ToDigit(ch)
 381 #define Py_UNICODE_TONUMERIC(ch) _PyUnicode_ToNumeric(ch)
 382
 383 #define Py_UNICODE_ISALPHA(ch) _PyUnicode_IsAlpha(ch)
 384
 385 #endif
 386
 387 #define Py_UNICODE_ISALNUM(ch) \
 388        (Py_UNICODE_ISALPHA(ch) || \
 389         Py_UNICODE_ISDECIMAL(ch) || \
 390         Py_UNICODE_ISDIGIT(ch) || \
 391         Py_UNICODE_ISNUMERIC(ch))
 392
 393 #define Py_UNICODE_COPY(target, source, length)                         \
 394         Py_MEMCPY((target), (source), (length)*sizeof(Py_UNICODE))
 395
 396 #define Py_UNICODE_FILL(target, value, length) \
 397     do {Py_ssize_t i_; Py_UNICODE *t_ = (target); Py_UNICODE v_ = (value);\
 398         for (i_ = 0; i_ < (length); i_++) t_[i_] = v_;\
 399     } while (0)
 400
 401 /* Check if substring matches at given offset.  the offset must be
 402    valid, and the substring must not be empty */
 403
 404 #define Py_UNICODE_MATCH(string, offset, substring) \
 405     ((*((string)->str + (offset)) == *((substring)->str)) && \
 406     ((*((string)->str + (offset) + (substring)->length-1) == *((substring)->str + (substring)->length-1))) && \
 407      !memcmp((string)->str + (offset), (substring)->str, (substring)->length*sizeof(Py_UNICODE)))
 408
 409 #ifdef __cplusplus
 410 extern "C" {
 411 #endif
 412
 413 /* --- Unicode Type ------------------------------------------------------- */
 414
 415 typedef struct {
 416     PyObject_HEAD
 417     Py_ssize_t length;          /* Length of raw Unicode data in buffer */
 418     Py_UNICODE *str;            /* Raw Unicode buffer */
 419     long hash;                  /* Hash value; -1 if not set */
 420     PyObject *defenc;           /* (Default) Encoded version as Python
 421                                    string, or NULL; this is used for
 422                                    implementing the buffer protocol */
 423 } PyUnicodeObject;
 424
 425 PyAPI_DATA(PyTypeObject) PyUnicode_Type;
 426
 427 #define PyUnicode_Check(op) \
 428                  PyType_FastSubclass(Py_TYPE(op), Py_TPFLAGS_UNICODE_SUBCLASS)
 429 #define PyUnicode_CheckExact(op) (Py_TYPE(op) == &PyUnicode_Type)
 430
 431 /* Fast access macros */
 432 #define PyUnicode_GET_SIZE(op) \
 433         (((PyUnicodeObject *)(op))->length)
 434 #define PyUnicode_GET_DATA_SIZE(op) \
 435         (((PyUnicodeObject *)(op))->length * sizeof(Py_UNICODE))
 436 #define PyUnicode_AS_UNICODE(op) \
 437         (((PyUnicodeObject *)(op))->str)
 438 #define PyUnicode_AS_DATA(op) \
 439         ((const char *)((PyUnicodeObject *)(op))->str)
 440
 441 /* --- Constants ---------------------------------------------------------- */
 442
 443 /* This Unicode character will be used as replacement character during
 444    decoding if the errors argument is set to "replace". Note: the
 445    Unicode character U+FFFD is the official REPLACEMENT CHARACTER in
 446    Unicode 3.0. */
 447
 448 #define Py_UNICODE_REPLACEMENT_CHARACTER ((Py_UNICODE) 0xFFFD)
 449
 450 /* === Public API ========================================================= */
 451
 452 /* --- Plain Py_UNICODE --------------------------------------------------- */
 453
 454 /* Create a Unicode Object from the Py_UNICODE buffer u of the given
 455    size.
 456
 457    u may be NULL which causes the contents to be undefined. It is the
 458    user's responsibility to fill in the needed data afterwards. Note
 459    that modifying the Unicode object contents after construction is
 460    only allowed if u was set to NULL.
 461
 462    The buffer is copied into the new object. */
 463
 464 PyAPI_FUNC(PyObject*) PyUnicode_FromUnicode(
 465     const Py_UNICODE *u,        /* Unicode buffer */
 466     Py_ssize_t size             /* size of buffer */
 467     );
 468
 469 /* Similar to PyUnicode_FromUnicode(), but u points to Latin-1 encoded bytes */
 470 PyAPI_FUNC(PyObject*) PyUnicode_FromStringAndSize(
 471     const char *u,        /* char buffer */
 472     Py_ssize_t size       /* size of buffer */
 473     );
 474
 475 /* Similar to PyUnicode_FromUnicode(), but u points to null-terminated
 476    Latin-1 encoded bytes */
 477 PyAPI_FUNC(PyObject*) PyUnicode_FromString(
 478     const char *u        /* string */
 479     );
 480
 481 /* Return a read-only pointer to the Unicode object's internal
 482    Py_UNICODE buffer. */
 483
 484 PyAPI_FUNC(Py_UNICODE *) PyUnicode_AsUnicode(
 485     PyObject *unicode           /* Unicode object */
 486     );
 487
 488 /* Get the length of the Unicode object. */
 489
 490 PyAPI_FUNC(Py_ssize_t) PyUnicode_GetSize(
 491     PyObject *unicode           /* Unicode object */
 492     );
 493
 494 /* Get the maximum ordinal for a Unicode character. */
 495 PyAPI_FUNC(Py_UNICODE) PyUnicode_GetMax(void);
 496
 497 /* Resize an already allocated Unicode object to the new size length.
 498
 499    *unicode is modified to point to the new (resized) object and 0
 500    returned on success.
 501
 502    This API may only be called by the function which also called the
 503    Unicode constructor. The refcount on the object must be 1. Otherwise,
 504    an error is returned.
 505
 506    Error handling is implemented as follows: an exception is set, -1
 507    is returned and *unicode left untouched.
 508
 509 */
 510
 511 PyAPI_FUNC(int) PyUnicode_Resize(
 512     PyObject **unicode,         /* Pointer to the Unicode object */
 513     Py_ssize_t length           /* New length */
 514     );
 515
 516 /* Coerce obj to an Unicode object and return a reference with
 517    *incremented* refcount.
 518
 519    Coercion is done in the following way:
 520
 521    1. String and other char buffer compatible objects are decoded
 522       under the assumptions that they contain data using the current
 523       default encoding. Decoding is done in "strict" mode.
 524
 525    2. All other objects (including Unicode objects) raise an
 526       exception.
 527
 528    The API returns NULL in case of an error. The caller is responsible
 529    for decref'ing the returned objects.
 530
 531 */
 532
 533 PyAPI_FUNC(PyObject*) PyUnicode_FromEncodedObject(
 534     register PyObject *obj,     /* Object */
 535     const char *encoding,       /* encoding */
 536     const char *errors          /* error handling */
 537     );
 538
 539 /* Coerce obj to an Unicode object and return a reference with
 540    *incremented* refcount.
 541
 542    Unicode objects are passed back as-is (subclasses are converted to
 543    true Unicode objects), all other objects are delegated to
 544    PyUnicode_FromEncodedObject(obj, NULL, "strict") which results in
 545    using the default encoding as basis for decoding the object.
 546
 547    The API returns NULL in case of an error. The caller is responsible
 548    for decref'ing the returned objects.
 549
 550 */
 551
 552 PyAPI_FUNC(PyObject*) PyUnicode_FromObject(
 553     register PyObject *obj      /* Object */
 554     );
 555
 556 PyAPI_FUNC(PyObject *) PyUnicode_FromFormatV(const char*, va_list);
 557 PyAPI_FUNC(PyObject *) PyUnicode_FromFormat(const char*, ...);
 558
 559 /* Format the object based on the format_spec, as defined in PEP 3101
 560    (Advanced String Formatting). */
 561 PyAPI_FUNC(PyObject *) _PyUnicode_FormatAdvanced(PyObject *obj,
 562                                                  Py_UNICODE *format_spec,
 563                                                  Py_ssize_t format_spec_len);
 564
 565 /* --- wchar_t support for platforms which support it --------------------- */
 566
 567 #ifdef HAVE_WCHAR_H
 568
 569 /* Create a Unicode Object from the whcar_t buffer w of the given
 570    size.
 571
 572    The buffer is copied into the new object. */
 573
 574 PyAPI_FUNC(PyObject*) PyUnicode_FromWideChar(
 575     register const wchar_t *w,  /* wchar_t buffer */
 576     Py_ssize_t size             /* size of buffer */
 577     );
 578
 579 /* Copies the Unicode Object contents into the wchar_t buffer w.  At
 580    most size wchar_t characters are copied.
 581
 582    Note that the resulting wchar_t string may or may not be
 583    0-terminated.  It is the responsibility of the caller to make sure
 584    that the wchar_t string is 0-terminated in case this is required by
 585    the application.
 586
 587    Returns the number of wchar_t characters copied (excluding a
 588    possibly trailing 0-termination character) or -1 in case of an
 589    error. */
 590
 591 PyAPI_FUNC(Py_ssize_t) PyUnicode_AsWideChar(
 592     PyUnicodeObject *unicode,   /* Unicode object */
 593     register wchar_t *w,        /* wchar_t buffer */
 594     Py_ssize_t size             /* size of buffer */
 595     );
 596
 597 #endif
 598
 599 /* --- Unicode ordinals --------------------------------------------------- */
 600
 601 /* Create a Unicode Object from the given Unicode code point ordinal.
 602
 603    The ordinal must be in range(0x10000) on narrow Python builds
 604    (UCS2), and range(0x110000) on wide builds (UCS4). A ValueError is
 605    raised in case it is not.
 606
 607 */
 608
 609 PyAPI_FUNC(PyObject*) PyUnicode_FromOrdinal(int ordinal);
 610
 611 /* --- Free-list management ----------------------------------------------- */
 612
 613 /* Clear the free list used by the Unicode implementation.
 614
 615    This can be used to release memory used for objects on the free
 616    list back to the Python memory allocator.
 617
 618 */
 619
 620 PyAPI_FUNC(int) PyUnicode_ClearFreeList(void);
 621
 622 /* === Builtin Codecs =====================================================
 623
 624    Many of these APIs take two arguments encoding and errors. These
 625    parameters encoding and errors have the same semantics as the ones
 626    of the builtin unicode() API.
 627
 628    Setting encoding to NULL causes the default encoding to be used.
 629
 630    Error handling is set by errors which may also be set to NULL
 631    meaning to use the default handling defined for the codec. Default
 632    error handling for all builtin codecs is "strict" (ValueErrors are
 633    raised).
 634
 635    The codecs all use a similar interface. Only deviation from the
 636    generic ones are documented.
 637
 638 */
 639
 640 /* --- Manage the default encoding ---------------------------------------- */
 641
 642 /* Return a Python string holding the default encoded value of the
 643    Unicode object.
 644
 645    The resulting string is cached in the Unicode object for subsequent
 646    usage by this function. The cached version is needed to implement
 647    the character buffer interface and will live (at least) as long as
 648    the Unicode object itself.
 649
 650    The refcount of the string is *not* incremented.
 651
 652    *** Exported for internal use by the interpreter only !!! ***
 653
 654 */
 655
 656 PyAPI_FUNC(PyObject *) _PyUnicode_AsDefaultEncodedString(
 657     PyObject *, const char *);
 658
 659 /* Returns the currently active default encoding.
 660
 661    The default encoding is currently implemented as run-time settable
 662    process global.  This may change in future versions of the
 663    interpreter to become a parameter which is managed on a per-thread
 664    basis.
 665
 666  */
 667
 668 PyAPI_FUNC(const char*) PyUnicode_GetDefaultEncoding(void);
 669
 670 /* Sets the currently active default encoding.
 671
 672    Returns 0 on success, -1 in case of an error.
 673
 674  */
 675
 676 PyAPI_FUNC(int) PyUnicode_SetDefaultEncoding(
 677     const char *encoding        /* Encoding name in standard form */
 678     );
 679
 680 /* --- Generic Codecs ----------------------------------------------------- */
 681
 682 /* Create a Unicode object by decoding the encoded string s of the
 683    given size. */
 684
 685 PyAPI_FUNC(PyObject*) PyUnicode_Decode(
 686     const char *s,              /* encoded string */
 687     Py_ssize_t size,            /* size of buffer */
 688     const char *encoding,       /* encoding */
 689     const char *errors          /* error handling */
 690     );
 691
 692 /* Encodes a Py_UNICODE buffer of the given size and returns a
 693    Python string object. */
 694
 695 PyAPI_FUNC(PyObject*) PyUnicode_Encode(
 696     const Py_UNICODE *s,        /* Unicode char buffer */
 697     Py_ssize_t size,            /* number of Py_UNICODE chars to encode */
 698     const char *encoding,       /* encoding */
 699     const char *errors          /* error handling */
 700     );
 701
 702 /* Encodes a Unicode object and returns the result as Python
 703    object. */
 704
 705 PyAPI_FUNC(PyObject*) PyUnicode_AsEncodedObject(
 706     PyObject *unicode,          /* Unicode object */
 707     const char *encoding,       /* encoding */
 708     const char *errors          /* error handling */
 709     );
 710
 711 /* Encodes a Unicode object and returns the result as Python string
 712    object. */
 713
 714 PyAPI_FUNC(PyObject*) PyUnicode_AsEncodedString(
 715     PyObject *unicode,          /* Unicode object */
 716     const char *encoding,       /* encoding */
 717     const char *errors          /* error handling */
 718     );
 719
 720 PyAPI_FUNC(PyObject*) PyUnicode_BuildEncodingMap(
 721     PyObject* string            /* 256 character map */
 722    );
 723
 724
 725 /* --- UTF-7 Codecs ------------------------------------------------------- */
 726
 727 PyAPI_FUNC(PyObject*) PyUnicode_DecodeUTF7(
 728     const char *string,         /* UTF-7 encoded string */
 729     Py_ssize_t length,          /* size of string */
 730     const char *errors          /* error handling */
 731     );
 732
 733 PyAPI_FUNC(PyObject*) PyUnicode_DecodeUTF7Stateful(
 734     const char *string,         /* UTF-7 encoded string */
 735     Py_ssize_t length,          /* size of string */
 736     const char *errors,         /* error handling */
 737     Py_ssize_t *consumed        /* bytes consumed */
 738     );
 739
 740 PyAPI_FUNC(PyObject*) PyUnicode_EncodeUTF7(
 741     const Py_UNICODE *data,     /* Unicode char buffer */
 742     Py_ssize_t length,                  /* number of Py_UNICODE chars to encode */
 743     int encodeSetO,             /* force the encoder to encode characters in
 744                                    Set O, as described in RFC2152 */
 745     int encodeWhiteSpace,       /* force the encoder to encode space, tab,
 746                                    carriage return and linefeed characters */
 747     const char *errors          /* error handling */
 748     );
 749
 750 /* --- UTF-8 Codecs ------------------------------------------------------- */
 751
 752 PyAPI_FUNC(PyObject*) PyUnicode_DecodeUTF8(
 753     const char *string,         /* UTF-8 encoded string */
 754     Py_ssize_t length,          /* size of string */
 755     const char *errors          /* error handling */
 756     );
 757
 758 PyAPI_FUNC(PyObject*) PyUnicode_DecodeUTF8Stateful(
 759     const char *string,         /* UTF-8 encoded string */
 760     Py_ssize_t length,          /* size of string */
 761     const char *errors,         /* error handling */
 762     Py_ssize_t *consumed                /* bytes consumed */
 763     );
 764
 765 PyAPI_FUNC(PyObject*) PyUnicode_AsUTF8String(
 766     PyObject *unicode           /* Unicode object */
 767     );
 768
 769 PyAPI_FUNC(PyObject*) PyUnicode_EncodeUTF8(
 770     const Py_UNICODE *data,     /* Unicode char buffer */
 771     Py_ssize_t length,                  /* number of Py_UNICODE chars to encode */
 772     const char *errors          /* error handling */
 773     );
 774
 775 /* --- UTF-32 Codecs ------------------------------------------------------ */
 776
 777 /* Decodes length bytes from a UTF-32 encoded buffer string and returns
 778    the corresponding Unicode object.
 779
 780    errors (if non-NULL) defines the error handling. It defaults
 781    to "strict".
 782
 783    If byteorder is non-NULL, the decoder starts decoding using the
 784    given byte order:
 785
 786         *byteorder == -1: little endian
 787         *byteorder == 0:  native order
 788         *byteorder == 1:  big endian
 789
 790    In native mode, the first four bytes of the stream are checked for a
 791    BOM mark. If found, the BOM mark is analysed, the byte order
 792    adjusted and the BOM skipped.  In the other modes, no BOM mark
 793    interpretation is done. After completion, *byteorder is set to the
 794    current byte order at the end of input data.
 795
 796    If byteorder is NULL, the codec starts in native order mode.
 797
 798 */
 799
 800 PyAPI_FUNC(PyObject*) PyUnicode_DecodeUTF32(
 801     const char *string,         /* UTF-32 encoded string */
 802     Py_ssize_t length,          /* size of string */
 803     const char *errors,         /* error handling */
 804     int *byteorder              /* pointer to byteorder to use
 805                                    0=native;-1=LE,1=BE; updated on
 806                                    exit */
 807     );
 808
 809 PyAPI_FUNC(PyObject*) PyUnicode_DecodeUTF32Stateful(
 810     const char *string,         /* UTF-32 encoded string */
 811     Py_ssize_t length,          /* size of string */
 812     const char *errors,         /* error handling */
 813     int *byteorder,             /* pointer to byteorder to use
 814                                    0=native;-1=LE,1=BE; updated on
 815                                    exit */
 816     Py_ssize_t *consumed        /* bytes consumed */
 817     );
 818
 819 /* Returns a Python string using the UTF-32 encoding in native byte
 820    order. The string always starts with a BOM mark.  */
 821
 822 PyAPI_FUNC(PyObject*) PyUnicode_AsUTF32String(
 823     PyObject *unicode           /* Unicode object */
 824     );
 825
 826 /* Returns a Python string object holding the UTF-32 encoded value of
 827    the Unicode data.
 828
 829    If byteorder is not 0, output is written according to the following
 830    byte order:
 831
 832    byteorder == -1: little endian
 833    byteorder == 0:  native byte order (writes a BOM mark)
 834    byteorder == 1:  big endian
 835
 836    If byteorder is 0, the output string will always start with the
 837    Unicode BOM mark (U+FEFF). In the other two modes, no BOM mark is
 838    prepended.
 839
 840 */
 841
 842 PyAPI_FUNC(PyObject*) PyUnicode_EncodeUTF32(
 843     const Py_UNICODE *data,     /* Unicode char buffer */
 844     Py_ssize_t length,          /* number of Py_UNICODE chars to encode */
 845     const char *errors,         /* error handling */
 846     int byteorder               /* byteorder to use 0=BOM+native;-1=LE,1=BE */
 847     );
 848
 849 /* --- UTF-16 Codecs ------------------------------------------------------ */
 850
 851 /* Decodes length bytes from a UTF-16 encoded buffer string and returns
 852    the corresponding Unicode object.
 853
 854    errors (if non-NULL) defines the error handling. It defaults
 855    to "strict".
 856
 857    If byteorder is non-NULL, the decoder starts decoding using the
 858    given byte order:
 859
 860         *byteorder == -1: little endian
 861         *byteorder == 0:  native order
 862         *byteorder == 1:  big endian
 863
 864    In native mode, the first two bytes of the stream are checked for a
 865    BOM mark. If found, the BOM mark is analysed, the byte order
 866    adjusted and the BOM skipped.  In the other modes, no BOM mark
 867    interpretation is done. After completion, *byteorder is set to the
 868    current byte order at the end of input data.
 869
 870    If byteorder is NULL, the codec starts in native order mode.
 871
 872 */
 873
 874 PyAPI_FUNC(PyObject*) PyUnicode_DecodeUTF16(
 875     const char *string,         /* UTF-16 encoded string */
 876     Py_ssize_t length,          /* size of string */
 877     const char *errors,         /* error handling */
 878     int *byteorder              /* pointer to byteorder to use
 879                                    0=native;-1=LE,1=BE; updated on
 880                                    exit */
 881     );
 882
 883 PyAPI_FUNC(PyObject*) PyUnicode_DecodeUTF16Stateful(
 884     const char *string,         /* UTF-16 encoded string */
 885     Py_ssize_t length,          /* size of string */
 886     const char *errors,         /* error handling */
 887     int *byteorder,             /* pointer to byteorder to use
 888                                    0=native;-1=LE,1=BE; updated on
 889                                    exit */
 890     Py_ssize_t *consumed                /* bytes consumed */
 891     );
 892
 893 /* Returns a Python string using the UTF-16 encoding in native byte
 894    order. The string always starts with a BOM mark.  */
 895
 896 PyAPI_FUNC(PyObject*) PyUnicode_AsUTF16String(
 897     PyObject *unicode           /* Unicode object */
 898     );
 899
 900 /* Returns a Python string object holding the UTF-16 encoded value of
 901    the Unicode data.
 902
 903    If byteorder is not 0, output is written according to the following
 904    byte order:
 905
 906    byteorder == -1: little endian
 907    byteorder == 0:  native byte order (writes a BOM mark)
 908    byteorder == 1:  big endian
 909
 910    If byteorder is 0, the output string will always start with the
 911    Unicode BOM mark (U+FEFF). In the other two modes, no BOM mark is
 912    prepended.
 913
 914    Note that Py_UNICODE data is being interpreted as UTF-16 reduced to
 915    UCS-2. This trick makes it possible to add full UTF-16 capabilities
 916    at a later point without compromising the APIs.
 917
 918 */
 919
 920 PyAPI_FUNC(PyObject*) PyUnicode_EncodeUTF16(
 921     const Py_UNICODE *data,     /* Unicode char buffer */
 922     Py_ssize_t length,                  /* number of Py_UNICODE chars to encode */
 923     const char *errors,         /* error handling */
 924     int byteorder               /* byteorder to use 0=BOM+native;-1=LE,1=BE */
 925     );
 926
 927 /* --- Unicode-Escape Codecs ---------------------------------------------- */
 928
 929 PyAPI_FUNC(PyObject*) PyUnicode_DecodeUnicodeEscape(
 930     const char *string,         /* Unicode-Escape encoded string */
 931     Py_ssize_t length,          /* size of string */
 932     const char *errors          /* error handling */
 933     );
 934
 935 PyAPI_FUNC(PyObject*) PyUnicode_AsUnicodeEscapeString(
 936     PyObject *unicode           /* Unicode object */
 937     );
 938
 939 PyAPI_FUNC(PyObject*) PyUnicode_EncodeUnicodeEscape(
 940     const Py_UNICODE *data,     /* Unicode char buffer */
 941     Py_ssize_t length                   /* Number of Py_UNICODE chars to encode */
 942     );
 943
 944 /* --- Raw-Unicode-Escape Codecs ------------------------------------------ */
 945
 946 PyAPI_FUNC(PyObject*) PyUnicode_DecodeRawUnicodeEscape(
 947     const char *string,         /* Raw-Unicode-Escape encoded string */
 948     Py_ssize_t length,          /* size of string */
 949     const char *errors          /* error handling */
 950     );
 951
 952 PyAPI_FUNC(PyObject*) PyUnicode_AsRawUnicodeEscapeString(
 953     PyObject *unicode           /* Unicode object */
 954     );
 955
 956 PyAPI_FUNC(PyObject*) PyUnicode_EncodeRawUnicodeEscape(
 957     const Py_UNICODE *data,     /* Unicode char buffer */
 958     Py_ssize_t length                   /* Number of Py_UNICODE chars to encode */
 959     );
 960
 961 /* --- Unicode Internal Codec ---------------------------------------------
 962
 963     Only for internal use in _codecsmodule.c */
 964
 965 PyObject *_PyUnicode_DecodeUnicodeInternal(
 966     const char *string,
 967     Py_ssize_t length,
 968     const char *errors
 969     );
 970
 971 /* --- Latin-1 Codecs -----------------------------------------------------
 972
 973    Note: Latin-1 corresponds to the first 256 Unicode ordinals.
 974
 975 */
 976
 977 PyAPI_FUNC(PyObject*) PyUnicode_DecodeLatin1(
 978     const char *string,         /* Latin-1 encoded string */
 979     Py_ssize_t length,          /* size of string */
 980     const char *errors          /* error handling */
 981     );
 982
 983 PyAPI_FUNC(PyObject*) PyUnicode_AsLatin1String(
 984     PyObject *unicode           /* Unicode object */
 985     );
 986
 987 PyAPI_FUNC(PyObject*) PyUnicode_EncodeLatin1(
 988     const Py_UNICODE *data,     /* Unicode char buffer */
 989     Py_ssize_t length,                  /* Number of Py_UNICODE chars to encode */
 990     const char *errors          /* error handling */
 991     );
 992
 993 /* --- ASCII Codecs -------------------------------------------------------
 994
 995    Only 7-bit ASCII data is excepted. All other codes generate errors.
 996
 997 */
 998
 999 PyAPI_FUNC(PyObject*) PyUnicode_DecodeASCII(
1000     const char *string,         /* ASCII encoded string */
1001     Py_ssize_t length,          /* size of string */
1002     const char *errors          /* error handling */
1003     );
1004
1005 PyAPI_FUNC(PyObject*) PyUnicode_AsASCIIString(
1006     PyObject *unicode           /* Unicode object */
1007     );
1008
1009 PyAPI_FUNC(PyObject*) PyUnicode_EncodeASCII(
1010     const Py_UNICODE *data,     /* Unicode char buffer */
1011     Py_ssize_t length,                  /* Number of Py_UNICODE chars to encode */
1012     const char *errors          /* error handling */
1013     );
1014
1015 /* --- Character Map Codecs -----------------------------------------------
1016
1017    This codec uses mappings to encode and decode characters.
1018
1019    Decoding mappings must map single string characters to single
1020    Unicode characters, integers (which are then interpreted as Unicode
1021    ordinals) or None (meaning "undefined mapping" and causing an
1022    error).
1023
1024    Encoding mappings must map single Unicode characters to single
1025    string characters, integers (which are then interpreted as Latin-1
1026    ordinals) or None (meaning "undefined mapping" and causing an
1027    error).
1028
1029    If a character lookup fails with a LookupError, the character is
1030    copied as-is meaning that its ordinal value will be interpreted as
1031    Unicode or Latin-1 ordinal resp. Because of this mappings only need
1032    to contain those mappings which map characters to different code
1033    points.
1034
1035 */
1036
1037 PyAPI_FUNC(PyObject*) PyUnicode_DecodeCharmap(
1038     const char *string,         /* Encoded string */
1039     Py_ssize_t length,          /* size of string */
1040     PyObject *mapping,          /* character mapping
1041                                    (char ordinal -> unicode ordinal) */
1042     const char *errors          /* error handling */
1043     );
1044
1045 PyAPI_FUNC(PyObject*) PyUnicode_AsCharmapString(
1046     PyObject *unicode,          /* Unicode object */
1047     PyObject *mapping           /* character mapping
1048                                    (unicode ordinal -> char ordinal) */
1049     );
1050
1051 PyAPI_FUNC(PyObject*) PyUnicode_EncodeCharmap(
1052     const Py_UNICODE *data,     /* Unicode char buffer */
1053     Py_ssize_t length,          /* Number of Py_UNICODE chars to encode */
1054     PyObject *mapping,          /* character mapping
1055                                    (unicode ordinal -> char ordinal) */
1056     const char *errors          /* error handling */
1057     );
1058
1059 /* Translate a Py_UNICODE buffer of the given length by applying a
1060    character mapping table to it and return the resulting Unicode
1061    object.
1062
1063    The mapping table must map Unicode ordinal integers to Unicode
1064    ordinal integers or None (causing deletion of the character).
1065
1066    Mapping tables may be dictionaries or sequences. Unmapped character
1067    ordinals (ones which cause a LookupError) are left untouched and
1068    are copied as-is.
1069
1070 */
1071
1072 PyAPI_FUNC(PyObject *) PyUnicode_TranslateCharmap(
1073     const Py_UNICODE *data,     /* Unicode char buffer */
1074     Py_ssize_t length,                  /* Number of Py_UNICODE chars to encode */
1075     PyObject *table,            /* Translate table */
1076     const char *errors          /* error handling */
1077     );
1078
1079 #ifdef MS_WIN32
1080
1081 /* --- MBCS codecs for Windows -------------------------------------------- */
1082
1083 PyAPI_FUNC(PyObject*) PyUnicode_DecodeMBCS(
1084     const char *string,         /* MBCS encoded string */
1085     Py_ssize_t length,              /* size of string */
1086     const char *errors          /* error handling */
1087     );
1088
1089 PyAPI_FUNC(PyObject*) PyUnicode_DecodeMBCSStateful(
1090     const char *string,         /* MBCS encoded string */
1091     Py_ssize_t length,          /* size of string */
1092     const char *errors,         /* error handling */
1093     Py_ssize_t *consumed        /* bytes consumed */
1094     );
1095
1096 PyAPI_FUNC(PyObject*) PyUnicode_AsMBCSString(
1097     PyObject *unicode           /* Unicode object */
1098     );
1099
1100 PyAPI_FUNC(PyObject*) PyUnicode_EncodeMBCS(
1101     const Py_UNICODE *data,     /* Unicode char buffer */
1102     Py_ssize_t length,              /* Number of Py_UNICODE chars to encode */
1103     const char *errors          /* error handling */
1104     );
1105
1106 #endif /* MS_WIN32 */
1107
1108 /* --- Decimal Encoder ---------------------------------------------------- */
1109
1110 /* Takes a Unicode string holding a decimal value and writes it into
1111    an output buffer using standard ASCII digit codes.
1112
1113    The output buffer has to provide at least length+1 bytes of storage
1114    area. The output string is 0-terminated.
1115
1116    The encoder converts whitespace to ' ', decimal characters to their
1117    corresponding ASCII digit and all other Latin-1 characters except
1118    \0 as-is. Characters outside this range (Unicode ordinals 1-256)
1119    are treated as errors. This includes embedded NULL bytes.
1120
1121    Error handling is defined by the errors argument:
1122
1123       NULL or "strict": raise a ValueError
1124       "ignore": ignore the wrong characters (these are not copied to the
1125                 output buffer)
1126       "replace": replaces illegal characters with '?'
1127
1128    Returns 0 on success, -1 on failure.
1129
1130 */
1131
1132 PyAPI_FUNC(int) PyUnicode_EncodeDecimal(
1133     Py_UNICODE *s,              /* Unicode buffer */
1134     Py_ssize_t length,                  /* Number of Py_UNICODE chars to encode */
1135     char *output,               /* Output buffer; must have size >= length */
1136     const char *errors          /* error handling */
1137     );
1138
1139 /* --- Methods & Slots ----------------------------------------------------
1140
1141    These are capable of handling Unicode objects and strings on input
1142    (we refer to them as strings in the descriptions) and return
1143    Unicode objects or integers as apporpriate. */
1144
1145 /* Concat two strings giving a new Unicode string. */
1146
1147 PyAPI_FUNC(PyObject*) PyUnicode_Concat(
1148     PyObject *left,             /* Left string */
1149     PyObject *right             /* Right string */
1150     );
1151
1152 /* Split a string giving a list of Unicode strings.
1153
1154    If sep is NULL, splitting will be done at all whitespace
1155    substrings. Otherwise, splits occur at the given separator.
1156
1157    At most maxsplit splits will be done. If negative, no limit is set.
1158
1159    Separators are not included in the resulting list.
1160
1161 */
1162
1163 PyAPI_FUNC(PyObject*) PyUnicode_Split(
1164     PyObject *s,                /* String to split */
1165     PyObject *sep,              /* String separator */
1166     Py_ssize_t maxsplit         /* Maxsplit count */
1167     );
1168
1169 /* Dito, but split at line breaks.
1170
1171    CRLF is considered to be one line break. Line breaks are not
1172    included in the resulting list. */
1173
1174 PyAPI_FUNC(PyObject*) PyUnicode_Splitlines(
1175     PyObject *s,                /* String to split */
1176     int keepends                /* If true, line end markers are included */
1177     );
1178
1179 /* Partition a string using a given separator. */
1180
1181 PyAPI_FUNC(PyObject*) PyUnicode_Partition(
1182     PyObject *s,                /* String to partition */
1183     PyObject *sep               /* String separator */
1184     );
1185
1186 /* Partition a string using a given separator, searching from the end of the
1187    string. */
1188
1189 PyAPI_FUNC(PyObject*) PyUnicode_RPartition(
1190     PyObject *s,                /* String to partition */
1191     PyObject *sep               /* String separator */
1192     );
1193
1194 /* Split a string giving a list of Unicode strings.
1195
1196    If sep is NULL, splitting will be done at all whitespace
1197    substrings. Otherwise, splits occur at the given separator.
1198
1199    At most maxsplit splits will be done. But unlike PyUnicode_Split
1200    PyUnicode_RSplit splits from the end of the string. If negative,
1201    no limit is set.
1202
1203    Separators are not included in the resulting list.
1204
1205 */
1206
1207 PyAPI_FUNC(PyObject*) PyUnicode_RSplit(
1208     PyObject *s,                /* String to split */
1209     PyObject *sep,              /* String separator */
1210     Py_ssize_t maxsplit         /* Maxsplit count */
1211     );
1212
1213 /* Translate a string by applying a character mapping table to it and
1214    return the resulting Unicode object.
1215
1216    The mapping table must map Unicode ordinal integers to Unicode
1217    ordinal integers or None (causing deletion of the character).
1218
1219    Mapping tables may be dictionaries or sequences. Unmapped character
1220    ordinals (ones which cause a LookupError) are left untouched and
1221    are copied as-is.
1222
1223 */
1224
1225 PyAPI_FUNC(PyObject *) PyUnicode_Translate(
1226     PyObject *str,              /* String */
1227     PyObject *table,            /* Translate table */
1228     const char *errors          /* error handling */
1229     );
1230
1231 /* Join a sequence of strings using the given separator and return
1232    the resulting Unicode string. */
1233
1234 PyAPI_FUNC(PyObject*) PyUnicode_Join(
1235     PyObject *separator,        /* Separator string */
1236     PyObject *seq               /* Sequence object */
1237     );
1238
1239 /* Return 1 if substr matches str[start:end] at the given tail end, 0
1240    otherwise. */
1241
1242 PyAPI_FUNC(Py_ssize_t) PyUnicode_Tailmatch(
1243     PyObject *str,              /* String */
1244     PyObject *substr,           /* Prefix or Suffix string */
1245     Py_ssize_t start,           /* Start index */
1246     Py_ssize_t end,             /* Stop index */
1247     int direction               /* Tail end: -1 prefix, +1 suffix */
1248     );
1249
1250 /* Return the first position of substr in str[start:end] using the
1251    given search direction or -1 if not found. -2 is returned in case
1252    an error occurred and an exception is set. */
1253
1254 PyAPI_FUNC(Py_ssize_t) PyUnicode_Find(
1255     PyObject *str,              /* String */
1256     PyObject *substr,           /* Substring to find */
1257     Py_ssize_t start,           /* Start index */
1258     Py_ssize_t end,             /* Stop index */
1259     int direction               /* Find direction: +1 forward, -1 backward */
1260     );
1261
1262 /* Count the number of occurrences of substr in str[start:end]. */
1263
1264 PyAPI_FUNC(Py_ssize_t) PyUnicode_Count(
1265     PyObject *str,              /* String */
1266     PyObject *substr,           /* Substring to count */
1267     Py_ssize_t start,           /* Start index */
1268     Py_ssize_t end              /* Stop index */
1269     );
1270
1271 /* Replace at most maxcount occurrences of substr in str with replstr
1272    and return the resulting Unicode object. */
1273
1274 PyAPI_FUNC(PyObject *) PyUnicode_Replace(
1275     PyObject *str,              /* String */
1276     PyObject *substr,           /* Substring to find */
1277     PyObject *replstr,          /* Substring to replace */
1278     Py_ssize_t maxcount         /* Max. number of replacements to apply;
1279                                    -1 = all */
1280     );
1281
1282 /* Compare two strings and return -1, 0, 1 for less than, equal,
1283    greater than resp. */
1284
1285 PyAPI_FUNC(int) PyUnicode_Compare(
1286     PyObject *left,             /* Left string */
1287     PyObject *right             /* Right string */
1288     );
1289
1290 /* Rich compare two strings and return one of the following:
1291
1292    - NULL in case an exception was raised
1293    - Py_True or Py_False for successfuly comparisons
1294    - Py_NotImplemented in case the type combination is unknown
1295
1296    Note that Py_EQ and Py_NE comparisons can cause a UnicodeWarning in
1297    case the conversion of the arguments to Unicode fails with a
1298    UnicodeDecodeError.
1299
1300    Possible values for op:
1301
1302      Py_GT, Py_GE, Py_EQ, Py_NE, Py_LT, Py_LE
1303
1304 */
1305
1306 PyAPI_FUNC(PyObject *) PyUnicode_RichCompare(
1307     PyObject *left,             /* Left string */
1308     PyObject *right,            /* Right string */
1309     int op                      /* Operation: Py_EQ, Py_NE, Py_GT, etc. */
1310     );
1311
1312 /* Apply a argument tuple or dictionary to a format string and return
1313    the resulting Unicode string. */
1314
1315 PyAPI_FUNC(PyObject *) PyUnicode_Format(
1316     PyObject *format,           /* Format string */
1317     PyObject *args              /* Argument tuple or dictionary */
1318     );
1319
1320 /* Checks whether element is contained in container and return 1/0
1321    accordingly.
1322
1323    element has to coerce to an one element Unicode string. -1 is
1324    returned in case of an error. */
1325
1326 PyAPI_FUNC(int) PyUnicode_Contains(
1327     PyObject *container,        /* Container string */
1328     PyObject *element           /* Element string */
1329     );
1330
1331 /* Externally visible for str.strip(unicode) */
1332 PyAPI_FUNC(PyObject *) _PyUnicode_XStrip(
1333     PyUnicodeObject *self,
1334     int striptype,
1335     PyObject *sepobj
1336     );
1337
1338 /* === Characters Type APIs =============================================== */
1339
1340 /* Helper array used by Py_UNICODE_ISSPACE(). */
1341
1342 PyAPI_DATA(const unsigned char) _Py_ascii_whitespace[];
1343
1344 /* These should not be used directly. Use the Py_UNICODE_IS* and
1345    Py_UNICODE_TO* macros instead.
1346
1347    These APIs are implemented in Objects/unicodectype.c.
1348
1349 */
1350
1351 PyAPI_FUNC(int) _PyUnicode_IsLowercase(
1352     Py_UNICODE ch       /* Unicode character */
1353     );
1354
1355 PyAPI_FUNC(int) _PyUnicode_IsUppercase(
1356     Py_UNICODE ch       /* Unicode character */
1357     );
1358
1359 PyAPI_FUNC(int) _PyUnicode_IsTitlecase(
1360     Py_UNICODE ch       /* Unicode character */
1361     );
1362
1363 PyAPI_FUNC(int) _PyUnicode_IsWhitespace(
1364     const Py_UNICODE ch         /* Unicode character */
1365     );
1366
1367 PyAPI_FUNC(int) _PyUnicode_IsLinebreak(
1368     const Py_UNICODE ch         /* Unicode character */
1369     );
1370
1371 PyAPI_FUNC(Py_UNICODE) _PyUnicode_ToLowercase(
1372     Py_UNICODE ch       /* Unicode character */
1373     );
1374
1375 PyAPI_FUNC(Py_UNICODE) _PyUnicode_ToUppercase(
1376     Py_UNICODE ch       /* Unicode character */
1377     );
1378
1379 PyAPI_FUNC(Py_UNICODE) _PyUnicode_ToTitlecase(
1380     Py_UNICODE ch       /* Unicode character */
1381     );
1382
1383 PyAPI_FUNC(int) _PyUnicode_ToDecimalDigit(
1384     Py_UNICODE ch       /* Unicode character */
1385     );
1386
1387 PyAPI_FUNC(int) _PyUnicode_ToDigit(
1388     Py_UNICODE ch       /* Unicode character */
1389     );
1390
1391 PyAPI_FUNC(double) _PyUnicode_ToNumeric(
1392     Py_UNICODE ch       /* Unicode character */
1393     );
1394
1395 PyAPI_FUNC(int) _PyUnicode_IsDecimalDigit(
1396     Py_UNICODE ch       /* Unicode character */
1397     );
1398
1399 PyAPI_FUNC(int) _PyUnicode_IsDigit(
1400     Py_UNICODE ch       /* Unicode character */
1401     );
1402
1403 PyAPI_FUNC(int) _PyUnicode_IsNumeric(
1404     Py_UNICODE ch       /* Unicode character */
1405     );
1406
1407 PyAPI_FUNC(int) _PyUnicode_IsAlpha(
1408     Py_UNICODE ch       /* Unicode character */
1409     );
1410
1411 #ifdef __cplusplus
1412 }
1413 #endif
1414 #endif /* Py_USING_UNICODE */
1415 #endif /* !Py_UNICODEOBJECT_H */