Include/unicodeobject.h

   1 #ifndef Py_UNICODEOBJECT_H
   2 #define Py_UNICODEOBJECT_H
   3
   4 #include <stdarg.h>
   5
   6 /*
   7
   8 Unicode implementation based on original code by Fredrik Lundh,
   9 modified by Marc-Andre Lemburg (mal@lemburg.com) according to the
  10 Unicode Integration Proposal (see file Misc/unicode.txt).
  11
  12 Copyright (c) Corporation for National Research Initiatives.
  13
  14
  15  Original header:
  16  --------------------------------------------------------------------
  17
  18  * Yet another Unicode string type for Python.  This type supports the
  19  * 16-bit Basic Multilingual Plane (BMP) only.
  20  *
  21  * Written by Fredrik Lundh, January 1999.
  22  *
  23  * Copyright (c) 1999 by Secret Labs AB.
  24  * Copyright (c) 1999 by Fredrik Lundh.
  25  *
  26  * fredrik@pythonware.com
  27  * http://www.pythonware.com
  28  *
  29  * --------------------------------------------------------------------
  30  * This Unicode String Type is
  31  *
  32  * Copyright (c) 1999 by Secret Labs AB
  33  * Copyright (c) 1999 by Fredrik Lundh
  34  *
  35  * By obtaining, using, and/or copying this software and/or its
  36  * associated documentation, you agree that you have read, understood,
  37  * and will comply with the following terms and conditions:
  38  *
  39  * Permission to use, copy, modify, and distribute this software and its
  40  * associated documentation for any purpose and without fee is hereby
  41  * granted, provided that the above copyright notice appears in all
  42  * copies, and that both that copyright notice and this permission notice
  43  * appear in supporting documentation, and that the name of Secret Labs
  44  * AB or the author not be used in advertising or publicity pertaining to
  45  * distribution of the software without specific, written prior
  46  * permission.
  47  *
  48  * SECRET LABS AB AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO
  49  * THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND
  50  * FITNESS.  IN NO EVENT SHALL SECRET LABS AB OR THE AUTHOR BE LIABLE FOR
  51  * ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
  52  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
  53  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT
  54  * OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
  55  * -------------------------------------------------------------------- */
  56
  57 #include <ctype.h>
  58
  59 /* === Internal API ======================================================= */
  60
  61 /* --- Internal Unicode Format -------------------------------------------- */
  62
  63 #ifndef Py_USING_UNICODE
  64
  65 #define PyUnicode_Check(op)                 0
  66 #define PyUnicode_CheckExact(op)            0
  67
  68 #else
  69
  70 /* FIXME: MvL's new implementation assumes that Py_UNICODE_SIZE is
  71    properly set, but the default rules below doesn't set it.  I'll
  72    sort this out some other day -- fredrik@pythonware.com */
  73
  74 #ifndef Py_UNICODE_SIZE
  75 #error Must define Py_UNICODE_SIZE
  76 #endif
  77
  78 /* Setting Py_UNICODE_WIDE enables UCS-4 storage.  Otherwise, Unicode
  79    strings are stored as UCS-2 (with limited support for UTF-16) */
  80
  81 #if Py_UNICODE_SIZE >= 4
  82 #define Py_UNICODE_WIDE
  83 #endif
  84
  85 /* Set these flags if the platform has "wchar.h", "wctype.h" and the
  86    wchar_t type is a 16-bit unsigned type */
  87 /* #define HAVE_WCHAR_H */
  88 /* #define HAVE_USABLE_WCHAR_T */
  89
  90 /* Defaults for various platforms */
  91 #ifndef PY_UNICODE_TYPE
  92
  93 /* Windows has a usable wchar_t type (unless we're using UCS-4) */
  94 # if defined(MS_WIN32) && Py_UNICODE_SIZE == 2
  95 #  define HAVE_USABLE_WCHAR_T
  96 #  define PY_UNICODE_TYPE wchar_t
  97 # endif
  98
  99 # if defined(Py_UNICODE_WIDE)
 100 #  define PY_UNICODE_TYPE Py_UCS4
 101 # endif
 102
 103 #endif
 104
 105 /* If the compiler provides a wchar_t type we try to support it
 106    through the interface functions PyUnicode_FromWideChar() and
 107    PyUnicode_AsWideChar(). */
 108
 109 #ifdef HAVE_USABLE_WCHAR_T
 110 # ifndef HAVE_WCHAR_H
 111 #  define HAVE_WCHAR_H
 112 # endif
 113 #endif
 114
 115 #ifdef HAVE_WCHAR_H
 116 /* Work around a cosmetic bug in BSDI 4.x wchar.h; thanks to Thomas Wouters */
 117 # ifdef _HAVE_BSDI
 118 #  include <time.h>
 119 # endif
 120 #  include <wchar.h>
 121 #endif
 122
 123 /*
 124  * Use this typedef when you need to represent a UTF-16 surrogate pair
 125  * as single unsigned integer.
 126  */
 127 #if SIZEOF_INT >= 4
 128 typedef unsigned int Py_UCS4;
 129 #elif SIZEOF_LONG >= 4
 130 typedef unsigned long Py_UCS4;
 131 #endif
 132
 133 /* Py_UNICODE is the native Unicode storage format (code unit) used by
 134    Python and represents a single Unicode element in the Unicode
 135    type. */
 136
 137 typedef PY_UNICODE_TYPE Py_UNICODE;
 138
 139 /* --- UCS-2/UCS-4 Name Mangling ------------------------------------------ */
 140
 141 /* Unicode API names are mangled to assure that UCS-2 and UCS-4 builds
 142    produce different external names and thus cause import errors in
 143    case Python interpreters and extensions with mixed compiled in
 144    Unicode width assumptions are combined. */
 145
 146 #ifndef Py_UNICODE_WIDE
 147
 148 # define PyUnicode_AsASCIIString PyUnicodeUCS2_AsASCIIString
 149 # define PyUnicode_AsCharmapString PyUnicodeUCS2_AsCharmapString
 150 # define PyUnicode_AsEncodedObject PyUnicodeUCS2_AsEncodedObject
 151 # define PyUnicode_AsEncodedString PyUnicodeUCS2_AsEncodedString
 152 # define PyUnicode_AsLatin1String PyUnicodeUCS2_AsLatin1String
 153 # define PyUnicode_AsRawUnicodeEscapeString PyUnicodeUCS2_AsRawUnicodeEscapeString
 154 # define PyUnicode_AsUTF32String PyUnicodeUCS2_AsUTF32String
 155 # define PyUnicode_AsUTF16String PyUnicodeUCS2_AsUTF16String
 156 # define PyUnicode_AsUTF8String PyUnicodeUCS2_AsUTF8String
 157 # define PyUnicode_AsUnicode PyUnicodeUCS2_AsUnicode
 158 # define PyUnicode_AsUnicodeEscapeString PyUnicodeUCS2_AsUnicodeEscapeString
 159 # define PyUnicode_AsWideChar PyUnicodeUCS2_AsWideChar
 160 # define PyUnicode_ClearFreeList PyUnicodeUCS2_ClearFreelist
 161 # define PyUnicode_Compare PyUnicodeUCS2_Compare
 162 # define PyUnicode_Concat PyUnicodeUCS2_Concat
 163 # define PyUnicode_Contains PyUnicodeUCS2_Contains
 164 # define PyUnicode_Count PyUnicodeUCS2_Count
 165 # define PyUnicode_Decode PyUnicodeUCS2_Decode
 166 # define PyUnicode_DecodeASCII PyUnicodeUCS2_DecodeASCII
 167 # define PyUnicode_DecodeCharmap PyUnicodeUCS2_DecodeCharmap
 168 # define PyUnicode_DecodeLatin1 PyUnicodeUCS2_DecodeLatin1
 169 # define PyUnicode_DecodeRawUnicodeEscape PyUnicodeUCS2_DecodeRawUnicodeEscape
 170 # define PyUnicode_DecodeUTF32 PyUnicodeUCS2_DecodeUTF32
 171 # define PyUnicode_DecodeUTF32Stateful PyUnicodeUCS2_DecodeUTF32Stateful
 172 # define PyUnicode_DecodeUTF16 PyUnicodeUCS2_DecodeUTF16
 173 # define PyUnicode_DecodeUTF16Stateful PyUnicodeUCS2_DecodeUTF16Stateful
 174 # define PyUnicode_DecodeUTF8 PyUnicodeUCS2_DecodeUTF8
 175 # define PyUnicode_DecodeUTF8Stateful PyUnicodeUCS2_DecodeUTF8Stateful
 176 # define PyUnicode_DecodeUnicodeEscape PyUnicodeUCS2_DecodeUnicodeEscape
 177 # define PyUnicode_Encode PyUnicodeUCS2_Encode
 178 # define PyUnicode_EncodeASCII PyUnicodeUCS2_EncodeASCII
 179 # define PyUnicode_EncodeCharmap PyUnicodeUCS2_EncodeCharmap
 180 # define PyUnicode_EncodeDecimal PyUnicodeUCS2_EncodeDecimal
 181 # define PyUnicode_EncodeLatin1 PyUnicodeUCS2_EncodeLatin1
 182 # define PyUnicode_EncodeRawUnicodeEscape PyUnicodeUCS2_EncodeRawUnicodeEscape
 183 # define PyUnicode_EncodeUTF32 PyUnicodeUCS2_EncodeUTF32
 184 # define PyUnicode_EncodeUTF16 PyUnicodeUCS2_EncodeUTF16
 185 # define PyUnicode_EncodeUTF8 PyUnicodeUCS2_EncodeUTF8
 186 # define PyUnicode_EncodeUnicodeEscape PyUnicodeUCS2_EncodeUnicodeEscape
 187 # define PyUnicode_Find PyUnicodeUCS2_Find
 188 # define PyUnicode_Format PyUnicodeUCS2_Format
 189 # define PyUnicode_FromEncodedObject PyUnicodeUCS2_FromEncodedObject
 190 # define PyUnicode_FromFormat PyUnicodeUCS2_FromFormat
 191 # define PyUnicode_FromFormatV PyUnicodeUCS2_FromFormatV
 192 # define PyUnicode_FromObject PyUnicodeUCS2_FromObject
 193 # define PyUnicode_FromOrdinal PyUnicodeUCS2_FromOrdinal
 194 # define PyUnicode_FromString PyUnicodeUCS2_FromString
 195 # define PyUnicode_FromStringAndSize PyUnicodeUCS2_FromStringAndSize
 196 # define PyUnicode_FromUnicode PyUnicodeUCS2_FromUnicode
 197 # define PyUnicode_FromWideChar PyUnicodeUCS2_FromWideChar
 198 # define PyUnicode_GetDefaultEncoding PyUnicodeUCS2_GetDefaultEncoding
 199 # define PyUnicode_GetMax PyUnicodeUCS2_GetMax
 200 # define PyUnicode_GetSize PyUnicodeUCS2_GetSize
 201 # define PyUnicode_Join PyUnicodeUCS2_Join
 202 # define PyUnicode_Partition PyUnicodeUCS2_Partition
 203 # define PyUnicode_RPartition PyUnicodeUCS2_RPartition
 204 # define PyUnicode_RSplit PyUnicodeUCS2_RSplit
 205 # define PyUnicode_Replace PyUnicodeUCS2_Replace
 206 # define PyUnicode_Resize PyUnicodeUCS2_Resize
 207 # define PyUnicode_RichCompare PyUnicodeUCS2_RichCompare
 208 # define PyUnicode_SetDefaultEncoding PyUnicodeUCS2_SetDefaultEncoding
 209 # define PyUnicode_Split PyUnicodeUCS2_Split
 210 # define PyUnicode_Splitlines PyUnicodeUCS2_Splitlines
 211 # define PyUnicode_Tailmatch PyUnicodeUCS2_Tailmatch
 212 # define PyUnicode_Translate PyUnicodeUCS2_Translate
 213 # define PyUnicode_TranslateCharmap PyUnicodeUCS2_TranslateCharmap
 214 # define _PyUnicode_AsDefaultEncodedString _PyUnicodeUCS2_AsDefaultEncodedString
 215 # define _PyUnicode_Fini _PyUnicodeUCS2_Fini
 216 # define _PyUnicode_Init _PyUnicodeUCS2_Init
 217 # define _PyUnicode_IsAlpha _PyUnicodeUCS2_IsAlpha
 218 # define _PyUnicode_IsDecimalDigit _PyUnicodeUCS2_IsDecimalDigit
 219 # define _PyUnicode_IsDigit _PyUnicodeUCS2_IsDigit
 220 # define _PyUnicode_IsLinebreak _PyUnicodeUCS2_IsLinebreak
 221 # define _PyUnicode_IsLowercase _PyUnicodeUCS2_IsLowercase
 222 # define _PyUnicode_IsNumeric _PyUnicodeUCS2_IsNumeric
 223 # define _PyUnicode_IsTitlecase _PyUnicodeUCS2_IsTitlecase
 224 # define _PyUnicode_IsUppercase _PyUnicodeUCS2_IsUppercase
 225 # define _PyUnicode_IsWhitespace _PyUnicodeUCS2_IsWhitespace
 226 # define _PyUnicode_ToDecimalDigit _PyUnicodeUCS2_ToDecimalDigit
 227 # define _PyUnicode_ToDigit _PyUnicodeUCS2_ToDigit
 228 # define _PyUnicode_ToLowercase _PyUnicodeUCS2_ToLowercase
 229 # define _PyUnicode_ToNumeric _PyUnicodeUCS2_ToNumeric
 230 # define _PyUnicode_ToTitlecase _PyUnicodeUCS2_ToTitlecase
 231 # define _PyUnicode_ToUppercase _PyUnicodeUCS2_ToUppercase
 232
 233 #else
 234
 235 # define PyUnicode_AsASCIIString PyUnicodeUCS4_AsASCIIString
 236 # define PyUnicode_AsCharmapString PyUnicodeUCS4_AsCharmapString
 237 # define PyUnicode_AsEncodedObject PyUnicodeUCS4_AsEncodedObject
 238 # define PyUnicode_AsEncodedString PyUnicodeUCS4_AsEncodedString
 239 # define PyUnicode_AsLatin1String PyUnicodeUCS4_AsLatin1String
 240 # define PyUnicode_AsRawUnicodeEscapeString PyUnicodeUCS4_AsRawUnicodeEscapeString
 241 # define PyUnicode_AsUTF32String PyUnicodeUCS4_AsUTF32String
 242 # define PyUnicode_AsUTF16String PyUnicodeUCS4_AsUTF16String
 243 # define PyUnicode_AsUTF8String PyUnicodeUCS4_AsUTF8String
 244 # define PyUnicode_AsUnicode PyUnicodeUCS4_AsUnicode
 245 # define PyUnicode_AsUnicodeEscapeString PyUnicodeUCS4_AsUnicodeEscapeString
 246 # define PyUnicode_AsWideChar PyUnicodeUCS4_AsWideChar
 247 # define PyUnicode_ClearFreeList PyUnicodeUCS4_ClearFreelist
 248 # define PyUnicode_Compare PyUnicodeUCS4_Compare
 249 # define PyUnicode_Concat PyUnicodeUCS4_Concat
 250 # define PyUnicode_Contains PyUnicodeUCS4_Contains
 251 # define PyUnicode_Count PyUnicodeUCS4_Count
 252 # define PyUnicode_Decode PyUnicodeUCS4_Decode
 253 # define PyUnicode_DecodeASCII PyUnicodeUCS4_DecodeASCII
 254 # define PyUnicode_DecodeCharmap PyUnicodeUCS4_DecodeCharmap
 255 # define PyUnicode_DecodeLatin1 PyUnicodeUCS4_DecodeLatin1
 256 # define PyUnicode_DecodeRawUnicodeEscape PyUnicodeUCS4_DecodeRawUnicodeEscape
 257 # define PyUnicode_DecodeUTF32 PyUnicodeUCS4_DecodeUTF32
 258 # define PyUnicode_DecodeUTF32Stateful PyUnicodeUCS4_DecodeUTF32Stateful
 259 # define PyUnicode_DecodeUTF16 PyUnicodeUCS4_DecodeUTF16
 260 # define PyUnicode_DecodeUTF16Stateful PyUnicodeUCS4_DecodeUTF16Stateful
 261 # define PyUnicode_DecodeUTF8 PyUnicodeUCS4_DecodeUTF8
 262 # define PyUnicode_DecodeUTF8Stateful PyUnicodeUCS4_DecodeUTF8Stateful
 263 # define PyUnicode_DecodeUnicodeEscape PyUnicodeUCS4_DecodeUnicodeEscape
 264 # define PyUnicode_Encode PyUnicodeUCS4_Encode
 265 # define PyUnicode_EncodeASCII PyUnicodeUCS4_EncodeASCII
 266 # define PyUnicode_EncodeCharmap PyUnicodeUCS4_EncodeCharmap
 267 # define PyUnicode_EncodeDecimal PyUnicodeUCS4_EncodeDecimal
 268 # define PyUnicode_EncodeLatin1 PyUnicodeUCS4_EncodeLatin1
 269 # define PyUnicode_EncodeRawUnicodeEscape PyUnicodeUCS4_EncodeRawUnicodeEscape
 270 # define PyUnicode_EncodeUTF32 PyUnicodeUCS4_EncodeUTF32
 271 # define PyUnicode_EncodeUTF16 PyUnicodeUCS4_EncodeUTF16
 272 # define PyUnicode_EncodeUTF8 PyUnicodeUCS4_EncodeUTF8
 273 # define PyUnicode_EncodeUnicodeEscape PyUnicodeUCS4_EncodeUnicodeEscape
 274 # define PyUnicode_Find PyUnicodeUCS4_Find
 275 # define PyUnicode_Format PyUnicodeUCS4_Format
 276 # define PyUnicode_FromEncodedObject PyUnicodeUCS4_FromEncodedObject
 277 # define PyUnicode_FromFormat PyUnicodeUCS4_FromFormat
 278 # define PyUnicode_FromFormatV PyUnicodeUCS4_FromFormatV
 279 # define PyUnicode_FromObject PyUnicodeUCS4_FromObject
 280 # define PyUnicode_FromOrdinal PyUnicodeUCS4_FromOrdinal
 281 # define PyUnicode_FromString PyUnicodeUCS4_FromString
 282 # define PyUnicode_FromStringAndSize PyUnicodeUCS4_FromStringAndSize
 283 # define PyUnicode_FromUnicode PyUnicodeUCS4_FromUnicode
 284 # define PyUnicode_FromWideChar PyUnicodeUCS4_FromWideChar
 285 # define PyUnicode_GetDefaultEncoding PyUnicodeUCS4_GetDefaultEncoding
 286 # define PyUnicode_GetMax PyUnicodeUCS4_GetMax
 287 # define PyUnicode_GetSize PyUnicodeUCS4_GetSize
 288 # define PyUnicode_Join PyUnicodeUCS4_Join
 289 # define PyUnicode_Partition PyUnicodeUCS4_Partition
 290 # define PyUnicode_RPartition PyUnicodeUCS4_RPartition
 291 # define PyUnicode_RSplit PyUnicodeUCS4_RSplit
 292 # define PyUnicode_Replace PyUnicodeUCS4_Replace
 293 # define PyUnicode_Resize PyUnicodeUCS4_Resize
 294 # define PyUnicode_RichCompare PyUnicodeUCS4_RichCompare
 295 # define PyUnicode_SetDefaultEncoding PyUnicodeUCS4_SetDefaultEncoding
 296 # define PyUnicode_Split PyUnicodeUCS4_Split
 297 # define PyUnicode_Splitlines PyUnicodeUCS4_Splitlines
 298 # define PyUnicode_Tailmatch PyUnicodeUCS4_Tailmatch
 299 # define PyUnicode_Translate PyUnicodeUCS4_Translate
 300 # define PyUnicode_TranslateCharmap PyUnicodeUCS4_TranslateCharmap
 301 # define _PyUnicode_AsDefaultEncodedString _PyUnicodeUCS4_AsDefaultEncodedString
 302 # define _PyUnicode_Fini _PyUnicodeUCS4_Fini
 303 # define _PyUnicode_Init _PyUnicodeUCS4_Init
 304 # define _PyUnicode_IsAlpha _PyUnicodeUCS4_IsAlpha
 305 # define _PyUnicode_IsDecimalDigit _PyUnicodeUCS4_IsDecimalDigit
 306 # define _PyUnicode_IsDigit _PyUnicodeUCS4_IsDigit
 307 # define _PyUnicode_IsLinebreak _PyUnicodeUCS4_IsLinebreak
 308 # define _PyUnicode_IsLowercase _PyUnicodeUCS4_IsLowercase
 309 # define _PyUnicode_IsNumeric _PyUnicodeUCS4_IsNumeric
 310 # define _PyUnicode_IsTitlecase _PyUnicodeUCS4_IsTitlecase
 311 # define _PyUnicode_IsUppercase _PyUnicodeUCS4_IsUppercase
 312 # define _PyUnicode_IsWhitespace _PyUnicodeUCS4_IsWhitespace
 313 # define _PyUnicode_ToDecimalDigit _PyUnicodeUCS4_ToDecimalDigit
 314 # define _PyUnicode_ToDigit _PyUnicodeUCS4_ToDigit
 315 # define _PyUnicode_ToLowercase _PyUnicodeUCS4_ToLowercase
 316 # define _PyUnicode_ToNumeric _PyUnicodeUCS4_ToNumeric
 317 # define _PyUnicode_ToTitlecase _PyUnicodeUCS4_ToTitlecase
 318 # define _PyUnicode_ToUppercase _PyUnicodeUCS4_ToUppercase
 319
 320
 321 #endif
 322
 323 /* --- Internal Unicode Operations ---------------------------------------- */
 324
 325 /* If you want Python to use the compiler's wctype.h functions instead
 326    of the ones supplied with Python, define WANT_WCTYPE_FUNCTIONS or
 327    configure Python using --with-wctype-functions.  This reduces the
 328    interpreter's code size. */
 329
 330 #if defined(HAVE_USABLE_WCHAR_T) && defined(WANT_WCTYPE_FUNCTIONS)
 331
 332 #include <wctype.h>
 333
 334 #define Py_UNICODE_ISSPACE(ch) iswspace(ch)
 335
 336 #define Py_UNICODE_ISLOWER(ch) iswlower(ch)
 337 #define Py_UNICODE_ISUPPER(ch) iswupper(ch)
 338 #define Py_UNICODE_ISTITLE(ch) _PyUnicode_IsTitlecase(ch)
 339 #define Py_UNICODE_ISLINEBREAK(ch) _PyUnicode_IsLinebreak(ch)
 340
 341 #define Py_UNICODE_TOLOWER(ch) towlower(ch)
 342 #define Py_UNICODE_TOUPPER(ch) towupper(ch)
 343 #define Py_UNICODE_TOTITLE(ch) _PyUnicode_ToTitlecase(ch)
 344
 345 #define Py_UNICODE_ISDECIMAL(ch) _PyUnicode_IsDecimalDigit(ch)
 346 #define Py_UNICODE_ISDIGIT(ch) _PyUnicode_IsDigit(ch)
 347 #define Py_UNICODE_ISNUMERIC(ch) _PyUnicode_IsNumeric(ch)
 348
 349 #define Py_UNICODE_TODECIMAL(ch) _PyUnicode_ToDecimalDigit(ch)
 350 #define Py_UNICODE_TODIGIT(ch) _PyUnicode_ToDigit(ch)
 351 #define Py_UNICODE_TONUMERIC(ch) _PyUnicode_ToNumeric(ch)
 352
 353 #define Py_UNICODE_ISALPHA(ch) iswalpha(ch)
 354
 355 #else
 356
 357 /* Since splitting on whitespace is an important use case, and
 358    whitespace in most situations is solely ASCII whitespace, we
 359    optimize for the common case by using a quick look-up table
 360    _Py_ascii_whitespace (see below) with an inlined check.
 361
 362  */
 363 #define Py_UNICODE_ISSPACE(ch) \
 364         ((ch) < 128U ? _Py_ascii_whitespace[(ch)] : _PyUnicode_IsWhitespace(ch))
 365
 366 #define Py_UNICODE_ISLOWER(ch) _PyUnicode_IsLowercase(ch)
 367 #define Py_UNICODE_ISUPPER(ch) _PyUnicode_IsUppercase(ch)
 368 #define Py_UNICODE_ISTITLE(ch) _PyUnicode_IsTitlecase(ch)
 369 #define Py_UNICODE_ISLINEBREAK(ch) _PyUnicode_IsLinebreak(ch)
 370
 371 #define Py_UNICODE_TOLOWER(ch) _PyUnicode_ToLowercase(ch)
 372 #define Py_UNICODE_TOUPPER(ch) _PyUnicode_ToUppercase(ch)
 373 #define Py_UNICODE_TOTITLE(ch) _PyUnicode_ToTitlecase(ch)
 374
 375 #define Py_UNICODE_ISDECIMAL(ch) _PyUnicode_IsDecimalDigit(ch)
 376 #define Py_UNICODE_ISDIGIT(ch) _PyUnicode_IsDigit(ch)
 377 #define Py_UNICODE_ISNUMERIC(ch) _PyUnicode_IsNumeric(ch)
 378
 379 #define Py_UNICODE_TODECIMAL(ch) _PyUnicode_ToDecimalDigit(ch)
 380 #define Py_UNICODE_TODIGIT(ch) _PyUnicode_ToDigit(ch)
 381 #define Py_UNICODE_TONUMERIC(ch) _PyUnicode_ToNumeric(ch)
 382
 383 #define Py_UNICODE_ISALPHA(ch) _PyUnicode_IsAlpha(ch)
 384
 385 #endif
 386
 387 #define Py_UNICODE_ISALNUM(ch) \
 388        (Py_UNICODE_ISALPHA(ch) || \
 389         Py_UNICODE_ISDECIMAL(ch) || \
 390         Py_UNICODE_ISDIGIT(ch) || \
 391         Py_UNICODE_ISNUMERIC(ch))
 392
 393 #define Py_UNICODE_COPY(target, source, length)                         \
 394         Py_MEMCPY((target), (source), (length)*sizeof(Py_UNICODE))
 395
 396 #define Py_UNICODE_FILL(target, value, length) \
 397     do {Py_ssize_t i_; Py_UNICODE *t_ = (target); Py_UNICODE v_ = (value);\
 398         for (i_ = 0; i_ < (length); i_++) t_[i_] = v_;\
 399     } while (0)
 400
 401 /* Check if substring matches at given offset.  the offset must be
 402    valid, and the substring must not be empty */
 403
 404 #define Py_UNICODE_MATCH(string, offset, substring) \
 405     ((*((string)->str + (offset)) == *((substring)->str)) && \
 406     ((*((string)->str + (offset) + (substring)->length-1) == *((substring)->str + (substring)->length-1))) && \
 407      !memcmp((string)->str + (offset), (substring)->str, (substring)->length*sizeof(Py_UNICODE)))
 408
 409 #ifdef __cplusplus
 410 extern "C" {
 411 #endif
 412
 413 /* --- Unicode Type ------------------------------------------------------- */
 414
 415 typedef struct {
 416     PyObject_HEAD
 417     Py_ssize_t length;          /* Length of raw Unicode data in buffer */
 418     Py_UNICODE *str;            /* Raw Unicode buffer */
 419     long hash;                  /* Hash value; -1 if not set */
 420     PyObject *defenc;           /* (Default) Encoded version as Python
 421                                    string, or NULL; this is used for
 422                                    implementing the buffer protocol */
 423 } PyUnicodeObject;
 424
 425 PyAPI_DATA(PyTypeObject) PyUnicode_Type;
 426
 427 #define PyUnicode_Check(op) \
 428                  PyType_FastSubclass(Py_TYPE(op), Py_TPFLAGS_UNICODE_SUBCLASS)
 429 #define PyUnicode_CheckExact(op) (Py_TYPE(op) == &PyUnicode_Type)
 430
 431 /* Fast access macros */
 432 #define PyUnicode_GET_SIZE(op) \
 433         (((PyUnicodeObject *)(op))->length)
 434 #define PyUnicode_GET_DATA_SIZE(op) \
 435         (((PyUnicodeObject *)(op))->length * sizeof(Py_UNICODE))
 436 #define PyUnicode_AS_UNICODE(op) \
 437         (((PyUnicodeObject *)(op))->str)
 438 #define PyUnicode_AS_DATA(op) \
 439         ((const char *)((PyUnicodeObject *)(op))->str)
 440
 441 /* --- Constants ---------------------------------------------------------- */
 442
 443 /* This Unicode character will be used as replacement character during
 444    decoding if the errors argument is set to "replace". Note: the
 445    Unicode character U+FFFD is the official REPLACEMENT CHARACTER in
 446    Unicode 3.0. */
 447
 448 #define Py_UNICODE_REPLACEMENT_CHARACTER ((Py_UNICODE) 0xFFFD)
 449
 450 /* === Public API ========================================================= */
 451
 452 /* --- Plain Py_UNICODE --------------------------------------------------- */
 453
 454 /* Create a Unicode Object from the Py_UNICODE buffer u of the given
 455    size.
 456
 457    u may be NULL which causes the contents to be undefined. It is the
 458    user's responsibility to fill in the needed data afterwards. Note
 459    that modifying the Unicode object contents after construction is
 460    only allowed if u was set to NULL.
 461
 462    The buffer is copied into the new object. */
 463
 464 PyAPI_FUNC(PyObject*) PyUnicode_FromUnicode(
 465     const Py_UNICODE *u,        /* Unicode buffer */
 466     Py_ssize_t size             /* size of buffer */
 467     );
 468
 469 /* Similar to PyUnicode_FromUnicode(), but u points to Latin-1 encoded bytes */
 470 PyAPI_FUNC(PyObject*) PyUnicode_FromStringAndSize(
 471     const char *u,        /* char buffer */
 472     Py_ssize_t size       /* size of buffer */
 473     );
 474
 475 /* Similar to PyUnicode_FromUnicode(), but u points to null-terminated
 476    Latin-1 encoded bytes */
 477 PyAPI_FUNC(PyObject*) PyUnicode_FromString(
 478     const char *u        /* string */
 479     );
 480
 481 /* Return a read-only pointer to the Unicode object's internal
 482    Py_UNICODE buffer. */
 483
 484 PyAPI_FUNC(Py_UNICODE *) PyUnicode_AsUnicode(
 485     PyObject *unicode           /* Unicode object */
 486     );
 487
 488 /* Get the length of the Unicode object. */
 489
 490 PyAPI_FUNC(Py_ssize_t) PyUnicode_GetSize(
 491     PyObject *unicode           /* Unicode object */
 492     );
 493
 494 /* Get the maximum ordinal for a Unicode character. */
 495 PyAPI_FUNC(Py_UNICODE) PyUnicode_GetMax(void);
 496
 497 /* Resize an already allocated Unicode object to the new size length.
 498
 499    *unicode is modified to point to the new (resized) object and 0
 500    returned on success.
 501
 502    This API may only be called by the function which also called the
 503    Unicode constructor. The refcount on the object must be 1. Otherwise,
 504    an error is returned.
 505
 506    Error handling is implemented as follows: an exception is set, -1
 507    is returned and *unicode left untouched.
 508
 509 */
 510
 511 PyAPI_FUNC(int) PyUnicode_Resize(
 512     PyObject **unicode,         /* Pointer to the Unicode object */
 513     Py_ssize_t length           /* New length */
 514     );
 515
 516 /* Coerce obj to an Unicode object and return a reference with
 517    *incremented* refcount.
 518
 519    Coercion is done in the following way:
 520
 521    1. String and other char buffer compatible objects are decoded
 522       under the assumptions that they contain data using the current
 523       default encoding. Decoding is done in "strict" mode.
 524
 525    2. All other objects (including Unicode objects) raise an
 526       exception.
 527
 528    The API returns NULL in case of an error. The caller is responsible
 529    for decref'ing the returned objects.
 530
 531 */
 532
 533 PyAPI_FUNC(PyObject*) PyUnicode_FromEncodedObject(
 534     register PyObject *obj,     /* Object */
 535     const char *encoding,       /* encoding */
 536     const char *errors          /* error handling */
 537     );
 538
 539 /* Coerce obj to an Unicode object and return a reference with
 540    *incremented* refcount.
 541
 542    Unicode objects are passed back as-is (subclasses are converted to
 543    true Unicode objects), all other objects are delegated to
 544    PyUnicode_FromEncodedObject(obj, NULL, "strict") which results in
 545    using the default encoding as basis for decoding the object.
 546
 547    The API returns NULL in case of an error. The caller is responsible
 548    for decref'ing the returned objects.
 549
 550 */
 551
 552 PyAPI_FUNC(PyObject*) PyUnicode_FromObject(
 553     register PyObject *obj      /* Object */
 554     );
 555
 556 PyAPI_FUNC(PyObject *) PyUnicode_FromFormatV(const char*, va_list);
 557 PyAPI_FUNC(PyObject *) PyUnicode_FromFormat(const char*, ...);
 558
 559 /* Format the object based on the format_spec, as defined in PEP 3101
 560    (Advanced String Formatting). */
 561 PyAPI_FUNC(PyObject *) _PyUnicode_FormatAdvanced(PyObject *obj,
 562                                                  Py_UNICODE *format_spec,
 563                                                  Py_ssize_t format_spec_len);
 564
 565 /* --- wchar_t support for platforms which support it --------------------- */
 566
 567 #ifdef HAVE_WCHAR_H
 568
 569 /* Create a Unicode Object from the whcar_t buffer w of the given
 570    size.
 571
 572    The buffer is copied into the new object. */
 573
 574 PyAPI_FUNC(PyObject*) PyUnicode_FromWideChar(
 575     register const wchar_t *w,  /* wchar_t buffer */
 576     Py_ssize_t size             /* size of buffer */
 577     );
 578
 579 /* Copies the Unicode Object contents into the wchar_t buffer w.  At
 580    most size wchar_t characters are copied.
 581
 582    Note that the resulting wchar_t string may or may not be
 583    0-terminated.  It is the responsibility of the caller to make sure
 584    that the wchar_t string is 0-terminated in case this is required by
 585    the application.
 586
 587    Returns the number of wchar_t characters copied (excluding a
 588    possibly trailing 0-termination character) or -1 in case of an
 589    error. */
 590
 591 PyAPI_FUNC(Py_ssize_t) PyUnicode_AsWideChar(
 592     PyUnicodeObject *unicode,   /* Unicode object */
 593     register wchar_t *w,        /* wchar_t buffer */
 594     Py_ssize_t size             /* size of buffer */
 595     );
 596
 597 #endif
 598
 599 /* --- Unicode ordinals --------------------------------------------------- */
 600
 601 /* Create a Unicode Object from the given Unicode code point ordinal.
 602
 603    The ordinal must be in range(0x10000) on narrow Python builds
 604    (UCS2), and range(0x110000) on wide builds (UCS4). A ValueError is
 605    raised in case it is not.
 606
 607 */
 608
 609 PyAPI_FUNC(PyObject*) PyUnicode_FromOrdinal(int ordinal);
 610
 611 /* --- Free-list management ----------------------------------------------- */
 612
 613 /* Clear the free list used by the Unicode implementation.
 614
 615    This can be used to release memory used for objects on the free
 616    list back to the Python memory allocator.
 617
 618 */
 619
 620 PyAPI_FUNC(int) PyUnicode_ClearFreeList(void);
 621
 622 /* === Builtin Codecs =====================================================
 623
 624    Many of these APIs take two arguments encoding and errors. These
 625    parameters encoding and errors have the same semantics as the ones
 626    of the builtin unicode() API.
 627
 628    Setting encoding to NULL causes the default encoding to be used.
 629
 630    Error handling is set by errors which may also be set to NULL
 631    meaning to use the default handling defined for the codec. Default
 632    error handling for all builtin codecs is "strict" (ValueErrors are
 633    raised).
 634
 635    The codecs all use a similar interface. Only deviation from the
 636    generic ones are documented.
 637
 638 */
 639
 640 /* --- Manage the default encoding ---------------------------------------- */
 641
 642 /* Return a Python string holding the default encoded value of the
 643    Unicode object.
 644
 645    The resulting string is cached in the Unicode object for subsequent
 646    usage by this function. The cached version is needed to implement
 647    the character buffer interface and will live (at least) as long as
 648    the Unicode object itself.
 649
 650    The refcount of the string is *not* incremented.
 651
 652    *** Exported for internal use by the interpreter only !!! ***
 653
 654 */
 655
 656 PyAPI_FUNC(PyObject *) _PyUnicode_AsDefaultEncodedString(
 657     PyObject *, const char *);
 658
 659 /* Returns the currently active default encoding.
 660
 661    The default encoding is currently implemented as run-time settable
 662    process global.  This may change in future versions of the
 663    interpreter to become a parameter which is managed on a per-thread
 664    basis.
 665
 666  */
 667
 668 PyAPI_FUNC(const char*) PyUnicode_GetDefaultEncoding(void);
 669
 670 /* Sets the currently active default encoding.
 671
 672    Returns 0 on success, -1 in case of an error.
 673
 674  */
 675
 676 PyAPI_FUNC(int) PyUnicode_SetDefaultEncoding(
 677     const char *encoding        /* Encoding name in standard form */
 678     );
 679
 680 /* --- Generic Codecs ----------------------------------------------------- */
 681
 682 /* Create a Unicode object by decoding the encoded string s of the
 683    given size. */
 684
 685 PyAPI_FUNC(PyObject*) PyUnicode_Decode(
 686     const char *s,              /* encoded string */
 687     Py_ssize_t size,            /* size of buffer */
 688     const char *encoding,       /* encoding */
 689     const char *errors          /* error handling */
 690     );
 691
 692 /* Encodes a Py_UNICODE buffer of the given size and returns a
 693    Python string object. */
 694
 695 PyAPI_FUNC(PyObject*) PyUnicode_Encode(
 696     const Py_UNICODE *s,        /* Unicode char buffer */
 697     Py_ssize_t size,            /* number of Py_UNICODE chars to encode */
 698     const char *encoding,       /* encoding */
 699     const char *errors          /* error handling */
 700     );
 701
 702 /* Encodes a Unicode object and returns the result as Python
 703    object. */
 704
 705 PyAPI_FUNC(PyObject*) PyUnicode_AsEncodedObject(
 706     PyObject *unicode,          /* Unicode object */
 707     const char *encoding,       /* encoding */
 708     const char *errors          /* error handling */
 709     );
 710
 711 /* Encodes a Unicode object and returns the result as Python string
 712    object. */
 713
 714 PyAPI_FUNC(PyObject*) PyUnicode_AsEncodedString(
 715     PyObject *unicode,          /* Unicode object */
 716     const char *encoding,       /* encoding */
 717     const char *errors          /* error handling */
 718     );
 719
 720 PyAPI_FUNC(PyObject*) PyUnicode_BuildEncodingMap(
 721     PyObject* string            /* 256 character map */
 722    );
 723
 724
 725 /* --- UTF-7 Codecs ------------------------------------------------------- */
 726
 727 PyAPI_FUNC(PyObject*) PyUnicode_DecodeUTF7(
 728     const char *string,         /* UTF-7 encoded string */
 729     Py_ssize_t length,          /* size of string */
 730     const char *errors          /* error handling */
 731     );
 732
 733 PyAPI_FUNC(PyObject*) PyUnicode_DecodeUTF7Stateful(
 734     const char *string,         /* UTF-7 encoded string */
 735     Py_ssize_t length,          /* size of string */
 736     const char *errors,         /* error handling */
 737     Py_ssize_t *consumed        /* bytes consumed */
 738     );
 739
 740 PyAPI_FUNC(PyObject*) PyUnicode_EncodeUTF7(
 741     const Py_UNICODE *data,     /* Unicode char buffer */
 742     Py_ssize_t length,                  /* number of Py_UNICODE chars to encode */
 743     int base64SetO,             /* Encode RFC2152 Set O characters in base64 */
 744     int base64WhiteSpace,       /* Encode whitespace (sp, ht, nl, cr) in base64 */
 745     const char *errors          /* error handling */
 746     );
 747
 748 /* --- UTF-8 Codecs ------------------------------------------------------- */
 749
 750 PyAPI_FUNC(PyObject*) PyUnicode_DecodeUTF8(
 751     const char *string,         /* UTF-8 encoded string */
 752     Py_ssize_t length,          /* size of string */
 753     const char *errors          /* error handling */
 754     );
 755
 756 PyAPI_FUNC(PyObject*) PyUnicode_DecodeUTF8Stateful(
 757     const char *string,         /* UTF-8 encoded string */
 758     Py_ssize_t length,          /* size of string */
 759     const char *errors,         /* error handling */
 760     Py_ssize_t *consumed                /* bytes consumed */
 761     );
 762
 763 PyAPI_FUNC(PyObject*) PyUnicode_AsUTF8String(
 764     PyObject *unicode           /* Unicode object */
 765     );
 766
 767 PyAPI_FUNC(PyObject*) PyUnicode_EncodeUTF8(
 768     const Py_UNICODE *data,     /* Unicode char buffer */
 769     Py_ssize_t length,                  /* number of Py_UNICODE chars to encode */
 770     const char *errors          /* error handling */
 771     );
 772
 773 /* --- UTF-32 Codecs ------------------------------------------------------ */
 774
 775 /* Decodes length bytes from a UTF-32 encoded buffer string and returns
 776    the corresponding Unicode object.
 777
 778    errors (if non-NULL) defines the error handling. It defaults
 779    to "strict".
 780
 781    If byteorder is non-NULL, the decoder starts decoding using the
 782    given byte order:
 783
 784         *byteorder == -1: little endian
 785         *byteorder == 0:  native order
 786         *byteorder == 1:  big endian
 787
 788    In native mode, the first four bytes of the stream are checked for a
 789    BOM mark. If found, the BOM mark is analysed, the byte order
 790    adjusted and the BOM skipped.  In the other modes, no BOM mark
 791    interpretation is done. After completion, *byteorder is set to the
 792    current byte order at the end of input data.
 793
 794    If byteorder is NULL, the codec starts in native order mode.
 795
 796 */
 797
 798 PyAPI_FUNC(PyObject*) PyUnicode_DecodeUTF32(
 799     const char *string,         /* UTF-32 encoded string */
 800     Py_ssize_t length,          /* size of string */
 801     const char *errors,         /* error handling */
 802     int *byteorder              /* pointer to byteorder to use
 803                                    0=native;-1=LE,1=BE; updated on
 804                                    exit */
 805     );
 806
 807 PyAPI_FUNC(PyObject*) PyUnicode_DecodeUTF32Stateful(
 808     const char *string,         /* UTF-32 encoded string */
 809     Py_ssize_t length,          /* size of string */
 810     const char *errors,         /* error handling */
 811     int *byteorder,             /* pointer to byteorder to use
 812                                    0=native;-1=LE,1=BE; updated on
 813                                    exit */
 814     Py_ssize_t *consumed        /* bytes consumed */
 815     );
 816
 817 /* Returns a Python string using the UTF-32 encoding in native byte
 818    order. The string always starts with a BOM mark.  */
 819
 820 PyAPI_FUNC(PyObject*) PyUnicode_AsUTF32String(
 821     PyObject *unicode           /* Unicode object */
 822     );
 823
 824 /* Returns a Python string object holding the UTF-32 encoded value of
 825    the Unicode data.
 826
 827    If byteorder is not 0, output is written according to the following
 828    byte order:
 829
 830    byteorder == -1: little endian
 831    byteorder == 0:  native byte order (writes a BOM mark)
 832    byteorder == 1:  big endian
 833
 834    If byteorder is 0, the output string will always start with the
 835    Unicode BOM mark (U+FEFF). In the other two modes, no BOM mark is
 836    prepended.
 837
 838 */
 839
 840 PyAPI_FUNC(PyObject*) PyUnicode_EncodeUTF32(
 841     const Py_UNICODE *data,     /* Unicode char buffer */
 842     Py_ssize_t length,          /* number of Py_UNICODE chars to encode */
 843     const char *errors,         /* error handling */
 844     int byteorder               /* byteorder to use 0=BOM+native;-1=LE,1=BE */
 845     );
 846
 847 /* --- UTF-16 Codecs ------------------------------------------------------ */
 848
 849 /* Decodes length bytes from a UTF-16 encoded buffer string and returns
 850    the corresponding Unicode object.
 851
 852    errors (if non-NULL) defines the error handling. It defaults
 853    to "strict".
 854
 855    If byteorder is non-NULL, the decoder starts decoding using the
 856    given byte order:
 857
 858         *byteorder == -1: little endian
 859         *byteorder == 0:  native order
 860         *byteorder == 1:  big endian
 861
 862    In native mode, the first two bytes of the stream are checked for a
 863    BOM mark. If found, the BOM mark is analysed, the byte order
 864    adjusted and the BOM skipped.  In the other modes, no BOM mark
 865    interpretation is done. After completion, *byteorder is set to the
 866    current byte order at the end of input data.
 867
 868    If byteorder is NULL, the codec starts in native order mode.
 869
 870 */
 871
 872 PyAPI_FUNC(PyObject*) PyUnicode_DecodeUTF16(
 873     const char *string,         /* UTF-16 encoded string */
 874     Py_ssize_t length,          /* size of string */
 875     const char *errors,         /* error handling */
 876     int *byteorder              /* pointer to byteorder to use
 877                                    0=native;-1=LE,1=BE; updated on
 878                                    exit */
 879     );
 880
 881 PyAPI_FUNC(PyObject*) PyUnicode_DecodeUTF16Stateful(
 882     const char *string,         /* UTF-16 encoded string */
 883     Py_ssize_t length,          /* size of string */
 884     const char *errors,         /* error handling */
 885     int *byteorder,             /* pointer to byteorder to use
 886                                    0=native;-1=LE,1=BE; updated on
 887                                    exit */
 888     Py_ssize_t *consumed                /* bytes consumed */
 889     );
 890
 891 /* Returns a Python string using the UTF-16 encoding in native byte
 892    order. The string always starts with a BOM mark.  */
 893
 894 PyAPI_FUNC(PyObject*) PyUnicode_AsUTF16String(
 895     PyObject *unicode           /* Unicode object */
 896     );
 897
 898 /* Returns a Python string object holding the UTF-16 encoded value of
 899    the Unicode data.
 900
 901    If byteorder is not 0, output is written according to the following
 902    byte order:
 903
 904    byteorder == -1: little endian
 905    byteorder == 0:  native byte order (writes a BOM mark)
 906    byteorder == 1:  big endian
 907
 908    If byteorder is 0, the output string will always start with the
 909    Unicode BOM mark (U+FEFF). In the other two modes, no BOM mark is
 910    prepended.
 911
 912    Note that Py_UNICODE data is being interpreted as UTF-16 reduced to
 913    UCS-2. This trick makes it possible to add full UTF-16 capabilities
 914    at a later point without compromising the APIs.
 915
 916 */
 917
 918 PyAPI_FUNC(PyObject*) PyUnicode_EncodeUTF16(
 919     const Py_UNICODE *data,     /* Unicode char buffer */
 920     Py_ssize_t length,                  /* number of Py_UNICODE chars to encode */
 921     const char *errors,         /* error handling */
 922     int byteorder               /* byteorder to use 0=BOM+native;-1=LE,1=BE */
 923     );
 924
 925 /* --- Unicode-Escape Codecs ---------------------------------------------- */
 926
 927 PyAPI_FUNC(PyObject*) PyUnicode_DecodeUnicodeEscape(
 928     const char *string,         /* Unicode-Escape encoded string */
 929     Py_ssize_t length,          /* size of string */
 930     const char *errors          /* error handling */
 931     );
 932
 933 PyAPI_FUNC(PyObject*) PyUnicode_AsUnicodeEscapeString(
 934     PyObject *unicode           /* Unicode object */
 935     );
 936
 937 PyAPI_FUNC(PyObject*) PyUnicode_EncodeUnicodeEscape(
 938     const Py_UNICODE *data,     /* Unicode char buffer */
 939     Py_ssize_t length                   /* Number of Py_UNICODE chars to encode */
 940     );
 941
 942 /* --- Raw-Unicode-Escape Codecs ------------------------------------------ */
 943
 944 PyAPI_FUNC(PyObject*) PyUnicode_DecodeRawUnicodeEscape(
 945     const char *string,         /* Raw-Unicode-Escape encoded string */
 946     Py_ssize_t length,          /* size of string */
 947     const char *errors          /* error handling */
 948     );
 949
 950 PyAPI_FUNC(PyObject*) PyUnicode_AsRawUnicodeEscapeString(
 951     PyObject *unicode           /* Unicode object */
 952     );
 953
 954 PyAPI_FUNC(PyObject*) PyUnicode_EncodeRawUnicodeEscape(
 955     const Py_UNICODE *data,     /* Unicode char buffer */
 956     Py_ssize_t length                   /* Number of Py_UNICODE chars to encode */
 957     );
 958
 959 /* --- Unicode Internal Codec ---------------------------------------------
 960
 961     Only for internal use in _codecsmodule.c */
 962
 963 PyObject *_PyUnicode_DecodeUnicodeInternal(
 964     const char *string,
 965     Py_ssize_t length,
 966     const char *errors
 967     );
 968
 969 /* --- Latin-1 Codecs -----------------------------------------------------
 970
 971    Note: Latin-1 corresponds to the first 256 Unicode ordinals.
 972
 973 */
 974
 975 PyAPI_FUNC(PyObject*) PyUnicode_DecodeLatin1(
 976     const char *string,         /* Latin-1 encoded string */
 977     Py_ssize_t length,          /* size of string */
 978     const char *errors          /* error handling */
 979     );
 980
 981 PyAPI_FUNC(PyObject*) PyUnicode_AsLatin1String(
 982     PyObject *unicode           /* Unicode object */
 983     );
 984
 985 PyAPI_FUNC(PyObject*) PyUnicode_EncodeLatin1(
 986     const Py_UNICODE *data,     /* Unicode char buffer */
 987     Py_ssize_t length,                  /* Number of Py_UNICODE chars to encode */
 988     const char *errors          /* error handling */
 989     );
 990
 991 /* --- ASCII Codecs -------------------------------------------------------
 992
 993    Only 7-bit ASCII data is excepted. All other codes generate errors.
 994
 995 */
 996
 997 PyAPI_FUNC(PyObject*) PyUnicode_DecodeASCII(
 998     const char *string,         /* ASCII encoded string */
 999     Py_ssize_t length,          /* size of string */
1000     const char *errors          /* error handling */
1001     );
1002
1003 PyAPI_FUNC(PyObject*) PyUnicode_AsASCIIString(
1004     PyObject *unicode           /* Unicode object */
1005     );
1006
1007 PyAPI_FUNC(PyObject*) PyUnicode_EncodeASCII(
1008     const Py_UNICODE *data,     /* Unicode char buffer */
1009     Py_ssize_t length,                  /* Number of Py_UNICODE chars to encode */
1010     const char *errors          /* error handling */
1011     );
1012
1013 /* --- Character Map Codecs -----------------------------------------------
1014
1015    This codec uses mappings to encode and decode characters.
1016
1017    Decoding mappings must map single string characters to single
1018    Unicode characters, integers (which are then interpreted as Unicode
1019    ordinals) or None (meaning "undefined mapping" and causing an
1020    error).
1021
1022    Encoding mappings must map single Unicode characters to single
1023    string characters, integers (which are then interpreted as Latin-1
1024    ordinals) or None (meaning "undefined mapping" and causing an
1025    error).
1026
1027    If a character lookup fails with a LookupError, the character is
1028    copied as-is meaning that its ordinal value will be interpreted as
1029    Unicode or Latin-1 ordinal resp. Because of this mappings only need
1030    to contain those mappings which map characters to different code
1031    points.
1032
1033 */
1034
1035 PyAPI_FUNC(PyObject*) PyUnicode_DecodeCharmap(
1036     const char *string,         /* Encoded string */
1037     Py_ssize_t length,          /* size of string */
1038     PyObject *mapping,          /* character mapping
1039                                    (char ordinal -> unicode ordinal) */
1040     const char *errors          /* error handling */
1041     );
1042
1043 PyAPI_FUNC(PyObject*) PyUnicode_AsCharmapString(
1044     PyObject *unicode,          /* Unicode object */
1045     PyObject *mapping           /* character mapping
1046                                    (unicode ordinal -> char ordinal) */
1047     );
1048
1049 PyAPI_FUNC(PyObject*) PyUnicode_EncodeCharmap(
1050     const Py_UNICODE *data,     /* Unicode char buffer */
1051     Py_ssize_t length,          /* Number of Py_UNICODE chars to encode */
1052     PyObject *mapping,          /* character mapping
1053                                    (unicode ordinal -> char ordinal) */
1054     const char *errors          /* error handling */
1055     );
1056
1057 /* Translate a Py_UNICODE buffer of the given length by applying a
1058    character mapping table to it and return the resulting Unicode
1059    object.
1060
1061    The mapping table must map Unicode ordinal integers to Unicode
1062    ordinal integers or None (causing deletion of the character).
1063
1064    Mapping tables may be dictionaries or sequences. Unmapped character
1065    ordinals (ones which cause a LookupError) are left untouched and
1066    are copied as-is.
1067
1068 */
1069
1070 PyAPI_FUNC(PyObject *) PyUnicode_TranslateCharmap(
1071     const Py_UNICODE *data,     /* Unicode char buffer */
1072     Py_ssize_t length,                  /* Number of Py_UNICODE chars to encode */
1073     PyObject *table,            /* Translate table */
1074     const char *errors          /* error handling */
1075     );
1076
1077 #ifdef MS_WIN32
1078
1079 /* --- MBCS codecs for Windows -------------------------------------------- */
1080
1081 PyAPI_FUNC(PyObject*) PyUnicode_DecodeMBCS(
1082     const char *string,         /* MBCS encoded string */
1083     Py_ssize_t length,              /* size of string */
1084     const char *errors          /* error handling */
1085     );
1086
1087 PyAPI_FUNC(PyObject*) PyUnicode_DecodeMBCSStateful(
1088     const char *string,         /* MBCS encoded string */
1089     Py_ssize_t length,          /* size of string */
1090     const char *errors,         /* error handling */
1091     Py_ssize_t *consumed        /* bytes consumed */
1092     );
1093
1094 PyAPI_FUNC(PyObject*) PyUnicode_AsMBCSString(
1095     PyObject *unicode           /* Unicode object */
1096     );
1097
1098 PyAPI_FUNC(PyObject*) PyUnicode_EncodeMBCS(
1099     const Py_UNICODE *data,     /* Unicode char buffer */
1100     Py_ssize_t length,              /* Number of Py_UNICODE chars to encode */
1101     const char *errors          /* error handling */
1102     );
1103
1104 #endif /* MS_WIN32 */
1105
1106 /* --- Decimal Encoder ---------------------------------------------------- */
1107
1108 /* Takes a Unicode string holding a decimal value and writes it into
1109    an output buffer using standard ASCII digit codes.
1110
1111    The output buffer has to provide at least length+1 bytes of storage
1112    area. The output string is 0-terminated.
1113
1114    The encoder converts whitespace to ' ', decimal characters to their
1115    corresponding ASCII digit and all other Latin-1 characters except
1116    \0 as-is. Characters outside this range (Unicode ordinals 1-256)
1117    are treated as errors. This includes embedded NULL bytes.
1118
1119    Error handling is defined by the errors argument:
1120
1121       NULL or "strict": raise a ValueError
1122       "ignore": ignore the wrong characters (these are not copied to the
1123                 output buffer)
1124       "replace": replaces illegal characters with '?'
1125
1126    Returns 0 on success, -1 on failure.
1127
1128 */
1129
1130 PyAPI_FUNC(int) PyUnicode_EncodeDecimal(
1131     Py_UNICODE *s,              /* Unicode buffer */
1132     Py_ssize_t length,                  /* Number of Py_UNICODE chars to encode */
1133     char *output,               /* Output buffer; must have size >= length */
1134     const char *errors          /* error handling */
1135     );
1136
1137 /* --- Methods & Slots ----------------------------------------------------
1138
1139    These are capable of handling Unicode objects and strings on input
1140    (we refer to them as strings in the descriptions) and return
1141    Unicode objects or integers as apporpriate. */
1142
1143 /* Concat two strings giving a new Unicode string. */
1144
1145 PyAPI_FUNC(PyObject*) PyUnicode_Concat(
1146     PyObject *left,             /* Left string */
1147     PyObject *right             /* Right string */
1148     );
1149
1150 /* Split a string giving a list of Unicode strings.
1151
1152    If sep is NULL, splitting will be done at all whitespace
1153    substrings. Otherwise, splits occur at the given separator.
1154
1155    At most maxsplit splits will be done. If negative, no limit is set.
1156
1157    Separators are not included in the resulting list.
1158
1159 */
1160
1161 PyAPI_FUNC(PyObject*) PyUnicode_Split(
1162     PyObject *s,                /* String to split */
1163     PyObject *sep,              /* String separator */
1164     Py_ssize_t maxsplit         /* Maxsplit count */
1165     );
1166
1167 /* Dito, but split at line breaks.
1168
1169    CRLF is considered to be one line break. Line breaks are not
1170    included in the resulting list. */
1171
1172 PyAPI_FUNC(PyObject*) PyUnicode_Splitlines(
1173     PyObject *s,                /* String to split */
1174     int keepends                /* If true, line end markers are included */
1175     );
1176
1177 /* Partition a string using a given separator. */
1178
1179 PyAPI_FUNC(PyObject*) PyUnicode_Partition(
1180     PyObject *s,                /* String to partition */
1181     PyObject *sep               /* String separator */
1182     );
1183
1184 /* Partition a string using a given separator, searching from the end of the
1185    string. */
1186
1187 PyAPI_FUNC(PyObject*) PyUnicode_RPartition(
1188     PyObject *s,                /* String to partition */
1189     PyObject *sep               /* String separator */
1190     );
1191
1192 /* Split a string giving a list of Unicode strings.
1193
1194    If sep is NULL, splitting will be done at all whitespace
1195    substrings. Otherwise, splits occur at the given separator.
1196
1197    At most maxsplit splits will be done. But unlike PyUnicode_Split
1198    PyUnicode_RSplit splits from the end of the string. If negative,
1199    no limit is set.
1200
1201    Separators are not included in the resulting list.
1202
1203 */
1204
1205 PyAPI_FUNC(PyObject*) PyUnicode_RSplit(
1206     PyObject *s,                /* String to split */
1207     PyObject *sep,              /* String separator */
1208     Py_ssize_t maxsplit         /* Maxsplit count */
1209     );
1210
1211 /* Translate a string by applying a character mapping table to it and
1212    return the resulting Unicode object.
1213
1214    The mapping table must map Unicode ordinal integers to Unicode
1215    ordinal integers or None (causing deletion of the character).
1216
1217    Mapping tables may be dictionaries or sequences. Unmapped character
1218    ordinals (ones which cause a LookupError) are left untouched and
1219    are copied as-is.
1220
1221 */
1222
1223 PyAPI_FUNC(PyObject *) PyUnicode_Translate(
1224     PyObject *str,              /* String */
1225     PyObject *table,            /* Translate table */
1226     const char *errors          /* error handling */
1227     );
1228
1229 /* Join a sequence of strings using the given separator and return
1230    the resulting Unicode string. */
1231
1232 PyAPI_FUNC(PyObject*) PyUnicode_Join(
1233     PyObject *separator,        /* Separator string */
1234     PyObject *seq               /* Sequence object */
1235     );
1236
1237 /* Return 1 if substr matches str[start:end] at the given tail end, 0
1238    otherwise. */
1239
1240 PyAPI_FUNC(Py_ssize_t) PyUnicode_Tailmatch(
1241     PyObject *str,              /* String */
1242     PyObject *substr,           /* Prefix or Suffix string */
1243     Py_ssize_t start,           /* Start index */
1244     Py_ssize_t end,             /* Stop index */
1245     int direction               /* Tail end: -1 prefix, +1 suffix */
1246     );
1247
1248 /* Return the first position of substr in str[start:end] using the
1249    given search direction or -1 if not found. -2 is returned in case
1250    an error occurred and an exception is set. */
1251
1252 PyAPI_FUNC(Py_ssize_t) PyUnicode_Find(
1253     PyObject *str,              /* String */
1254     PyObject *substr,           /* Substring to find */
1255     Py_ssize_t start,           /* Start index */
1256     Py_ssize_t end,             /* Stop index */
1257     int direction               /* Find direction: +1 forward, -1 backward */
1258     );
1259
1260 /* Count the number of occurrences of substr in str[start:end]. */
1261
1262 PyAPI_FUNC(Py_ssize_t) PyUnicode_Count(
1263     PyObject *str,              /* String */
1264     PyObject *substr,           /* Substring to count */
1265     Py_ssize_t start,           /* Start index */
1266     Py_ssize_t end              /* Stop index */
1267     );
1268
1269 /* Replace at most maxcount occurrences of substr in str with replstr
1270    and return the resulting Unicode object. */
1271
1272 PyAPI_FUNC(PyObject *) PyUnicode_Replace(
1273     PyObject *str,              /* String */
1274     PyObject *substr,           /* Substring to find */
1275     PyObject *replstr,          /* Substring to replace */
1276     Py_ssize_t maxcount         /* Max. number of replacements to apply;
1277                                    -1 = all */
1278     );
1279
1280 /* Compare two strings and return -1, 0, 1 for less than, equal,
1281    greater than resp. */
1282
1283 PyAPI_FUNC(int) PyUnicode_Compare(
1284     PyObject *left,             /* Left string */
1285     PyObject *right             /* Right string */
1286     );
1287
1288 /* Rich compare two strings and return one of the following:
1289
1290    - NULL in case an exception was raised
1291    - Py_True or Py_False for successfuly comparisons
1292    - Py_NotImplemented in case the type combination is unknown
1293
1294    Note that Py_EQ and Py_NE comparisons can cause a UnicodeWarning in
1295    case the conversion of the arguments to Unicode fails with a
1296    UnicodeDecodeError.
1297
1298    Possible values for op:
1299
1300      Py_GT, Py_GE, Py_EQ, Py_NE, Py_LT, Py_LE
1301
1302 */
1303
1304 PyAPI_FUNC(PyObject *) PyUnicode_RichCompare(
1305     PyObject *left,             /* Left string */
1306     PyObject *right,            /* Right string */
1307     int op                      /* Operation: Py_EQ, Py_NE, Py_GT, etc. */
1308     );
1309
1310 /* Apply a argument tuple or dictionary to a format string and return
1311    the resulting Unicode string. */
1312
1313 PyAPI_FUNC(PyObject *) PyUnicode_Format(
1314     PyObject *format,           /* Format string */
1315     PyObject *args              /* Argument tuple or dictionary */
1316     );
1317
1318 /* Checks whether element is contained in container and return 1/0
1319    accordingly.
1320
1321    element has to coerce to an one element Unicode string. -1 is
1322    returned in case of an error. */
1323
1324 PyAPI_FUNC(int) PyUnicode_Contains(
1325     PyObject *container,        /* Container string */
1326     PyObject *element           /* Element string */
1327     );
1328
1329 /* Externally visible for str.strip(unicode) */
1330 PyAPI_FUNC(PyObject *) _PyUnicode_XStrip(
1331     PyUnicodeObject *self,
1332     int striptype,
1333     PyObject *sepobj
1334     );
1335
1336 /* === Characters Type APIs =============================================== */
1337
1338 /* Helper array used by Py_UNICODE_ISSPACE(). */
1339
1340 PyAPI_DATA(const unsigned char) _Py_ascii_whitespace[];
1341
1342 /* These should not be used directly. Use the Py_UNICODE_IS* and
1343    Py_UNICODE_TO* macros instead.
1344
1345    These APIs are implemented in Objects/unicodectype.c.
1346
1347 */
1348
1349 PyAPI_FUNC(int) _PyUnicode_IsLowercase(
1350     Py_UNICODE ch       /* Unicode character */
1351     );
1352
1353 PyAPI_FUNC(int) _PyUnicode_IsUppercase(
1354     Py_UNICODE ch       /* Unicode character */
1355     );
1356
1357 PyAPI_FUNC(int) _PyUnicode_IsTitlecase(
1358     Py_UNICODE ch       /* Unicode character */
1359     );
1360
1361 PyAPI_FUNC(int) _PyUnicode_IsWhitespace(
1362     const Py_UNICODE ch         /* Unicode character */
1363     );
1364
1365 PyAPI_FUNC(int) _PyUnicode_IsLinebreak(
1366     const Py_UNICODE ch         /* Unicode character */
1367     );
1368
1369 PyAPI_FUNC(Py_UNICODE) _PyUnicode_ToLowercase(
1370     Py_UNICODE ch       /* Unicode character */
1371     );
1372
1373 PyAPI_FUNC(Py_UNICODE) _PyUnicode_ToUppercase(
1374     Py_UNICODE ch       /* Unicode character */
1375     );
1376
1377 PyAPI_FUNC(Py_UNICODE) _PyUnicode_ToTitlecase(
1378     Py_UNICODE ch       /* Unicode character */
1379     );
1380
1381 PyAPI_FUNC(int) _PyUnicode_ToDecimalDigit(
1382     Py_UNICODE ch       /* Unicode character */
1383     );
1384
1385 PyAPI_FUNC(int) _PyUnicode_ToDigit(
1386     Py_UNICODE ch       /* Unicode character */
1387     );
1388
1389 PyAPI_FUNC(double) _PyUnicode_ToNumeric(
1390     Py_UNICODE ch       /* Unicode character */
1391     );
1392
1393 PyAPI_FUNC(int) _PyUnicode_IsDecimalDigit(
1394     Py_UNICODE ch       /* Unicode character */
1395     );
1396
1397 PyAPI_FUNC(int) _PyUnicode_IsDigit(
1398     Py_UNICODE ch       /* Unicode character */
1399     );
1400
1401 PyAPI_FUNC(int) _PyUnicode_IsNumeric(
1402     Py_UNICODE ch       /* Unicode character */
1403     );
1404
1405 PyAPI_FUNC(int) _PyUnicode_IsAlpha(
1406     Py_UNICODE ch       /* Unicode character */
1407     );
1408
1409 #ifdef __cplusplus
1410 }
1411 #endif
1412 #endif /* Py_USING_UNICODE */
1413 #endif /* !Py_UNICODEOBJECT_H */