Modules/_csv.c

   1 /* csv module */
   2
   3 /*
   4
   5 This module provides the low-level underpinnings of a CSV reading/writing
   6 module.  Users should not use this module directly, but import the csv.py
   7 module instead.
   8
   9 **** For people modifying this code, please note that as of this writing
  10 **** (2003-03-23), it is intended that this code should work with Python
  11 **** 2.2.
  12
  13 */
  14
  15 #define MODULE_VERSION "1.0"
  16
  17 #include "Python.h"
  18 #include "structmember.h"
  19
  20 #define IS_BASESTRING(o) \
  21         PyUnicode_Check(o)
  22
  23 static PyObject *error_obj;     /* CSV exception */
  24 static PyObject *dialects;      /* Dialect registry */
  25 static long field_limit = 128 * 1024;   /* max parsed field size */
  26
  27 typedef enum {
  28         START_RECORD, START_FIELD, ESCAPED_CHAR, IN_FIELD,
  29         IN_QUOTED_FIELD, ESCAPE_IN_QUOTED_FIELD, QUOTE_IN_QUOTED_FIELD,
  30         EAT_CRNL
  31 } ParserState;
  32
  33 typedef enum {
  34         QUOTE_MINIMAL, QUOTE_ALL, QUOTE_NONNUMERIC, QUOTE_NONE
  35 } QuoteStyle;
  36
  37 typedef struct {
  38         QuoteStyle style;
  39         char *name;
  40 } StyleDesc;
  41
  42 static StyleDesc quote_styles[] = {
  43         { QUOTE_MINIMAL,    "QUOTE_MINIMAL" },
  44         { QUOTE_ALL,        "QUOTE_ALL" },
  45         { QUOTE_NONNUMERIC, "QUOTE_NONNUMERIC" },
  46         { QUOTE_NONE,       "QUOTE_NONE" },
  47         { 0 }
  48 };
  49
  50 typedef struct {
  51         PyObject_HEAD
  52
  53         int doublequote;        /* is " represented by ""? */
  54         Py_UNICODE delimiter;   /* field separator */
  55         Py_UNICODE quotechar;   /* quote character */
  56         Py_UNICODE escapechar;  /* escape character */
  57         int skipinitialspace;   /* ignore spaces following delimiter? */
  58         PyObject *lineterminator; /* string to write between records */
  59         int quoting;            /* style of quoting to write */
  60
  61         int strict;             /* raise exception on bad CSV */
  62 } DialectObj;
  63
  64 static PyTypeObject Dialect_Type;
  65
  66 typedef struct {
  67         PyObject_HEAD
  68
  69         PyObject *input_iter;   /* iterate over this for input lines */
  70
  71         DialectObj *dialect;    /* parsing dialect */
  72
  73         PyObject *fields;       /* field list for current record */
  74         ParserState state;      /* current CSV parse state */
  75         Py_UNICODE *field;      /* build current field in here */
  76         int field_size;         /* size of allocated buffer */
  77     Py_ssize_t field_len;       /* length of current field */
  78         int numeric_field;      /* treat field as numeric */
  79         unsigned long line_num; /* Source-file line number */
  80 } ReaderObj;
  81
  82 static PyTypeObject Reader_Type;
  83
  84 #define ReaderObject_Check(v)   (Py_TYPE(v) == &Reader_Type)
  85
  86 typedef struct {
  87         PyObject_HEAD
  88
  89         PyObject *writeline;    /* write output lines to this file */
  90
  91         DialectObj *dialect;    /* parsing dialect */
  92
  93         Py_UNICODE *rec;        /* buffer for parser.join */
  94         int rec_size;           /* size of allocated record */
  95         Py_ssize_t rec_len;     /* length of record */
  96         int num_fields;         /* number of fields in record */
  97 } WriterObj;
  98
  99 static PyTypeObject Writer_Type;
 100
 101 /*
 102  * DIALECT class
 103  */
 104
 105 static PyObject *
 106 get_dialect_from_registry(PyObject * name_obj)
 107 {
 108         PyObject *dialect_obj;
 109
 110         dialect_obj = PyDict_GetItem(dialects, name_obj);
 111         if (dialect_obj == NULL) {
 112                 if (!PyErr_Occurred())
 113                         PyErr_Format(error_obj, "unknown dialect");
 114         }
 115         else
 116                 Py_INCREF(dialect_obj);
 117         return dialect_obj;
 118 }
 119
 120 static PyObject *
 121 get_string(PyObject *str)
 122 {
 123         Py_XINCREF(str);
 124         return str;
 125 }
 126
 127 static PyObject *
 128 get_nullchar_as_None(Py_UNICODE c)
 129 {
 130         if (c == '\0') {
 131                 Py_INCREF(Py_None);
 132                 return Py_None;
 133         }
 134         else
 135                 return PyUnicode_FromUnicode((Py_UNICODE *)&c, 1);
 136 }
 137
 138 static PyObject *
 139 Dialect_get_lineterminator(DialectObj *self)
 140 {
 141         return get_string(self->lineterminator);
 142 }
 143
 144 static PyObject *
 145 Dialect_get_delimiter(DialectObj *self)
 146 {
 147         return get_nullchar_as_None(self->delimiter);
 148 }
 149
 150 static PyObject *
 151 Dialect_get_escapechar(DialectObj *self)
 152 {
 153         return get_nullchar_as_None(self->escapechar);
 154 }
 155
 156 static PyObject *
 157 Dialect_get_quotechar(DialectObj *self)
 158 {
 159         return get_nullchar_as_None(self->quotechar);
 160 }
 161
 162 static PyObject *
 163 Dialect_get_quoting(DialectObj *self)
 164 {
 165         return PyLong_FromLong(self->quoting);
 166 }
 167
 168 static int
 169 _set_bool(const char *name, int *target, PyObject *src, int dflt)
 170 {
 171         if (src == NULL)
 172                 *target = dflt;
 173         else
 174                 *target = PyObject_IsTrue(src);
 175         return 0;
 176 }
 177
 178 static int
 179 _set_int(const char *name, int *target, PyObject *src, int dflt)
 180 {
 181         if (src == NULL)
 182                 *target = dflt;
 183         else {
 184                 long value;
 185                 if (!PyLong_CheckExact(src)) {
 186                         PyErr_Format(PyExc_TypeError,
 187                                      "\"%s\" must be an integer", name);
 188                         return -1;
 189                 }
 190                 value = PyLong_AsLong(src);
 191                 if (value == -1 && PyErr_Occurred())
 192                         return -1;
 193 #if SIZEOF_LONG > SIZEOF_INT
 194                 if (value > INT_MAX || value < INT_MIN) {
 195                         PyErr_Format(PyExc_ValueError,
 196                                      "integer out of range for \"%s\"", name);
 197                         return -1;
 198                 }
 199 #endif
 200                 *target = (int)value;
 201         }
 202         return 0;
 203 }
 204
 205 static int
 206 _set_char(const char *name, Py_UNICODE *target, PyObject *src, Py_UNICODE dflt)
 207 {
 208         if (src == NULL)
 209                 *target = dflt;
 210         else {
 211                 *target = '\0';
 212                 if (src != Py_None) {
 213                         Py_UNICODE *buf;
 214                         Py_ssize_t len;
 215                         buf = PyUnicode_AsUnicode(src);
 216                         len = PyUnicode_GetSize(src);
 217                         if (buf == NULL || len > 1) {
 218                                 PyErr_Format(PyExc_TypeError,
 219                                         "\"%s\" must be an 1-character string",
 220                                         name);
 221                                 return -1;
 222                         }
 223                         if (len > 0)
 224                                 *target = buf[0];
 225                 }
 226         }
 227         return 0;
 228 }
 229
 230 static int
 231 _set_str(const char *name, PyObject **target, PyObject *src, const char *dflt)
 232 {
 233         if (src == NULL)
 234                 *target = PyUnicode_DecodeASCII(dflt, strlen(dflt), NULL);
 235         else {
 236                 if (src == Py_None)
 237                         *target = NULL;
 238                 else if (!IS_BASESTRING(src)) {
 239                         PyErr_Format(PyExc_TypeError,
 240                                      "\"%s\" must be a string", name);
 241                         return -1;
 242                 }
 243                 else {
 244                         Py_XDECREF(*target);
 245                         Py_INCREF(src);
 246                         *target = src;
 247                 }
 248         }
 249         return 0;
 250 }
 251
 252 static int
 253 dialect_check_quoting(int quoting)
 254 {
 255         StyleDesc *qs = quote_styles;
 256
 257         for (qs = quote_styles; qs->name; qs++) {
 258                 if (qs->style == quoting)
 259                         return 0;
 260         }
 261         PyErr_Format(PyExc_TypeError, "bad \"quoting\" value");
 262         return -1;
 263 }
 264
 265 #define D_OFF(x) offsetof(DialectObj, x)
 266
 267 static struct PyMemberDef Dialect_memberlist[] = {
 268         { "skipinitialspace",   T_INT, D_OFF(skipinitialspace), READONLY },
 269         { "doublequote",        T_INT, D_OFF(doublequote), READONLY },
 270         { "strict",             T_INT, D_OFF(strict), READONLY },
 271         { NULL }
 272 };
 273
 274 static PyGetSetDef Dialect_getsetlist[] = {
 275         { "delimiter",          (getter)Dialect_get_delimiter},
 276         { "escapechar",         (getter)Dialect_get_escapechar},
 277         { "lineterminator",     (getter)Dialect_get_lineterminator},
 278         { "quotechar",          (getter)Dialect_get_quotechar},
 279         { "quoting",            (getter)Dialect_get_quoting},
 280         {NULL},
 281 };
 282
 283 static void
 284 Dialect_dealloc(DialectObj *self)
 285 {
 286         Py_XDECREF(self->lineterminator);
 287         Py_TYPE(self)->tp_free((PyObject *)self);
 288 }
 289
 290 static char *dialect_kws[] = {
 291         "dialect",
 292         "delimiter",
 293         "doublequote",
 294         "escapechar",
 295         "lineterminator",
 296         "quotechar",
 297         "quoting",
 298         "skipinitialspace",
 299         "strict",
 300         NULL
 301 };
 302
 303 static PyObject *
 304 dialect_new(PyTypeObject *type, PyObject *args, PyObject *kwargs)
 305 {
 306         DialectObj *self;
 307         PyObject *ret = NULL;
 308         PyObject *dialect = NULL;
 309         PyObject *delimiter = NULL;
 310         PyObject *doublequote = NULL;
 311         PyObject *escapechar = NULL;
 312         PyObject *lineterminator = NULL;
 313         PyObject *quotechar = NULL;
 314         PyObject *quoting = NULL;
 315         PyObject *skipinitialspace = NULL;
 316         PyObject *strict = NULL;
 317
 318         if (!PyArg_ParseTupleAndKeywords(args, kwargs,
 319                                          "|OOOOOOOOO", dialect_kws,
 320                                          &dialect,
 321                                          &delimiter,
 322                                          &doublequote,
 323                                          &escapechar,
 324                                          &lineterminator,
 325                                          &quotechar,
 326                                          &quoting,
 327                                          &skipinitialspace,
 328                                          &strict))
 329                 return NULL;
 330
 331         if (dialect != NULL) {
 332                 if (IS_BASESTRING(dialect)) {
 333                         dialect = get_dialect_from_registry(dialect);
 334                         if (dialect == NULL)
 335                                 return NULL;
 336                 }
 337                 else
 338                         Py_INCREF(dialect);
 339                 /* Can we reuse this instance? */
 340                 if (PyObject_TypeCheck(dialect, &Dialect_Type) &&
 341                     delimiter == 0 &&
 342                     doublequote == 0 &&
 343                     escapechar == 0 &&
 344                     lineterminator == 0 &&
 345                     quotechar == 0 &&
 346                     quoting == 0 &&
 347                     skipinitialspace == 0 &&
 348                     strict == 0)
 349                         return dialect;
 350         }
 351
 352         self = (DialectObj *)type->tp_alloc(type, 0);
 353         if (self == NULL) {
 354                 Py_XDECREF(dialect);
 355                 return NULL;
 356         }
 357         self->lineterminator = NULL;
 358
 359         Py_XINCREF(delimiter);
 360         Py_XINCREF(doublequote);
 361         Py_XINCREF(escapechar);
 362         Py_XINCREF(lineterminator);
 363         Py_XINCREF(quotechar);
 364         Py_XINCREF(quoting);
 365         Py_XINCREF(skipinitialspace);
 366         Py_XINCREF(strict);
 367         if (dialect != NULL) {
 368 #define DIALECT_GETATTR(v, n) \
 369                 if (v == NULL) \
 370                         v = PyObject_GetAttrString(dialect, n)
 371                 DIALECT_GETATTR(delimiter, "delimiter");
 372                 DIALECT_GETATTR(doublequote, "doublequote");
 373                 DIALECT_GETATTR(escapechar, "escapechar");
 374                 DIALECT_GETATTR(lineterminator, "lineterminator");
 375                 DIALECT_GETATTR(quotechar, "quotechar");
 376                 DIALECT_GETATTR(quoting, "quoting");
 377                 DIALECT_GETATTR(skipinitialspace, "skipinitialspace");
 378                 DIALECT_GETATTR(strict, "strict");
 379                 PyErr_Clear();
 380         }
 381
 382         /* check types and convert to C values */
 383 #define DIASET(meth, name, target, src, dflt) \
 384         if (meth(name, target, src, dflt)) \
 385                 goto err
 386         DIASET(_set_char, "delimiter", &self->delimiter, delimiter, ',');
 387         DIASET(_set_bool, "doublequote", &self->doublequote, doublequote, 1);
 388         DIASET(_set_char, "escapechar", &self->escapechar, escapechar, 0);
 389         DIASET(_set_str, "lineterminator", &self->lineterminator, lineterminator, "\r\n");
 390         DIASET(_set_char, "quotechar", &self->quotechar, quotechar, '"');
 391         DIASET(_set_int, "quoting", &self->quoting, quoting, QUOTE_MINIMAL);
 392         DIASET(_set_bool, "skipinitialspace", &self->skipinitialspace, skipinitialspace, 0);
 393         DIASET(_set_bool, "strict", &self->strict, strict, 0);
 394
 395         /* validate options */
 396         if (dialect_check_quoting(self->quoting))
 397                 goto err;
 398         if (self->delimiter == 0) {
 399                 PyErr_SetString(PyExc_TypeError, "delimiter must be set");
 400                 goto err;
 401         }
 402         if (quotechar == Py_None && quoting == NULL)
 403                 self->quoting = QUOTE_NONE;
 404         if (self->quoting != QUOTE_NONE && self->quotechar == 0) {
 405                 PyErr_SetString(PyExc_TypeError,
 406                                 "quotechar must be set if quoting enabled");
 407                 goto err;
 408         }
 409         if (self->lineterminator == 0) {
 410                 PyErr_SetString(PyExc_TypeError, "lineterminator must be set");
 411                 goto err;
 412         }
 413
 414         ret = (PyObject *)self;
 415         Py_INCREF(self);
 416 err:
 417         Py_XDECREF(self);
 418         Py_XDECREF(dialect);
 419         Py_XDECREF(delimiter);
 420         Py_XDECREF(doublequote);
 421         Py_XDECREF(escapechar);
 422         Py_XDECREF(lineterminator);
 423         Py_XDECREF(quotechar);
 424         Py_XDECREF(quoting);
 425         Py_XDECREF(skipinitialspace);
 426         Py_XDECREF(strict);
 427         return ret;
 428 }
 429
 430
 431 PyDoc_STRVAR(Dialect_Type_doc,
 432 "CSV dialect\n"
 433 "\n"
 434 "The Dialect type records CSV parsing and generation options.\n");
 435
 436 static PyTypeObject Dialect_Type = {
 437         PyVarObject_HEAD_INIT(NULL, 0)
 438         "_csv.Dialect",                         /* tp_name */
 439         sizeof(DialectObj),                     /* tp_basicsize */
 440         0,                                      /* tp_itemsize */
 441         /*  methods  */
 442         (destructor)Dialect_dealloc,            /* tp_dealloc */
 443         (printfunc)0,                           /* tp_print */
 444         (getattrfunc)0,                         /* tp_getattr */
 445         (setattrfunc)0,                         /* tp_setattr */
 446         0,                                      /* tp_reserved */
 447         (reprfunc)0,                            /* tp_repr */
 448         0,                                      /* tp_as_number */
 449         0,                                      /* tp_as_sequence */
 450         0,                                      /* tp_as_mapping */
 451         (hashfunc)0,                            /* tp_hash */
 452         (ternaryfunc)0,                         /* tp_call */
 453         (reprfunc)0,                            /* tp_str */
 454         0,                                      /* tp_getattro */
 455         0,                                      /* tp_setattro */
 456         0,                                      /* tp_as_buffer */
 457         Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */
 458         Dialect_Type_doc,                       /* tp_doc */
 459         0,                                      /* tp_traverse */
 460         0,                                      /* tp_clear */
 461         0,                                      /* tp_richcompare */
 462         0,                                      /* tp_weaklistoffset */
 463         0,                                      /* tp_iter */
 464         0,                                      /* tp_iternext */
 465         0,                                      /* tp_methods */
 466         Dialect_memberlist,                     /* tp_members */
 467         Dialect_getsetlist,                     /* tp_getset */
 468         0,                                      /* tp_base */
 469         0,                                      /* tp_dict */
 470         0,                                      /* tp_descr_get */
 471         0,                                      /* tp_descr_set */
 472         0,                                      /* tp_dictoffset */
 473         0,                                      /* tp_init */
 474         0,                                      /* tp_alloc */
 475         dialect_new,                            /* tp_new */
 476         0,                                      /* tp_free */
 477 };
 478
 479 /*
 480  * Return an instance of the dialect type, given a Python instance or kwarg
 481  * description of the dialect
 482  */
 483 static PyObject *
 484 _call_dialect(PyObject *dialect_inst, PyObject *kwargs)
 485 {
 486         PyObject *ctor_args;
 487         PyObject *dialect;
 488
 489         ctor_args = Py_BuildValue(dialect_inst ? "(O)" : "()", dialect_inst);
 490         if (ctor_args == NULL)
 491                 return NULL;
 492         dialect = PyObject_Call((PyObject *)&Dialect_Type, ctor_args, kwargs);
 493         Py_DECREF(ctor_args);
 494         return dialect;
 495 }
 496
 497 /*
 498  * READER
 499  */
 500 static int
 501 parse_save_field(ReaderObj *self)
 502 {
 503         PyObject *field;
 504
 505         field = PyUnicode_FromUnicode(self->field, self->field_len);
 506         if (field == NULL)
 507                 return -1;
 508         self->field_len = 0;
 509         if (self->numeric_field) {
 510                 PyObject *tmp;
 511
 512                 self->numeric_field = 0;
 513                 tmp = PyNumber_Float(field);
 514                 if (tmp == NULL) {
 515                         Py_DECREF(field);
 516                         return -1;
 517                 }
 518                 Py_DECREF(field);
 519                 field = tmp;
 520         }
 521         PyList_Append(self->fields, field);
 522         Py_DECREF(field);
 523         return 0;
 524 }
 525
 526 static int
 527 parse_grow_buff(ReaderObj *self)
 528 {
 529         if (self->field_size == 0) {
 530                 self->field_size = 4096;
 531                 if (self->field != NULL)
 532                         PyMem_Free(self->field);
 533                 self->field = PyMem_New(Py_UNICODE, self->field_size);
 534         }
 535         else {
 536                 if (self->field_size > INT_MAX / 2) {
 537                         PyErr_NoMemory();
 538                         return 0;
 539                 }
 540                 self->field_size *= 2;
 541                 self->field = PyMem_Resize(self->field, Py_UNICODE,
 542                                            self->field_size);
 543         }
 544         if (self->field == NULL) {
 545                 PyErr_NoMemory();
 546                 return 0;
 547         }
 548         return 1;
 549 }
 550
 551 static int
 552 parse_add_char(ReaderObj *self, Py_UNICODE c)
 553 {
 554         if (self->field_len >= field_limit) {
 555                 PyErr_Format(error_obj, "field larger than field limit (%ld)",
 556                              field_limit);
 557                 return -1;
 558         }
 559         if (self->field_len == self->field_size && !parse_grow_buff(self))
 560                 return -1;
 561         self->field[self->field_len++] = c;
 562         return 0;
 563 }
 564
 565 static int
 566 parse_process_char(ReaderObj *self, Py_UNICODE c)
 567 {
 568         DialectObj *dialect = self->dialect;
 569
 570         switch (self->state) {
 571         case START_RECORD:
 572                 /* start of record */
 573                 if (c == '\0')
 574                         /* empty line - return [] */
 575                         break;
 576                 else if (c == '\n' || c == '\r') {
 577                         self->state = EAT_CRNL;
 578                         break;
 579                 }
 580                 /* normal character - handle as START_FIELD */
 581                 self->state = START_FIELD;
 582                 /* fallthru */
 583         case START_FIELD:
 584                 /* expecting field */
 585                 if (c == '\n' || c == '\r' || c == '\0') {
 586                         /* save empty field - return [fields] */
 587                         if (parse_save_field(self) < 0)
 588                                 return -1;
 589                         self->state = (c == '\0' ? START_RECORD : EAT_CRNL);
 590                 }
 591                 else if (c == dialect->quotechar &&
 592                          dialect->quoting != QUOTE_NONE) {
 593                         /* start quoted field */
 594                         self->state = IN_QUOTED_FIELD;
 595                 }
 596                 else if (c == dialect->escapechar) {
 597                         /* possible escaped character */
 598                         self->state = ESCAPED_CHAR;
 599                 }
 600                 else if (c == ' ' && dialect->skipinitialspace)
 601                         /* ignore space at start of field */
 602                         ;
 603                 else if (c == dialect->delimiter) {
 604                         /* save empty field */
 605                         if (parse_save_field(self) < 0)
 606                                 return -1;
 607                 }
 608                 else {
 609                         /* begin new unquoted field */
 610                         if (dialect->quoting == QUOTE_NONNUMERIC)
 611                                 self->numeric_field = 1;
 612                         if (parse_add_char(self, c) < 0)
 613                                 return -1;
 614                         self->state = IN_FIELD;
 615                 }
 616                 break;
 617
 618         case ESCAPED_CHAR:
 619                 if (c == '\0')
 620                         c = '\n';
 621                 if (parse_add_char(self, c) < 0)
 622                         return -1;
 623                 self->state = IN_FIELD;
 624                 break;
 625
 626         case IN_FIELD:
 627                 /* in unquoted field */
 628                 if (c == '\n' || c == '\r' || c == '\0') {
 629                         /* end of line - return [fields] */
 630                         if (parse_save_field(self) < 0)
 631                                 return -1;
 632                         self->state = (c == '\0' ? START_RECORD : EAT_CRNL);
 633                 }
 634                 else if (c == dialect->escapechar) {
 635                         /* possible escaped character */
 636                         self->state = ESCAPED_CHAR;
 637                 }
 638                 else if (c == dialect->delimiter) {
 639                         /* save field - wait for new field */
 640                         if (parse_save_field(self) < 0)
 641                                 return -1;
 642                         self->state = START_FIELD;
 643                 }
 644                 else {
 645                         /* normal character - save in field */
 646                         if (parse_add_char(self, c) < 0)
 647                                 return -1;
 648                 }
 649                 break;
 650
 651         case IN_QUOTED_FIELD:
 652                 /* in quoted field */
 653                 if (c == '\0')
 654                         ;
 655                 else if (c == dialect->escapechar) {
 656                         /* Possible escape character */
 657                         self->state = ESCAPE_IN_QUOTED_FIELD;
 658                 }
 659                 else if (c == dialect->quotechar &&
 660                          dialect->quoting != QUOTE_NONE) {
 661                         if (dialect->doublequote) {
 662                                 /* doublequote; " represented by "" */
 663                                 self->state = QUOTE_IN_QUOTED_FIELD;
 664                         }
 665                         else {
 666                                 /* end of quote part of field */
 667                                 self->state = IN_FIELD;
 668                         }
 669                 }
 670                 else {
 671                         /* normal character - save in field */
 672                         if (parse_add_char(self, c) < 0)
 673                                 return -1;
 674                 }
 675                 break;
 676
 677         case ESCAPE_IN_QUOTED_FIELD:
 678                 if (c == '\0')
 679                         c = '\n';
 680                 if (parse_add_char(self, c) < 0)
 681                         return -1;
 682                 self->state = IN_QUOTED_FIELD;
 683                 break;
 684
 685         case QUOTE_IN_QUOTED_FIELD:
 686                 /* doublequote - seen a quote in an quoted field */
 687                 if (dialect->quoting != QUOTE_NONE &&
 688                     c == dialect->quotechar) {
 689                         /* save "" as " */
 690                         if (parse_add_char(self, c) < 0)
 691                                 return -1;
 692                         self->state = IN_QUOTED_FIELD;
 693                 }
 694                 else if (c == dialect->delimiter) {
 695                         /* save field - wait for new field */
 696                         if (parse_save_field(self) < 0)
 697                                 return -1;
 698                         self->state = START_FIELD;
 699                 }
 700                 else if (c == '\n' || c == '\r' || c == '\0') {
 701                         /* end of line - return [fields] */
 702                         if (parse_save_field(self) < 0)
 703                                 return -1;
 704                         self->state = (c == '\0' ? START_RECORD : EAT_CRNL);
 705                 }
 706                 else if (!dialect->strict) {
 707                         if (parse_add_char(self, c) < 0)
 708                                 return -1;
 709                         self->state = IN_FIELD;
 710                 }
 711                 else {
 712                         /* illegal */
 713                         PyErr_Format(error_obj, "'%c' expected after '%c'",
 714                                         dialect->delimiter,
 715                                         dialect->quotechar);
 716                         return -1;
 717                 }
 718                 break;
 719
 720         case EAT_CRNL:
 721                 if (c == '\n' || c == '\r')
 722                         ;
 723                 else if (c == '\0')
 724                         self->state = START_RECORD;
 725                 else {
 726                         PyErr_Format(error_obj, "new-line character seen in unquoted field - do you need to open the file in universal-newline mode?");
 727                         return -1;
 728                 }
 729                 break;
 730
 731         }
 732         return 0;
 733 }
 734
 735 static int
 736 parse_reset(ReaderObj *self)
 737 {
 738         Py_XDECREF(self->fields);
 739         self->fields = PyList_New(0);
 740         if (self->fields == NULL)
 741                 return -1;
 742         self->field_len = 0;
 743         self->state = START_RECORD;
 744         self->numeric_field = 0;
 745         return 0;
 746 }
 747
 748 static PyObject *
 749 Reader_iternext(ReaderObj *self)
 750 {
 751         PyObject *lineobj;
 752         PyObject *fields = NULL;
 753         Py_UNICODE *line, c;
 754         Py_ssize_t linelen;
 755
 756         if (parse_reset(self) < 0)
 757                 return NULL;
 758         do {
 759                 lineobj = PyIter_Next(self->input_iter);
 760                 if (lineobj == NULL) {
 761                         /* End of input OR exception */
 762                         if (!PyErr_Occurred() && self->field_len != 0)
 763                                 PyErr_Format(error_obj,
 764                                              "newline inside string");
 765                         return NULL;
 766                 }
 767                 if (!PyUnicode_Check(lineobj)) {
 768                         PyErr_Format(error_obj,
 769                                      "iterator should return strings, "
 770                                      "not %.200s "
 771                                      "(did you open the file in text mode?)",
 772                                      lineobj->ob_type->tp_name
 773                                 );
 774                         Py_DECREF(lineobj);
 775                         return NULL;
 776                 }
 777                 ++self->line_num;
 778                 line = PyUnicode_AsUnicode(lineobj);
 779                 linelen = PyUnicode_GetSize(lineobj);
 780                 if (line == NULL || linelen < 0) {
 781                         Py_DECREF(lineobj);
 782                         return NULL;
 783                 }
 784                 while (linelen--) {
 785                         c = *line++;
 786                         if (c == '\0') {
 787                                 Py_DECREF(lineobj);
 788                                 PyErr_Format(error_obj,
 789                                              "line contains NULL byte");
 790                                 goto err;
 791                         }
 792                         if (parse_process_char(self, c) < 0) {
 793                                 Py_DECREF(lineobj);
 794                                 goto err;
 795                         }
 796                 }
 797                 Py_DECREF(lineobj);
 798                 if (parse_process_char(self, 0) < 0)
 799                         goto err;
 800         } while (self->state != START_RECORD);
 801
 802         fields = self->fields;
 803         self->fields = NULL;
 804 err:
 805         return fields;
 806 }
 807
 808 static void
 809 Reader_dealloc(ReaderObj *self)
 810 {
 811         PyObject_GC_UnTrack(self);
 812         Py_XDECREF(self->dialect);
 813         Py_XDECREF(self->input_iter);
 814         Py_XDECREF(self->fields);
 815         if (self->field != NULL)
 816                 PyMem_Free(self->field);
 817         PyObject_GC_Del(self);
 818 }
 819
 820 static int
 821 Reader_traverse(ReaderObj *self, visitproc visit, void *arg)
 822 {
 823         Py_VISIT(self->dialect);
 824         Py_VISIT(self->input_iter);
 825         Py_VISIT(self->fields);
 826         return 0;
 827 }
 828
 829 static int
 830 Reader_clear(ReaderObj *self)
 831 {
 832         Py_CLEAR(self->dialect);
 833         Py_CLEAR(self->input_iter);
 834         Py_CLEAR(self->fields);
 835         return 0;
 836 }
 837
 838 PyDoc_STRVAR(Reader_Type_doc,
 839 "CSV reader\n"
 840 "\n"
 841 "Reader objects are responsible for reading and parsing tabular data\n"
 842 "in CSV format.\n"
 843 );
 844
 845 static struct PyMethodDef Reader_methods[] = {
 846         { NULL, NULL }
 847 };
 848 #define R_OFF(x) offsetof(ReaderObj, x)
 849
 850 static struct PyMemberDef Reader_memberlist[] = {
 851         { "dialect", T_OBJECT, R_OFF(dialect), READONLY },
 852         { "line_num", T_ULONG, R_OFF(line_num), READONLY },
 853         { NULL }
 854 };
 855
 856
 857 static PyTypeObject Reader_Type = {
 858         PyVarObject_HEAD_INIT(NULL, 0)
 859         "_csv.reader",                          /*tp_name*/
 860         sizeof(ReaderObj),                      /*tp_basicsize*/
 861         0,                                      /*tp_itemsize*/
 862         /* methods */
 863         (destructor)Reader_dealloc,             /*tp_dealloc*/
 864         (printfunc)0,                           /*tp_print*/
 865         (getattrfunc)0,                         /*tp_getattr*/
 866         (setattrfunc)0,                         /*tp_setattr*/
 867         0,                                     /*tp_reserved*/
 868         (reprfunc)0,                            /*tp_repr*/
 869         0,                                      /*tp_as_number*/
 870         0,                                      /*tp_as_sequence*/
 871         0,                                      /*tp_as_mapping*/
 872         (hashfunc)0,                            /*tp_hash*/
 873         (ternaryfunc)0,                         /*tp_call*/
 874         (reprfunc)0,                            /*tp_str*/
 875         0,                                      /*tp_getattro*/
 876         0,                                      /*tp_setattro*/
 877         0,                                      /*tp_as_buffer*/
 878         Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE |
 879                 Py_TPFLAGS_HAVE_GC,             /*tp_flags*/
 880         Reader_Type_doc,                        /*tp_doc*/
 881         (traverseproc)Reader_traverse,          /*tp_traverse*/
 882         (inquiry)Reader_clear,                  /*tp_clear*/
 883         0,                                      /*tp_richcompare*/
 884         0,                                      /*tp_weaklistoffset*/
 885         PyObject_SelfIter,                      /*tp_iter*/
 886         (getiterfunc)Reader_iternext,           /*tp_iternext*/
 887         Reader_methods,                         /*tp_methods*/
 888         Reader_memberlist,                      /*tp_members*/
 889         0,                                      /*tp_getset*/
 890
 891 };
 892
 893 static PyObject *
 894 csv_reader(PyObject *module, PyObject *args, PyObject *keyword_args)
 895 {
 896         PyObject * iterator, * dialect = NULL;
 897         ReaderObj * self = PyObject_GC_New(ReaderObj, &Reader_Type);
 898
 899         if (!self)
 900                 return NULL;
 901
 902         self->dialect = NULL;
 903         self->fields = NULL;
 904         self->input_iter = NULL;
 905         self->field = NULL;
 906         self->field_size = 0;
 907         self->line_num = 0;
 908
 909         if (parse_reset(self) < 0) {
 910                 Py_DECREF(self);
 911                 return NULL;
 912         }
 913
 914         if (!PyArg_UnpackTuple(args, "", 1, 2, &iterator, &dialect)) {
 915                 Py_DECREF(self);
 916                 return NULL;
 917         }
 918         self->input_iter = PyObject_GetIter(iterator);
 919         if (self->input_iter == NULL) {
 920                 PyErr_SetString(PyExc_TypeError,
 921                                 "argument 1 must be an iterator");
 922                 Py_DECREF(self);
 923                 return NULL;
 924         }
 925         self->dialect = (DialectObj *)_call_dialect(dialect, keyword_args);
 926         if (self->dialect == NULL) {
 927                 Py_DECREF(self);
 928                 return NULL;
 929         }
 930
 931         PyObject_GC_Track(self);
 932         return (PyObject *)self;
 933 }
 934
 935 /*
 936  * WRITER
 937  */
 938 /* ---------------------------------------------------------------- */
 939 static void
 940 join_reset(WriterObj *self)
 941 {
 942         self->rec_len = 0;
 943         self->num_fields = 0;
 944 }
 945
 946 #define MEM_INCR 32768
 947
 948 /* Calculate new record length or append field to record.  Return new
 949  * record length.
 950  */
 951 static int
 952 join_append_data(WriterObj *self, Py_UNICODE *field, int quote_empty,
 953                  int *quoted, int copy_phase)
 954 {
 955         DialectObj *dialect = self->dialect;
 956         int i;
 957         int rec_len;
 958         Py_UNICODE *lineterm;
 959
 960 #define ADDCH(c) \
 961         do {\
 962                 if (copy_phase) \
 963                         self->rec[rec_len] = c;\
 964                 rec_len++;\
 965         } while(0)
 966
 967         lineterm = PyUnicode_AsUnicode(dialect->lineterminator);
 968         if (lineterm == NULL)
 969                 return -1;
 970
 971         rec_len = self->rec_len;
 972
 973         /* If this is not the first field we need a field separator */
 974         if (self->num_fields > 0)
 975                 ADDCH(dialect->delimiter);
 976
 977         /* Handle preceding quote */
 978         if (copy_phase && *quoted)
 979                 ADDCH(dialect->quotechar);
 980
 981         /* Copy/count field data */
 982         /* If field is null just pass over */
 983         for (i = 0; field; i++) {
 984                 Py_UNICODE c = field[i];
 985                 int want_escape = 0;
 986
 987                 if (c == '\0')
 988                         break;
 989
 990                 if (c == dialect->delimiter ||
 991                     c == dialect->escapechar ||
 992                     c == dialect->quotechar  ||
 993                     Py_UNICODE_strchr(lineterm, c)) {
 994                         if (dialect->quoting == QUOTE_NONE)
 995                                 want_escape = 1;
 996                         else {
 997                                 if (c == dialect->quotechar) {
 998                                         if (dialect->doublequote)
 999                                                 ADDCH(dialect->quotechar);
1000                                         else
1001                                                 want_escape = 1;
1002                                 }
1003                                 if (!want_escape)
1004                                         *quoted = 1;
1005                         }
1006                         if (want_escape) {
1007                                 if (!dialect->escapechar) {
1008                                         PyErr_Format(error_obj,
1009                                                      "need to escape, but no escapechar set");
1010                                         return -1;
1011                                 }
1012                                 ADDCH(dialect->escapechar);
1013                         }
1014                 }
1015                 /* Copy field character into record buffer.
1016                  */
1017                 ADDCH(c);
1018         }
1019
1020         /* If field is empty check if it needs to be quoted.
1021          */
1022         if (i == 0 && quote_empty) {
1023                 if (dialect->quoting == QUOTE_NONE) {
1024                         PyErr_Format(error_obj,
1025                                 "single empty field record must be quoted");
1026                         return -1;
1027                 }
1028                 else
1029                         *quoted = 1;
1030         }
1031
1032         if (*quoted) {
1033                 if (copy_phase)
1034                         ADDCH(dialect->quotechar);
1035                 else
1036                         rec_len += 2;
1037         }
1038         return rec_len;
1039 #undef ADDCH
1040 }
1041
1042 static int
1043 join_check_rec_size(WriterObj *self, int rec_len)
1044 {
1045
1046         if (rec_len < 0 || rec_len > INT_MAX - MEM_INCR) {
1047                 PyErr_NoMemory();
1048                 return 0;
1049         }
1050
1051         if (rec_len > self->rec_size) {
1052                 if (self->rec_size == 0) {
1053                         self->rec_size = (rec_len / MEM_INCR + 1) * MEM_INCR;
1054                         if (self->rec != NULL)
1055                                 PyMem_Free(self->rec);
1056                         self->rec = PyMem_New(Py_UNICODE, self->rec_size);
1057                 }
1058                 else {
1059                         Py_UNICODE* old_rec = self->rec;
1060
1061                         self->rec_size = (rec_len / MEM_INCR + 1) * MEM_INCR;
1062                         self->rec = PyMem_Resize(self->rec, Py_UNICODE,
1063                                      self->rec_size);
1064                         if (self->rec == NULL)
1065                                 PyMem_Free(old_rec);
1066                 }
1067                 if (self->rec == NULL) {
1068                         PyErr_NoMemory();
1069                         return 0;
1070                 }
1071         }
1072         return 1;
1073 }
1074
1075 static int
1076 join_append(WriterObj *self, Py_UNICODE *field, int *quoted, int quote_empty)
1077 {
1078         int rec_len;
1079
1080         rec_len = join_append_data(self, field, quote_empty, quoted, 0);
1081         if (rec_len < 0)
1082                 return 0;
1083
1084         /* grow record buffer if necessary */
1085         if (!join_check_rec_size(self, rec_len))
1086                 return 0;
1087
1088         self->rec_len = join_append_data(self, field, quote_empty, quoted, 1);
1089         self->num_fields++;
1090
1091         return 1;
1092 }
1093
1094 static int
1095 join_append_lineterminator(WriterObj *self)
1096 {
1097         int terminator_len;
1098         Py_UNICODE *terminator;
1099
1100         terminator_len = PyUnicode_GetSize(self->dialect->lineterminator);
1101         if (terminator_len == -1)
1102                 return 0;
1103
1104         /* grow record buffer if necessary */
1105         if (!join_check_rec_size(self, self->rec_len + terminator_len))
1106                 return 0;
1107
1108         terminator = PyUnicode_AsUnicode(self->dialect->lineterminator);
1109         if (terminator == NULL)
1110                 return 0;
1111         memmove(self->rec + self->rec_len, terminator,
1112             sizeof(Py_UNICODE)*terminator_len);
1113         self->rec_len += terminator_len;
1114
1115         return 1;
1116 }
1117
1118 PyDoc_STRVAR(csv_writerow_doc,
1119 "writerow(sequence)\n"
1120 "\n"
1121 "Construct and write a CSV record from a sequence of fields.  Non-string\n"
1122 "elements will be converted to string.");
1123
1124 static PyObject *
1125 csv_writerow(WriterObj *self, PyObject *seq)
1126 {
1127         DialectObj *dialect = self->dialect;
1128         int len, i;
1129
1130         if (!PySequence_Check(seq))
1131                 return PyErr_Format(error_obj, "sequence expected");
1132
1133         len = PySequence_Length(seq);
1134         if (len < 0)
1135                 return NULL;
1136
1137         /* Join all fields in internal buffer.
1138          */
1139         join_reset(self);
1140         for (i = 0; i < len; i++) {
1141                 PyObject *field;
1142                 int append_ok;
1143                 int quoted;
1144
1145                 field = PySequence_GetItem(seq, i);
1146                 if (field == NULL)
1147                         return NULL;
1148
1149                 switch (dialect->quoting) {
1150                 case QUOTE_NONNUMERIC:
1151                         quoted = !PyNumber_Check(field);
1152                         break;
1153                 case QUOTE_ALL:
1154                         quoted = 1;
1155                         break;
1156                 default:
1157                         quoted = 0;
1158                         break;
1159                 }
1160
1161                 if (PyUnicode_Check(field)) {
1162                         append_ok = join_append(self,
1163                                                 PyUnicode_AS_UNICODE(field),
1164                                                 &quoted, len == 1);
1165                         Py_DECREF(field);
1166                 }
1167                 else if (field == Py_None) {
1168                         append_ok = join_append(self, NULL,
1169                                     &quoted, len == 1);
1170                         Py_DECREF(field);
1171                 }
1172                 else {
1173                         PyObject *str;
1174
1175                         str = PyObject_Str(field);
1176                         Py_DECREF(field);
1177                         if (str == NULL)
1178                                 return NULL;
1179                         append_ok = join_append(self,
1180                                                 PyUnicode_AS_UNICODE(str),
1181                                                 &quoted, len == 1);
1182                         Py_DECREF(str);
1183                 }
1184                 if (!append_ok)
1185                         return NULL;
1186         }
1187
1188         /* Add line terminator.
1189          */
1190         if (!join_append_lineterminator(self))
1191                 return 0;
1192
1193         return PyObject_CallFunction(self->writeline,
1194                                      "(u#)", self->rec,
1195                                      self->rec_len);
1196 }
1197
1198 PyDoc_STRVAR(csv_writerows_doc,
1199 "writerows(sequence of sequences)\n"
1200 "\n"
1201 "Construct and write a series of sequences to a csv file.  Non-string\n"
1202 "elements will be converted to string.");
1203
1204 static PyObject *
1205 csv_writerows(WriterObj *self, PyObject *seqseq)
1206 {
1207         PyObject *row_iter, *row_obj, *result;
1208
1209         row_iter = PyObject_GetIter(seqseq);
1210         if (row_iter == NULL) {
1211                 PyErr_SetString(PyExc_TypeError,
1212                                 "writerows() argument must be iterable");
1213                 return NULL;
1214         }
1215         while ((row_obj = PyIter_Next(row_iter))) {
1216                 result = csv_writerow(self, row_obj);
1217                 Py_DECREF(row_obj);
1218                 if (!result) {
1219                         Py_DECREF(row_iter);
1220                         return NULL;
1221                 }
1222                 else
1223                      Py_DECREF(result);
1224         }
1225         Py_DECREF(row_iter);
1226         if (PyErr_Occurred())
1227                 return NULL;
1228         Py_INCREF(Py_None);
1229         return Py_None;
1230 }
1231
1232 static struct PyMethodDef Writer_methods[] = {
1233         { "writerow", (PyCFunction)csv_writerow, METH_O, csv_writerow_doc},
1234         { "writerows", (PyCFunction)csv_writerows, METH_O, csv_writerows_doc},
1235         { NULL, NULL }
1236 };
1237
1238 #define W_OFF(x) offsetof(WriterObj, x)
1239
1240 static struct PyMemberDef Writer_memberlist[] = {
1241         { "dialect", T_OBJECT, W_OFF(dialect), READONLY },
1242         { NULL }
1243 };
1244
1245 static void
1246 Writer_dealloc(WriterObj *self)
1247 {
1248         PyObject_GC_UnTrack(self);
1249         Py_XDECREF(self->dialect);
1250         Py_XDECREF(self->writeline);
1251         if (self->rec != NULL)
1252                 PyMem_Free(self->rec);
1253         PyObject_GC_Del(self);
1254 }
1255
1256 static int
1257 Writer_traverse(WriterObj *self, visitproc visit, void *arg)
1258 {
1259         Py_VISIT(self->dialect);
1260         Py_VISIT(self->writeline);
1261         return 0;
1262 }
1263
1264 static int
1265 Writer_clear(WriterObj *self)
1266 {
1267         Py_CLEAR(self->dialect);
1268         Py_CLEAR(self->writeline);
1269         return 0;
1270 }
1271
1272 PyDoc_STRVAR(Writer_Type_doc,
1273 "CSV writer\n"
1274 "\n"
1275 "Writer objects are responsible for generating tabular data\n"
1276 "in CSV format from sequence input.\n"
1277 );
1278
1279 static PyTypeObject Writer_Type = {
1280         PyVarObject_HEAD_INIT(NULL, 0)
1281         "_csv.writer",                          /*tp_name*/
1282         sizeof(WriterObj),                      /*tp_basicsize*/
1283         0,                                      /*tp_itemsize*/
1284         /* methods */
1285         (destructor)Writer_dealloc,             /*tp_dealloc*/
1286         (printfunc)0,                           /*tp_print*/
1287         (getattrfunc)0,                         /*tp_getattr*/
1288         (setattrfunc)0,                         /*tp_setattr*/
1289         0,                                      /*tp_reserved*/
1290         (reprfunc)0,                            /*tp_repr*/
1291         0,                                      /*tp_as_number*/
1292         0,                                      /*tp_as_sequence*/
1293         0,                                      /*tp_as_mapping*/
1294         (hashfunc)0,                            /*tp_hash*/
1295         (ternaryfunc)0,                         /*tp_call*/
1296         (reprfunc)0,                            /*tp_str*/
1297         0,                                      /*tp_getattro*/
1298         0,                                      /*tp_setattro*/
1299         0,                                      /*tp_as_buffer*/
1300         Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE |
1301                 Py_TPFLAGS_HAVE_GC,             /*tp_flags*/
1302         Writer_Type_doc,
1303         (traverseproc)Writer_traverse,          /*tp_traverse*/
1304         (inquiry)Writer_clear,                  /*tp_clear*/
1305         0,                                      /*tp_richcompare*/
1306         0,                                      /*tp_weaklistoffset*/
1307         (getiterfunc)0,                         /*tp_iter*/
1308         (getiterfunc)0,                         /*tp_iternext*/
1309         Writer_methods,                         /*tp_methods*/
1310         Writer_memberlist,                      /*tp_members*/
1311         0,                                      /*tp_getset*/
1312 };
1313
1314 static PyObject *
1315 csv_writer(PyObject *module, PyObject *args, PyObject *keyword_args)
1316 {
1317         PyObject * output_file, * dialect = NULL;
1318         WriterObj * self = PyObject_GC_New(WriterObj, &Writer_Type);
1319
1320         if (!self)
1321                 return NULL;
1322
1323         self->dialect = NULL;
1324         self->writeline = NULL;
1325
1326         self->rec = NULL;
1327         self->rec_size = 0;
1328         self->rec_len = 0;
1329         self->num_fields = 0;
1330
1331         if (!PyArg_UnpackTuple(args, "", 1, 2, &output_file, &dialect)) {
1332                 Py_DECREF(self);
1333                 return NULL;
1334         }
1335         self->writeline = PyObject_GetAttrString(output_file, "write");
1336         if (self->writeline == NULL || !PyCallable_Check(self->writeline)) {
1337                 PyErr_SetString(PyExc_TypeError,
1338                                 "argument 1 must have a \"write\" method");
1339                 Py_DECREF(self);
1340                 return NULL;
1341         }
1342         self->dialect = (DialectObj *)_call_dialect(dialect, keyword_args);
1343         if (self->dialect == NULL) {
1344                 Py_DECREF(self);
1345                 return NULL;
1346         }
1347         PyObject_GC_Track(self);
1348         return (PyObject *)self;
1349 }
1350
1351 /*
1352  * DIALECT REGISTRY
1353  */
1354 static PyObject *
1355 csv_list_dialects(PyObject *module, PyObject *args)
1356 {
1357         return PyDict_Keys(dialects);
1358 }
1359
1360 static PyObject *
1361 csv_register_dialect(PyObject *module, PyObject *args, PyObject *kwargs)
1362 {
1363         PyObject *name_obj, *dialect_obj = NULL;
1364         PyObject *dialect;
1365
1366         if (!PyArg_UnpackTuple(args, "", 1, 2, &name_obj, &dialect_obj))
1367                 return NULL;
1368         if (!IS_BASESTRING(name_obj)) {
1369                 PyErr_SetString(PyExc_TypeError,
1370                                 "dialect name must be a string or unicode");
1371                 return NULL;
1372         }
1373         dialect = _call_dialect(dialect_obj, kwargs);
1374         if (dialect == NULL)
1375                 return NULL;
1376         if (PyDict_SetItem(dialects, name_obj, dialect) < 0) {
1377                 Py_DECREF(dialect);
1378                 return NULL;
1379         }
1380         Py_DECREF(dialect);
1381         Py_INCREF(Py_None);
1382         return Py_None;
1383 }
1384
1385 static PyObject *
1386 csv_unregister_dialect(PyObject *module, PyObject *name_obj)
1387 {
1388         if (PyDict_DelItem(dialects, name_obj) < 0)
1389                 return PyErr_Format(error_obj, "unknown dialect");
1390         Py_INCREF(Py_None);
1391         return Py_None;
1392 }
1393
1394 static PyObject *
1395 csv_get_dialect(PyObject *module, PyObject *name_obj)
1396 {
1397         return get_dialect_from_registry(name_obj);
1398 }
1399
1400 static PyObject *
1401 csv_field_size_limit(PyObject *module, PyObject *args)
1402 {
1403         PyObject *new_limit = NULL;
1404         long old_limit = field_limit;
1405
1406         if (!PyArg_UnpackTuple(args, "field_size_limit", 0, 1, &new_limit))
1407                 return NULL;
1408         if (new_limit != NULL) {
1409                 if (!PyLong_CheckExact(new_limit)) {
1410                         PyErr_Format(PyExc_TypeError,
1411                                      "limit must be an integer");
1412                         return NULL;
1413                 }
1414                 field_limit = PyLong_AsLong(new_limit);
1415                 if (field_limit == -1 && PyErr_Occurred()) {
1416                         field_limit = old_limit;
1417                         return NULL;
1418                 }
1419         }
1420         return PyLong_FromLong(old_limit);
1421 }
1422
1423 /*
1424  * MODULE
1425  */
1426
1427 PyDoc_STRVAR(csv_module_doc,
1428 "CSV parsing and writing.\n"
1429 "\n"
1430 "This module provides classes that assist in the reading and writing\n"
1431 "of Comma Separated Value (CSV) files, and implements the interface\n"
1432 "described by PEP 305.  Although many CSV files are simple to parse,\n"
1433 "the format is not formally defined by a stable specification and\n"
1434 "is subtle enough that parsing lines of a CSV file with something\n"
1435 "like line.split(\",\") is bound to fail.  The module supports three\n"
1436 "basic APIs: reading, writing, and registration of dialects.\n"
1437 "\n"
1438 "\n"
1439 "DIALECT REGISTRATION:\n"
1440 "\n"
1441 "Readers and writers support a dialect argument, which is a convenient\n"
1442 "handle on a group of settings.  When the dialect argument is a string,\n"
1443 "it identifies one of the dialects previously registered with the module.\n"
1444 "If it is a class or instance, the attributes of the argument are used as\n"
1445 "the settings for the reader or writer:\n"
1446 "\n"
1447 "    class excel:\n"
1448 "        delimiter = ','\n"
1449 "        quotechar = '\"'\n"
1450 "        escapechar = None\n"
1451 "        doublequote = True\n"
1452 "        skipinitialspace = False\n"
1453 "        lineterminator = '\\r\\n'\n"
1454 "        quoting = QUOTE_MINIMAL\n"
1455 "\n"
1456 "SETTINGS:\n"
1457 "\n"
1458 "    * quotechar - specifies a one-character string to use as the \n"
1459 "        quoting character.  It defaults to '\"'.\n"
1460 "    * delimiter - specifies a one-character string to use as the \n"
1461 "        field separator.  It defaults to ','.\n"
1462 "    * skipinitialspace - specifies how to interpret whitespace which\n"
1463 "        immediately follows a delimiter.  It defaults to False, which\n"
1464 "        means that whitespace immediately following a delimiter is part\n"
1465 "        of the following field.\n"
1466 "    * lineterminator -  specifies the character sequence which should \n"
1467 "        terminate rows.\n"
1468 "    * quoting - controls when quotes should be generated by the writer.\n"
1469 "        It can take on any of the following module constants:\n"
1470 "\n"
1471 "        csv.QUOTE_MINIMAL means only when required, for example, when a\n"
1472 "            field contains either the quotechar or the delimiter\n"
1473 "        csv.QUOTE_ALL means that quotes are always placed around fields.\n"
1474 "        csv.QUOTE_NONNUMERIC means that quotes are always placed around\n"
1475 "            fields which do not parse as integers or floating point\n"
1476 "            numbers.\n"
1477 "        csv.QUOTE_NONE means that quotes are never placed around fields.\n"
1478 "    * escapechar - specifies a one-character string used to escape \n"
1479 "        the delimiter when quoting is set to QUOTE_NONE.\n"
1480 "    * doublequote - controls the handling of quotes inside fields.  When\n"
1481 "        True, two consecutive quotes are interpreted as one during read,\n"
1482 "        and when writing, each quote character embedded in the data is\n"
1483 "        written as two quotes\n");
1484
1485 PyDoc_STRVAR(csv_reader_doc,
1486 "    csv_reader = reader(iterable [, dialect='excel']\n"
1487 "                        [optional keyword args])\n"
1488 "    for row in csv_reader:\n"
1489 "        process(row)\n"
1490 "\n"
1491 "The \"iterable\" argument can be any object that returns a line\n"
1492 "of input for each iteration, such as a file object or a list.  The\n"
1493 "optional \"dialect\" parameter is discussed below.  The function\n"
1494 "also accepts optional keyword arguments which override settings\n"
1495 "provided by the dialect.\n"
1496 "\n"
1497 "The returned object is an iterator.  Each iteration returns a row\n"
1498 "of the CSV file (which can span multiple input lines):\n");
1499
1500 PyDoc_STRVAR(csv_writer_doc,
1501 "    csv_writer = csv.writer(fileobj [, dialect='excel']\n"
1502 "                            [optional keyword args])\n"
1503 "    for row in sequence:\n"
1504 "        csv_writer.writerow(row)\n"
1505 "\n"
1506 "    [or]\n"
1507 "\n"
1508 "    csv_writer = csv.writer(fileobj [, dialect='excel']\n"
1509 "                            [optional keyword args])\n"
1510 "    csv_writer.writerows(rows)\n"
1511 "\n"
1512 "The \"fileobj\" argument can be any object that supports the file API.\n");
1513
1514 PyDoc_STRVAR(csv_list_dialects_doc,
1515 "Return a list of all know dialect names.\n"
1516 "    names = csv.list_dialects()");
1517
1518 PyDoc_STRVAR(csv_get_dialect_doc,
1519 "Return the dialect instance associated with name.\n"
1520 "    dialect = csv.get_dialect(name)");
1521
1522 PyDoc_STRVAR(csv_register_dialect_doc,
1523 "Create a mapping from a string name to a dialect class.\n"
1524 "    dialect = csv.register_dialect(name, dialect)");
1525
1526 PyDoc_STRVAR(csv_unregister_dialect_doc,
1527 "Delete the name/dialect mapping associated with a string name.\n"
1528 "    csv.unregister_dialect(name)");
1529
1530 PyDoc_STRVAR(csv_field_size_limit_doc,
1531 "Sets an upper limit on parsed fields.\n"
1532 "    csv.field_size_limit([limit])\n"
1533 "\n"
1534 "Returns old limit. If limit is not given, no new limit is set and\n"
1535 "the old limit is returned");
1536
1537 static struct PyMethodDef csv_methods[] = {
1538         { "reader", (PyCFunction)csv_reader,
1539                 METH_VARARGS | METH_KEYWORDS, csv_reader_doc},
1540         { "writer", (PyCFunction)csv_writer,
1541                 METH_VARARGS | METH_KEYWORDS, csv_writer_doc},
1542         { "list_dialects", (PyCFunction)csv_list_dialects,
1543                 METH_NOARGS, csv_list_dialects_doc},
1544         { "register_dialect", (PyCFunction)csv_register_dialect,
1545                 METH_VARARGS | METH_KEYWORDS, csv_register_dialect_doc},
1546         { "unregister_dialect", (PyCFunction)csv_unregister_dialect,
1547                 METH_O, csv_unregister_dialect_doc},
1548         { "get_dialect", (PyCFunction)csv_get_dialect,
1549                 METH_O, csv_get_dialect_doc},
1550         { "field_size_limit", (PyCFunction)csv_field_size_limit,
1551                 METH_VARARGS, csv_field_size_limit_doc},
1552         { NULL, NULL }
1553 };
1554
1555
1556 static struct PyModuleDef _csvmodule = {
1557         PyModuleDef_HEAD_INIT,
1558         "_csv",
1559         csv_module_doc,
1560         -1,
1561         csv_methods,
1562         NULL,
1563         NULL,
1564         NULL,
1565         NULL
1566 };
1567
1568 PyMODINIT_FUNC
1569 PyInit__csv(void)
1570 {
1571         PyObject *module;
1572         StyleDesc *style;
1573
1574         if (PyType_Ready(&Dialect_Type) < 0)
1575                 return NULL;
1576
1577         if (PyType_Ready(&Reader_Type) < 0)
1578                 return NULL;
1579
1580         if (PyType_Ready(&Writer_Type) < 0)
1581                 return NULL;
1582
1583         /* Create the module and add the functions */
1584         module = PyModule_Create(&_csvmodule);
1585         if (module == NULL)
1586                 return NULL;
1587
1588         /* Add version to the module. */
1589         if (PyModule_AddStringConstant(module, "__version__",
1590                                        MODULE_VERSION) == -1)
1591                 return NULL;
1592
1593         /* Add _dialects dictionary */
1594         dialects = PyDict_New();
1595         if (dialects == NULL)
1596                 return NULL;
1597         if (PyModule_AddObject(module, "_dialects", dialects))
1598                 return NULL;
1599
1600         /* Add quote styles into dictionary */
1601         for (style = quote_styles; style->name; style++) {
1602                 if (PyModule_AddIntConstant(module, style->name,
1603                                             style->style) == -1)
1604                         return NULL;
1605         }
1606
1607         /* Add the Dialect type */
1608         Py_INCREF(&Dialect_Type);
1609         if (PyModule_AddObject(module, "Dialect", (PyObject *)&Dialect_Type))
1610                 return NULL;
1611
1612         /* Add the CSV exception object to the module. */
1613         error_obj = PyErr_NewException("_csv.Error", NULL, NULL);
1614         if (error_obj == NULL)
1615                 return NULL;
1616         PyModule_AddObject(module, "Error", error_obj);
1617         return module;
1618 }