?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Updated with fix for #3126.
[python.git] / Objects / fileobject.c
blob7c49afff0699eeff10ade07391a7495f2a5f98dd
1 /* File object implementation */
2
3 #define PY_SSIZE_T_CLEAN
4 #include "Python.h"
5 #include "structmember.h"
6
7 #ifdef HAVE_SYS_TYPES_H
8 #include <sys/types.h>
9 #endif /* HAVE_SYS_TYPES_H */
10
11 #ifdef MS_WINDOWS
12 #define fileno _fileno
13 /* can simulate truncate with Win32 API functions; see file_truncate */
14 #define HAVE_FTRUNCATE
15 #define WIN32_LEAN_AND_MEAN
16 #include <windows.h>
17 #endif
18
19 #ifdef _MSC_VER
20 /* Need GetVersion to see if on NT so safe to use _wfopen */
21 #define WIN32_LEAN_AND_MEAN
22 #include <windows.h>
23 #endif /* _MSC_VER */
24
25 #if defined(PYOS_OS2) && defined(PYCC_GCC)
26 #include <io.h>
27 #endif
28
29 #define BUF(v) PyString_AS_STRING((PyStringObject *)v)
30
31 #ifndef DONT_HAVE_ERRNO_H
32 #include <errno.h>
33 #endif
34
35 #ifdef HAVE_GETC_UNLOCKED
36 #define GETC(f) getc_unlocked(f)
37 #define FLOCKFILE(f) flockfile(f)
38 #define FUNLOCKFILE(f) funlockfile(f)
39 #else
40 #define GETC(f) getc(f)
41 #define FLOCKFILE(f)
42 #define FUNLOCKFILE(f)
43 #endif
44
45 /* Bits in f_newlinetypes */
46 #define NEWLINE_UNKNOWN 0 /* No newline seen, yet */
47 #define NEWLINE_CR 1 /* \r newline seen */
48 #define NEWLINE_LF 2 /* \n newline seen */
49 #define NEWLINE_CRLF 4 /* \r\n newline seen */
50
51 /*
52 * These macros release the GIL while preventing the f_close() function being
53 * called in the interval between them. For that purpose, a running total of
54 * the number of currently running unlocked code sections is kept in
55 * the unlocked_count field of the PyFileObject. The close() method raises
56 * an IOError if that field is non-zero. See issue #815646, #595601.
57 */
58
59 #define FILE_BEGIN_ALLOW_THREADS(fobj) \
60 { \
61 fobj->unlocked_count++; \
62 Py_BEGIN_ALLOW_THREADS
63
64 #define FILE_END_ALLOW_THREADS(fobj) \
65 Py_END_ALLOW_THREADS \
66 fobj->unlocked_count--; \
67 assert(fobj->unlocked_count >= 0); \
68 }
69
70 #define FILE_ABORT_ALLOW_THREADS(fobj) \
71 Py_BLOCK_THREADS \
72 fobj->unlocked_count--; \
73 assert(fobj->unlocked_count >= 0);
74
75 #ifdef __cplusplus
76 extern "C" {
77 #endif
78
79 FILE *
80 PyFile_AsFile(PyObject *f)
81 {
82 if (f == NULL || !PyFile_Check(f))
83 return NULL;
84 else
85 return ((PyFileObject *)f)->f_fp;
86 }
87
88 void PyFile_IncUseCount(PyFileObject *fobj)
89 {
90 fobj->unlocked_count++;
91 }
92
93 void PyFile_DecUseCount(PyFileObject *fobj)
94 {
95 fobj->unlocked_count--;
96 assert(fobj->unlocked_count >= 0);
97 }
98
99 PyObject *
100 PyFile_Name(PyObject *f)
101 {
102 if (f == NULL || !PyFile_Check(f))
103 return NULL;
104 else
105 return ((PyFileObject *)f)->f_name;
106 }
107
108 /* This is a safe wrapper around PyObject_Print to print to the FILE
109 of a PyFileObject. PyObject_Print releases the GIL but knows nothing
110 about PyFileObject. */
111 static int
112 file_PyObject_Print(PyObject *op, PyFileObject *f, int flags)
113 {
114 int result;
115 PyFile_IncUseCount(f);
116 result = PyObject_Print(op, f->f_fp, flags);
117 PyFile_DecUseCount(f);
118 return result;
119 }
120
121 /* On Unix, fopen will succeed for directories.
122 In Python, there should be no file objects referring to
123 directories, so we need a check. */
124
125 static PyFileObject*
126 dircheck(PyFileObject* f)
127 {
128 #if defined(HAVE_FSTAT) && defined(S_IFDIR) && defined(EISDIR)
129 struct stat buf;
130 if (f->f_fp == NULL)
131 return f;
132 if (fstat(fileno(f->f_fp), &buf) == 0 &&
133 S_ISDIR(buf.st_mode)) {
134 char *msg = strerror(EISDIR);
135 PyObject *exc = PyObject_CallFunction(PyExc_IOError, "(is)",
136 EISDIR, msg);
137 PyErr_SetObject(PyExc_IOError, exc);
138 Py_XDECREF(exc);
139 return NULL;
140 }
141 #endif
142 return f;
143 }
144
145
146 static PyObject *
147 fill_file_fields(PyFileObject *f, FILE *fp, PyObject *name, char *mode,
148 int (*close)(FILE *))
149 {
150 assert(name != NULL);
151 assert(f != NULL);
152 assert(PyFile_Check(f));
153 assert(f->f_fp == NULL);
154
155 Py_DECREF(f->f_name);
156 Py_DECREF(f->f_mode);
157 Py_DECREF(f->f_encoding);
158 Py_DECREF(f->f_errors);
159
160 Py_INCREF(name);
161 f->f_name = name;
162
163 f->f_mode = PyString_FromString(mode);
164
165 f->f_close = close;
166 f->f_softspace = 0;
167 f->f_binary = strchr(mode,'b') != NULL;
168 f->f_buf = NULL;
169 f->f_univ_newline = (strchr(mode, 'U') != NULL);
170 f->f_newlinetypes = NEWLINE_UNKNOWN;
171 f->f_skipnextlf = 0;
172 Py_INCREF(Py_None);
173 f->f_encoding = Py_None;
174 Py_INCREF(Py_None);
175 f->f_errors = Py_None;
176
177 if (f->f_mode == NULL)
178 return NULL;
179 f->f_fp = fp;
180 f = dircheck(f);
181 return (PyObject *) f;
182 }
183
184 /* check for known incorrect mode strings - problem is, platforms are
185 free to accept any mode characters they like and are supposed to
186 ignore stuff they don't understand... write or append mode with
187 universal newline support is expressly forbidden by PEP 278.
188 Additionally, remove the 'U' from the mode string as platforms
189 won't know what it is. Non-zero return signals an exception */
190 int
191 _PyFile_SanitizeMode(char *mode)
192 {
193 char *upos;
194 size_t len = strlen(mode);
195
196 if (!len) {
197 PyErr_SetString(PyExc_ValueError, "empty mode string");
198 return -1;
199 }
200
201 upos = strchr(mode, 'U');
202 if (upos) {
203 memmove(upos, upos+1, len-(upos-mode)); /* incl null char */
204
205 if (mode[0] == 'w' || mode[0] == 'a') {
206 PyErr_Format(PyExc_ValueError, "universal newline "
207 "mode can only be used with modes "
208 "starting with 'r'");
209 return -1;
210 }
211
212 if (mode[0] != 'r') {
213 memmove(mode+1, mode, strlen(mode)+1);
214 mode[0] = 'r';
215 }
216
217 if (!strchr(mode, 'b')) {
218 memmove(mode+2, mode+1, strlen(mode));
219 mode[1] = 'b';
220 }
221 } else if (mode[0] != 'r' && mode[0] != 'w' && mode[0] != 'a') {
222 PyErr_Format(PyExc_ValueError, "mode string must begin with "
223 "one of 'r', 'w', 'a' or 'U', not '%.200s'", mode);
224 return -1;
225 }
226
227 return 0;
228 }
229
230 static PyObject *
231 open_the_file(PyFileObject *f, char *name, char *mode)
232 {
233 char *newmode;
234 assert(f != NULL);
235 assert(PyFile_Check(f));
236 #ifdef MS_WINDOWS
237 /* windows ignores the passed name in order to support Unicode */
238 assert(f->f_name != NULL);
239 #else
240 assert(name != NULL);
241 #endif
242 assert(mode != NULL);
243 assert(f->f_fp == NULL);
244
245 /* probably need to replace 'U' by 'rb' */
246 newmode = PyMem_MALLOC(strlen(mode) + 3);
247 if (!newmode) {
248 PyErr_NoMemory();
249 return NULL;
250 }
251 strcpy(newmode, mode);
252
253 if (_PyFile_SanitizeMode(newmode)) {
254 f = NULL;
255 goto cleanup;
256 }
257
258 /* rexec.py can't stop a user from getting the file() constructor --
259 all they have to do is get *any* file object f, and then do
260 type(f). Here we prevent them from doing damage with it. */
261 if (PyEval_GetRestricted()) {
262 PyErr_SetString(PyExc_IOError,
263 "file() constructor not accessible in restricted mode");
264 f = NULL;
265 goto cleanup;
266 }
267 errno = 0;
268
269 #ifdef MS_WINDOWS
270 if (PyUnicode_Check(f->f_name)) {
271 PyObject *wmode;
272 wmode = PyUnicode_DecodeASCII(newmode, strlen(newmode), NULL);
273 if (f->f_name && wmode) {
274 FILE_BEGIN_ALLOW_THREADS(f)
275 /* PyUnicode_AS_UNICODE OK without thread
276 lock as it is a simple dereference. */
277 f->f_fp = _wfopen(PyUnicode_AS_UNICODE(f->f_name),
278 PyUnicode_AS_UNICODE(wmode));
279 FILE_END_ALLOW_THREADS(f)
280 }
281 Py_XDECREF(wmode);
282 }
283 #endif
284 if (NULL == f->f_fp && NULL != name) {
285 FILE_BEGIN_ALLOW_THREADS(f)
286 f->f_fp = fopen(name, newmode);
287 FILE_END_ALLOW_THREADS(f)
288 }
289
290 if (f->f_fp == NULL) {
291 #if defined _MSC_VER && (_MSC_VER < 1400 || !defined(__STDC_SECURE_LIB__))
292 /* MSVC 6 (Microsoft) leaves errno at 0 for bad mode strings,
293 * across all Windows flavors. When it sets EINVAL varies
294 * across Windows flavors, the exact conditions aren't
295 * documented, and the answer lies in the OS's implementation
296 * of Win32's CreateFile function (whose source is secret).
297 * Seems the best we can do is map EINVAL to ENOENT.
298 * Starting with Visual Studio .NET 2005, EINVAL is correctly
299 * set by our CRT error handler (set in exceptions.c.)
300 */
301 if (errno == 0) /* bad mode string */
302 errno = EINVAL;
303 else if (errno == EINVAL) /* unknown, but not a mode string */
304 errno = ENOENT;
305 #endif
306 /* EINVAL is returned when an invalid filename or
307 * an invalid mode is supplied. */
308 if (errno == EINVAL)
309 PyErr_Format(PyExc_IOError,
310 "invalid filename: %s or mode: %s",
311 name, mode);
312 else
313 PyErr_SetFromErrnoWithFilenameObject(PyExc_IOError, f->f_name);
314 f = NULL;
315 }
316 if (f != NULL)
317 f = dircheck(f);
318
319 cleanup:
320 PyMem_FREE(newmode);
321
322 return (PyObject *)f;
323 }
324
325 static PyObject *
326 close_the_file(PyFileObject *f)
327 {
328 int sts = 0;
329 int (*local_close)(FILE *);
330 FILE *local_fp = f->f_fp;
331 if (local_fp != NULL) {
332 local_close = f->f_close;
333 if (local_close != NULL && f->unlocked_count > 0) {
334 if (f->ob_refcnt > 0) {
335 PyErr_SetString(PyExc_IOError,
336 "close() called during concurrent "
337 "operation on the same file object.");
338 } else {
339 /* This should not happen unless someone is
340 * carelessly playing with the PyFileObject
341 * struct fields and/or its associated FILE
342 * pointer. */
343 PyErr_SetString(PyExc_SystemError,
344 "PyFileObject locking error in "
345 "destructor (refcnt <= 0 at close).");
346 }
347 return NULL;
348 }
349 /* NULL out the FILE pointer before releasing the GIL, because
350 * it will not be valid anymore after the close() function is
351 * called. */
352 f->f_fp = NULL;
353 if (local_close != NULL) {
354 Py_BEGIN_ALLOW_THREADS
355 errno = 0;
356 sts = (*local_close)(local_fp);
357 Py_END_ALLOW_THREADS
358 if (sts == EOF)
359 return PyErr_SetFromErrno(PyExc_IOError);
360 if (sts != 0)
361 return PyInt_FromLong((long)sts);
362 }
363 }
364 Py_RETURN_NONE;
365 }
366
367 PyObject *
368 PyFile_FromFile(FILE *fp, char *name, char *mode, int (*close)(FILE *))
369 {
370 PyFileObject *f = (PyFileObject *)PyFile_Type.tp_new(&PyFile_Type,
371 NULL, NULL);
372 if (f != NULL) {
373 PyObject *o_name = PyString_FromString(name);
374 if (o_name == NULL)
375 return NULL;
376 if (fill_file_fields(f, fp, o_name, mode, close) == NULL) {
377 Py_DECREF(f);
378 f = NULL;
379 }
380 Py_DECREF(o_name);
381 }
382 return (PyObject *) f;
383 }
384
385 PyObject *
386 PyFile_FromString(char *name, char *mode)
387 {
388 extern int fclose(FILE *);
389 PyFileObject *f;
390
391 f = (PyFileObject *)PyFile_FromFile((FILE *)NULL, name, mode, fclose);
392 if (f != NULL) {
393 if (open_the_file(f, name, mode) == NULL) {
394 Py_DECREF(f);
395 f = NULL;
396 }
397 }
398 return (PyObject *)f;
399 }
400
401 void
402 PyFile_SetBufSize(PyObject *f, int bufsize)
403 {
404 PyFileObject *file = (PyFileObject *)f;
405 if (bufsize >= 0) {
406 int type;
407 switch (bufsize) {
408 case 0:
409 type = _IONBF;
410 break;
411 #ifdef HAVE_SETVBUF
412 case 1:
413 type = _IOLBF;
414 bufsize = BUFSIZ;
415 break;
416 #endif
417 default:
418 type = _IOFBF;
419 #ifndef HAVE_SETVBUF
420 bufsize = BUFSIZ;
421 #endif
422 break;
423 }
424 fflush(file->f_fp);
425 if (type == _IONBF) {
426 PyMem_Free(file->f_setbuf);
427 file->f_setbuf = NULL;
428 } else {
429 file->f_setbuf = (char *)PyMem_Realloc(file->f_setbuf,
430 bufsize);
431 }
432 #ifdef HAVE_SETVBUF
433 setvbuf(file->f_fp, file->f_setbuf, type, bufsize);
434 #else /* !HAVE_SETVBUF */
435 setbuf(file->f_fp, file->f_setbuf);
436 #endif /* !HAVE_SETVBUF */
437 }
438 }
439
440 /* Set the encoding used to output Unicode strings.
441 Return 1 on success, 0 on failure. */
442
443 int
444 PyFile_SetEncoding(PyObject *f, const char *enc)
445 {
446 return PyFile_SetEncodingAndErrors(f, enc, NULL);
447 }
448
449 int
450 PyFile_SetEncodingAndErrors(PyObject *f, const char *enc, char* errors)
451 {
452 PyFileObject *file = (PyFileObject*)f;
453 PyObject *str, *oerrors;
454
455 assert(PyFile_Check(f));
456 str = PyString_FromString(enc);
457 if (!str)
458 return 0;
459 if (errors) {
460 oerrors = PyString_FromString(errors);
461 if (!oerrors) {
462 Py_DECREF(str);
463 return 0;
464 }
465 } else {
466 oerrors = Py_None;
467 Py_INCREF(Py_None);
468 }
469 Py_DECREF(file->f_encoding);
470 file->f_encoding = str;
471 Py_DECREF(file->f_errors);
472 file->f_errors = oerrors;
473 return 1;
474 }
475
476 static PyObject *
477 err_closed(void)
478 {
479 PyErr_SetString(PyExc_ValueError, "I/O operation on closed file");
480 return NULL;
481 }
482
483 /* Refuse regular file I/O if there's data in the iteration-buffer.
484 * Mixing them would cause data to arrive out of order, as the read*
485 * methods don't use the iteration buffer. */
486 static PyObject *
487 err_iterbuffered(void)
488 {
489 PyErr_SetString(PyExc_ValueError,
490 "Mixing iteration and read methods would lose data");
491 return NULL;
492 }
493
494 static void drop_readahead(PyFileObject *);
495
496 /* Methods */
497
498 static void
499 file_dealloc(PyFileObject *f)
500 {
501 PyObject *ret;
502 if (f->weakreflist != NULL)
503 PyObject_ClearWeakRefs((PyObject *) f);
504 ret = close_the_file(f);
505 if (!ret) {
506 PySys_WriteStderr("close failed in file object destructor:\n");
507 PyErr_Print();
508 }
509 else {
510 Py_DECREF(ret);
511 }
512 PyMem_Free(f->f_setbuf);
513 Py_XDECREF(f->f_name);
514 Py_XDECREF(f->f_mode);
515 Py_XDECREF(f->f_encoding);
516 Py_XDECREF(f->f_errors);
517 drop_readahead(f);
518 Py_TYPE(f)->tp_free((PyObject *)f);
519 }
520
521 static PyObject *
522 file_repr(PyFileObject *f)
523 {
524 if (PyUnicode_Check(f->f_name)) {
525 #ifdef Py_USING_UNICODE
526 PyObject *ret = NULL;
527 PyObject *name = PyUnicode_AsUnicodeEscapeString(f->f_name);
528 const char *name_str = name ? PyString_AsString(name) : "?";
529 ret = PyString_FromFormat("<%s file u'%s', mode '%s' at %p>",
530 f->f_fp == NULL ? "closed" : "open",
531 name_str,
532 PyString_AsString(f->f_mode),
533 f);
534 Py_XDECREF(name);
535 return ret;
536 #endif
537 } else {
538 return PyString_FromFormat("<%s file '%s', mode '%s' at %p>",
539 f->f_fp == NULL ? "closed" : "open",
540 PyString_AsString(f->f_name),
541 PyString_AsString(f->f_mode),
542 f);
543 }
544 }
545
546 static PyObject *
547 file_close(PyFileObject *f)
548 {
549 PyObject *sts = close_the_file(f);
550 PyMem_Free(f->f_setbuf);
551 f->f_setbuf = NULL;
552 return sts;
553 }
554
555
556 /* Our very own off_t-like type, 64-bit if possible */
557 #if !defined(HAVE_LARGEFILE_SUPPORT)
558 typedef off_t Py_off_t;
559 #elif SIZEOF_OFF_T >= 8
560 typedef off_t Py_off_t;
561 #elif SIZEOF_FPOS_T >= 8
562 typedef fpos_t Py_off_t;
563 #else
564 #error "Large file support, but neither off_t nor fpos_t is large enough."
565 #endif
566
567
568 /* a portable fseek() function
569 return 0 on success, non-zero on failure (with errno set) */
570 static int
571 _portable_fseek(FILE *fp, Py_off_t offset, int whence)
572 {
573 #if !defined(HAVE_LARGEFILE_SUPPORT)
574 return fseek(fp, offset, whence);
575 #elif defined(HAVE_FSEEKO) && SIZEOF_OFF_T >= 8
576 return fseeko(fp, offset, whence);
577 #elif defined(HAVE_FSEEK64)
578 return fseek64(fp, offset, whence);
579 #elif defined(__BEOS__)
580 return _fseek(fp, offset, whence);
581 #elif SIZEOF_FPOS_T >= 8
582 /* lacking a 64-bit capable fseek(), use a 64-bit capable fsetpos()
583 and fgetpos() to implement fseek()*/
584 fpos_t pos;
585 switch (whence) {
586 case SEEK_END:
587 #ifdef MS_WINDOWS
588 fflush(fp);
589 if (_lseeki64(fileno(fp), 0, 2) == -1)
590 return -1;
591 #else
592 if (fseek(fp, 0, SEEK_END) != 0)
593 return -1;
594 #endif
595 /* fall through */
596 case SEEK_CUR:
597 if (fgetpos(fp, &pos) != 0)
598 return -1;
599 offset += pos;
600 break;
601 /* case SEEK_SET: break; */
602 }
603 return fsetpos(fp, &offset);
604 #else
605 #error "Large file support, but no way to fseek."
606 #endif
607 }
608
609
610 /* a portable ftell() function
611 Return -1 on failure with errno set appropriately, current file
612 position on success */
613 static Py_off_t
614 _portable_ftell(FILE* fp)
615 {
616 #if !defined(HAVE_LARGEFILE_SUPPORT)
617 return ftell(fp);
618 #elif defined(HAVE_FTELLO) && SIZEOF_OFF_T >= 8
619 return ftello(fp);
620 #elif defined(HAVE_FTELL64)
621 return ftell64(fp);
622 #elif SIZEOF_FPOS_T >= 8
623 fpos_t pos;
624 if (fgetpos(fp, &pos) != 0)
625 return -1;
626 return pos;
627 #else
628 #error "Large file support, but no way to ftell."
629 #endif
630 }
631
632
633 static PyObject *
634 file_seek(PyFileObject *f, PyObject *args)
635 {
636 int whence;
637 int ret;
638 Py_off_t offset;
639 PyObject *offobj, *off_index;
640
641 if (f->f_fp == NULL)
642 return err_closed();
643 drop_readahead(f);
644 whence = 0;
645 if (!PyArg_ParseTuple(args, "O|i:seek", &offobj, &whence))
646 return NULL;
647 off_index = PyNumber_Index(offobj);
648 if (!off_index) {
649 if (!PyFloat_Check(offobj))
650 return NULL;
651 /* Deprecated in 2.6 */
652 PyErr_Clear();
653 if (PyErr_WarnEx(PyExc_DeprecationWarning,
654 "integer argument expected, got float",
655 1) < 0)
656 return NULL;
657 off_index = offobj;
658 Py_INCREF(offobj);
659 }
660 #if !defined(HAVE_LARGEFILE_SUPPORT)
661 offset = PyInt_AsLong(off_index);
662 #else
663 offset = PyLong_Check(off_index) ?
664 PyLong_AsLongLong(off_index) : PyInt_AsLong(off_index);
665 #endif
666 Py_DECREF(off_index);
667 if (PyErr_Occurred())
668 return NULL;
669
670 FILE_BEGIN_ALLOW_THREADS(f)
671 errno = 0;
672 ret = _portable_fseek(f->f_fp, offset, whence);
673 FILE_END_ALLOW_THREADS(f)
674
675 if (ret != 0) {
676 PyErr_SetFromErrno(PyExc_IOError);
677 clearerr(f->f_fp);
678 return NULL;
679 }
680 f->f_skipnextlf = 0;
681 Py_INCREF(Py_None);
682 return Py_None;
683 }
684
685
686 #ifdef HAVE_FTRUNCATE
687 static PyObject *
688 file_truncate(PyFileObject *f, PyObject *args)
689 {
690 Py_off_t newsize;
691 PyObject *newsizeobj = NULL;
692 Py_off_t initialpos;
693 int ret;
694
695 if (f->f_fp == NULL)
696 return err_closed();
697 if (!PyArg_UnpackTuple(args, "truncate", 0, 1, &newsizeobj))
698 return NULL;
699
700 /* Get current file position. If the file happens to be open for
701 * update and the last operation was an input operation, C doesn't
702 * define what the later fflush() will do, but we promise truncate()
703 * won't change the current position (and fflush() *does* change it
704 * then at least on Windows). The easiest thing is to capture
705 * current pos now and seek back to it at the end.
706 */
707 FILE_BEGIN_ALLOW_THREADS(f)
708 errno = 0;
709 initialpos = _portable_ftell(f->f_fp);
710 FILE_END_ALLOW_THREADS(f)
711 if (initialpos == -1)
712 goto onioerror;
713
714 /* Set newsize to current postion if newsizeobj NULL, else to the
715 * specified value.
716 */
717 if (newsizeobj != NULL) {
718 #if !defined(HAVE_LARGEFILE_SUPPORT)
719 newsize = PyInt_AsLong(newsizeobj);
720 #else
721 newsize = PyLong_Check(newsizeobj) ?
722 PyLong_AsLongLong(newsizeobj) :
723 PyInt_AsLong(newsizeobj);
724 #endif
725 if (PyErr_Occurred())
726 return NULL;
727 }
728 else /* default to current position */
729 newsize = initialpos;
730
731 /* Flush the stream. We're mixing stream-level I/O with lower-level
732 * I/O, and a flush may be necessary to synch both platform views
733 * of the current file state.
734 */
735 FILE_BEGIN_ALLOW_THREADS(f)
736 errno = 0;
737 ret = fflush(f->f_fp);
738 FILE_END_ALLOW_THREADS(f)
739 if (ret != 0)
740 goto onioerror;
741
742 #ifdef MS_WINDOWS
743 /* MS _chsize doesn't work if newsize doesn't fit in 32 bits,
744 so don't even try using it. */
745 {
746 HANDLE hFile;
747
748 /* Have to move current pos to desired endpoint on Windows. */
749 FILE_BEGIN_ALLOW_THREADS(f)
750 errno = 0;
751 ret = _portable_fseek(f->f_fp, newsize, SEEK_SET) != 0;
752 FILE_END_ALLOW_THREADS(f)
753 if (ret)
754 goto onioerror;
755
756 /* Truncate. Note that this may grow the file! */
757 FILE_BEGIN_ALLOW_THREADS(f)
758 errno = 0;
759 hFile = (HANDLE)_get_osfhandle(fileno(f->f_fp));
760 ret = hFile == (HANDLE)-1;
761 if (ret == 0) {
762 ret = SetEndOfFile(hFile) == 0;
763 if (ret)
764 errno = EACCES;
765 }
766 FILE_END_ALLOW_THREADS(f)
767 if (ret)
768 goto onioerror;
769 }
770 #else
771 FILE_BEGIN_ALLOW_THREADS(f)
772 errno = 0;
773 ret = ftruncate(fileno(f->f_fp), newsize);
774 FILE_END_ALLOW_THREADS(f)
775 if (ret != 0)
776 goto onioerror;
777 #endif /* !MS_WINDOWS */
778
779 /* Restore original file position. */
780 FILE_BEGIN_ALLOW_THREADS(f)
781 errno = 0;
782 ret = _portable_fseek(f->f_fp, initialpos, SEEK_SET) != 0;
783 FILE_END_ALLOW_THREADS(f)
784 if (ret)
785 goto onioerror;
786
787 Py_INCREF(Py_None);
788 return Py_None;
789
790 onioerror:
791 PyErr_SetFromErrno(PyExc_IOError);
792 clearerr(f->f_fp);
793 return NULL;
794 }
795 #endif /* HAVE_FTRUNCATE */
796
797 static PyObject *
798 file_tell(PyFileObject *f)
799 {
800 Py_off_t pos;
801
802 if (f->f_fp == NULL)
803 return err_closed();
804 FILE_BEGIN_ALLOW_THREADS(f)
805 errno = 0;
806 pos = _portable_ftell(f->f_fp);
807 FILE_END_ALLOW_THREADS(f)
808
809 if (pos == -1) {
810 PyErr_SetFromErrno(PyExc_IOError);
811 clearerr(f->f_fp);
812 return NULL;
813 }
814 if (f->f_skipnextlf) {
815 int c;
816 c = GETC(f->f_fp);
817 if (c == '\n') {
818 f->f_newlinetypes |= NEWLINE_CRLF;
819 pos++;
820 f->f_skipnextlf = 0;
821 } else if (c != EOF) ungetc(c, f->f_fp);
822 }
823 #if !defined(HAVE_LARGEFILE_SUPPORT)
824 return PyInt_FromLong(pos);
825 #else
826 return PyLong_FromLongLong(pos);
827 #endif
828 }
829
830 static PyObject *
831 file_fileno(PyFileObject *f)
832 {
833 if (f->f_fp == NULL)
834 return err_closed();
835 return PyInt_FromLong((long) fileno(f->f_fp));
836 }
837
838 static PyObject *
839 file_flush(PyFileObject *f)
840 {
841 int res;
842
843 if (f->f_fp == NULL)
844 return err_closed();
845 FILE_BEGIN_ALLOW_THREADS(f)
846 errno = 0;
847 res = fflush(f->f_fp);
848 FILE_END_ALLOW_THREADS(f)
849 if (res != 0) {
850 PyErr_SetFromErrno(PyExc_IOError);
851 clearerr(f->f_fp);
852 return NULL;
853 }
854 Py_INCREF(Py_None);
855 return Py_None;
856 }
857
858 static PyObject *
859 file_isatty(PyFileObject *f)
860 {
861 long res;
862 if (f->f_fp == NULL)
863 return err_closed();
864 FILE_BEGIN_ALLOW_THREADS(f)
865 res = isatty((int)fileno(f->f_fp));
866 FILE_END_ALLOW_THREADS(f)
867 return PyBool_FromLong(res);
868 }
869
870
871 #if BUFSIZ < 8192
872 #define SMALLCHUNK 8192
873 #else
874 #define SMALLCHUNK BUFSIZ
875 #endif
876
877 #if SIZEOF_INT < 4
878 #define BIGCHUNK (512 * 32)
879 #else
880 #define BIGCHUNK (512 * 1024)
881 #endif
882
883 static size_t
884 new_buffersize(PyFileObject *f, size_t currentsize)
885 {
886 #ifdef HAVE_FSTAT
887 off_t pos, end;
888 struct stat st;
889 if (fstat(fileno(f->f_fp), &st) == 0) {
890 end = st.st_size;
891 /* The following is not a bug: we really need to call lseek()
892 *and* ftell(). The reason is that some stdio libraries
893 mistakenly flush their buffer when ftell() is called and
894 the lseek() call it makes fails, thereby throwing away
895 data that cannot be recovered in any way. To avoid this,
896 we first test lseek(), and only call ftell() if lseek()
897 works. We can't use the lseek() value either, because we
898 need to take the amount of buffered data into account.
899 (Yet another reason why stdio stinks. :-) */
900 pos = lseek(fileno(f->f_fp), 0L, SEEK_CUR);
901 if (pos >= 0) {
902 pos = ftell(f->f_fp);
903 }
904 if (pos < 0)
905 clearerr(f->f_fp);
906 if (end > pos && pos >= 0)
907 return currentsize + end - pos + 1;
908 /* Add 1 so if the file were to grow we'd notice. */
909 }
910 #endif
911 if (currentsize > SMALLCHUNK) {
912 /* Keep doubling until we reach BIGCHUNK;
913 then keep adding BIGCHUNK. */
914 if (currentsize <= BIGCHUNK)
915 return currentsize + currentsize;
916 else
917 return currentsize + BIGCHUNK;
918 }
919 return currentsize + SMALLCHUNK;
920 }
921
922 #if defined(EWOULDBLOCK) && defined(EAGAIN) && EWOULDBLOCK != EAGAIN
923 #define BLOCKED_ERRNO(x) ((x) == EWOULDBLOCK || (x) == EAGAIN)
924 #else
925 #ifdef EWOULDBLOCK
926 #define BLOCKED_ERRNO(x) ((x) == EWOULDBLOCK)
927 #else
928 #ifdef EAGAIN
929 #define BLOCKED_ERRNO(x) ((x) == EAGAIN)
930 #else
931 #define BLOCKED_ERRNO(x) 0
932 #endif
933 #endif
934 #endif
935
936 static PyObject *
937 file_read(PyFileObject *f, PyObject *args)
938 {
939 long bytesrequested = -1;
940 size_t bytesread, buffersize, chunksize;
941 PyObject *v;
942
943 if (f->f_fp == NULL)
944 return err_closed();
945 /* refuse to mix with f.next() */
946 if (f->f_buf != NULL &&
947 (f->f_bufend - f->f_bufptr) > 0 &&
948 f->f_buf[0] != '\0')
949 return err_iterbuffered();
950 if (!PyArg_ParseTuple(args, "|l:read", &bytesrequested))
951 return NULL;
952 if (bytesrequested < 0)
953 buffersize = new_buffersize(f, (size_t)0);
954 else
955 buffersize = bytesrequested;
956 if (buffersize > PY_SSIZE_T_MAX) {
957 PyErr_SetString(PyExc_OverflowError,
958 "requested number of bytes is more than a Python string can hold");
959 return NULL;
960 }
961 v = PyString_FromStringAndSize((char *)NULL, buffersize);
962 if (v == NULL)
963 return NULL;
964 bytesread = 0;
965 for (;;) {
966 FILE_BEGIN_ALLOW_THREADS(f)
967 errno = 0;
968 chunksize = Py_UniversalNewlineFread(BUF(v) + bytesread,
969 buffersize - bytesread, f->f_fp, (PyObject *)f);
970 FILE_END_ALLOW_THREADS(f)
971 if (chunksize == 0) {
972 if (!ferror(f->f_fp))
973 break;
974 clearerr(f->f_fp);
975 /* When in non-blocking mode, data shouldn't
976 * be discarded if a blocking signal was
977 * received. That will also happen if
978 * chunksize != 0, but bytesread < buffersize. */
979 if (bytesread > 0 && BLOCKED_ERRNO(errno))
980 break;
981 PyErr_SetFromErrno(PyExc_IOError);
982 Py_DECREF(v);
983 return NULL;
984 }
985 bytesread += chunksize;
986 if (bytesread < buffersize) {
987 clearerr(f->f_fp);
988 break;
989 }
990 if (bytesrequested < 0) {
991 buffersize = new_buffersize(f, buffersize);
992 if (_PyString_Resize(&v, buffersize) < 0)
993 return NULL;
994 } else {
995 /* Got what was requested. */
996 break;
997 }
998 }
999 if (bytesread != buffersize)
1000 _PyString_Resize(&v, bytesread);
1001 return v;
1002 }
1003
1004 static PyObject *
1005 file_readinto(PyFileObject *f, PyObject *args)
1006 {
1007 char *ptr;
1008 Py_ssize_t ntodo;
1009 Py_ssize_t ndone, nnow;
1010
1011 if (f->f_fp == NULL)
1012 return err_closed();
1013 /* refuse to mix with f.next() */
1014 if (f->f_buf != NULL &&
1015 (f->f_bufend - f->f_bufptr) > 0 &&
1016 f->f_buf[0] != '\0')
1017 return err_iterbuffered();
1018 if (!PyArg_ParseTuple(args, "w#", &ptr, &ntodo))
1019 return NULL;
1020 ndone = 0;
1021 while (ntodo > 0) {
1022 FILE_BEGIN_ALLOW_THREADS(f)
1023 errno = 0;
1024 nnow = Py_UniversalNewlineFread(ptr+ndone, ntodo, f->f_fp,
1025 (PyObject *)f);
1026 FILE_END_ALLOW_THREADS(f)
1027 if (nnow == 0) {
1028 if (!ferror(f->f_fp))
1029 break;
1030 PyErr_SetFromErrno(PyExc_IOError);
1031 clearerr(f->f_fp);
1032 return NULL;
1033 }
1034 ndone += nnow;
1035 ntodo -= nnow;
1036 }
1037 return PyInt_FromSsize_t(ndone);
1038 }
1039
1040 /**************************************************************************
1041 Routine to get next line using platform fgets().
1042
1043 Under MSVC 6:
1044
1045 + MS threadsafe getc is very slow (multiple layers of function calls before+
1046 after each character, to lock+unlock the stream).
1047 + The stream-locking functions are MS-internal -- can't access them from user
1048 code.
1049 + There's nothing Tim could find in the MS C or platform SDK libraries that
1050 can worm around this.
1051 + MS fgets locks/unlocks only once per line; it's the only hook we have.
1052
1053 So we use fgets for speed(!), despite that it's painful.
1054
1055 MS realloc is also slow.
1056
1057 Reports from other platforms on this method vs getc_unlocked (which MS doesn't
1058 have):
1059 Linux a wash
1060 Solaris a wash
1061 Tru64 Unix getline_via_fgets significantly faster
1062
1063 CAUTION: The C std isn't clear about this: in those cases where fgets
1064 writes something into the buffer, can it write into any position beyond the
1065 required trailing null byte? MSVC 6 fgets does not, and no platform is (yet)
1066 known on which it does; and it would be a strange way to code fgets. Still,
1067 getline_via_fgets may not work correctly if it does. The std test
1068 test_bufio.py should fail if platform fgets() routinely writes beyond the
1069 trailing null byte. #define DONT_USE_FGETS_IN_GETLINE to disable this code.
1070 **************************************************************************/
1071
1072 /* Use this routine if told to, or by default on non-get_unlocked()
1073 * platforms unless told not to. Yikes! Let's spell that out:
1074 * On a platform with getc_unlocked():
1075 * By default, use getc_unlocked().
1076 * If you want to use fgets() instead, #define USE_FGETS_IN_GETLINE.
1077 * On a platform without getc_unlocked():
1078 * By default, use fgets().
1079 * If you don't want to use fgets(), #define DONT_USE_FGETS_IN_GETLINE.
1080 */
1081 #if !defined(USE_FGETS_IN_GETLINE) && !defined(HAVE_GETC_UNLOCKED)
1082 #define USE_FGETS_IN_GETLINE
1083 #endif
1084
1085 #if defined(DONT_USE_FGETS_IN_GETLINE) && defined(USE_FGETS_IN_GETLINE)
1086 #undef USE_FGETS_IN_GETLINE
1087 #endif
1088
1089 #ifdef USE_FGETS_IN_GETLINE
1090 static PyObject*
1091 getline_via_fgets(PyFileObject *f, FILE *fp)
1092 {
1093 /* INITBUFSIZE is the maximum line length that lets us get away with the fast
1094 * no-realloc, one-fgets()-call path. Boosting it isn't free, because we have
1095 * to fill this much of the buffer with a known value in order to figure out
1096 * how much of the buffer fgets() overwrites. So if INITBUFSIZE is larger
1097 * than "most" lines, we waste time filling unused buffer slots. 100 is
1098 * surely adequate for most peoples' email archives, chewing over source code,
1099 * etc -- "regular old text files".
1100 * MAXBUFSIZE is the maximum line length that lets us get away with the less
1101 * fast (but still zippy) no-realloc, two-fgets()-call path. See above for
1102 * cautions about boosting that. 300 was chosen because the worst real-life
1103 * text-crunching job reported on Python-Dev was a mail-log crawler where over
1104 * half the lines were 254 chars.
1105 */
1106 #define INITBUFSIZE 100
1107 #define MAXBUFSIZE 300
1108 char* p; /* temp */
1109 char buf[MAXBUFSIZE];
1110 PyObject* v; /* the string object result */
1111 char* pvfree; /* address of next free slot */
1112 char* pvend; /* address one beyond last free slot */
1113 size_t nfree; /* # of free buffer slots; pvend-pvfree */
1114 size_t total_v_size; /* total # of slots in buffer */
1115 size_t increment; /* amount to increment the buffer */
1116 size_t prev_v_size;
1117
1118 /* Optimize for normal case: avoid _PyString_Resize if at all
1119 * possible via first reading into stack buffer "buf".
1120 */
1121 total_v_size = INITBUFSIZE; /* start small and pray */
1122 pvfree = buf;
1123 for (;;) {
1124 FILE_BEGIN_ALLOW_THREADS(f)
1125 pvend = buf + total_v_size;
1126 nfree = pvend - pvfree;
1127 memset(pvfree, '\n', nfree);
1128 assert(nfree < INT_MAX); /* Should be atmost MAXBUFSIZE */
1129 p = fgets(pvfree, (int)nfree, fp);
1130 FILE_END_ALLOW_THREADS(f)
1131
1132 if (p == NULL) {
1133 clearerr(fp);
1134 if (PyErr_CheckSignals())
1135 return NULL;
1136 v = PyString_FromStringAndSize(buf, pvfree - buf);
1137 return v;
1138 }
1139 /* fgets read *something* */
1140 p = memchr(pvfree, '\n', nfree);
1141 if (p != NULL) {
1142 /* Did the \n come from fgets or from us?
1143 * Since fgets stops at the first \n, and then writes
1144 * \0, if it's from fgets a \0 must be next. But if
1145 * that's so, it could not have come from us, since
1146 * the \n's we filled the buffer with have only more
1147 * \n's to the right.
1148 */
1149 if (p+1 < pvend && *(p+1) == '\0') {
1150 /* It's from fgets: we win! In particular,
1151 * we haven't done any mallocs yet, and can
1152 * build the final result on the first try.
1153 */
1154 ++p; /* include \n from fgets */
1155 }
1156 else {
1157 /* Must be from us: fgets didn't fill the
1158 * buffer and didn't find a newline, so it
1159 * must be the last and newline-free line of
1160 * the file.
1161 */
1162 assert(p > pvfree && *(p-1) == '\0');
1163 --p; /* don't include \0 from fgets */
1164 }
1165 v = PyString_FromStringAndSize(buf, p - buf);
1166 return v;
1167 }
1168 /* yuck: fgets overwrote all the newlines, i.e. the entire
1169 * buffer. So this line isn't over yet, or maybe it is but
1170 * we're exactly at EOF. If we haven't already, try using the
1171 * rest of the stack buffer.
1172 */
1173 assert(*(pvend-1) == '\0');
1174 if (pvfree == buf) {
1175 pvfree = pvend - 1; /* overwrite trailing null */
1176 total_v_size = MAXBUFSIZE;
1177 }
1178 else
1179 break;
1180 }
1181
1182 /* The stack buffer isn't big enough; malloc a string object and read
1183 * into its buffer.
1184 */
1185 total_v_size = MAXBUFSIZE << 1;
1186 v = PyString_FromStringAndSize((char*)NULL, (int)total_v_size);
1187 if (v == NULL)
1188 return v;
1189 /* copy over everything except the last null byte */
1190 memcpy(BUF(v), buf, MAXBUFSIZE-1);
1191 pvfree = BUF(v) + MAXBUFSIZE - 1;
1192
1193 /* Keep reading stuff into v; if it ever ends successfully, break
1194 * after setting p one beyond the end of the line. The code here is
1195 * very much like the code above, except reads into v's buffer; see
1196 * the code above for detailed comments about the logic.
1197 */
1198 for (;;) {
1199 FILE_BEGIN_ALLOW_THREADS(f)
1200 pvend = BUF(v) + total_v_size;
1201 nfree = pvend - pvfree;
1202 memset(pvfree, '\n', nfree);
1203 assert(nfree < INT_MAX);
1204 p = fgets(pvfree, (int)nfree, fp);
1205 FILE_END_ALLOW_THREADS(f)
1206
1207 if (p == NULL) {
1208 clearerr(fp);
1209 if (PyErr_CheckSignals()) {
1210 Py_DECREF(v);
1211 return NULL;
1212 }
1213 p = pvfree;
1214 break;
1215 }
1216 p = memchr(pvfree, '\n', nfree);
1217 if (p != NULL) {
1218 if (p+1 < pvend && *(p+1) == '\0') {
1219 /* \n came from fgets */
1220 ++p;
1221 break;
1222 }
1223 /* \n came from us; last line of file, no newline */
1224 assert(p > pvfree && *(p-1) == '\0');
1225 --p;
1226 break;
1227 }
1228 /* expand buffer and try again */
1229 assert(*(pvend-1) == '\0');
1230 increment = total_v_size >> 2; /* mild exponential growth */
1231 prev_v_size = total_v_size;
1232 total_v_size += increment;
1233 /* check for overflow */
1234 if (total_v_size <= prev_v_size ||
1235 total_v_size > PY_SSIZE_T_MAX) {
1236 PyErr_SetString(PyExc_OverflowError,
1237 "line is longer than a Python string can hold");
1238 Py_DECREF(v);
1239 return NULL;
1240 }
1241 if (_PyString_Resize(&v, (int)total_v_size) < 0)
1242 return NULL;
1243 /* overwrite the trailing null byte */
1244 pvfree = BUF(v) + (prev_v_size - 1);
1245 }
1246 if (BUF(v) + total_v_size != p)
1247 _PyString_Resize(&v, p - BUF(v));
1248 return v;
1249 #undef INITBUFSIZE
1250 #undef MAXBUFSIZE
1251 }
1252 #endif /* ifdef USE_FGETS_IN_GETLINE */
1253
1254 /* Internal routine to get a line.
1255 Size argument interpretation:
1256 > 0: max length;
1257 <= 0: read arbitrary line
1258 */
1259
1260 static PyObject *
1261 get_line(PyFileObject *f, int n)
1262 {
1263 FILE *fp = f->f_fp;
1264 int c;
1265 char *buf, *end;
1266 size_t total_v_size; /* total # of slots in buffer */
1267 size_t used_v_size; /* # used slots in buffer */
1268 size_t increment; /* amount to increment the buffer */
1269 PyObject *v;
1270 int newlinetypes = f->f_newlinetypes;
1271 int skipnextlf = f->f_skipnextlf;
1272 int univ_newline = f->f_univ_newline;
1273
1274 #if defined(USE_FGETS_IN_GETLINE)
1275 if (n <= 0 && !univ_newline )
1276 return getline_via_fgets(f, fp);
1277 #endif
1278 total_v_size = n > 0 ? n : 100;
1279 v = PyString_FromStringAndSize((char *)NULL, total_v_size);
1280 if (v == NULL)
1281 return NULL;
1282 buf = BUF(v);
1283 end = buf + total_v_size;
1284
1285 for (;;) {
1286 FILE_BEGIN_ALLOW_THREADS(f)
1287 FLOCKFILE(fp);
1288 if (univ_newline) {
1289 c = 'x'; /* Shut up gcc warning */
1290 while ( buf != end && (c = GETC(fp)) != EOF ) {
1291 if (skipnextlf ) {
1292 skipnextlf = 0;
1293 if (c == '\n') {
1294 /* Seeing a \n here with
1295 * skipnextlf true means we
1296 * saw a \r before.
1297 */
1298 newlinetypes |= NEWLINE_CRLF;
1299 c = GETC(fp);
1300 if (c == EOF) break;
1301 } else {
1302 newlinetypes |= NEWLINE_CR;
1303 }
1304 }
1305 if (c == '\r') {
1306 skipnextlf = 1;
1307 c = '\n';
1308 } else if ( c == '\n')
1309 newlinetypes |= NEWLINE_LF;
1310 *buf++ = c;
1311 if (c == '\n') break;
1312 }
1313 if ( c == EOF && skipnextlf )
1314 newlinetypes |= NEWLINE_CR;
1315 } else /* If not universal newlines use the normal loop */
1316 while ((c = GETC(fp)) != EOF &&
1317 (*buf++ = c) != '\n' &&
1318 buf != end)
1319 ;
1320 FUNLOCKFILE(fp);
1321 FILE_END_ALLOW_THREADS(f)
1322 f->f_newlinetypes = newlinetypes;
1323 f->f_skipnextlf = skipnextlf;
1324 if (c == '\n')
1325 break;
1326 if (c == EOF) {
1327 if (ferror(fp)) {
1328 PyErr_SetFromErrno(PyExc_IOError);
1329 clearerr(fp);
1330 Py_DECREF(v);
1331 return NULL;
1332 }
1333 clearerr(fp);
1334 if (PyErr_CheckSignals()) {
1335 Py_DECREF(v);
1336 return NULL;
1337 }
1338 break;
1339 }
1340 /* Must be because buf == end */
1341 if (n > 0)
1342 break;
1343 used_v_size = total_v_size;
1344 increment = total_v_size >> 2; /* mild exponential growth */
1345 total_v_size += increment;
1346 if (total_v_size > PY_SSIZE_T_MAX) {
1347 PyErr_SetString(PyExc_OverflowError,
1348 "line is longer than a Python string can hold");
1349 Py_DECREF(v);
1350 return NULL;
1351 }
1352 if (_PyString_Resize(&v, total_v_size) < 0)
1353 return NULL;
1354 buf = BUF(v) + used_v_size;
1355 end = BUF(v) + total_v_size;
1356 }
1357
1358 used_v_size = buf - BUF(v);
1359 if (used_v_size != total_v_size)
1360 _PyString_Resize(&v, used_v_size);
1361 return v;
1362 }
1363
1364 /* External C interface */
1365
1366 PyObject *
1367 PyFile_GetLine(PyObject *f, int n)
1368 {
1369 PyObject *result;
1370
1371 if (f == NULL) {
1372 PyErr_BadInternalCall();
1373 return NULL;
1374 }
1375
1376 if (PyFile_Check(f)) {
1377 PyFileObject *fo = (PyFileObject *)f;
1378 if (fo->f_fp == NULL)
1379 return err_closed();
1380 /* refuse to mix with f.next() */
1381 if (fo->f_buf != NULL &&
1382 (fo->f_bufend - fo->f_bufptr) > 0 &&
1383 fo->f_buf[0] != '\0')
1384 return err_iterbuffered();
1385 result = get_line(fo, n);
1386 }
1387 else {
1388 PyObject *reader;
1389 PyObject *args;
1390
1391 reader = PyObject_GetAttrString(f, "readline");
1392 if (reader == NULL)
1393 return NULL;
1394 if (n <= 0)
1395 args = PyTuple_New(0);
1396 else
1397 args = Py_BuildValue("(i)", n);
1398 if (args == NULL) {
1399 Py_DECREF(reader);
1400 return NULL;
1401 }
1402 result = PyEval_CallObject(reader, args);
1403 Py_DECREF(reader);
1404 Py_DECREF(args);
1405 if (result != NULL && !PyString_Check(result) &&
1406 !PyUnicode_Check(result)) {
1407 Py_DECREF(result);
1408 result = NULL;
1409 PyErr_SetString(PyExc_TypeError,
1410 "object.readline() returned non-string");
1411 }
1412 }
1413
1414 if (n < 0 && result != NULL && PyString_Check(result)) {
1415 char *s = PyString_AS_STRING(result);
1416 Py_ssize_t len = PyString_GET_SIZE(result);
1417 if (len == 0) {
1418 Py_DECREF(result);
1419 result = NULL;
1420 PyErr_SetString(PyExc_EOFError,
1421 "EOF when reading a line");
1422 }
1423 else if (s[len-1] == '\n') {
1424 if (result->ob_refcnt == 1)
1425 _PyString_Resize(&result, len-1);
1426 else {
1427 PyObject *v;
1428 v = PyString_FromStringAndSize(s, len-1);
1429 Py_DECREF(result);
1430 result = v;
1431 }
1432 }
1433 }
1434 #ifdef Py_USING_UNICODE
1435 if (n < 0 && result != NULL && PyUnicode_Check(result)) {
1436 Py_UNICODE *s = PyUnicode_AS_UNICODE(result);
1437 Py_ssize_t len = PyUnicode_GET_SIZE(result);
1438 if (len == 0) {
1439 Py_DECREF(result);
1440 result = NULL;
1441 PyErr_SetString(PyExc_EOFError,
1442 "EOF when reading a line");
1443 }
1444 else if (s[len-1] == '\n') {
1445 if (result->ob_refcnt == 1)
1446 PyUnicode_Resize(&result, len-1);
1447 else {
1448 PyObject *v;
1449 v = PyUnicode_FromUnicode(s, len-1);
1450 Py_DECREF(result);
1451 result = v;
1452 }
1453 }
1454 }
1455 #endif
1456 return result;
1457 }
1458
1459 /* Python method */
1460
1461 static PyObject *
1462 file_readline(PyFileObject *f, PyObject *args)
1463 {
1464 int n = -1;
1465
1466 if (f->f_fp == NULL)
1467 return err_closed();
1468 /* refuse to mix with f.next() */
1469 if (f->f_buf != NULL &&
1470 (f->f_bufend - f->f_bufptr) > 0 &&
1471 f->f_buf[0] != '\0')
1472 return err_iterbuffered();
1473 if (!PyArg_ParseTuple(args, "|i:readline", &n))
1474 return NULL;
1475 if (n == 0)
1476 return PyString_FromString("");
1477 if (n < 0)
1478 n = 0;
1479 return get_line(f, n);
1480 }
1481
1482 static PyObject *
1483 file_readlines(PyFileObject *f, PyObject *args)
1484 {
1485 long sizehint = 0;
1486 PyObject *list = NULL;
1487 PyObject *line;
1488 char small_buffer[SMALLCHUNK];
1489 char *buffer = small_buffer;
1490 size_t buffersize = SMALLCHUNK;
1491 PyObject *big_buffer = NULL;
1492 size_t nfilled = 0;
1493 size_t nread;
1494 size_t totalread = 0;
1495 char *p, *q, *end;
1496 int err;
1497 int shortread = 0;
1498
1499 if (f->f_fp == NULL)
1500 return err_closed();
1501 /* refuse to mix with f.next() */
1502 if (f->f_buf != NULL &&
1503 (f->f_bufend - f->f_bufptr) > 0 &&
1504 f->f_buf[0] != '\0')
1505 return err_iterbuffered();
1506 if (!PyArg_ParseTuple(args, "|l:readlines", &sizehint))
1507 return NULL;
1508 if ((list = PyList_New(0)) == NULL)
1509 return NULL;
1510 for (;;) {
1511 if (shortread)
1512 nread = 0;
1513 else {
1514 FILE_BEGIN_ALLOW_THREADS(f)
1515 errno = 0;
1516 nread = Py_UniversalNewlineFread(buffer+nfilled,
1517 buffersize-nfilled, f->f_fp, (PyObject *)f);
1518 FILE_END_ALLOW_THREADS(f)
1519 shortread = (nread < buffersize-nfilled);
1520 }
1521 if (nread == 0) {
1522 sizehint = 0;
1523 if (!ferror(f->f_fp))
1524 break;
1525 PyErr_SetFromErrno(PyExc_IOError);
1526 clearerr(f->f_fp);
1527 goto error;
1528 }
1529 totalread += nread;
1530 p = (char *)memchr(buffer+nfilled, '\n', nread);
1531 if (p == NULL) {
1532 /* Need a larger buffer to fit this line */
1533 nfilled += nread;
1534 buffersize *= 2;
1535 if (buffersize > PY_SSIZE_T_MAX) {
1536 PyErr_SetString(PyExc_OverflowError,
1537 "line is longer than a Python string can hold");
1538 goto error;
1539 }
1540 if (big_buffer == NULL) {
1541 /* Create the big buffer */
1542 big_buffer = PyString_FromStringAndSize(
1543 NULL, buffersize);
1544 if (big_buffer == NULL)
1545 goto error;
1546 buffer = PyString_AS_STRING(big_buffer);
1547 memcpy(buffer, small_buffer, nfilled);
1548 }
1549 else {
1550 /* Grow the big buffer */
1551 if ( _PyString_Resize(&big_buffer, buffersize) < 0 )
1552 goto error;
1553 buffer = PyString_AS_STRING(big_buffer);
1554 }
1555 continue;
1556 }
1557 end = buffer+nfilled+nread;
1558 q = buffer;
1559 do {
1560 /* Process complete lines */
1561 p++;
1562 line = PyString_FromStringAndSize(q, p-q);
1563 if (line == NULL)
1564 goto error;
1565 err = PyList_Append(list, line);
1566 Py_DECREF(line);
1567 if (err != 0)
1568 goto error;
1569 q = p;
1570 p = (char *)memchr(q, '\n', end-q);
1571 } while (p != NULL);
1572 /* Move the remaining incomplete line to the start */
1573 nfilled = end-q;
1574 memmove(buffer, q, nfilled);
1575 if (sizehint > 0)
1576 if (totalread >= (size_t)sizehint)
1577 break;
1578 }
1579 if (nfilled != 0) {
1580 /* Partial last line */
1581 line = PyString_FromStringAndSize(buffer, nfilled);
1582 if (line == NULL)
1583 goto error;
1584 if (sizehint > 0) {
1585 /* Need to complete the last line */
1586 PyObject *rest = get_line(f, 0);
1587 if (rest == NULL) {
1588 Py_DECREF(line);
1589 goto error;
1590 }
1591 PyString_Concat(&line, rest);
1592 Py_DECREF(rest);
1593 if (line == NULL)
1594 goto error;
1595 }
1596 err = PyList_Append(list, line);
1597 Py_DECREF(line);
1598 if (err != 0)
1599 goto error;
1600 }
1601
1602 cleanup:
1603 Py_XDECREF(big_buffer);
1604 return list;
1605
1606 error:
1607 Py_CLEAR(list);
1608 goto cleanup;
1609 }
1610
1611 static PyObject *
1612 file_write(PyFileObject *f, PyObject *args)
1613 {
1614 char *s;
1615 Py_ssize_t n, n2;
1616 if (f->f_fp == NULL)
1617 return err_closed();
1618 if (!PyArg_ParseTuple(args, f->f_binary ? "s#" : "t#", &s, &n))
1619 return NULL;
1620 f->f_softspace = 0;
1621 FILE_BEGIN_ALLOW_THREADS(f)
1622 errno = 0;
1623 n2 = fwrite(s, 1, n, f->f_fp);
1624 FILE_END_ALLOW_THREADS(f)
1625 if (n2 != n) {
1626 PyErr_SetFromErrno(PyExc_IOError);
1627 clearerr(f->f_fp);
1628 return NULL;
1629 }
1630 Py_INCREF(Py_None);
1631 return Py_None;
1632 }
1633
1634 static PyObject *
1635 file_writelines(PyFileObject *f, PyObject *seq)
1636 {
1637 #define CHUNKSIZE 1000
1638 PyObject *list, *line;
1639 PyObject *it; /* iter(seq) */
1640 PyObject *result;
1641 int index, islist;
1642 Py_ssize_t i, j, nwritten, len;
1643
1644 assert(seq != NULL);
1645 if (f->f_fp == NULL)
1646 return err_closed();
1647
1648 result = NULL;
1649 list = NULL;
1650 islist = PyList_Check(seq);
1651 if (islist)
1652 it = NULL;
1653 else {
1654 it = PyObject_GetIter(seq);
1655 if (it == NULL) {
1656 PyErr_SetString(PyExc_TypeError,
1657 "writelines() requires an iterable argument");
1658 return NULL;
1659 }
1660 /* From here on, fail by going to error, to reclaim "it". */
1661 list = PyList_New(CHUNKSIZE);
1662 if (list == NULL)
1663 goto error;
1664 }
1665
1666 /* Strategy: slurp CHUNKSIZE lines into a private list,
1667 checking that they are all strings, then write that list
1668 without holding the interpreter lock, then come back for more. */
1669 for (index = 0; ; index += CHUNKSIZE) {
1670 if (islist) {
1671 Py_XDECREF(list);
1672 list = PyList_GetSlice(seq, index, index+CHUNKSIZE);
1673 if (list == NULL)
1674 goto error;
1675 j = PyList_GET_SIZE(list);
1676 }
1677 else {
1678 for (j = 0; j < CHUNKSIZE; j++) {
1679 line = PyIter_Next(it);
1680 if (line == NULL) {
1681 if (PyErr_Occurred())
1682 goto error;
1683 break;
1684 }
1685 PyList_SetItem(list, j, line);
1686 }
1687 }
1688 if (j == 0)
1689 break;
1690
1691 /* Check that all entries are indeed strings. If not,
1692 apply the same rules as for file.write() and
1693 convert the results to strings. This is slow, but
1694 seems to be the only way since all conversion APIs
1695 could potentially execute Python code. */
1696 for (i = 0; i < j; i++) {
1697 PyObject *v = PyList_GET_ITEM(list, i);
1698 if (!PyString_Check(v)) {
1699 const char *buffer;
1700 if (((f->f_binary &&
1701 PyObject_AsReadBuffer(v,
1702 (const void**)&buffer,
1703 &len)) ||
1704 PyObject_AsCharBuffer(v,
1705 &buffer,
1706 &len))) {
1707 PyErr_SetString(PyExc_TypeError,
1708 "writelines() argument must be a sequence of strings");
1709 goto error;
1710 }
1711 line = PyString_FromStringAndSize(buffer,
1712 len);
1713 if (line == NULL)
1714 goto error;
1715 Py_DECREF(v);
1716 PyList_SET_ITEM(list, i, line);
1717 }
1718 }
1719
1720 /* Since we are releasing the global lock, the
1721 following code may *not* execute Python code. */
1722 f->f_softspace = 0;
1723 FILE_BEGIN_ALLOW_THREADS(f)
1724 errno = 0;
1725 for (i = 0; i < j; i++) {
1726 line = PyList_GET_ITEM(list, i);
1727 len = PyString_GET_SIZE(line);
1728 nwritten = fwrite(PyString_AS_STRING(line),
1729 1, len, f->f_fp);
1730 if (nwritten != len) {
1731 FILE_ABORT_ALLOW_THREADS(f)
1732 PyErr_SetFromErrno(PyExc_IOError);
1733 clearerr(f->f_fp);
1734 goto error;
1735 }
1736 }
1737 FILE_END_ALLOW_THREADS(f)
1738
1739 if (j < CHUNKSIZE)
1740 break;
1741 }
1742
1743 Py_INCREF(Py_None);
1744 result = Py_None;
1745 error:
1746 Py_XDECREF(list);
1747 Py_XDECREF(it);
1748 return result;
1749 #undef CHUNKSIZE
1750 }
1751
1752 static PyObject *
1753 file_self(PyFileObject *f)
1754 {
1755 if (f->f_fp == NULL)
1756 return err_closed();
1757 Py_INCREF(f);
1758 return (PyObject *)f;
1759 }
1760
1761 static PyObject *
1762 file_xreadlines(PyFileObject *f)
1763 {
1764 if (PyErr_WarnPy3k("f.xreadlines() not supported in 3.x, "
1765 "try 'for line in f' instead", 1) < 0)
1766 return NULL;
1767 return file_self(f);
1768 }
1769
1770 static PyObject *
1771 file_exit(PyObject *f, PyObject *args)
1772 {
1773 PyObject *ret = PyObject_CallMethod(f, "close", NULL);
1774 if (!ret)
1775 /* If error occurred, pass through */
1776 return NULL;
1777 Py_DECREF(ret);
1778 /* We cannot return the result of close since a true
1779 * value will be interpreted as "yes, swallow the
1780 * exception if one was raised inside the with block". */
1781 Py_RETURN_NONE;
1782 }
1783
1784 PyDoc_STRVAR(readline_doc,
1785 "readline([size]) -> next line from the file, as a string.\n"
1786 "\n"
1787 "Retain newline. A non-negative size argument limits the maximum\n"
1788 "number of bytes to return (an incomplete line may be returned then).\n"
1789 "Return an empty string at EOF.");
1790
1791 PyDoc_STRVAR(read_doc,
1792 "read([size]) -> read at most size bytes, returned as a string.\n"
1793 "\n"
1794 "If the size argument is negative or omitted, read until EOF is reached.\n"
1795 "Notice that when in non-blocking mode, less data than what was requested\n"
1796 "may be returned, even if no size parameter was given.");
1797
1798 PyDoc_STRVAR(write_doc,
1799 "write(str) -> None. Write string str to file.\n"
1800 "\n"
1801 "Note that due to buffering, flush() or close() may be needed before\n"
1802 "the file on disk reflects the data written.");
1803
1804 PyDoc_STRVAR(fileno_doc,
1805 "fileno() -> integer \"file descriptor\".\n"
1806 "\n"
1807 "This is needed for lower-level file interfaces, such os.read().");
1808
1809 PyDoc_STRVAR(seek_doc,
1810 "seek(offset[, whence]) -> None. Move to new file position.\n"
1811 "\n"
1812 "Argument offset is a byte count. Optional argument whence defaults to\n"
1813 "0 (offset from start of file, offset should be >= 0); other values are 1\n"
1814 "(move relative to current position, positive or negative), and 2 (move\n"
1815 "relative to end of file, usually negative, although many platforms allow\n"
1816 "seeking beyond the end of a file). If the file is opened in text mode,\n"
1817 "only offsets returned by tell() are legal. Use of other offsets causes\n"
1818 "undefined behavior."
1819 "\n"
1820 "Note that not all file objects are seekable.");
1821
1822 #ifdef HAVE_FTRUNCATE
1823 PyDoc_STRVAR(truncate_doc,
1824 "truncate([size]) -> None. Truncate the file to at most size bytes.\n"
1825 "\n"
1826 "Size defaults to the current file position, as returned by tell().");
1827 #endif
1828
1829 PyDoc_STRVAR(tell_doc,
1830 "tell() -> current file position, an integer (may be a long integer).");
1831
1832 PyDoc_STRVAR(readinto_doc,
1833 "readinto() -> Undocumented. Don't use this; it may go away.");
1834
1835 PyDoc_STRVAR(readlines_doc,
1836 "readlines([size]) -> list of strings, each a line from the file.\n"
1837 "\n"
1838 "Call readline() repeatedly and return a list of the lines so read.\n"
1839 "The optional size argument, if given, is an approximate bound on the\n"
1840 "total number of bytes in the lines returned.");
1841
1842 PyDoc_STRVAR(xreadlines_doc,
1843 "xreadlines() -> returns self.\n"
1844 "\n"
1845 "For backward compatibility. File objects now include the performance\n"
1846 "optimizations previously implemented in the xreadlines module.");
1847
1848 PyDoc_STRVAR(writelines_doc,
1849 "writelines(sequence_of_strings) -> None. Write the strings to the file.\n"
1850 "\n"
1851 "Note that newlines are not added. The sequence can be any iterable object\n"
1852 "producing strings. This is equivalent to calling write() for each string.");
1853
1854 PyDoc_STRVAR(flush_doc,
1855 "flush() -> None. Flush the internal I/O buffer.");
1856
1857 PyDoc_STRVAR(close_doc,
1858 "close() -> None or (perhaps) an integer. Close the file.\n"
1859 "\n"
1860 "Sets data attribute .closed to True. A closed file cannot be used for\n"
1861 "further I/O operations. close() may be called more than once without\n"
1862 "error. Some kinds of file objects (for example, opened by popen())\n"
1863 "may return an exit status upon closing.");
1864
1865 PyDoc_STRVAR(isatty_doc,
1866 "isatty() -> true or false. True if the file is connected to a tty device.");
1867
1868 PyDoc_STRVAR(enter_doc,
1869 "__enter__() -> self.");
1870
1871 PyDoc_STRVAR(exit_doc,
1872 "__exit__(*excinfo) -> None. Closes the file.");
1873
1874 static PyMethodDef file_methods[] = {
1875 {"readline", (PyCFunction)file_readline, METH_VARARGS, readline_doc},
1876 {"read", (PyCFunction)file_read, METH_VARARGS, read_doc},
1877 {"write", (PyCFunction)file_write, METH_VARARGS, write_doc},
1878 {"fileno", (PyCFunction)file_fileno, METH_NOARGS, fileno_doc},
1879 {"seek", (PyCFunction)file_seek, METH_VARARGS, seek_doc},
1880 #ifdef HAVE_FTRUNCATE
1881 {"truncate", (PyCFunction)file_truncate, METH_VARARGS, truncate_doc},
1882 #endif
1883 {"tell", (PyCFunction)file_tell, METH_NOARGS, tell_doc},
1884 {"readinto", (PyCFunction)file_readinto, METH_VARARGS, readinto_doc},
1885 {"readlines", (PyCFunction)file_readlines, METH_VARARGS, readlines_doc},
1886 {"xreadlines",(PyCFunction)file_xreadlines, METH_NOARGS, xreadlines_doc},
1887 {"writelines",(PyCFunction)file_writelines, METH_O, writelines_doc},
1888 {"flush", (PyCFunction)file_flush, METH_NOARGS, flush_doc},
1889 {"close", (PyCFunction)file_close, METH_NOARGS, close_doc},
1890 {"isatty", (PyCFunction)file_isatty, METH_NOARGS, isatty_doc},
1891 {"__enter__", (PyCFunction)file_self, METH_NOARGS, enter_doc},
1892 {"__exit__", (PyCFunction)file_exit, METH_VARARGS, exit_doc},
1893 {NULL, NULL} /* sentinel */
1894 };
1895
1896 #define OFF(x) offsetof(PyFileObject, x)
1897
1898 static PyMemberDef file_memberlist[] = {
1899 {"mode", T_OBJECT, OFF(f_mode), RO,
1900 "file mode ('r', 'U', 'w', 'a', possibly with 'b' or '+' added)"},
1901 {"name", T_OBJECT, OFF(f_name), RO,
1902 "file name"},
1903 {"encoding", T_OBJECT, OFF(f_encoding), RO,
1904 "file encoding"},
1905 {"errors", T_OBJECT, OFF(f_errors), RO,
1906 "Unicode error handler"},
1907 /* getattr(f, "closed") is implemented without this table */
1908 {NULL} /* Sentinel */
1909 };
1910
1911 static PyObject *
1912 get_closed(PyFileObject *f, void *closure)
1913 {
1914 return PyBool_FromLong((long)(f->f_fp == 0));
1915 }
1916 static PyObject *
1917 get_newlines(PyFileObject *f, void *closure)
1918 {
1919 switch (f->f_newlinetypes) {
1920 case NEWLINE_UNKNOWN:
1921 Py_INCREF(Py_None);
1922 return Py_None;
1923 case NEWLINE_CR:
1924 return PyString_FromString("\r");
1925 case NEWLINE_LF:
1926 return PyString_FromString("\n");
1927 case NEWLINE_CR|NEWLINE_LF:
1928 return Py_BuildValue("(ss)", "\r", "\n");
1929 case NEWLINE_CRLF:
1930 return PyString_FromString("\r\n");
1931 case NEWLINE_CR|NEWLINE_CRLF:
1932 return Py_BuildValue("(ss)", "\r", "\r\n");
1933 case NEWLINE_LF|NEWLINE_CRLF:
1934 return Py_BuildValue("(ss)", "\n", "\r\n");
1935 case NEWLINE_CR|NEWLINE_LF|NEWLINE_CRLF:
1936 return Py_BuildValue("(sss)", "\r", "\n", "\r\n");
1937 default:
1938 PyErr_Format(PyExc_SystemError,
1939 "Unknown newlines value 0x%x\n",
1940 f->f_newlinetypes);
1941 return NULL;
1942 }
1943 }
1944
1945 static PyObject *
1946 get_softspace(PyFileObject *f, void *closure)
1947 {
1948 if (PyErr_WarnPy3k("file.softspace not supported in 3.x", 1) < 0)
1949 return NULL;
1950 return PyInt_FromLong(f->f_softspace);
1951 }
1952
1953 static int
1954 set_softspace(PyFileObject *f, PyObject *value)
1955 {
1956 int new;
1957 if (PyErr_WarnPy3k("file.softspace not supported in 3.x", 1) < 0)
1958 return -1;
1959
1960 if (value == NULL) {
1961 PyErr_SetString(PyExc_TypeError,
1962 "can't delete softspace attribute");
1963 return -1;
1964 }
1965
1966 new = PyInt_AsLong(value);
1967 if (new == -1 && PyErr_Occurred())
1968 return -1;
1969 f->f_softspace = new;
1970 return 0;
1971 }
1972
1973 static PyGetSetDef file_getsetlist[] = {
1974 {"closed", (getter)get_closed, NULL, "True if the file is closed"},
1975 {"newlines", (getter)get_newlines, NULL,
1976 "end-of-line convention used in this file"},
1977 {"softspace", (getter)get_softspace, (setter)set_softspace,
1978 "flag indicating that a space needs to be printed; used by print"},
1979 {0},
1980 };
1981
1982 static void
1983 drop_readahead(PyFileObject *f)
1984 {
1985 if (f->f_buf != NULL) {
1986 PyMem_Free(f->f_buf);
1987 f->f_buf = NULL;
1988 }
1989 }
1990
1991 /* Make sure that file has a readahead buffer with at least one byte
1992 (unless at EOF) and no more than bufsize. Returns negative value on
1993 error, will set MemoryError if bufsize bytes cannot be allocated. */
1994 static int
1995 readahead(PyFileObject *f, int bufsize)
1996 {
1997 Py_ssize_t chunksize;
1998
1999 if (f->f_buf != NULL) {
2000 if( (f->f_bufend - f->f_bufptr) >= 1)
2001 return 0;
2002 else
2003 drop_readahead(f);
2004 }
2005 if ((f->f_buf = (char *)PyMem_Malloc(bufsize)) == NULL) {
2006 PyErr_NoMemory();
2007 return -1;
2008 }
2009 FILE_BEGIN_ALLOW_THREADS(f)
2010 errno = 0;
2011 chunksize = Py_UniversalNewlineFread(
2012 f->f_buf, bufsize, f->f_fp, (PyObject *)f);
2013 FILE_END_ALLOW_THREADS(f)
2014 if (chunksize == 0) {
2015 if (ferror(f->f_fp)) {
2016 PyErr_SetFromErrno(PyExc_IOError);
2017 clearerr(f->f_fp);
2018 drop_readahead(f);
2019 return -1;
2020 }
2021 }
2022 f->f_bufptr = f->f_buf;
2023 f->f_bufend = f->f_buf + chunksize;
2024 return 0;
2025 }
2026
2027 /* Used by file_iternext. The returned string will start with 'skip'
2028 uninitialized bytes followed by the remainder of the line. Don't be
2029 horrified by the recursive call: maximum recursion depth is limited by
2030 logarithmic buffer growth to about 50 even when reading a 1gb line. */
2031
2032 static PyStringObject *
2033 readahead_get_line_skip(PyFileObject *f, int skip, int bufsize)
2034 {
2035 PyStringObject* s;
2036 char *bufptr;
2037 char *buf;
2038 Py_ssize_t len;
2039
2040 if (f->f_buf == NULL)
2041 if (readahead(f, bufsize) < 0)
2042 return NULL;
2043
2044 len = f->f_bufend - f->f_bufptr;
2045 if (len == 0)
2046 return (PyStringObject *)
2047 PyString_FromStringAndSize(NULL, skip);
2048 bufptr = (char *)memchr(f->f_bufptr, '\n', len);
2049 if (bufptr != NULL) {
2050 bufptr++; /* Count the '\n' */
2051 len = bufptr - f->f_bufptr;
2052 s = (PyStringObject *)
2053 PyString_FromStringAndSize(NULL, skip+len);
2054 if (s == NULL)
2055 return NULL;
2056 memcpy(PyString_AS_STRING(s)+skip, f->f_bufptr, len);
2057 f->f_bufptr = bufptr;
2058 if (bufptr == f->f_bufend)
2059 drop_readahead(f);
2060 } else {
2061 bufptr = f->f_bufptr;
2062 buf = f->f_buf;
2063 f->f_buf = NULL; /* Force new readahead buffer */
2064 assert(skip+len < INT_MAX);
2065 s = readahead_get_line_skip(
2066 f, (int)(skip+len), bufsize + (bufsize>>2) );
2067 if (s == NULL) {
2068 PyMem_Free(buf);
2069 return NULL;
2070 }
2071 memcpy(PyString_AS_STRING(s)+skip, bufptr, len);
2072 PyMem_Free(buf);
2073 }
2074 return s;
2075 }
2076
2077 /* A larger buffer size may actually decrease performance. */
2078 #define READAHEAD_BUFSIZE 8192
2079
2080 static PyObject *
2081 file_iternext(PyFileObject *f)
2082 {
2083 PyStringObject* l;
2084
2085 if (f->f_fp == NULL)
2086 return err_closed();
2087
2088 l = readahead_get_line_skip(f, 0, READAHEAD_BUFSIZE);
2089 if (l == NULL || PyString_GET_SIZE(l) == 0) {
2090 Py_XDECREF(l);
2091 return NULL;
2092 }
2093 return (PyObject *)l;
2094 }
2095
2096
2097 static PyObject *
2098 file_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
2099 {
2100 PyObject *self;
2101 static PyObject *not_yet_string;
2102
2103 assert(type != NULL && type->tp_alloc != NULL);
2104
2105 if (not_yet_string == NULL) {
2106 not_yet_string = PyString_InternFromString("<uninitialized file>");
2107 if (not_yet_string == NULL)
2108 return NULL;
2109 }
2110
2111 self = type->tp_alloc(type, 0);
2112 if (self != NULL) {
2113 /* Always fill in the name and mode, so that nobody else
2114 needs to special-case NULLs there. */
2115 Py_INCREF(not_yet_string);
2116 ((PyFileObject *)self)->f_name = not_yet_string;
2117 Py_INCREF(not_yet_string);
2118 ((PyFileObject *)self)->f_mode = not_yet_string;
2119 Py_INCREF(Py_None);
2120 ((PyFileObject *)self)->f_encoding = Py_None;
2121 Py_INCREF(Py_None);
2122 ((PyFileObject *)self)->f_errors = Py_None;
2123 ((PyFileObject *)self)->weakreflist = NULL;
2124 ((PyFileObject *)self)->unlocked_count = 0;
2125 }
2126 return self;
2127 }
2128
2129 static int
2130 file_init(PyObject *self, PyObject *args, PyObject *kwds)
2131 {
2132 PyFileObject *foself = (PyFileObject *)self;
2133 int ret = 0;
2134 static char *kwlist[] = {"name", "mode", "buffering", 0};
2135 char *name = NULL;
2136 char *mode = "r";
2137 int bufsize = -1;
2138 int wideargument = 0;
2139
2140 assert(PyFile_Check(self));
2141 if (foself->f_fp != NULL) {
2142 /* Have to close the existing file first. */
2143 PyObject *closeresult = file_close(foself);
2144 if (closeresult == NULL)
2145 return -1;
2146 Py_DECREF(closeresult);
2147 }
2148
2149 #ifdef Py_WIN_WIDE_FILENAMES
2150 if (GetVersion() < 0x80000000) { /* On NT, so wide API available */
2151 PyObject *po;
2152 if (PyArg_ParseTupleAndKeywords(args, kwds, "U|si:file",
2153 kwlist, &po, &mode, &bufsize)) {
2154 wideargument = 1;
2155 if (fill_file_fields(foself, NULL, po, mode,
2156 fclose) == NULL)
2157 goto Error;
2158 } else {
2159 /* Drop the argument parsing error as narrow
2160 strings are also valid. */
2161 PyErr_Clear();
2162 }
2163 }
2164 #endif
2165
2166 if (!wideargument) {
2167 PyObject *o_name;
2168
2169 if (!PyArg_ParseTupleAndKeywords(args, kwds, "et|si:file", kwlist,
2170 Py_FileSystemDefaultEncoding,
2171 &name,
2172 &mode, &bufsize))
2173 return -1;
2174
2175 /* We parse again to get the name as a PyObject */
2176 if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|si:file",
2177 kwlist, &o_name, &mode,
2178 &bufsize))
2179 goto Error;
2180
2181 if (fill_file_fields(foself, NULL, o_name, mode,
2182 fclose) == NULL)
2183 goto Error;
2184 }
2185 if (open_the_file(foself, name, mode) == NULL)
2186 goto Error;
2187 foself->f_setbuf = NULL;
2188 PyFile_SetBufSize(self, bufsize);
2189 goto Done;
2190
2191 Error:
2192 ret = -1;
2193 /* fall through */
2194 Done:
2195 PyMem_Free(name); /* free the encoded string */
2196 return ret;
2197 }
2198
2199 PyDoc_VAR(file_doc) =
2200 PyDoc_STR(
2201 "file(name[, mode[, buffering]]) -> file object\n"
2202 "\n"
2203 "Open a file. The mode can be 'r', 'w' or 'a' for reading (default),\n"
2204 "writing or appending. The file will be created if it doesn't exist\n"
2205 "when opened for writing or appending; it will be truncated when\n"
2206 "opened for writing. Add a 'b' to the mode for binary files.\n"
2207 "Add a '+' to the mode to allow simultaneous reading and writing.\n"
2208 "If the buffering argument is given, 0 means unbuffered, 1 means line\n"
2209 "buffered, and larger numbers specify the buffer size. The preferred way\n"
2210 "to open a file is with the builtin open() function.\n"
2211 )
2212 PyDoc_STR(
2213 "Add a 'U' to mode to open the file for input with universal newline\n"
2214 "support. Any line ending in the input file will be seen as a '\\n'\n"
2215 "in Python. Also, a file so opened gains the attribute 'newlines';\n"
2216 "the value for this attribute is one of None (no newline read yet),\n"
2217 "'\\r', '\\n', '\\r\\n' or a tuple containing all the newline types seen.\n"
2218 "\n"
2219 "'U' cannot be combined with 'w' or '+' mode.\n"
2220 );
2221
2222 PyTypeObject PyFile_Type = {
2223 PyVarObject_HEAD_INIT(&PyType_Type, 0)
2224 "file",
2225 sizeof(PyFileObject),
2226 0,
2227 (destructor)file_dealloc, /* tp_dealloc */
2228 0, /* tp_print */
2229 0, /* tp_getattr */
2230 0, /* tp_setattr */
2231 0, /* tp_compare */
2232 (reprfunc)file_repr, /* tp_repr */
2233 0, /* tp_as_number */
2234 0, /* tp_as_sequence */
2235 0, /* tp_as_mapping */
2236 0, /* tp_hash */
2237 0, /* tp_call */
2238 0, /* tp_str */
2239 PyObject_GenericGetAttr, /* tp_getattro */
2240 /* softspace is writable: we must supply tp_setattro */
2241 PyObject_GenericSetAttr, /* tp_setattro */
2242 0, /* tp_as_buffer */
2243 Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_WEAKREFS, /* tp_flags */
2244 file_doc, /* tp_doc */
2245 0, /* tp_traverse */
2246 0, /* tp_clear */
2247 0, /* tp_richcompare */
2248 offsetof(PyFileObject, weakreflist), /* tp_weaklistoffset */
2249 (getiterfunc)file_self, /* tp_iter */
2250 (iternextfunc)file_iternext, /* tp_iternext */
2251 file_methods, /* tp_methods */
2252 file_memberlist, /* tp_members */
2253 file_getsetlist, /* tp_getset */
2254 0, /* tp_base */
2255 0, /* tp_dict */
2256 0, /* tp_descr_get */
2257 0, /* tp_descr_set */
2258 0, /* tp_dictoffset */
2259 file_init, /* tp_init */
2260 PyType_GenericAlloc, /* tp_alloc */
2261 file_new, /* tp_new */
2262 PyObject_Del, /* tp_free */
2263 };
2264
2265 /* Interface for the 'soft space' between print items. */
2266
2267 int
2268 PyFile_SoftSpace(PyObject *f, int newflag)
2269 {
2270 long oldflag = 0;
2271 if (f == NULL) {
2272 /* Do nothing */
2273 }
2274 else if (PyFile_Check(f)) {
2275 oldflag = ((PyFileObject *)f)->f_softspace;
2276 ((PyFileObject *)f)->f_softspace = newflag;
2277 }
2278 else {
2279 PyObject *v;
2280 v = PyObject_GetAttrString(f, "softspace");
2281 if (v == NULL)
2282 PyErr_Clear();
2283 else {
2284 if (PyInt_Check(v))
2285 oldflag = PyInt_AsLong(v);
2286 assert(oldflag < INT_MAX);
2287 Py_DECREF(v);
2288 }
2289 v = PyInt_FromLong((long)newflag);
2290 if (v == NULL)
2291 PyErr_Clear();
2292 else {
2293 if (PyObject_SetAttrString(f, "softspace", v) != 0)
2294 PyErr_Clear();
2295 Py_DECREF(v);
2296 }
2297 }
2298 return (int)oldflag;
2299 }
2300
2301 /* Interfaces to write objects/strings to file-like objects */
2302
2303 int
2304 PyFile_WriteObject(PyObject *v, PyObject *f, int flags)
2305 {
2306 PyObject *writer, *value, *args, *result;
2307 if (f == NULL) {
2308 PyErr_SetString(PyExc_TypeError, "writeobject with NULL file");
2309 return -1;
2310 }
2311 else if (PyFile_Check(f)) {
2312 PyFileObject *fobj = (PyFileObject *) f;
2313 #ifdef Py_USING_UNICODE
2314 PyObject *enc = fobj->f_encoding;
2315 int result;
2316 #endif
2317 if (fobj->f_fp == NULL) {
2318 err_closed();
2319 return -1;
2320 }
2321 #ifdef Py_USING_UNICODE
2322 if ((flags & Py_PRINT_RAW) &&
2323 PyUnicode_Check(v) && enc != Py_None) {
2324 char *cenc = PyString_AS_STRING(enc);
2325 char *errors = fobj->f_errors == Py_None ?
2326 "strict" : PyString_AS_STRING(fobj->f_errors);
2327 value = PyUnicode_AsEncodedString(v, cenc, errors);
2328 if (value == NULL)
2329 return -1;
2330 } else {
2331 value = v;
2332 Py_INCREF(value);
2333 }
2334 result = file_PyObject_Print(value, fobj, flags);
2335 Py_DECREF(value);
2336 return result;
2337 #else
2338 return file_PyObject_Print(v, fobj, flags);
2339 #endif
2340 }
2341 writer = PyObject_GetAttrString(f, "write");
2342 if (writer == NULL)
2343 return -1;
2344 if (flags & Py_PRINT_RAW) {
2345 if (PyUnicode_Check(v)) {
2346 value = v;
2347 Py_INCREF(value);
2348 } else
2349 value = PyObject_Str(v);
2350 }
2351 else
2352 value = PyObject_Repr(v);
2353 if (value == NULL) {
2354 Py_DECREF(writer);
2355 return -1;
2356 }
2357 args = PyTuple_Pack(1, value);
2358 if (args == NULL) {
2359 Py_DECREF(value);
2360 Py_DECREF(writer);
2361 return -1;
2362 }
2363 result = PyEval_CallObject(writer, args);
2364 Py_DECREF(args);
2365 Py_DECREF(value);
2366 Py_DECREF(writer);
2367 if (result == NULL)
2368 return -1;
2369 Py_DECREF(result);
2370 return 0;
2371 }
2372
2373 int
2374 PyFile_WriteString(const char *s, PyObject *f)
2375 {
2376
2377 if (f == NULL) {
2378 /* Should be caused by a pre-existing error */
2379 if (!PyErr_Occurred())
2380 PyErr_SetString(PyExc_SystemError,
2381 "null file for PyFile_WriteString");
2382 return -1;
2383 }
2384 else if (PyFile_Check(f)) {
2385 PyFileObject *fobj = (PyFileObject *) f;
2386 FILE *fp = PyFile_AsFile(f);
2387 if (fp == NULL) {
2388 err_closed();
2389 return -1;
2390 }
2391 FILE_BEGIN_ALLOW_THREADS(fobj)
2392 fputs(s, fp);
2393 FILE_END_ALLOW_THREADS(fobj)
2394 return 0;
2395 }
2396 else if (!PyErr_Occurred()) {
2397 PyObject *v = PyString_FromString(s);
2398 int err;
2399 if (v == NULL)
2400 return -1;
2401 err = PyFile_WriteObject(v, f, Py_PRINT_RAW);
2402 Py_DECREF(v);
2403 return err;
2404 }
2405 else
2406 return -1;
2407 }
2408
2409 /* Try to get a file-descriptor from a Python object. If the object
2410 is an integer or long integer, its value is returned. If not, the
2411 object's fileno() method is called if it exists; the method must return
2412 an integer or long integer, which is returned as the file descriptor value.
2413 -1 is returned on failure.
2414 */
2415
2416 int PyObject_AsFileDescriptor(PyObject *o)
2417 {
2418 int fd;
2419 PyObject *meth;
2420
2421 if (PyInt_Check(o)) {
2422 fd = PyInt_AsLong(o);
2423 }
2424 else if (PyLong_Check(o)) {
2425 fd = PyLong_AsLong(o);
2426 }
2427 else if ((meth = PyObject_GetAttrString(o, "fileno")) != NULL)
2428 {
2429 PyObject *fno = PyEval_CallObject(meth, NULL);
2430 Py_DECREF(meth);
2431 if (fno == NULL)
2432 return -1;
2433
2434 if (PyInt_Check(fno)) {
2435 fd = PyInt_AsLong(fno);
2436 Py_DECREF(fno);
2437 }
2438 else if (PyLong_Check(fno)) {
2439 fd = PyLong_AsLong(fno);
2440 Py_DECREF(fno);
2441 }
2442 else {
2443 PyErr_SetString(PyExc_TypeError,
2444 "fileno() returned a non-integer");
2445 Py_DECREF(fno);
2446 return -1;
2447 }
2448 }
2449 else {
2450 PyErr_SetString(PyExc_TypeError,
2451 "argument must be an int, or have a fileno() method.");
2452 return -1;
2453 }
2454
2455 if (fd < 0) {
2456 PyErr_Format(PyExc_ValueError,
2457 "file descriptor cannot be a negative integer (%i)",
2458 fd);
2459 return -1;
2460 }
2461 return fd;
2462 }
2463
2464 /* From here on we need access to the real fgets and fread */
2465 #undef fgets
2466 #undef fread
2467
2468 /*
2469 ** Py_UniversalNewlineFgets is an fgets variation that understands
2470 ** all of \r, \n and \r\n conventions.
2471 ** The stream should be opened in binary mode.
2472 ** If fobj is NULL the routine always does newline conversion, and
2473 ** it may peek one char ahead to gobble the second char in \r\n.
2474 ** If fobj is non-NULL it must be a PyFileObject. In this case there
2475 ** is no readahead but in stead a flag is used to skip a following
2476 ** \n on the next read. Also, if the file is open in binary mode
2477 ** the whole conversion is skipped. Finally, the routine keeps track of
2478 ** the different types of newlines seen.
2479 ** Note that we need no error handling: fgets() treats error and eof
2480 ** identically.
2481 */
2482 char *
2483 Py_UniversalNewlineFgets(char *buf, int n, FILE *stream, PyObject *fobj)
2484 {
2485 char *p = buf;
2486 int c;
2487 int newlinetypes = 0;
2488 int skipnextlf = 0;
2489 int univ_newline = 1;
2490
2491 if (fobj) {
2492 if (!PyFile_Check(fobj)) {
2493 errno = ENXIO; /* What can you do... */
2494 return NULL;
2495 }
2496 univ_newline = ((PyFileObject *)fobj)->f_univ_newline;
2497 if ( !univ_newline )
2498 return fgets(buf, n, stream);
2499 newlinetypes = ((PyFileObject *)fobj)->f_newlinetypes;
2500 skipnextlf = ((PyFileObject *)fobj)->f_skipnextlf;
2501 }
2502 FLOCKFILE(stream);
2503 c = 'x'; /* Shut up gcc warning */
2504 while (--n > 0 && (c = GETC(stream)) != EOF ) {
2505 if (skipnextlf ) {
2506 skipnextlf = 0;
2507 if (c == '\n') {
2508 /* Seeing a \n here with skipnextlf true
2509 ** means we saw a \r before.
2510 */
2511 newlinetypes |= NEWLINE_CRLF;
2512 c = GETC(stream);
2513 if (c == EOF) break;
2514 } else {
2515 /*
2516 ** Note that c == EOF also brings us here,
2517 ** so we're okay if the last char in the file
2518 ** is a CR.
2519 */
2520 newlinetypes |= NEWLINE_CR;
2521 }
2522 }
2523 if (c == '\r') {
2524 /* A \r is translated into a \n, and we skip
2525 ** an adjacent \n, if any. We don't set the
2526 ** newlinetypes flag until we've seen the next char.
2527 */
2528 skipnextlf = 1;
2529 c = '\n';
2530 } else if ( c == '\n') {
2531 newlinetypes |= NEWLINE_LF;
2532 }
2533 *p++ = c;
2534 if (c == '\n') break;
2535 }
2536 if ( c == EOF && skipnextlf )
2537 newlinetypes |= NEWLINE_CR;
2538 FUNLOCKFILE(stream);
2539 *p = '\0';
2540 if (fobj) {
2541 ((PyFileObject *)fobj)->f_newlinetypes = newlinetypes;
2542 ((PyFileObject *)fobj)->f_skipnextlf = skipnextlf;
2543 } else if ( skipnextlf ) {
2544 /* If we have no file object we cannot save the
2545 ** skipnextlf flag. We have to readahead, which
2546 ** will cause a pause if we're reading from an
2547 ** interactive stream, but that is very unlikely
2548 ** unless we're doing something silly like
2549 ** execfile("/dev/tty").
2550 */
2551 c = GETC(stream);
2552 if ( c != '\n' )
2553 ungetc(c, stream);
2554 }
2555 if (p == buf)
2556 return NULL;
2557 return buf;
2558 }
2559
2560 /*
2561 ** Py_UniversalNewlineFread is an fread variation that understands
2562 ** all of \r, \n and \r\n conventions.
2563 ** The stream should be opened in binary mode.
2564 ** fobj must be a PyFileObject. In this case there
2565 ** is no readahead but in stead a flag is used to skip a following
2566 ** \n on the next read. Also, if the file is open in binary mode
2567 ** the whole conversion is skipped. Finally, the routine keeps track of
2568 ** the different types of newlines seen.
2569 */
2570 size_t
2571 Py_UniversalNewlineFread(char *buf, size_t n,
2572 FILE *stream, PyObject *fobj)
2573 {
2574 char *dst = buf;
2575 PyFileObject *f = (PyFileObject *)fobj;
2576 int newlinetypes, skipnextlf;
2577
2578 assert(buf != NULL);
2579 assert(stream != NULL);
2580
2581 if (!fobj || !PyFile_Check(fobj)) {
2582 errno = ENXIO; /* What can you do... */
2583 return 0;
2584 }
2585 if (!f->f_univ_newline)
2586 return fread(buf, 1, n, stream);
2587 newlinetypes = f->f_newlinetypes;
2588 skipnextlf = f->f_skipnextlf;
2589 /* Invariant: n is the number of bytes remaining to be filled
2590 * in the buffer.
2591 */
2592 while (n) {
2593 size_t nread;
2594 int shortread;
2595 char *src = dst;
2596
2597 nread = fread(dst, 1, n, stream);
2598 assert(nread <= n);
2599 if (nread == 0)
2600 break;
2601
2602 n -= nread; /* assuming 1 byte out for each in; will adjust */
2603 shortread = n != 0; /* true iff EOF or error */
2604 while (nread--) {
2605 char c = *src++;
2606 if (c == '\r') {
2607 /* Save as LF and set flag to skip next LF. */
2608 *dst++ = '\n';
2609 skipnextlf = 1;
2610 }
2611 else if (skipnextlf && c == '\n') {
2612 /* Skip LF, and remember we saw CR LF. */
2613 skipnextlf = 0;
2614 newlinetypes |= NEWLINE_CRLF;
2615 ++n;
2616 }
2617 else {
2618 /* Normal char to be stored in buffer. Also
2619 * update the newlinetypes flag if either this
2620 * is an LF or the previous char was a CR.
2621 */
2622 if (c == '\n')
2623 newlinetypes |= NEWLINE_LF;
2624 else if (skipnextlf)
2625 newlinetypes |= NEWLINE_CR;
2626 *dst++ = c;
2627 skipnextlf = 0;
2628 }
2629 }
2630 if (shortread) {
2631 /* If this is EOF, update type flags. */
2632 if (skipnextlf && feof(stream))
2633 newlinetypes |= NEWLINE_CR;
2634 break;
2635 }
2636 }
2637 f->f_newlinetypes = newlinetypes;
2638 f->f_skipnextlf = skipnextlf;
2639 return dst - buf;
2640 }
2641
2642 #ifdef __cplusplus
2643 }
2644 #endif
python git mirror