Fix a bug introduced in r62627. see issue2760 and issue2632.
[python.git] / Modules / mmapmodule.c
blobc71d8402cbd495ea6ce7eadcf1143a971d8c8efd
1 /*
2 / Author: Sam Rushing <rushing@nightmare.com>
3 / Hacked for Unix by AMK
4 / $Id$
6 / Modified to support mmap with offset - to map a 'window' of a file
7 / Author: Yotam Medini yotamm@mellanox.co.il
9 / mmapmodule.cpp -- map a view of a file into memory
11 / todo: need permission flags, perhaps a 'chsize' analog
12 / not all functions check range yet!!!
15 / This version of mmapmodule.c has been changed significantly
16 / from the original mmapfile.c on which it was based.
17 / The original version of mmapfile is maintained by Sam at
18 / ftp://squirl.nightmare.com/pub/python/python-ext.
21 #define PY_SSIZE_T_CLEAN
22 #include <Python.h>
24 #ifndef MS_WINDOWS
25 #define UNIX
26 #endif
28 #ifdef MS_WINDOWS
29 #include <windows.h>
30 static int
31 my_getpagesize(void)
33 SYSTEM_INFO si;
34 GetSystemInfo(&si);
35 return si.dwPageSize;
38 static int
39 my_getallocationgranularity (void)
42 SYSTEM_INFO si;
43 GetSystemInfo(&si);
44 return si.dwAllocationGranularity;
47 #endif
49 #ifdef UNIX
50 #include <sys/mman.h>
51 #include <sys/stat.h>
53 #if defined(HAVE_SYSCONF) && defined(_SC_PAGESIZE)
54 static int
55 my_getpagesize(void)
57 return sysconf(_SC_PAGESIZE);
60 #define my_getallocationgranularity my_getpagesize
61 #else
62 #define my_getpagesize getpagesize
63 #endif
65 #endif /* UNIX */
67 #include <string.h>
69 #ifdef HAVE_SYS_TYPES_H
70 #include <sys/types.h>
71 #endif /* HAVE_SYS_TYPES_H */
73 /* Prefer MAP_ANONYMOUS since MAP_ANON is deprecated according to man page. */
74 #if !defined(MAP_ANONYMOUS) && defined(MAP_ANON)
75 # define MAP_ANONYMOUS MAP_ANON
76 #endif
78 static PyObject *mmap_module_error;
80 typedef enum
82 ACCESS_DEFAULT,
83 ACCESS_READ,
84 ACCESS_WRITE,
85 ACCESS_COPY
86 } access_mode;
88 typedef struct {
89 PyObject_HEAD
90 char * data;
91 size_t size;
92 size_t pos; /* relative to offset */
93 size_t offset;
95 #ifdef MS_WINDOWS
96 HANDLE map_handle;
97 HANDLE file_handle;
98 char * tagname;
99 #endif
101 #ifdef UNIX
102 int fd;
103 #endif
105 access_mode access;
106 } mmap_object;
109 static void
110 mmap_object_dealloc(mmap_object *m_obj)
112 #ifdef MS_WINDOWS
113 if (m_obj->data != NULL)
114 UnmapViewOfFile (m_obj->data);
115 if (m_obj->map_handle != INVALID_HANDLE_VALUE)
116 CloseHandle (m_obj->map_handle);
117 if (m_obj->file_handle != INVALID_HANDLE_VALUE)
118 CloseHandle (m_obj->file_handle);
119 if (m_obj->tagname)
120 PyMem_Free(m_obj->tagname);
121 #endif /* MS_WINDOWS */
123 #ifdef UNIX
124 if (m_obj->fd >= 0)
125 (void) close(m_obj->fd);
126 if (m_obj->data!=NULL) {
127 msync(m_obj->data, m_obj->size, MS_SYNC);
128 munmap(m_obj->data, m_obj->size);
130 #endif /* UNIX */
132 Py_TYPE(m_obj)->tp_free((PyObject*)m_obj);
135 static PyObject *
136 mmap_close_method(mmap_object *self, PyObject *unused)
138 #ifdef MS_WINDOWS
139 /* For each resource we maintain, we need to check
140 the value is valid, and if so, free the resource
141 and set the member value to an invalid value so
142 the dealloc does not attempt to resource clearing
143 again.
144 TODO - should we check for errors in the close operations???
146 if (self->data != NULL) {
147 UnmapViewOfFile(self->data);
148 self->data = NULL;
150 if (self->map_handle != INVALID_HANDLE_VALUE) {
151 CloseHandle(self->map_handle);
152 self->map_handle = INVALID_HANDLE_VALUE;
154 if (self->file_handle != INVALID_HANDLE_VALUE) {
155 CloseHandle(self->file_handle);
156 self->file_handle = INVALID_HANDLE_VALUE;
158 #endif /* MS_WINDOWS */
160 #ifdef UNIX
161 (void) close(self->fd);
162 self->fd = -1;
163 if (self->data != NULL) {
164 munmap(self->data, self->size);
165 self->data = NULL;
167 #endif
169 Py_INCREF(Py_None);
170 return Py_None;
173 #ifdef MS_WINDOWS
174 #define CHECK_VALID(err) \
175 do { \
176 if (self->map_handle == INVALID_HANDLE_VALUE) { \
177 PyErr_SetString(PyExc_ValueError, "mmap closed or invalid"); \
178 return err; \
180 } while (0)
181 #endif /* MS_WINDOWS */
183 #ifdef UNIX
184 #define CHECK_VALID(err) \
185 do { \
186 if (self->data == NULL) { \
187 PyErr_SetString(PyExc_ValueError, "mmap closed or invalid"); \
188 return err; \
190 } while (0)
191 #endif /* UNIX */
193 static PyObject *
194 mmap_read_byte_method(mmap_object *self,
195 PyObject *unused)
197 CHECK_VALID(NULL);
198 if (self->pos < self->size) {
199 char value = self->data[self->pos];
200 self->pos += 1;
201 return Py_BuildValue("c", value);
202 } else {
203 PyErr_SetString(PyExc_ValueError, "read byte out of range");
204 return NULL;
208 static PyObject *
209 mmap_read_line_method(mmap_object *self,
210 PyObject *unused)
212 char *start = self->data+self->pos;
213 char *eof = self->data+self->size;
214 char *eol;
215 PyObject *result;
217 CHECK_VALID(NULL);
219 eol = memchr(start, '\n', self->size - self->pos);
220 if (!eol)
221 eol = eof;
222 else
223 ++eol; /* we're interested in the position after the
224 newline. */
225 result = PyString_FromStringAndSize(start, (eol - start));
226 self->pos += (eol - start);
227 return result;
230 static PyObject *
231 mmap_read_method(mmap_object *self,
232 PyObject *args)
234 Py_ssize_t num_bytes;
235 PyObject *result;
237 CHECK_VALID(NULL);
238 if (!PyArg_ParseTuple(args, "n:read", &num_bytes))
239 return(NULL);
241 /* silently 'adjust' out-of-range requests */
242 if ((self->pos + num_bytes) > self->size) {
243 num_bytes -= (self->pos+num_bytes) - self->size;
245 result = Py_BuildValue("s#", self->data+self->pos, num_bytes);
246 self->pos += num_bytes;
247 return result;
250 static PyObject *
251 mmap_gfind(mmap_object *self,
252 PyObject *args,
253 int reverse)
255 Py_ssize_t start = self->pos;
256 Py_ssize_t end = self->size;
257 const char *needle;
258 Py_ssize_t len;
260 CHECK_VALID(NULL);
261 if (!PyArg_ParseTuple(args, reverse ? "s#|nn:rfind" : "s#|nn:find",
262 &needle, &len, &start, &end)) {
263 return NULL;
264 } else {
265 const char *p, *start_p, *end_p;
266 int sign = reverse ? -1 : 1;
268 if (start < 0)
269 start += self->size;
270 if (start < 0)
271 start = 0;
272 else if ((size_t)start > self->size)
273 start = self->size;
275 if (end < 0)
276 end += self->size;
277 if (end < 0)
278 end = 0;
279 else if ((size_t)end > self->size)
280 end = self->size;
282 start_p = self->data + start;
283 end_p = self->data + end;
285 for (p = (reverse ? end_p - len : start_p);
286 (p >= start_p) && (p + len <= end_p); p += sign) {
287 Py_ssize_t i;
288 for (i = 0; i < len && needle[i] == p[i]; ++i)
289 /* nothing */;
290 if (i == len) {
291 return PyInt_FromSsize_t(p - self->data);
294 return PyInt_FromLong(-1);
298 static PyObject *
299 mmap_find_method(mmap_object *self,
300 PyObject *args)
302 return mmap_gfind(self, args, 0);
305 static PyObject *
306 mmap_rfind_method(mmap_object *self,
307 PyObject *args)
309 return mmap_gfind(self, args, 1);
312 static int
313 is_writeable(mmap_object *self)
315 if (self->access != ACCESS_READ)
316 return 1;
317 PyErr_Format(PyExc_TypeError, "mmap can't modify a readonly memory map.");
318 return 0;
321 static int
322 is_resizeable(mmap_object *self)
324 if ((self->access == ACCESS_WRITE) || (self->access == ACCESS_DEFAULT))
325 return 1;
326 PyErr_Format(PyExc_TypeError,
327 "mmap can't resize a readonly or copy-on-write memory map.");
328 return 0;
332 static PyObject *
333 mmap_write_method(mmap_object *self,
334 PyObject *args)
336 Py_ssize_t length;
337 char *data;
339 CHECK_VALID(NULL);
340 if (!PyArg_ParseTuple(args, "s#:write", &data, &length))
341 return(NULL);
343 if (!is_writeable(self))
344 return NULL;
346 if ((self->pos + length) > self->size) {
347 PyErr_SetString(PyExc_ValueError, "data out of range");
348 return NULL;
350 memcpy(self->data+self->pos, data, length);
351 self->pos = self->pos+length;
352 Py_INCREF(Py_None);
353 return Py_None;
356 static PyObject *
357 mmap_write_byte_method(mmap_object *self,
358 PyObject *args)
360 char value;
362 CHECK_VALID(NULL);
363 if (!PyArg_ParseTuple(args, "c:write_byte", &value))
364 return(NULL);
366 if (!is_writeable(self))
367 return NULL;
368 *(self->data+self->pos) = value;
369 self->pos += 1;
370 Py_INCREF(Py_None);
371 return Py_None;
374 static PyObject *
375 mmap_size_method(mmap_object *self,
376 PyObject *unused)
378 CHECK_VALID(NULL);
380 #ifdef MS_WINDOWS
381 if (self->file_handle != INVALID_HANDLE_VALUE) {
382 DWORD low,high;
383 PY_LONG_LONG size;
384 low = GetFileSize(self->file_handle, &high);
385 if (low == INVALID_FILE_SIZE) {
386 /* It might be that the function appears to have failed,
387 when indeed its size equals INVALID_FILE_SIZE */
388 DWORD error = GetLastError();
389 if (error != NO_ERROR)
390 return PyErr_SetFromWindowsErr(error);
392 if (!high && low < LONG_MAX)
393 return PyInt_FromLong((long)low);
394 size = (((PY_LONG_LONG)high)<<32) + low;
395 return PyLong_FromLongLong(size);
396 } else {
397 return PyInt_FromSsize_t(self->size);
399 #endif /* MS_WINDOWS */
401 #ifdef UNIX
403 struct stat buf;
404 if (-1 == fstat(self->fd, &buf)) {
405 PyErr_SetFromErrno(mmap_module_error);
406 return NULL;
408 return PyInt_FromSsize_t(buf.st_size);
410 #endif /* UNIX */
413 /* This assumes that you want the entire file mapped,
414 / and when recreating the map will make the new file
415 / have the new size
417 / Is this really necessary? This could easily be done
418 / from python by just closing and re-opening with the
419 / new size?
422 static PyObject *
423 mmap_resize_method(mmap_object *self,
424 PyObject *args)
426 Py_ssize_t new_size;
427 CHECK_VALID(NULL);
428 if (!PyArg_ParseTuple(args, "n:resize", &new_size) ||
429 !is_resizeable(self)) {
430 return NULL;
431 #ifdef MS_WINDOWS
432 } else {
433 DWORD dwErrCode = 0;
434 DWORD off_hi, off_lo, newSizeLow, newSizeHigh;
435 /* First, unmap the file view */
436 UnmapViewOfFile(self->data);
437 /* Close the mapping object */
438 CloseHandle(self->map_handle);
439 /* Move to the desired EOF position */
440 #if SIZEOF_SIZE_T > 4
441 newSizeHigh = (DWORD)((self->offset + new_size) >> 32);
442 newSizeLow = (DWORD)((self->offset + new_size) & 0xFFFFFFFF);
443 off_hi = (DWORD)(self->offset >> 32);
444 off_lo = (DWORD)(self->offset & 0xFFFFFFFF);
445 #else
446 newSizeHigh = 0;
447 newSizeLow = (DWORD)new_size;
448 off_hi = 0;
449 off_lo = (DWORD)self->offset;
450 #endif
451 SetFilePointer(self->file_handle,
452 newSizeLow, &newSizeHigh, FILE_BEGIN);
453 /* Change the size of the file */
454 SetEndOfFile(self->file_handle);
455 /* Create another mapping object and remap the file view */
456 self->map_handle = CreateFileMapping(
457 self->file_handle,
458 NULL,
459 PAGE_READWRITE,
462 self->tagname);
463 if (self->map_handle != NULL) {
464 self->data = (char *) MapViewOfFile(self->map_handle,
465 FILE_MAP_WRITE,
466 off_hi,
467 off_lo,
468 new_size);
469 if (self->data != NULL) {
470 self->size = new_size;
471 Py_INCREF(Py_None);
472 return Py_None;
473 } else {
474 dwErrCode = GetLastError();
476 } else {
477 dwErrCode = GetLastError();
479 PyErr_SetFromWindowsErr(dwErrCode);
480 return NULL;
481 #endif /* MS_WINDOWS */
483 #ifdef UNIX
484 #ifndef HAVE_MREMAP
485 } else {
486 PyErr_SetString(PyExc_SystemError,
487 "mmap: resizing not available--no mremap()");
488 return NULL;
489 #else
490 } else {
491 void *newmap;
493 if (ftruncate(self->fd, new_size) == -1) {
494 PyErr_SetFromErrno(mmap_module_error);
495 return NULL;
498 #ifdef MREMAP_MAYMOVE
499 newmap = mremap(self->data, self->size, new_size, MREMAP_MAYMOVE);
500 #else
501 newmap = mremap(self->data, self->size, new_size, 0);
502 #endif
503 if (newmap == (void *)-1)
505 PyErr_SetFromErrno(mmap_module_error);
506 return NULL;
508 self->data = newmap;
509 self->size = new_size;
510 Py_INCREF(Py_None);
511 return Py_None;
512 #endif /* HAVE_MREMAP */
513 #endif /* UNIX */
517 static PyObject *
518 mmap_tell_method(mmap_object *self, PyObject *unused)
520 CHECK_VALID(NULL);
521 return PyInt_FromSize_t(self->pos);
524 static PyObject *
525 mmap_flush_method(mmap_object *self, PyObject *args)
527 Py_ssize_t offset = 0;
528 Py_ssize_t size = self->size;
529 CHECK_VALID(NULL);
530 if (!PyArg_ParseTuple(args, "|nn:flush", &offset, &size))
531 return NULL;
532 if ((size_t)(offset + size) > self->size) {
533 PyErr_SetString(PyExc_ValueError, "flush values out of range");
534 return NULL;
536 #ifdef MS_WINDOWS
537 return PyInt_FromLong((long) FlushViewOfFile(self->data+offset, size));
538 #elif defined(UNIX)
539 /* XXX semantics of return value? */
540 /* XXX flags for msync? */
541 if (-1 == msync(self->data + offset, size, MS_SYNC)) {
542 PyErr_SetFromErrno(mmap_module_error);
543 return NULL;
545 return PyInt_FromLong(0);
546 #else
547 PyErr_SetString(PyExc_ValueError, "flush not supported on this system");
548 return NULL;
549 #endif
552 static PyObject *
553 mmap_seek_method(mmap_object *self, PyObject *args)
555 Py_ssize_t dist;
556 int how=0;
557 CHECK_VALID(NULL);
558 if (!PyArg_ParseTuple(args, "n|i:seek", &dist, &how))
559 return NULL;
560 else {
561 size_t where;
562 switch (how) {
563 case 0: /* relative to start */
564 if (dist < 0)
565 goto onoutofrange;
566 where = dist;
567 break;
568 case 1: /* relative to current position */
569 if ((Py_ssize_t)self->pos + dist < 0)
570 goto onoutofrange;
571 where = self->pos + dist;
572 break;
573 case 2: /* relative to end */
574 if ((Py_ssize_t)self->size + dist < 0)
575 goto onoutofrange;
576 where = self->size + dist;
577 break;
578 default:
579 PyErr_SetString(PyExc_ValueError, "unknown seek type");
580 return NULL;
582 if (where > self->size)
583 goto onoutofrange;
584 self->pos = where;
585 Py_INCREF(Py_None);
586 return Py_None;
589 onoutofrange:
590 PyErr_SetString(PyExc_ValueError, "seek out of range");
591 return NULL;
594 static PyObject *
595 mmap_move_method(mmap_object *self, PyObject *args)
597 unsigned long dest, src, count;
598 CHECK_VALID(NULL);
599 if (!PyArg_ParseTuple(args, "kkk:move", &dest, &src, &count) ||
600 !is_writeable(self)) {
601 return NULL;
602 } else {
603 /* bounds check the values */
604 if (/* end of source after end of data?? */
605 ((src+count) > self->size)
606 /* dest will fit? */
607 || (dest+count > self->size)) {
608 PyErr_SetString(PyExc_ValueError,
609 "source or destination out of range");
610 return NULL;
611 } else {
612 memmove(self->data+dest, self->data+src, count);
613 Py_INCREF(Py_None);
614 return Py_None;
619 static struct PyMethodDef mmap_object_methods[] = {
620 {"close", (PyCFunction) mmap_close_method, METH_NOARGS},
621 {"find", (PyCFunction) mmap_find_method, METH_VARARGS},
622 {"rfind", (PyCFunction) mmap_rfind_method, METH_VARARGS},
623 {"flush", (PyCFunction) mmap_flush_method, METH_VARARGS},
624 {"move", (PyCFunction) mmap_move_method, METH_VARARGS},
625 {"read", (PyCFunction) mmap_read_method, METH_VARARGS},
626 {"read_byte", (PyCFunction) mmap_read_byte_method, METH_NOARGS},
627 {"readline", (PyCFunction) mmap_read_line_method, METH_NOARGS},
628 {"resize", (PyCFunction) mmap_resize_method, METH_VARARGS},
629 {"seek", (PyCFunction) mmap_seek_method, METH_VARARGS},
630 {"size", (PyCFunction) mmap_size_method, METH_NOARGS},
631 {"tell", (PyCFunction) mmap_tell_method, METH_NOARGS},
632 {"write", (PyCFunction) mmap_write_method, METH_VARARGS},
633 {"write_byte", (PyCFunction) mmap_write_byte_method, METH_VARARGS},
634 {NULL, NULL} /* sentinel */
637 /* Functions for treating an mmap'ed file as a buffer */
639 static Py_ssize_t
640 mmap_buffer_getreadbuf(mmap_object *self, Py_ssize_t index, const void **ptr)
642 CHECK_VALID(-1);
643 if (index != 0) {
644 PyErr_SetString(PyExc_SystemError,
645 "Accessing non-existent mmap segment");
646 return -1;
648 *ptr = self->data;
649 return self->size;
652 static Py_ssize_t
653 mmap_buffer_getwritebuf(mmap_object *self, Py_ssize_t index, const void **ptr)
655 CHECK_VALID(-1);
656 if (index != 0) {
657 PyErr_SetString(PyExc_SystemError,
658 "Accessing non-existent mmap segment");
659 return -1;
661 if (!is_writeable(self))
662 return -1;
663 *ptr = self->data;
664 return self->size;
667 static Py_ssize_t
668 mmap_buffer_getsegcount(mmap_object *self, Py_ssize_t *lenp)
670 CHECK_VALID(-1);
671 if (lenp)
672 *lenp = self->size;
673 return 1;
676 static Py_ssize_t
677 mmap_buffer_getcharbuffer(mmap_object *self, Py_ssize_t index, const void **ptr)
679 if (index != 0) {
680 PyErr_SetString(PyExc_SystemError,
681 "accessing non-existent buffer segment");
682 return -1;
684 *ptr = (const char *)self->data;
685 return self->size;
688 static Py_ssize_t
689 mmap_length(mmap_object *self)
691 CHECK_VALID(-1);
692 return self->size;
695 static PyObject *
696 mmap_item(mmap_object *self, Py_ssize_t i)
698 CHECK_VALID(NULL);
699 if (i < 0 || (size_t)i >= self->size) {
700 PyErr_SetString(PyExc_IndexError, "mmap index out of range");
701 return NULL;
703 return PyString_FromStringAndSize(self->data + i, 1);
706 static PyObject *
707 mmap_slice(mmap_object *self, Py_ssize_t ilow, Py_ssize_t ihigh)
709 CHECK_VALID(NULL);
710 if (ilow < 0)
711 ilow = 0;
712 else if ((size_t)ilow > self->size)
713 ilow = self->size;
714 if (ihigh < 0)
715 ihigh = 0;
716 if (ihigh < ilow)
717 ihigh = ilow;
718 else if ((size_t)ihigh > self->size)
719 ihigh = self->size;
721 return PyString_FromStringAndSize(self->data + ilow, ihigh-ilow);
724 static PyObject *
725 mmap_subscript(mmap_object *self, PyObject *item)
727 CHECK_VALID(NULL);
728 if (PyIndex_Check(item)) {
729 Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError);
730 if (i == -1 && PyErr_Occurred())
731 return NULL;
732 if (i < 0)
733 i += self->size;
734 if (i < 0 || (size_t)i > self->size) {
735 PyErr_SetString(PyExc_IndexError,
736 "mmap index out of range");
737 return NULL;
739 return PyString_FromStringAndSize(self->data + i, 1);
741 else if (PySlice_Check(item)) {
742 Py_ssize_t start, stop, step, slicelen;
744 if (PySlice_GetIndicesEx((PySliceObject *)item, self->size,
745 &start, &stop, &step, &slicelen) < 0) {
746 return NULL;
749 if (slicelen <= 0)
750 return PyString_FromStringAndSize("", 0);
751 else if (step == 1)
752 return PyString_FromStringAndSize(self->data + start,
753 slicelen);
754 else {
755 char *result_buf = (char *)PyMem_Malloc(slicelen);
756 Py_ssize_t cur, i;
757 PyObject *result;
759 if (result_buf == NULL)
760 return PyErr_NoMemory();
761 for (cur = start, i = 0; i < slicelen;
762 cur += step, i++) {
763 result_buf[i] = self->data[cur];
765 result = PyString_FromStringAndSize(result_buf,
766 slicelen);
767 PyMem_Free(result_buf);
768 return result;
771 else {
772 PyErr_SetString(PyExc_TypeError,
773 "mmap indices must be integers");
774 return NULL;
778 static PyObject *
779 mmap_concat(mmap_object *self, PyObject *bb)
781 CHECK_VALID(NULL);
782 PyErr_SetString(PyExc_SystemError,
783 "mmaps don't support concatenation");
784 return NULL;
787 static PyObject *
788 mmap_repeat(mmap_object *self, Py_ssize_t n)
790 CHECK_VALID(NULL);
791 PyErr_SetString(PyExc_SystemError,
792 "mmaps don't support repeat operation");
793 return NULL;
796 static int
797 mmap_ass_slice(mmap_object *self, Py_ssize_t ilow, Py_ssize_t ihigh, PyObject *v)
799 const char *buf;
801 CHECK_VALID(-1);
802 if (ilow < 0)
803 ilow = 0;
804 else if ((size_t)ilow > self->size)
805 ilow = self->size;
806 if (ihigh < 0)
807 ihigh = 0;
808 if (ihigh < ilow)
809 ihigh = ilow;
810 else if ((size_t)ihigh > self->size)
811 ihigh = self->size;
813 if (v == NULL) {
814 PyErr_SetString(PyExc_TypeError,
815 "mmap object doesn't support slice deletion");
816 return -1;
818 if (! (PyString_Check(v)) ) {
819 PyErr_SetString(PyExc_IndexError,
820 "mmap slice assignment must be a string");
821 return -1;
823 if (PyString_Size(v) != (ihigh - ilow)) {
824 PyErr_SetString(PyExc_IndexError,
825 "mmap slice assignment is wrong size");
826 return -1;
828 if (!is_writeable(self))
829 return -1;
830 buf = PyString_AsString(v);
831 memcpy(self->data + ilow, buf, ihigh-ilow);
832 return 0;
835 static int
836 mmap_ass_item(mmap_object *self, Py_ssize_t i, PyObject *v)
838 const char *buf;
840 CHECK_VALID(-1);
841 if (i < 0 || (size_t)i >= self->size) {
842 PyErr_SetString(PyExc_IndexError, "mmap index out of range");
843 return -1;
845 if (v == NULL) {
846 PyErr_SetString(PyExc_TypeError,
847 "mmap object doesn't support item deletion");
848 return -1;
850 if (! (PyString_Check(v) && PyString_Size(v)==1) ) {
851 PyErr_SetString(PyExc_IndexError,
852 "mmap assignment must be single-character string");
853 return -1;
855 if (!is_writeable(self))
856 return -1;
857 buf = PyString_AsString(v);
858 self->data[i] = buf[0];
859 return 0;
862 static int
863 mmap_ass_subscript(mmap_object *self, PyObject *item, PyObject *value)
865 CHECK_VALID(-1);
867 if (PyIndex_Check(item)) {
868 Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError);
869 const char *buf;
871 if (i == -1 && PyErr_Occurred())
872 return -1;
873 if (i < 0)
874 i += self->size;
875 if (i < 0 || (size_t)i > self->size) {
876 PyErr_SetString(PyExc_IndexError,
877 "mmap index out of range");
878 return -1;
880 if (value == NULL) {
881 PyErr_SetString(PyExc_TypeError,
882 "mmap object doesn't support item deletion");
883 return -1;
885 if (!PyString_Check(value) || PyString_Size(value) != 1) {
886 PyErr_SetString(PyExc_IndexError,
887 "mmap assignment must be single-character string");
888 return -1;
890 if (!is_writeable(self))
891 return -1;
892 buf = PyString_AsString(value);
893 self->data[i] = buf[0];
894 return 0;
896 else if (PySlice_Check(item)) {
897 Py_ssize_t start, stop, step, slicelen;
899 if (PySlice_GetIndicesEx((PySliceObject *)item,
900 self->size, &start, &stop,
901 &step, &slicelen) < 0) {
902 return -1;
904 if (value == NULL) {
905 PyErr_SetString(PyExc_TypeError,
906 "mmap object doesn't support slice deletion");
907 return -1;
909 if (!PyString_Check(value)) {
910 PyErr_SetString(PyExc_IndexError,
911 "mmap slice assignment must be a string");
912 return -1;
914 if (PyString_Size(value) != slicelen) {
915 PyErr_SetString(PyExc_IndexError,
916 "mmap slice assignment is wrong size");
917 return -1;
919 if (!is_writeable(self))
920 return -1;
922 if (slicelen == 0)
923 return 0;
924 else if (step == 1) {
925 const char *buf = PyString_AsString(value);
927 if (buf == NULL)
928 return -1;
929 memcpy(self->data + start, buf, slicelen);
930 return 0;
932 else {
933 Py_ssize_t cur, i;
934 const char *buf = PyString_AsString(value);
936 if (buf == NULL)
937 return -1;
938 for (cur = start, i = 0; i < slicelen;
939 cur += step, i++) {
940 self->data[cur] = buf[i];
942 return 0;
945 else {
946 PyErr_SetString(PyExc_TypeError,
947 "mmap indices must be integer");
948 return -1;
952 static PySequenceMethods mmap_as_sequence = {
953 (lenfunc)mmap_length, /*sq_length*/
954 (binaryfunc)mmap_concat, /*sq_concat*/
955 (ssizeargfunc)mmap_repeat, /*sq_repeat*/
956 (ssizeargfunc)mmap_item, /*sq_item*/
957 (ssizessizeargfunc)mmap_slice, /*sq_slice*/
958 (ssizeobjargproc)mmap_ass_item, /*sq_ass_item*/
959 (ssizessizeobjargproc)mmap_ass_slice, /*sq_ass_slice*/
962 static PyMappingMethods mmap_as_mapping = {
963 (lenfunc)mmap_length,
964 (binaryfunc)mmap_subscript,
965 (objobjargproc)mmap_ass_subscript,
968 static PyBufferProcs mmap_as_buffer = {
969 (readbufferproc)mmap_buffer_getreadbuf,
970 (writebufferproc)mmap_buffer_getwritebuf,
971 (segcountproc)mmap_buffer_getsegcount,
972 (charbufferproc)mmap_buffer_getcharbuffer,
975 static PyObject *
976 new_mmap_object(PyTypeObject *type, PyObject *args, PyObject *kwdict);
978 PyDoc_STRVAR(mmap_doc,
979 "Windows: mmap(fileno, length[, tagname[, access[, offset]]])\n\
981 Maps length bytes from the file specified by the file handle fileno,\n\
982 and returns a mmap object. If length is larger than the current size\n\
983 of the file, the file is extended to contain length bytes. If length\n\
984 is 0, the maximum length of the map is the current size of the file,\n\
985 except that if the file is empty Windows raises an exception (you cannot\n\
986 create an empty mapping on Windows).\n\
988 Unix: mmap(fileno, length[, flags[, prot[, access[, offset]]]])\n\
990 Maps length bytes from the file specified by the file descriptor fileno,\n\
991 and returns a mmap object. If length is 0, the maximum length of the map\n\
992 will be the current size of the file when mmap is called.\n\
993 flags specifies the nature of the mapping. MAP_PRIVATE creates a\n\
994 private copy-on-write mapping, so changes to the contents of the mmap\n\
995 object will be private to this process, and MAP_SHARED`creates a mapping\n\
996 that's shared with all other processes mapping the same areas of the file.\n\
997 The default value is MAP_SHARED.\n\
999 To map anonymous memory, pass -1 as the fileno (both versions).");
1002 static PyTypeObject mmap_object_type = {
1003 PyVarObject_HEAD_INIT(NULL, 0)
1004 "mmap.mmap", /* tp_name */
1005 sizeof(mmap_object), /* tp_size */
1006 0, /* tp_itemsize */
1007 /* methods */
1008 (destructor) mmap_object_dealloc, /* tp_dealloc */
1009 0, /* tp_print */
1010 0, /* tp_getattr */
1011 0, /* tp_setattr */
1012 0, /* tp_compare */
1013 0, /* tp_repr */
1014 0, /* tp_as_number */
1015 &mmap_as_sequence, /*tp_as_sequence*/
1016 &mmap_as_mapping, /*tp_as_mapping*/
1017 0, /*tp_hash*/
1018 0, /*tp_call*/
1019 0, /*tp_str*/
1020 PyObject_GenericGetAttr, /*tp_getattro*/
1021 0, /*tp_setattro*/
1022 &mmap_as_buffer, /*tp_as_buffer*/
1023 Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GETCHARBUFFER, /*tp_flags*/
1024 mmap_doc, /*tp_doc*/
1025 0, /* tp_traverse */
1026 0, /* tp_clear */
1027 0, /* tp_richcompare */
1028 0, /* tp_weaklistoffset */
1029 0, /* tp_iter */
1030 0, /* tp_iternext */
1031 mmap_object_methods, /* tp_methods */
1032 0, /* tp_members */
1033 0, /* tp_getset */
1034 0, /* tp_base */
1035 0, /* tp_dict */
1036 0, /* tp_descr_get */
1037 0, /* tp_descr_set */
1038 0, /* tp_dictoffset */
1039 0, /* tp_init */
1040 PyType_GenericAlloc, /* tp_alloc */
1041 new_mmap_object, /* tp_new */
1042 PyObject_Del, /* tp_free */
1046 /* extract the map size from the given PyObject
1048 Returns -1 on error, with an appropriate Python exception raised. On
1049 success, the map size is returned. */
1050 static Py_ssize_t
1051 _GetMapSize(PyObject *o, const char* param)
1053 if (o == NULL)
1054 return 0;
1055 if (PyIndex_Check(o)) {
1056 Py_ssize_t i = PyNumber_AsSsize_t(o, PyExc_OverflowError);
1057 if (i==-1 && PyErr_Occurred())
1058 return -1;
1059 if (i < 0) {
1060 PyErr_Format(PyExc_OverflowError,
1061 "memory mapped %s must be positive",
1062 param);
1063 return -1;
1065 return i;
1068 PyErr_SetString(PyExc_TypeError, "map size must be an integral value");
1069 return -1;
1072 #ifdef UNIX
1073 static PyObject *
1074 new_mmap_object(PyTypeObject *type, PyObject *args, PyObject *kwdict)
1076 #ifdef HAVE_FSTAT
1077 struct stat st;
1078 #endif
1079 mmap_object *m_obj;
1080 PyObject *map_size_obj = NULL, *offset_obj = NULL;
1081 Py_ssize_t map_size, offset;
1082 int fd, flags = MAP_SHARED, prot = PROT_WRITE | PROT_READ;
1083 int devzero = -1;
1084 int access = (int)ACCESS_DEFAULT;
1085 static char *keywords[] = {"fileno", "length",
1086 "flags", "prot",
1087 "access", "offset", NULL};
1089 if (!PyArg_ParseTupleAndKeywords(args, kwdict, "iO|iiiO", keywords,
1090 &fd, &map_size_obj, &flags, &prot,
1091 &access, &offset_obj))
1092 return NULL;
1093 map_size = _GetMapSize(map_size_obj, "size");
1094 if (map_size < 0)
1095 return NULL;
1096 offset = _GetMapSize(offset_obj, "offset");
1097 if (offset < 0)
1098 return NULL;
1100 if ((access != (int)ACCESS_DEFAULT) &&
1101 ((flags != MAP_SHARED) || (prot != (PROT_WRITE | PROT_READ))))
1102 return PyErr_Format(PyExc_ValueError,
1103 "mmap can't specify both access and flags, prot.");
1104 switch ((access_mode)access) {
1105 case ACCESS_READ:
1106 flags = MAP_SHARED;
1107 prot = PROT_READ;
1108 break;
1109 case ACCESS_WRITE:
1110 flags = MAP_SHARED;
1111 prot = PROT_READ | PROT_WRITE;
1112 break;
1113 case ACCESS_COPY:
1114 flags = MAP_PRIVATE;
1115 prot = PROT_READ | PROT_WRITE;
1116 break;
1117 case ACCESS_DEFAULT:
1118 /* use the specified or default values of flags and prot */
1119 break;
1120 default:
1121 return PyErr_Format(PyExc_ValueError,
1122 "mmap invalid access parameter.");
1125 if (prot == PROT_READ) {
1126 access = ACCESS_READ;
1129 #ifdef HAVE_FSTAT
1130 # ifdef __VMS
1131 /* on OpenVMS we must ensure that all bytes are written to the file */
1132 if (fd != -1) {
1133 fsync(fd);
1135 # endif
1136 if (fd != -1 && fstat(fd, &st) == 0 && S_ISREG(st.st_mode)) {
1137 if (map_size == 0) {
1138 map_size = st.st_size;
1139 } else if ((size_t)offset + (size_t)map_size > st.st_size) {
1140 PyErr_SetString(PyExc_ValueError,
1141 "mmap length is greater than file size");
1142 return NULL;
1145 #endif
1146 m_obj = (mmap_object *)type->tp_alloc(type, 0);
1147 if (m_obj == NULL) {return NULL;}
1148 m_obj->data = NULL;
1149 m_obj->size = (size_t) map_size;
1150 m_obj->pos = (size_t) 0;
1151 m_obj->offset = offset;
1152 if (fd == -1) {
1153 m_obj->fd = -1;
1154 /* Assume the caller wants to map anonymous memory.
1155 This is the same behaviour as Windows. mmap.mmap(-1, size)
1156 on both Windows and Unix map anonymous memory.
1158 #ifdef MAP_ANONYMOUS
1159 /* BSD way to map anonymous memory */
1160 flags |= MAP_ANONYMOUS;
1161 #else
1162 /* SVR4 method to map anonymous memory is to open /dev/zero */
1163 fd = devzero = open("/dev/zero", O_RDWR);
1164 if (devzero == -1) {
1165 Py_DECREF(m_obj);
1166 PyErr_SetFromErrno(mmap_module_error);
1167 return NULL;
1169 #endif
1170 } else {
1171 m_obj->fd = dup(fd);
1172 if (m_obj->fd == -1) {
1173 Py_DECREF(m_obj);
1174 PyErr_SetFromErrno(mmap_module_error);
1175 return NULL;
1179 m_obj->data = mmap(NULL, map_size,
1180 prot, flags,
1181 fd, offset);
1183 if (devzero != -1) {
1184 close(devzero);
1187 if (m_obj->data == (char *)-1) {
1188 m_obj->data = NULL;
1189 Py_DECREF(m_obj);
1190 PyErr_SetFromErrno(mmap_module_error);
1191 return NULL;
1193 m_obj->access = (access_mode)access;
1194 return (PyObject *)m_obj;
1196 #endif /* UNIX */
1198 #ifdef MS_WINDOWS
1199 static PyObject *
1200 new_mmap_object(PyTypeObject *type, PyObject *args, PyObject *kwdict)
1202 mmap_object *m_obj;
1203 PyObject *map_size_obj = NULL, *offset_obj = NULL;
1204 Py_ssize_t map_size, offset;
1205 DWORD off_hi; /* upper 32 bits of offset */
1206 DWORD off_lo; /* lower 32 bits of offset */
1207 DWORD size_hi; /* upper 32 bits of size */
1208 DWORD size_lo; /* lower 32 bits of size */
1209 char *tagname = "";
1210 DWORD dwErr = 0;
1211 int fileno;
1212 HANDLE fh = 0;
1213 int access = (access_mode)ACCESS_DEFAULT;
1214 DWORD flProtect, dwDesiredAccess;
1215 static char *keywords[] = { "fileno", "length",
1216 "tagname",
1217 "access", "offset", NULL };
1219 if (!PyArg_ParseTupleAndKeywords(args, kwdict, "iO|ziO", keywords,
1220 &fileno, &map_size_obj,
1221 &tagname, &access, &offset_obj)) {
1222 return NULL;
1225 switch((access_mode)access) {
1226 case ACCESS_READ:
1227 flProtect = PAGE_READONLY;
1228 dwDesiredAccess = FILE_MAP_READ;
1229 break;
1230 case ACCESS_DEFAULT: case ACCESS_WRITE:
1231 flProtect = PAGE_READWRITE;
1232 dwDesiredAccess = FILE_MAP_WRITE;
1233 break;
1234 case ACCESS_COPY:
1235 flProtect = PAGE_WRITECOPY;
1236 dwDesiredAccess = FILE_MAP_COPY;
1237 break;
1238 default:
1239 return PyErr_Format(PyExc_ValueError,
1240 "mmap invalid access parameter.");
1243 map_size = _GetMapSize(map_size_obj, "size");
1244 if (map_size < 0)
1245 return NULL;
1246 offset = _GetMapSize(offset_obj, "offset");
1247 if (offset < 0)
1248 return NULL;
1250 /* assume -1 and 0 both mean invalid filedescriptor
1251 to 'anonymously' map memory.
1252 XXX: fileno == 0 is a valid fd, but was accepted prior to 2.5.
1253 XXX: Should this code be added?
1254 if (fileno == 0)
1255 PyErr_Warn(PyExc_DeprecationWarning,
1256 "don't use 0 for anonymous memory");
1258 if (fileno != -1 && fileno != 0) {
1259 fh = (HANDLE)_get_osfhandle(fileno);
1260 if (fh==(HANDLE)-1) {
1261 PyErr_SetFromErrno(mmap_module_error);
1262 return NULL;
1264 /* Win9x appears to need us seeked to zero */
1265 lseek(fileno, 0, SEEK_SET);
1268 m_obj = (mmap_object *)type->tp_alloc(type, 0);
1269 if (m_obj == NULL)
1270 return NULL;
1271 /* Set every field to an invalid marker, so we can safely
1272 destruct the object in the face of failure */
1273 m_obj->data = NULL;
1274 m_obj->file_handle = INVALID_HANDLE_VALUE;
1275 m_obj->map_handle = INVALID_HANDLE_VALUE;
1276 m_obj->tagname = NULL;
1277 m_obj->offset = offset;
1279 if (fh) {
1280 /* It is necessary to duplicate the handle, so the
1281 Python code can close it on us */
1282 if (!DuplicateHandle(
1283 GetCurrentProcess(), /* source process handle */
1284 fh, /* handle to be duplicated */
1285 GetCurrentProcess(), /* target proc handle */
1286 (LPHANDLE)&m_obj->file_handle, /* result */
1287 0, /* access - ignored due to options value */
1288 FALSE, /* inherited by child processes? */
1289 DUPLICATE_SAME_ACCESS)) { /* options */
1290 dwErr = GetLastError();
1291 Py_DECREF(m_obj);
1292 PyErr_SetFromWindowsErr(dwErr);
1293 return NULL;
1295 if (!map_size) {
1296 DWORD low,high;
1297 low = GetFileSize(fh, &high);
1298 /* low might just happen to have the value INVALID_FILE_SIZE;
1299 so we need to check the last error also. */
1300 if (low == INVALID_FILE_SIZE &&
1301 (dwErr = GetLastError()) != NO_ERROR) {
1302 Py_DECREF(m_obj);
1303 return PyErr_SetFromWindowsErr(dwErr);
1306 #if SIZEOF_SIZE_T > 4
1307 m_obj->size = (((size_t)high)<<32) + low;
1308 #else
1309 if (high)
1310 /* File is too large to map completely */
1311 m_obj->size = (size_t)-1;
1312 else
1313 m_obj->size = low;
1314 #endif
1315 } else {
1316 m_obj->size = map_size;
1319 else {
1320 m_obj->size = map_size;
1323 /* set the initial position */
1324 m_obj->pos = (size_t) 0;
1326 /* set the tag name */
1327 if (tagname != NULL && *tagname != '\0') {
1328 m_obj->tagname = PyMem_Malloc(strlen(tagname)+1);
1329 if (m_obj->tagname == NULL) {
1330 PyErr_NoMemory();
1331 Py_DECREF(m_obj);
1332 return NULL;
1334 strcpy(m_obj->tagname, tagname);
1336 else
1337 m_obj->tagname = NULL;
1339 m_obj->access = (access_mode)access;
1340 /* DWORD is a 4-byte int. If we're on a box where size_t consumes
1341 * more than 4 bytes, we need to break it apart. Else (size_t
1342 * consumes 4 bytes), C doesn't define what happens if we shift
1343 * right by 32, so we need different code.
1345 #if SIZEOF_SIZE_T > 4
1346 size_hi = (DWORD)((offset + m_obj->size) >> 32);
1347 size_lo = (DWORD)((offset + m_obj->size) & 0xFFFFFFFF);
1348 off_hi = (DWORD)(offset >> 32);
1349 off_lo = (DWORD)(offset & 0xFFFFFFFF);
1350 #else
1351 size_hi = 0;
1352 size_lo = (DWORD)(offset + m_obj->size);
1353 off_hi = 0;
1354 off_lo = (DWORD)offset;
1355 #endif
1356 /* For files, it would be sufficient to pass 0 as size.
1357 For anonymous maps, we have to pass the size explicitly. */
1358 m_obj->map_handle = CreateFileMapping(m_obj->file_handle,
1359 NULL,
1360 flProtect,
1361 size_hi,
1362 size_lo,
1363 m_obj->tagname);
1364 if (m_obj->map_handle != NULL) {
1365 m_obj->data = (char *) MapViewOfFile(m_obj->map_handle,
1366 dwDesiredAccess,
1367 off_hi,
1368 off_lo,
1370 if (m_obj->data != NULL)
1371 return (PyObject *)m_obj;
1372 else
1373 dwErr = GetLastError();
1374 } else
1375 dwErr = GetLastError();
1376 Py_DECREF(m_obj);
1377 PyErr_SetFromWindowsErr(dwErr);
1378 return NULL;
1380 #endif /* MS_WINDOWS */
1382 static void
1383 setint(PyObject *d, const char *name, long value)
1385 PyObject *o = PyInt_FromLong(value);
1386 if (o && PyDict_SetItemString(d, name, o) == 0) {
1387 Py_DECREF(o);
1391 PyMODINIT_FUNC
1392 initmmap(void)
1394 PyObject *dict, *module;
1396 if (PyType_Ready(&mmap_object_type) < 0)
1397 return;
1399 module = Py_InitModule("mmap", NULL);
1400 if (module == NULL)
1401 return;
1402 dict = PyModule_GetDict(module);
1403 if (!dict)
1404 return;
1405 mmap_module_error = PyErr_NewException("mmap.error",
1406 PyExc_EnvironmentError , NULL);
1407 if (mmap_module_error == NULL)
1408 return;
1409 PyDict_SetItemString(dict, "error", mmap_module_error);
1410 PyDict_SetItemString(dict, "mmap", (PyObject*) &mmap_object_type);
1411 #ifdef PROT_EXEC
1412 setint(dict, "PROT_EXEC", PROT_EXEC);
1413 #endif
1414 #ifdef PROT_READ
1415 setint(dict, "PROT_READ", PROT_READ);
1416 #endif
1417 #ifdef PROT_WRITE
1418 setint(dict, "PROT_WRITE", PROT_WRITE);
1419 #endif
1421 #ifdef MAP_SHARED
1422 setint(dict, "MAP_SHARED", MAP_SHARED);
1423 #endif
1424 #ifdef MAP_PRIVATE
1425 setint(dict, "MAP_PRIVATE", MAP_PRIVATE);
1426 #endif
1427 #ifdef MAP_DENYWRITE
1428 setint(dict, "MAP_DENYWRITE", MAP_DENYWRITE);
1429 #endif
1430 #ifdef MAP_EXECUTABLE
1431 setint(dict, "MAP_EXECUTABLE", MAP_EXECUTABLE);
1432 #endif
1433 #ifdef MAP_ANONYMOUS
1434 setint(dict, "MAP_ANON", MAP_ANONYMOUS);
1435 setint(dict, "MAP_ANONYMOUS", MAP_ANONYMOUS);
1436 #endif
1438 setint(dict, "PAGESIZE", (long)my_getpagesize());
1440 setint(dict, "ALLOCATIONGRANULARITY", (long)my_getallocationgranularity());
1442 setint(dict, "ACCESS_READ", ACCESS_READ);
1443 setint(dict, "ACCESS_WRITE", ACCESS_WRITE);
1444 setint(dict, "ACCESS_COPY", ACCESS_COPY);