Added section about adding contextual information to log output.
[python.git] / Modules / mmapmodule.c
blob2d745c130d559dd61aa56d6ea2822d2991b476c5
1 /*
2 / Author: Sam Rushing <rushing@nightmare.com>
3 / Hacked for Unix by AMK
4 / $Id$
6 / Modified to support mmap with offset - to map a 'window' of a file
7 / Author: Yotam Medini yotamm@mellanox.co.il
9 / mmapmodule.cpp -- map a view of a file into memory
11 / todo: need permission flags, perhaps a 'chsize' analog
12 / not all functions check range yet!!!
15 / This version of mmapmodule.c has been changed significantly
16 / from the original mmapfile.c on which it was based.
17 / The original version of mmapfile is maintained by Sam at
18 / ftp://squirl.nightmare.com/pub/python/python-ext.
21 #define PY_SSIZE_T_CLEAN
22 #include <Python.h>
24 #ifndef MS_WINDOWS
25 #define UNIX
26 #endif
28 #ifdef MS_WINDOWS
29 #include <windows.h>
30 static int
31 my_getpagesize(void)
33 SYSTEM_INFO si;
34 GetSystemInfo(&si);
35 return si.dwPageSize;
38 static int
39 my_getallocationgranularity (void)
42 SYSTEM_INFO si;
43 GetSystemInfo(&si);
44 return si.dwAllocationGranularity;
47 #endif
49 #ifdef UNIX
50 #include <sys/mman.h>
51 #include <sys/stat.h>
53 #if defined(HAVE_SYSCONF) && defined(_SC_PAGESIZE)
54 static int
55 my_getpagesize(void)
57 return sysconf(_SC_PAGESIZE);
60 #define my_getallocationgranularity my_getpagesize
61 #else
62 #define my_getpagesize getpagesize
63 #endif
65 #endif /* UNIX */
67 #include <string.h>
69 #ifdef HAVE_SYS_TYPES_H
70 #include <sys/types.h>
71 #endif /* HAVE_SYS_TYPES_H */
73 /* Prefer MAP_ANONYMOUS since MAP_ANON is deprecated according to man page. */
74 #if !defined(MAP_ANONYMOUS) && defined(MAP_ANON)
75 # define MAP_ANONYMOUS MAP_ANON
76 #endif
78 static PyObject *mmap_module_error;
80 typedef enum
82 ACCESS_DEFAULT,
83 ACCESS_READ,
84 ACCESS_WRITE,
85 ACCESS_COPY
86 } access_mode;
88 typedef struct {
89 PyObject_HEAD
90 char * data;
91 size_t size;
92 size_t pos; /* relative to offset */
93 size_t offset;
95 #ifdef MS_WINDOWS
96 HANDLE map_handle;
97 HANDLE file_handle;
98 char * tagname;
99 #endif
101 #ifdef UNIX
102 int fd;
103 #endif
105 access_mode access;
106 } mmap_object;
109 static void
110 mmap_object_dealloc(mmap_object *m_obj)
112 #ifdef MS_WINDOWS
113 if (m_obj->data != NULL)
114 UnmapViewOfFile (m_obj->data);
115 if (m_obj->map_handle != INVALID_HANDLE_VALUE)
116 CloseHandle (m_obj->map_handle);
117 if (m_obj->file_handle != INVALID_HANDLE_VALUE)
118 CloseHandle (m_obj->file_handle);
119 if (m_obj->tagname)
120 PyMem_Free(m_obj->tagname);
121 #endif /* MS_WINDOWS */
123 #ifdef UNIX
124 if (m_obj->fd >= 0)
125 (void) close(m_obj->fd);
126 if (m_obj->data!=NULL) {
127 msync(m_obj->data, m_obj->size, MS_SYNC);
128 munmap(m_obj->data, m_obj->size);
130 #endif /* UNIX */
132 PyObject_Del(m_obj);
135 static PyObject *
136 mmap_close_method(mmap_object *self, PyObject *unused)
138 #ifdef MS_WINDOWS
139 /* For each resource we maintain, we need to check
140 the value is valid, and if so, free the resource
141 and set the member value to an invalid value so
142 the dealloc does not attempt to resource clearing
143 again.
144 TODO - should we check for errors in the close operations???
146 if (self->data != NULL) {
147 UnmapViewOfFile(self->data);
148 self->data = NULL;
150 if (self->map_handle != INVALID_HANDLE_VALUE) {
151 CloseHandle(self->map_handle);
152 self->map_handle = INVALID_HANDLE_VALUE;
154 if (self->file_handle != INVALID_HANDLE_VALUE) {
155 CloseHandle(self->file_handle);
156 self->file_handle = INVALID_HANDLE_VALUE;
158 #endif /* MS_WINDOWS */
160 #ifdef UNIX
161 (void) close(self->fd);
162 self->fd = -1;
163 if (self->data != NULL) {
164 munmap(self->data, self->size);
165 self->data = NULL;
167 #endif
169 Py_INCREF(Py_None);
170 return Py_None;
173 #ifdef MS_WINDOWS
174 #define CHECK_VALID(err) \
175 do { \
176 if (self->map_handle == INVALID_HANDLE_VALUE) { \
177 PyErr_SetString(PyExc_ValueError, "mmap closed or invalid"); \
178 return err; \
180 } while (0)
181 #endif /* MS_WINDOWS */
183 #ifdef UNIX
184 #define CHECK_VALID(err) \
185 do { \
186 if (self->data == NULL) { \
187 PyErr_SetString(PyExc_ValueError, "mmap closed or invalid"); \
188 return err; \
190 } while (0)
191 #endif /* UNIX */
193 static PyObject *
194 mmap_read_byte_method(mmap_object *self,
195 PyObject *unused)
197 CHECK_VALID(NULL);
198 if (self->pos < self->size) {
199 char value = self->data[self->pos];
200 self->pos += 1;
201 return Py_BuildValue("c", value);
202 } else {
203 PyErr_SetString(PyExc_ValueError, "read byte out of range");
204 return NULL;
208 static PyObject *
209 mmap_read_line_method(mmap_object *self,
210 PyObject *unused)
212 char *start = self->data+self->pos;
213 char *eof = self->data+self->size;
214 char *eol;
215 PyObject *result;
217 CHECK_VALID(NULL);
219 eol = memchr(start, '\n', self->size - self->pos);
220 if (!eol)
221 eol = eof;
222 else
223 ++eol; /* we're interested in the position after the
224 newline. */
225 result = PyString_FromStringAndSize(start, (eol - start));
226 self->pos += (eol - start);
227 return result;
230 static PyObject *
231 mmap_read_method(mmap_object *self,
232 PyObject *args)
234 Py_ssize_t num_bytes;
235 PyObject *result;
237 CHECK_VALID(NULL);
238 if (!PyArg_ParseTuple(args, "n:read", &num_bytes))
239 return(NULL);
241 /* silently 'adjust' out-of-range requests */
242 if ((self->pos + num_bytes) > self->size) {
243 num_bytes -= (self->pos+num_bytes) - self->size;
245 result = Py_BuildValue("s#", self->data+self->pos, num_bytes);
246 self->pos += num_bytes;
247 return result;
250 static PyObject *
251 mmap_find_method(mmap_object *self,
252 PyObject *args)
254 Py_ssize_t start = self->pos;
255 char *needle;
256 Py_ssize_t len;
258 CHECK_VALID(NULL);
259 if (!PyArg_ParseTuple(args, "s#|n:find", &needle, &len, &start)) {
260 return NULL;
261 } else {
262 char *p;
263 char *e = self->data + self->size;
265 if (start < 0)
266 start += self->size;
267 if (start < 0)
268 start = 0;
269 else if ((size_t)start > self->size)
270 start = self->size;
272 for (p = self->data + start; p + len <= e; ++p) {
273 Py_ssize_t i;
274 for (i = 0; i < len && needle[i] == p[i]; ++i)
275 /* nothing */;
276 if (i == len) {
277 return PyInt_FromSsize_t(p - self->data);
280 return PyInt_FromLong(-1);
284 static int
285 is_writeable(mmap_object *self)
287 if (self->access != ACCESS_READ)
288 return 1;
289 PyErr_Format(PyExc_TypeError, "mmap can't modify a readonly memory map.");
290 return 0;
293 static int
294 is_resizeable(mmap_object *self)
296 if ((self->access == ACCESS_WRITE) || (self->access == ACCESS_DEFAULT))
297 return 1;
298 PyErr_Format(PyExc_TypeError,
299 "mmap can't resize a readonly or copy-on-write memory map.");
300 return 0;
304 static PyObject *
305 mmap_write_method(mmap_object *self,
306 PyObject *args)
308 Py_ssize_t length;
309 char *data;
311 CHECK_VALID(NULL);
312 if (!PyArg_ParseTuple(args, "s#:write", &data, &length))
313 return(NULL);
315 if (!is_writeable(self))
316 return NULL;
318 if ((self->pos + length) > self->size) {
319 PyErr_SetString(PyExc_ValueError, "data out of range");
320 return NULL;
322 memcpy(self->data+self->pos, data, length);
323 self->pos = self->pos+length;
324 Py_INCREF(Py_None);
325 return Py_None;
328 static PyObject *
329 mmap_write_byte_method(mmap_object *self,
330 PyObject *args)
332 char value;
334 CHECK_VALID(NULL);
335 if (!PyArg_ParseTuple(args, "c:write_byte", &value))
336 return(NULL);
338 if (!is_writeable(self))
339 return NULL;
340 *(self->data+self->pos) = value;
341 self->pos += 1;
342 Py_INCREF(Py_None);
343 return Py_None;
346 static PyObject *
347 mmap_size_method(mmap_object *self,
348 PyObject *unused)
350 CHECK_VALID(NULL);
352 #ifdef MS_WINDOWS
353 if (self->file_handle != INVALID_HANDLE_VALUE) {
354 DWORD low,high;
355 PY_LONG_LONG size;
356 low = GetFileSize(self->file_handle, &high);
357 if (low == INVALID_FILE_SIZE) {
358 /* It might be that the function appears to have failed,
359 when indeed its size equals INVALID_FILE_SIZE */
360 DWORD error = GetLastError();
361 if (error != NO_ERROR)
362 return PyErr_SetFromWindowsErr(error);
364 if (!high && low < LONG_MAX)
365 return PyInt_FromLong((long)low);
366 size = (((PY_LONG_LONG)high)<<32) + low;
367 return PyLong_FromLongLong(size);
368 } else {
369 return PyInt_FromSsize_t(self->size);
371 #endif /* MS_WINDOWS */
373 #ifdef UNIX
375 struct stat buf;
376 if (-1 == fstat(self->fd, &buf)) {
377 PyErr_SetFromErrno(mmap_module_error);
378 return NULL;
380 return PyInt_FromSsize_t(buf.st_size);
382 #endif /* UNIX */
385 /* This assumes that you want the entire file mapped,
386 / and when recreating the map will make the new file
387 / have the new size
389 / Is this really necessary? This could easily be done
390 / from python by just closing and re-opening with the
391 / new size?
394 static PyObject *
395 mmap_resize_method(mmap_object *self,
396 PyObject *args)
398 Py_ssize_t new_size;
399 CHECK_VALID(NULL);
400 if (!PyArg_ParseTuple(args, "n:resize", &new_size) ||
401 !is_resizeable(self)) {
402 return NULL;
403 #ifdef MS_WINDOWS
404 } else {
405 DWORD dwErrCode = 0;
406 DWORD off_hi, off_lo, newSizeLow, newSizeHigh;
407 /* First, unmap the file view */
408 UnmapViewOfFile(self->data);
409 /* Close the mapping object */
410 CloseHandle(self->map_handle);
411 /* Move to the desired EOF position */
412 #if SIZEOF_SIZE_T > 4
413 newSizeHigh = (DWORD)((self->offset + new_size) >> 32);
414 newSizeLow = (DWORD)((self->offset + new_size) & 0xFFFFFFFF);
415 off_hi = (DWORD)(self->offset >> 32);
416 off_lo = (DWORD)(self->offset & 0xFFFFFFFF);
417 #else
418 newSizeHigh = 0;
419 newSizeLow = (DWORD)new_size;
420 off_hi = 0;
421 off_lo = (DWORD)self->offset;
422 #endif
423 SetFilePointer(self->file_handle,
424 newSizeLow, &newSizeHigh, FILE_BEGIN);
425 /* Change the size of the file */
426 SetEndOfFile(self->file_handle);
427 /* Create another mapping object and remap the file view */
428 self->map_handle = CreateFileMapping(
429 self->file_handle,
430 NULL,
431 PAGE_READWRITE,
434 self->tagname);
435 if (self->map_handle != NULL) {
436 self->data = (char *) MapViewOfFile(self->map_handle,
437 FILE_MAP_WRITE,
438 off_hi,
439 off_lo,
440 new_size);
441 if (self->data != NULL) {
442 self->size = new_size;
443 Py_INCREF(Py_None);
444 return Py_None;
445 } else {
446 dwErrCode = GetLastError();
448 } else {
449 dwErrCode = GetLastError();
451 PyErr_SetFromWindowsErr(dwErrCode);
452 return NULL;
453 #endif /* MS_WINDOWS */
455 #ifdef UNIX
456 #ifndef HAVE_MREMAP
457 } else {
458 PyErr_SetString(PyExc_SystemError,
459 "mmap: resizing not available--no mremap()");
460 return NULL;
461 #else
462 } else {
463 void *newmap;
465 if (ftruncate(self->fd, new_size) == -1) {
466 PyErr_SetFromErrno(mmap_module_error);
467 return NULL;
470 #ifdef MREMAP_MAYMOVE
471 newmap = mremap(self->data, self->size, new_size, MREMAP_MAYMOVE);
472 #else
473 newmap = mremap(self->data, self->size, new_size, 0);
474 #endif
475 if (newmap == (void *)-1)
477 PyErr_SetFromErrno(mmap_module_error);
478 return NULL;
480 self->data = newmap;
481 self->size = new_size;
482 Py_INCREF(Py_None);
483 return Py_None;
484 #endif /* HAVE_MREMAP */
485 #endif /* UNIX */
489 static PyObject *
490 mmap_tell_method(mmap_object *self, PyObject *unused)
492 CHECK_VALID(NULL);
493 return PyInt_FromSize_t(self->pos);
496 static PyObject *
497 mmap_flush_method(mmap_object *self, PyObject *args)
499 Py_ssize_t offset = 0;
500 Py_ssize_t size = self->size;
501 CHECK_VALID(NULL);
502 if (!PyArg_ParseTuple(args, "|nn:flush", &offset, &size))
503 return NULL;
504 if ((size_t)(offset + size) > self->size) {
505 PyErr_SetString(PyExc_ValueError, "flush values out of range");
506 return NULL;
507 } else {
508 #ifdef MS_WINDOWS
509 return PyInt_FromLong((long)
510 FlushViewOfFile(self->data+offset, size));
511 #endif /* MS_WINDOWS */
512 #ifdef UNIX
513 /* XXX semantics of return value? */
514 /* XXX flags for msync? */
515 if (-1 == msync(self->data + offset, size,
516 MS_SYNC))
518 PyErr_SetFromErrno(mmap_module_error);
519 return NULL;
521 return PyInt_FromLong(0);
522 #endif /* UNIX */
526 static PyObject *
527 mmap_seek_method(mmap_object *self, PyObject *args)
529 Py_ssize_t dist;
530 int how=0;
531 CHECK_VALID(NULL);
532 if (!PyArg_ParseTuple(args, "n|i:seek", &dist, &how))
533 return NULL;
534 else {
535 size_t where;
536 switch (how) {
537 case 0: /* relative to start */
538 if (dist < 0)
539 goto onoutofrange;
540 where = dist;
541 break;
542 case 1: /* relative to current position */
543 if ((Py_ssize_t)self->pos + dist < 0)
544 goto onoutofrange;
545 where = self->pos + dist;
546 break;
547 case 2: /* relative to end */
548 if ((Py_ssize_t)self->size + dist < 0)
549 goto onoutofrange;
550 where = self->size + dist;
551 break;
552 default:
553 PyErr_SetString(PyExc_ValueError, "unknown seek type");
554 return NULL;
556 if (where > self->size)
557 goto onoutofrange;
558 self->pos = where;
559 Py_INCREF(Py_None);
560 return Py_None;
563 onoutofrange:
564 PyErr_SetString(PyExc_ValueError, "seek out of range");
565 return NULL;
568 static PyObject *
569 mmap_move_method(mmap_object *self, PyObject *args)
571 unsigned long dest, src, count;
572 CHECK_VALID(NULL);
573 if (!PyArg_ParseTuple(args, "kkk:move", &dest, &src, &count) ||
574 !is_writeable(self)) {
575 return NULL;
576 } else {
577 /* bounds check the values */
578 if (/* end of source after end of data?? */
579 ((src+count) > self->size)
580 /* dest will fit? */
581 || (dest+count > self->size)) {
582 PyErr_SetString(PyExc_ValueError,
583 "source or destination out of range");
584 return NULL;
585 } else {
586 memmove(self->data+dest, self->data+src, count);
587 Py_INCREF(Py_None);
588 return Py_None;
593 static struct PyMethodDef mmap_object_methods[] = {
594 {"close", (PyCFunction) mmap_close_method, METH_NOARGS},
595 {"find", (PyCFunction) mmap_find_method, METH_VARARGS},
596 {"flush", (PyCFunction) mmap_flush_method, METH_VARARGS},
597 {"move", (PyCFunction) mmap_move_method, METH_VARARGS},
598 {"read", (PyCFunction) mmap_read_method, METH_VARARGS},
599 {"read_byte", (PyCFunction) mmap_read_byte_method, METH_NOARGS},
600 {"readline", (PyCFunction) mmap_read_line_method, METH_NOARGS},
601 {"resize", (PyCFunction) mmap_resize_method, METH_VARARGS},
602 {"seek", (PyCFunction) mmap_seek_method, METH_VARARGS},
603 {"size", (PyCFunction) mmap_size_method, METH_NOARGS},
604 {"tell", (PyCFunction) mmap_tell_method, METH_NOARGS},
605 {"write", (PyCFunction) mmap_write_method, METH_VARARGS},
606 {"write_byte", (PyCFunction) mmap_write_byte_method, METH_VARARGS},
607 {NULL, NULL} /* sentinel */
610 /* Functions for treating an mmap'ed file as a buffer */
612 static Py_ssize_t
613 mmap_buffer_getreadbuf(mmap_object *self, Py_ssize_t index, const void **ptr)
615 CHECK_VALID(-1);
616 if (index != 0) {
617 PyErr_SetString(PyExc_SystemError,
618 "Accessing non-existent mmap segment");
619 return -1;
621 *ptr = self->data;
622 return self->size;
625 static Py_ssize_t
626 mmap_buffer_getwritebuf(mmap_object *self, Py_ssize_t index, const void **ptr)
628 CHECK_VALID(-1);
629 if (index != 0) {
630 PyErr_SetString(PyExc_SystemError,
631 "Accessing non-existent mmap segment");
632 return -1;
634 if (!is_writeable(self))
635 return -1;
636 *ptr = self->data;
637 return self->size;
640 static Py_ssize_t
641 mmap_buffer_getsegcount(mmap_object *self, Py_ssize_t *lenp)
643 CHECK_VALID(-1);
644 if (lenp)
645 *lenp = self->size;
646 return 1;
649 static Py_ssize_t
650 mmap_buffer_getcharbuffer(mmap_object *self, Py_ssize_t index, const void **ptr)
652 if (index != 0) {
653 PyErr_SetString(PyExc_SystemError,
654 "accessing non-existent buffer segment");
655 return -1;
657 *ptr = (const char *)self->data;
658 return self->size;
661 static PyObject *
662 mmap_object_getattr(mmap_object *self, char *name)
664 return Py_FindMethod(mmap_object_methods, (PyObject *)self, name);
667 static Py_ssize_t
668 mmap_length(mmap_object *self)
670 CHECK_VALID(-1);
671 return self->size;
674 static PyObject *
675 mmap_item(mmap_object *self, Py_ssize_t i)
677 CHECK_VALID(NULL);
678 if (i < 0 || (size_t)i >= self->size) {
679 PyErr_SetString(PyExc_IndexError, "mmap index out of range");
680 return NULL;
682 return PyString_FromStringAndSize(self->data + i, 1);
685 static PyObject *
686 mmap_slice(mmap_object *self, Py_ssize_t ilow, Py_ssize_t ihigh)
688 CHECK_VALID(NULL);
689 if (ilow < 0)
690 ilow = 0;
691 else if ((size_t)ilow > self->size)
692 ilow = self->size;
693 if (ihigh < 0)
694 ihigh = 0;
695 if (ihigh < ilow)
696 ihigh = ilow;
697 else if ((size_t)ihigh > self->size)
698 ihigh = self->size;
700 return PyString_FromStringAndSize(self->data + ilow, ihigh-ilow);
703 static PyObject *
704 mmap_subscript(mmap_object *self, PyObject *item)
706 CHECK_VALID(NULL);
707 if (PyIndex_Check(item)) {
708 Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError);
709 if (i == -1 && PyErr_Occurred())
710 return NULL;
711 if (i < 0)
712 i += self->size;
713 if (i < 0 || (size_t)i > self->size) {
714 PyErr_SetString(PyExc_IndexError,
715 "mmap index out of range");
716 return NULL;
718 return PyString_FromStringAndSize(self->data + i, 1);
720 else if (PySlice_Check(item)) {
721 Py_ssize_t start, stop, step, slicelen;
723 if (PySlice_GetIndicesEx((PySliceObject *)item, self->size,
724 &start, &stop, &step, &slicelen) < 0) {
725 return NULL;
728 if (slicelen <= 0)
729 return PyString_FromStringAndSize("", 0);
730 else if (step == 1)
731 return PyString_FromStringAndSize(self->data + start,
732 slicelen);
733 else {
734 char *result_buf = (char *)PyMem_Malloc(slicelen);
735 Py_ssize_t cur, i;
736 PyObject *result;
738 if (result_buf == NULL)
739 return PyErr_NoMemory();
740 for (cur = start, i = 0; i < slicelen;
741 cur += step, i++) {
742 result_buf[i] = self->data[cur];
744 result = PyString_FromStringAndSize(result_buf,
745 slicelen);
746 PyMem_Free(result_buf);
747 return result;
750 else {
751 PyErr_SetString(PyExc_TypeError,
752 "mmap indices must be integers");
753 return NULL;
757 static PyObject *
758 mmap_concat(mmap_object *self, PyObject *bb)
760 CHECK_VALID(NULL);
761 PyErr_SetString(PyExc_SystemError,
762 "mmaps don't support concatenation");
763 return NULL;
766 static PyObject *
767 mmap_repeat(mmap_object *self, Py_ssize_t n)
769 CHECK_VALID(NULL);
770 PyErr_SetString(PyExc_SystemError,
771 "mmaps don't support repeat operation");
772 return NULL;
775 static int
776 mmap_ass_slice(mmap_object *self, Py_ssize_t ilow, Py_ssize_t ihigh, PyObject *v)
778 const char *buf;
780 CHECK_VALID(-1);
781 if (ilow < 0)
782 ilow = 0;
783 else if ((size_t)ilow > self->size)
784 ilow = self->size;
785 if (ihigh < 0)
786 ihigh = 0;
787 if (ihigh < ilow)
788 ihigh = ilow;
789 else if ((size_t)ihigh > self->size)
790 ihigh = self->size;
792 if (v == NULL) {
793 PyErr_SetString(PyExc_TypeError,
794 "mmap object doesn't support slice deletion");
795 return -1;
797 if (! (PyString_Check(v)) ) {
798 PyErr_SetString(PyExc_IndexError,
799 "mmap slice assignment must be a string");
800 return -1;
802 if (PyString_Size(v) != (ihigh - ilow)) {
803 PyErr_SetString(PyExc_IndexError,
804 "mmap slice assignment is wrong size");
805 return -1;
807 if (!is_writeable(self))
808 return -1;
809 buf = PyString_AsString(v);
810 memcpy(self->data + ilow, buf, ihigh-ilow);
811 return 0;
814 static int
815 mmap_ass_item(mmap_object *self, Py_ssize_t i, PyObject *v)
817 const char *buf;
819 CHECK_VALID(-1);
820 if (i < 0 || (size_t)i >= self->size) {
821 PyErr_SetString(PyExc_IndexError, "mmap index out of range");
822 return -1;
824 if (v == NULL) {
825 PyErr_SetString(PyExc_TypeError,
826 "mmap object doesn't support item deletion");
827 return -1;
829 if (! (PyString_Check(v) && PyString_Size(v)==1) ) {
830 PyErr_SetString(PyExc_IndexError,
831 "mmap assignment must be single-character string");
832 return -1;
834 if (!is_writeable(self))
835 return -1;
836 buf = PyString_AsString(v);
837 self->data[i] = buf[0];
838 return 0;
841 static int
842 mmap_ass_subscript(mmap_object *self, PyObject *item, PyObject *value)
844 CHECK_VALID(-1);
846 if (PyIndex_Check(item)) {
847 Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError);
848 const char *buf;
850 if (i == -1 && PyErr_Occurred())
851 return -1;
852 if (i < 0)
853 i += self->size;
854 if (i < 0 || (size_t)i > self->size) {
855 PyErr_SetString(PyExc_IndexError,
856 "mmap index out of range");
857 return -1;
859 if (value == NULL) {
860 PyErr_SetString(PyExc_TypeError,
861 "mmap object doesn't support item deletion");
862 return -1;
864 if (!PyString_Check(value) || PyString_Size(value) != 1) {
865 PyErr_SetString(PyExc_IndexError,
866 "mmap assignment must be single-character string");
867 return -1;
869 if (!is_writeable(self))
870 return -1;
871 buf = PyString_AsString(value);
872 self->data[i] = buf[0];
873 return 0;
875 else if (PySlice_Check(item)) {
876 Py_ssize_t start, stop, step, slicelen;
878 if (PySlice_GetIndicesEx((PySliceObject *)item,
879 self->size, &start, &stop,
880 &step, &slicelen) < 0) {
881 return -1;
883 if (value == NULL) {
884 PyErr_SetString(PyExc_TypeError,
885 "mmap object doesn't support slice deletion");
886 return -1;
888 if (!PyString_Check(value)) {
889 PyErr_SetString(PyExc_IndexError,
890 "mmap slice assignment must be a string");
891 return -1;
893 if (PyString_Size(value) != slicelen) {
894 PyErr_SetString(PyExc_IndexError,
895 "mmap slice assignment is wrong size");
896 return -1;
898 if (!is_writeable(self))
899 return -1;
901 if (slicelen == 0)
902 return 0;
903 else if (step == 1) {
904 const char *buf = PyString_AsString(value);
906 if (buf == NULL)
907 return -1;
908 memcpy(self->data + start, buf, slicelen);
909 return 0;
911 else {
912 Py_ssize_t cur, i;
913 const char *buf = PyString_AsString(value);
915 if (buf == NULL)
916 return -1;
917 for (cur = start, i = 0; i < slicelen;
918 cur += step, i++) {
919 self->data[cur] = buf[i];
921 return 0;
924 else {
925 PyErr_SetString(PyExc_TypeError,
926 "mmap indices must be integer");
927 return -1;
931 static PySequenceMethods mmap_as_sequence = {
932 (lenfunc)mmap_length, /*sq_length*/
933 (binaryfunc)mmap_concat, /*sq_concat*/
934 (ssizeargfunc)mmap_repeat, /*sq_repeat*/
935 (ssizeargfunc)mmap_item, /*sq_item*/
936 (ssizessizeargfunc)mmap_slice, /*sq_slice*/
937 (ssizeobjargproc)mmap_ass_item, /*sq_ass_item*/
938 (ssizessizeobjargproc)mmap_ass_slice, /*sq_ass_slice*/
941 static PyMappingMethods mmap_as_mapping = {
942 (lenfunc)mmap_length,
943 (binaryfunc)mmap_subscript,
944 (objobjargproc)mmap_ass_subscript,
947 static PyBufferProcs mmap_as_buffer = {
948 (readbufferproc)mmap_buffer_getreadbuf,
949 (writebufferproc)mmap_buffer_getwritebuf,
950 (segcountproc)mmap_buffer_getsegcount,
951 (charbufferproc)mmap_buffer_getcharbuffer,
954 static PyTypeObject mmap_object_type = {
955 PyVarObject_HEAD_INIT(0, 0) /* patched in module init */
956 "mmap.mmap", /* tp_name */
957 sizeof(mmap_object), /* tp_size */
958 0, /* tp_itemsize */
959 /* methods */
960 (destructor) mmap_object_dealloc, /* tp_dealloc */
961 0, /* tp_print */
962 (getattrfunc) mmap_object_getattr, /* tp_getattr */
963 0, /* tp_setattr */
964 0, /* tp_compare */
965 0, /* tp_repr */
966 0, /* tp_as_number */
967 &mmap_as_sequence, /*tp_as_sequence*/
968 &mmap_as_mapping, /*tp_as_mapping*/
969 0, /*tp_hash*/
970 0, /*tp_call*/
971 0, /*tp_str*/
972 0, /*tp_getattro*/
973 0, /*tp_setattro*/
974 &mmap_as_buffer, /*tp_as_buffer*/
975 Py_TPFLAGS_HAVE_GETCHARBUFFER, /*tp_flags*/
976 0, /*tp_doc*/
980 /* extract the map size from the given PyObject
982 Returns -1 on error, with an appropriate Python exception raised. On
983 success, the map size is returned. */
984 static Py_ssize_t
985 _GetMapSize(PyObject *o, const char* param)
987 if (o == NULL)
988 return 0;
989 if (PyIndex_Check(o)) {
990 Py_ssize_t i = PyNumber_AsSsize_t(o, PyExc_OverflowError);
991 if (i==-1 && PyErr_Occurred())
992 return -1;
993 if (i < 0) {
994 PyErr_Format(PyExc_OverflowError,
995 "memory mapped %s must be positive",
996 param);
997 return -1;
999 return i;
1002 PyErr_SetString(PyExc_TypeError, "map size must be an integral value");
1003 return -1;
1006 #ifdef UNIX
1007 static PyObject *
1008 new_mmap_object(PyObject *self, PyObject *args, PyObject *kwdict)
1010 #ifdef HAVE_FSTAT
1011 struct stat st;
1012 #endif
1013 mmap_object *m_obj;
1014 PyObject *map_size_obj = NULL, *offset_obj = NULL;
1015 Py_ssize_t map_size, offset;
1016 int fd, flags = MAP_SHARED, prot = PROT_WRITE | PROT_READ;
1017 int devzero = -1;
1018 int access = (int)ACCESS_DEFAULT;
1019 static char *keywords[] = {"fileno", "length",
1020 "flags", "prot",
1021 "access", "offset", NULL};
1023 if (!PyArg_ParseTupleAndKeywords(args, kwdict, "iO|iiiO", keywords,
1024 &fd, &map_size_obj, &flags, &prot,
1025 &access, &offset_obj))
1026 return NULL;
1027 map_size = _GetMapSize(map_size_obj, "size");
1028 if (map_size < 0)
1029 return NULL;
1030 offset = _GetMapSize(offset_obj, "offset");
1031 if (offset < 0)
1032 return NULL;
1034 if ((access != (int)ACCESS_DEFAULT) &&
1035 ((flags != MAP_SHARED) || (prot != (PROT_WRITE | PROT_READ))))
1036 return PyErr_Format(PyExc_ValueError,
1037 "mmap can't specify both access and flags, prot.");
1038 switch ((access_mode)access) {
1039 case ACCESS_READ:
1040 flags = MAP_SHARED;
1041 prot = PROT_READ;
1042 break;
1043 case ACCESS_WRITE:
1044 flags = MAP_SHARED;
1045 prot = PROT_READ | PROT_WRITE;
1046 break;
1047 case ACCESS_COPY:
1048 flags = MAP_PRIVATE;
1049 prot = PROT_READ | PROT_WRITE;
1050 break;
1051 case ACCESS_DEFAULT:
1052 /* use the specified or default values of flags and prot */
1053 break;
1054 default:
1055 return PyErr_Format(PyExc_ValueError,
1056 "mmap invalid access parameter.");
1059 #ifdef HAVE_FSTAT
1060 # ifdef __VMS
1061 /* on OpenVMS we must ensure that all bytes are written to the file */
1062 fsync(fd);
1063 # endif
1064 if (fstat(fd, &st) == 0 && S_ISREG(st.st_mode)) {
1065 if (map_size == 0) {
1066 map_size = st.st_size;
1067 } else if ((size_t)offset + (size_t)map_size > st.st_size) {
1068 PyErr_SetString(PyExc_ValueError,
1069 "mmap length is greater than file size");
1070 return NULL;
1073 #endif
1074 m_obj = PyObject_New(mmap_object, &mmap_object_type);
1075 if (m_obj == NULL) {return NULL;}
1076 m_obj->data = NULL;
1077 m_obj->size = (size_t) map_size;
1078 m_obj->pos = (size_t) 0;
1079 m_obj->offset = offset;
1080 if (fd == -1) {
1081 m_obj->fd = -1;
1082 /* Assume the caller wants to map anonymous memory.
1083 This is the same behaviour as Windows. mmap.mmap(-1, size)
1084 on both Windows and Unix map anonymous memory.
1086 #ifdef MAP_ANONYMOUS
1087 /* BSD way to map anonymous memory */
1088 flags |= MAP_ANONYMOUS;
1089 #else
1090 /* SVR4 method to map anonymous memory is to open /dev/zero */
1091 fd = devzero = open("/dev/zero", O_RDWR);
1092 if (devzero == -1) {
1093 Py_DECREF(m_obj);
1094 PyErr_SetFromErrno(mmap_module_error);
1095 return NULL;
1097 #endif
1098 } else {
1099 m_obj->fd = dup(fd);
1100 if (m_obj->fd == -1) {
1101 Py_DECREF(m_obj);
1102 PyErr_SetFromErrno(mmap_module_error);
1103 return NULL;
1107 m_obj->data = mmap(NULL, map_size,
1108 prot, flags,
1109 fd, offset);
1111 if (devzero != -1) {
1112 close(devzero);
1115 if (m_obj->data == (char *)-1) {
1116 m_obj->data = NULL;
1117 Py_DECREF(m_obj);
1118 PyErr_SetFromErrno(mmap_module_error);
1119 return NULL;
1121 m_obj->access = (access_mode)access;
1122 return (PyObject *)m_obj;
1124 #endif /* UNIX */
1126 #ifdef MS_WINDOWS
1127 static PyObject *
1128 new_mmap_object(PyObject *self, PyObject *args, PyObject *kwdict)
1130 mmap_object *m_obj;
1131 PyObject *map_size_obj = NULL, *offset_obj = NULL;
1132 Py_ssize_t map_size, offset;
1133 DWORD off_hi; /* upper 32 bits of offset */
1134 DWORD off_lo; /* lower 32 bits of offset */
1135 DWORD size_hi; /* upper 32 bits of size */
1136 DWORD size_lo; /* lower 32 bits of size */
1137 char *tagname = "";
1138 DWORD dwErr = 0;
1139 int fileno;
1140 HANDLE fh = 0;
1141 int access = (access_mode)ACCESS_DEFAULT;
1142 DWORD flProtect, dwDesiredAccess;
1143 static char *keywords[] = { "fileno", "length",
1144 "tagname",
1145 "access", "offset", NULL };
1147 if (!PyArg_ParseTupleAndKeywords(args, kwdict, "iO|ziO", keywords,
1148 &fileno, &map_size_obj,
1149 &tagname, &access, &offset_obj)) {
1150 return NULL;
1153 switch((access_mode)access) {
1154 case ACCESS_READ:
1155 flProtect = PAGE_READONLY;
1156 dwDesiredAccess = FILE_MAP_READ;
1157 break;
1158 case ACCESS_DEFAULT: case ACCESS_WRITE:
1159 flProtect = PAGE_READWRITE;
1160 dwDesiredAccess = FILE_MAP_WRITE;
1161 break;
1162 case ACCESS_COPY:
1163 flProtect = PAGE_WRITECOPY;
1164 dwDesiredAccess = FILE_MAP_COPY;
1165 break;
1166 default:
1167 return PyErr_Format(PyExc_ValueError,
1168 "mmap invalid access parameter.");
1171 map_size = _GetMapSize(map_size_obj, "size");
1172 if (map_size < 0)
1173 return NULL;
1174 offset = _GetMapSize(offset_obj, "offset");
1175 if (offset < 0)
1176 return NULL;
1178 /* assume -1 and 0 both mean invalid filedescriptor
1179 to 'anonymously' map memory.
1180 XXX: fileno == 0 is a valid fd, but was accepted prior to 2.5.
1181 XXX: Should this code be added?
1182 if (fileno == 0)
1183 PyErr_Warn(PyExc_DeprecationWarning,
1184 "don't use 0 for anonymous memory");
1186 if (fileno != -1 && fileno != 0) {
1187 fh = (HANDLE)_get_osfhandle(fileno);
1188 if (fh==(HANDLE)-1) {
1189 PyErr_SetFromErrno(mmap_module_error);
1190 return NULL;
1192 /* Win9x appears to need us seeked to zero */
1193 lseek(fileno, 0, SEEK_SET);
1196 m_obj = PyObject_New(mmap_object, &mmap_object_type);
1197 if (m_obj == NULL)
1198 return NULL;
1199 /* Set every field to an invalid marker, so we can safely
1200 destruct the object in the face of failure */
1201 m_obj->data = NULL;
1202 m_obj->file_handle = INVALID_HANDLE_VALUE;
1203 m_obj->map_handle = INVALID_HANDLE_VALUE;
1204 m_obj->tagname = NULL;
1205 m_obj->offset = offset;
1207 if (fh) {
1208 /* It is necessary to duplicate the handle, so the
1209 Python code can close it on us */
1210 if (!DuplicateHandle(
1211 GetCurrentProcess(), /* source process handle */
1212 fh, /* handle to be duplicated */
1213 GetCurrentProcess(), /* target proc handle */
1214 (LPHANDLE)&m_obj->file_handle, /* result */
1215 0, /* access - ignored due to options value */
1216 FALSE, /* inherited by child processes? */
1217 DUPLICATE_SAME_ACCESS)) { /* options */
1218 dwErr = GetLastError();
1219 Py_DECREF(m_obj);
1220 PyErr_SetFromWindowsErr(dwErr);
1221 return NULL;
1223 if (!map_size) {
1224 DWORD low,high;
1225 low = GetFileSize(fh, &high);
1226 /* low might just happen to have the value INVALID_FILE_SIZE;
1227 so we need to check the last error also. */
1228 if (low == INVALID_FILE_SIZE &&
1229 (dwErr = GetLastError()) != NO_ERROR) {
1230 Py_DECREF(m_obj);
1231 return PyErr_SetFromWindowsErr(dwErr);
1234 #if SIZEOF_SIZE_T > 4
1235 m_obj->size = (((size_t)high)<<32) + low;
1236 #else
1237 if (high)
1238 /* File is too large to map completely */
1239 m_obj->size = (size_t)-1;
1240 else
1241 m_obj->size = low;
1242 #endif
1243 } else {
1244 m_obj->size = map_size;
1247 else {
1248 m_obj->size = map_size;
1251 /* set the initial position */
1252 m_obj->pos = (size_t) 0;
1254 /* set the tag name */
1255 if (tagname != NULL && *tagname != '\0') {
1256 m_obj->tagname = PyMem_Malloc(strlen(tagname)+1);
1257 if (m_obj->tagname == NULL) {
1258 PyErr_NoMemory();
1259 Py_DECREF(m_obj);
1260 return NULL;
1262 strcpy(m_obj->tagname, tagname);
1264 else
1265 m_obj->tagname = NULL;
1267 m_obj->access = (access_mode)access;
1268 /* DWORD is a 4-byte int. If we're on a box where size_t consumes
1269 * more than 4 bytes, we need to break it apart. Else (size_t
1270 * consumes 4 bytes), C doesn't define what happens if we shift
1271 * right by 32, so we need different code.
1273 #if SIZEOF_SIZE_T > 4
1274 size_hi = (DWORD)((offset + m_obj->size) >> 32);
1275 size_lo = (DWORD)((offset + m_obj->size) & 0xFFFFFFFF);
1276 off_hi = (DWORD)(offset >> 32);
1277 off_lo = (DWORD)(offset & 0xFFFFFFFF);
1278 #else
1279 size_hi = 0;
1280 size_lo = (DWORD)(offset + m_obj->size);
1281 off_hi = 0;
1282 off_lo = (DWORD)offset;
1283 #endif
1284 /* For files, it would be sufficient to pass 0 as size.
1285 For anonymous maps, we have to pass the size explicitly. */
1286 m_obj->map_handle = CreateFileMapping(m_obj->file_handle,
1287 NULL,
1288 flProtect,
1289 size_hi,
1290 size_lo,
1291 m_obj->tagname);
1292 if (m_obj->map_handle != NULL) {
1293 m_obj->data = (char *) MapViewOfFile(m_obj->map_handle,
1294 dwDesiredAccess,
1295 off_hi,
1296 off_lo,
1298 if (m_obj->data != NULL)
1299 return (PyObject *)m_obj;
1300 else
1301 dwErr = GetLastError();
1302 } else
1303 dwErr = GetLastError();
1304 Py_DECREF(m_obj);
1305 PyErr_SetFromWindowsErr(dwErr);
1306 return NULL;
1308 #endif /* MS_WINDOWS */
1310 /* List of functions exported by this module */
1311 static struct PyMethodDef mmap_functions[] = {
1312 {"mmap", (PyCFunction) new_mmap_object,
1313 METH_VARARGS|METH_KEYWORDS},
1314 {NULL, NULL} /* Sentinel */
1317 static void
1318 setint(PyObject *d, const char *name, long value)
1320 PyObject *o = PyInt_FromLong(value);
1321 if (o && PyDict_SetItemString(d, name, o) == 0) {
1322 Py_DECREF(o);
1326 PyMODINIT_FUNC
1327 initmmap(void)
1329 PyObject *dict, *module;
1331 /* Patch the object type */
1332 Py_TYPE(&mmap_object_type) = &PyType_Type;
1334 module = Py_InitModule("mmap", mmap_functions);
1335 if (module == NULL)
1336 return;
1337 dict = PyModule_GetDict(module);
1338 if (!dict)
1339 return;
1340 mmap_module_error = PyExc_EnvironmentError;
1341 PyDict_SetItemString(dict, "error", mmap_module_error);
1342 #ifdef PROT_EXEC
1343 setint(dict, "PROT_EXEC", PROT_EXEC);
1344 #endif
1345 #ifdef PROT_READ
1346 setint(dict, "PROT_READ", PROT_READ);
1347 #endif
1348 #ifdef PROT_WRITE
1349 setint(dict, "PROT_WRITE", PROT_WRITE);
1350 #endif
1352 #ifdef MAP_SHARED
1353 setint(dict, "MAP_SHARED", MAP_SHARED);
1354 #endif
1355 #ifdef MAP_PRIVATE
1356 setint(dict, "MAP_PRIVATE", MAP_PRIVATE);
1357 #endif
1358 #ifdef MAP_DENYWRITE
1359 setint(dict, "MAP_DENYWRITE", MAP_DENYWRITE);
1360 #endif
1361 #ifdef MAP_EXECUTABLE
1362 setint(dict, "MAP_EXECUTABLE", MAP_EXECUTABLE);
1363 #endif
1364 #ifdef MAP_ANONYMOUS
1365 setint(dict, "MAP_ANON", MAP_ANONYMOUS);
1366 setint(dict, "MAP_ANONYMOUS", MAP_ANONYMOUS);
1367 #endif
1369 setint(dict, "PAGESIZE", (long)my_getpagesize());
1371 setint(dict, "ALLOCATIONGRANULARITY", (long)my_getallocationgranularity());
1373 setint(dict, "ACCESS_READ", ACCESS_READ);
1374 setint(dict, "ACCESS_WRITE", ACCESS_WRITE);
1375 setint(dict, "ACCESS_COPY", ACCESS_COPY);