Added section on passing contextual information to logging and documentation for...
[python.git] / Modules / mmapmodule.c
blob9a39b2424e8e1c694962af6fa5b509fd2ea686b3
1 /*
2 / Author: Sam Rushing <rushing@nightmare.com>
3 / Hacked for Unix by AMK
4 / $Id$
6 / Modified to support mmap with offset - to map a 'window' of a file
7 / Author: Yotam Medini yotamm@mellanox.co.il
9 / mmapmodule.cpp -- map a view of a file into memory
11 / todo: need permission flags, perhaps a 'chsize' analog
12 / not all functions check range yet!!!
15 / This version of mmapmodule.c has been changed significantly
16 / from the original mmapfile.c on which it was based.
17 / The original version of mmapfile is maintained by Sam at
18 / ftp://squirl.nightmare.com/pub/python/python-ext.
21 #define PY_SSIZE_T_CLEAN
22 #include <Python.h>
24 #ifndef MS_WINDOWS
25 #define UNIX
26 #endif
28 #ifdef MS_WINDOWS
29 #include <windows.h>
30 static int
31 my_getpagesize(void)
33 SYSTEM_INFO si;
34 GetSystemInfo(&si);
35 return si.dwPageSize;
38 static int
39 my_getallocationgranularity (void)
42 SYSTEM_INFO si;
43 GetSystemInfo(&si);
44 return si.dwAllocationGranularity;
47 #endif
49 #ifdef UNIX
50 #include <sys/mman.h>
51 #include <sys/stat.h>
53 #if defined(HAVE_SYSCONF) && defined(_SC_PAGESIZE)
54 static int
55 my_getpagesize(void)
57 return sysconf(_SC_PAGESIZE);
60 #define my_getallocationgranularity my_getpagesize
61 #else
62 #define my_getpagesize getpagesize
63 #endif
65 #endif /* UNIX */
67 #include <string.h>
69 #ifdef HAVE_SYS_TYPES_H
70 #include <sys/types.h>
71 #endif /* HAVE_SYS_TYPES_H */
73 /* Prefer MAP_ANONYMOUS since MAP_ANON is deprecated according to man page. */
74 #if !defined(MAP_ANONYMOUS) && defined(MAP_ANON)
75 # define MAP_ANONYMOUS MAP_ANON
76 #endif
78 static PyObject *mmap_module_error;
80 typedef enum
82 ACCESS_DEFAULT,
83 ACCESS_READ,
84 ACCESS_WRITE,
85 ACCESS_COPY
86 } access_mode;
88 typedef struct {
89 PyObject_HEAD
90 char * data;
91 size_t size;
92 size_t pos; /* relative to offset */
93 size_t offset;
95 #ifdef MS_WINDOWS
96 HANDLE map_handle;
97 HANDLE file_handle;
98 char * tagname;
99 #endif
101 #ifdef UNIX
102 int fd;
103 #endif
105 access_mode access;
106 } mmap_object;
109 static void
110 mmap_object_dealloc(mmap_object *m_obj)
112 #ifdef MS_WINDOWS
113 if (m_obj->data != NULL)
114 UnmapViewOfFile (m_obj->data);
115 if (m_obj->map_handle != INVALID_HANDLE_VALUE)
116 CloseHandle (m_obj->map_handle);
117 if (m_obj->file_handle != INVALID_HANDLE_VALUE)
118 CloseHandle (m_obj->file_handle);
119 if (m_obj->tagname)
120 PyMem_Free(m_obj->tagname);
121 #endif /* MS_WINDOWS */
123 #ifdef UNIX
124 if (m_obj->fd >= 0)
125 (void) close(m_obj->fd);
126 if (m_obj->data!=NULL) {
127 msync(m_obj->data, m_obj->size, MS_SYNC);
128 munmap(m_obj->data, m_obj->size);
130 #endif /* UNIX */
132 PyObject_Del(m_obj);
135 static PyObject *
136 mmap_close_method(mmap_object *self, PyObject *unused)
138 #ifdef MS_WINDOWS
139 /* For each resource we maintain, we need to check
140 the value is valid, and if so, free the resource
141 and set the member value to an invalid value so
142 the dealloc does not attempt to resource clearing
143 again.
144 TODO - should we check for errors in the close operations???
146 if (self->data != NULL) {
147 UnmapViewOfFile(self->data);
148 self->data = NULL;
150 if (self->map_handle != INVALID_HANDLE_VALUE) {
151 CloseHandle(self->map_handle);
152 self->map_handle = INVALID_HANDLE_VALUE;
154 if (self->file_handle != INVALID_HANDLE_VALUE) {
155 CloseHandle(self->file_handle);
156 self->file_handle = INVALID_HANDLE_VALUE;
158 #endif /* MS_WINDOWS */
160 #ifdef UNIX
161 (void) close(self->fd);
162 self->fd = -1;
163 if (self->data != NULL) {
164 munmap(self->data, self->size);
165 self->data = NULL;
167 #endif
169 Py_INCREF(Py_None);
170 return Py_None;
173 #ifdef MS_WINDOWS
174 #define CHECK_VALID(err) \
175 do { \
176 if (self->map_handle == INVALID_HANDLE_VALUE) { \
177 PyErr_SetString(PyExc_ValueError, "mmap closed or invalid"); \
178 return err; \
180 } while (0)
181 #endif /* MS_WINDOWS */
183 #ifdef UNIX
184 #define CHECK_VALID(err) \
185 do { \
186 if (self->data == NULL) { \
187 PyErr_SetString(PyExc_ValueError, "mmap closed or invalid"); \
188 return err; \
190 } while (0)
191 #endif /* UNIX */
193 static PyObject *
194 mmap_read_byte_method(mmap_object *self,
195 PyObject *unused)
197 CHECK_VALID(NULL);
198 if (self->pos < self->size) {
199 char value = self->data[self->pos];
200 self->pos += 1;
201 return Py_BuildValue("c", value);
202 } else {
203 PyErr_SetString(PyExc_ValueError, "read byte out of range");
204 return NULL;
208 static PyObject *
209 mmap_read_line_method(mmap_object *self,
210 PyObject *unused)
212 char *start = self->data+self->pos;
213 char *eof = self->data+self->size;
214 char *eol;
215 PyObject *result;
217 CHECK_VALID(NULL);
219 eol = memchr(start, '\n', self->size - self->pos);
220 if (!eol)
221 eol = eof;
222 else
223 ++eol; /* we're interested in the position after the
224 newline. */
225 result = PyString_FromStringAndSize(start, (eol - start));
226 self->pos += (eol - start);
227 return result;
230 static PyObject *
231 mmap_read_method(mmap_object *self,
232 PyObject *args)
234 Py_ssize_t num_bytes;
235 PyObject *result;
237 CHECK_VALID(NULL);
238 if (!PyArg_ParseTuple(args, "n:read", &num_bytes))
239 return(NULL);
241 /* silently 'adjust' out-of-range requests */
242 if ((self->pos + num_bytes) > self->size) {
243 num_bytes -= (self->pos+num_bytes) - self->size;
245 result = Py_BuildValue("s#", self->data+self->pos, num_bytes);
246 self->pos += num_bytes;
247 return result;
250 static PyObject *
251 mmap_find_method(mmap_object *self,
252 PyObject *args)
254 Py_ssize_t start = self->pos;
255 char *needle;
256 Py_ssize_t len;
258 CHECK_VALID(NULL);
259 if (!PyArg_ParseTuple(args, "s#|n:find", &needle, &len, &start)) {
260 return NULL;
261 } else {
262 char *p;
263 char *e = self->data + self->size;
265 if (start < 0)
266 start += self->size;
267 if (start < 0)
268 start = 0;
269 else if ((size_t)start > self->size)
270 start = self->size;
272 for (p = self->data + start; p + len <= e; ++p) {
273 Py_ssize_t i;
274 for (i = 0; i < len && needle[i] == p[i]; ++i)
275 /* nothing */;
276 if (i == len) {
277 return PyInt_FromSsize_t(p - self->data);
280 return PyInt_FromLong(-1);
284 static int
285 is_writeable(mmap_object *self)
287 if (self->access != ACCESS_READ)
288 return 1;
289 PyErr_Format(PyExc_TypeError, "mmap can't modify a readonly memory map.");
290 return 0;
293 static int
294 is_resizeable(mmap_object *self)
296 if ((self->access == ACCESS_WRITE) || (self->access == ACCESS_DEFAULT))
297 return 1;
298 PyErr_Format(PyExc_TypeError,
299 "mmap can't resize a readonly or copy-on-write memory map.");
300 return 0;
304 static PyObject *
305 mmap_write_method(mmap_object *self,
306 PyObject *args)
308 Py_ssize_t length;
309 char *data;
311 CHECK_VALID(NULL);
312 if (!PyArg_ParseTuple(args, "s#:write", &data, &length))
313 return(NULL);
315 if (!is_writeable(self))
316 return NULL;
318 if ((self->pos + length) > self->size) {
319 PyErr_SetString(PyExc_ValueError, "data out of range");
320 return NULL;
322 memcpy(self->data+self->pos, data, length);
323 self->pos = self->pos+length;
324 Py_INCREF(Py_None);
325 return Py_None;
328 static PyObject *
329 mmap_write_byte_method(mmap_object *self,
330 PyObject *args)
332 char value;
334 CHECK_VALID(NULL);
335 if (!PyArg_ParseTuple(args, "c:write_byte", &value))
336 return(NULL);
338 if (!is_writeable(self))
339 return NULL;
340 *(self->data+self->pos) = value;
341 self->pos += 1;
342 Py_INCREF(Py_None);
343 return Py_None;
346 static PyObject *
347 mmap_size_method(mmap_object *self,
348 PyObject *unused)
350 CHECK_VALID(NULL);
352 #ifdef MS_WINDOWS
353 if (self->file_handle != INVALID_HANDLE_VALUE) {
354 DWORD low,high;
355 PY_LONG_LONG size;
356 low = GetFileSize(self->file_handle, &high);
357 if (low == INVALID_FILE_SIZE) {
358 /* It might be that the function appears to have failed,
359 when indeed its size equals INVALID_FILE_SIZE */
360 DWORD error = GetLastError();
361 if (error != NO_ERROR)
362 return PyErr_SetFromWindowsErr(error);
364 if (!high && low < LONG_MAX)
365 return PyInt_FromLong((long)low);
366 size = (((PY_LONG_LONG)high)<<32) + low;
367 return PyLong_FromLongLong(size);
368 } else {
369 return PyInt_FromSsize_t(self->size);
371 #endif /* MS_WINDOWS */
373 #ifdef UNIX
375 struct stat buf;
376 if (-1 == fstat(self->fd, &buf)) {
377 PyErr_SetFromErrno(mmap_module_error);
378 return NULL;
380 return PyInt_FromSsize_t(buf.st_size);
382 #endif /* UNIX */
385 /* This assumes that you want the entire file mapped,
386 / and when recreating the map will make the new file
387 / have the new size
389 / Is this really necessary? This could easily be done
390 / from python by just closing and re-opening with the
391 / new size?
394 static PyObject *
395 mmap_resize_method(mmap_object *self,
396 PyObject *args)
398 Py_ssize_t new_size;
399 CHECK_VALID(NULL);
400 if (!PyArg_ParseTuple(args, "n:resize", &new_size) ||
401 !is_resizeable(self)) {
402 return NULL;
403 #ifdef MS_WINDOWS
404 } else {
405 DWORD dwErrCode = 0;
406 DWORD off_hi, off_lo, newSizeLow, newSizeHigh;
407 /* First, unmap the file view */
408 UnmapViewOfFile(self->data);
409 /* Close the mapping object */
410 CloseHandle(self->map_handle);
411 /* Move to the desired EOF position */
412 #if SIZEOF_SIZE_T > 4
413 newSizeHigh = (DWORD)((self->offset + new_size) >> 32);
414 newSizeLow = (DWORD)((self->offset + new_size) & 0xFFFFFFFF);
415 off_hi = (DWORD)(self->offset >> 32);
416 off_lo = (DWORD)(self->offset & 0xFFFFFFFF);
417 #else
418 newSizeHigh = 0;
419 newSizeLow = (DWORD)new_size;
420 off_hi = 0;
421 off_lo = (DWORD)self->offset;
422 #endif
423 SetFilePointer(self->file_handle,
424 newSizeLow, &newSizeHigh, FILE_BEGIN);
425 /* Change the size of the file */
426 SetEndOfFile(self->file_handle);
427 /* Create another mapping object and remap the file view */
428 self->map_handle = CreateFileMapping(
429 self->file_handle,
430 NULL,
431 PAGE_READWRITE,
434 self->tagname);
435 if (self->map_handle != NULL) {
436 self->data = (char *) MapViewOfFile(self->map_handle,
437 FILE_MAP_WRITE,
438 off_hi,
439 off_lo,
440 new_size);
441 if (self->data != NULL) {
442 self->size = new_size;
443 Py_INCREF(Py_None);
444 return Py_None;
445 } else {
446 dwErrCode = GetLastError();
448 } else {
449 dwErrCode = GetLastError();
451 PyErr_SetFromWindowsErr(dwErrCode);
452 return NULL;
453 #endif /* MS_WINDOWS */
455 #ifdef UNIX
456 #ifndef HAVE_MREMAP
457 } else {
458 PyErr_SetString(PyExc_SystemError,
459 "mmap: resizing not available--no mremap()");
460 return NULL;
461 #else
462 } else {
463 void *newmap;
465 if (ftruncate(self->fd, new_size) == -1) {
466 PyErr_SetFromErrno(mmap_module_error);
467 return NULL;
470 #ifdef MREMAP_MAYMOVE
471 newmap = mremap(self->data, self->size, new_size, MREMAP_MAYMOVE);
472 #else
473 newmap = mremap(self->data, self->size, new_size, 0);
474 #endif
475 if (newmap == (void *)-1)
477 PyErr_SetFromErrno(mmap_module_error);
478 return NULL;
480 self->data = newmap;
481 self->size = new_size;
482 Py_INCREF(Py_None);
483 return Py_None;
484 #endif /* HAVE_MREMAP */
485 #endif /* UNIX */
489 static PyObject *
490 mmap_tell_method(mmap_object *self, PyObject *unused)
492 CHECK_VALID(NULL);
493 return PyInt_FromSize_t(self->pos);
496 static PyObject *
497 mmap_flush_method(mmap_object *self, PyObject *args)
499 Py_ssize_t offset = 0;
500 Py_ssize_t size = self->size;
501 CHECK_VALID(NULL);
502 if (!PyArg_ParseTuple(args, "|nn:flush", &offset, &size))
503 return NULL;
504 if ((size_t)(offset + size) > self->size) {
505 PyErr_SetString(PyExc_ValueError, "flush values out of range");
506 return NULL;
507 } else {
508 #ifdef MS_WINDOWS
509 return PyInt_FromLong((long)
510 FlushViewOfFile(self->data+offset, size));
511 #endif /* MS_WINDOWS */
512 #ifdef UNIX
513 /* XXX semantics of return value? */
514 /* XXX flags for msync? */
515 if (-1 == msync(self->data + offset, size,
516 MS_SYNC))
518 PyErr_SetFromErrno(mmap_module_error);
519 return NULL;
521 return PyInt_FromLong(0);
522 #endif /* UNIX */
526 static PyObject *
527 mmap_seek_method(mmap_object *self, PyObject *args)
529 Py_ssize_t dist;
530 int how=0;
531 CHECK_VALID(NULL);
532 if (!PyArg_ParseTuple(args, "n|i:seek", &dist, &how))
533 return NULL;
534 else {
535 size_t where;
536 switch (how) {
537 case 0: /* relative to start */
538 if (dist < 0)
539 goto onoutofrange;
540 where = dist;
541 break;
542 case 1: /* relative to current position */
543 if ((Py_ssize_t)self->pos + dist < 0)
544 goto onoutofrange;
545 where = self->pos + dist;
546 break;
547 case 2: /* relative to end */
548 if ((Py_ssize_t)self->size + dist < 0)
549 goto onoutofrange;
550 where = self->size + dist;
551 break;
552 default:
553 PyErr_SetString(PyExc_ValueError, "unknown seek type");
554 return NULL;
556 if (where > self->size)
557 goto onoutofrange;
558 self->pos = where;
559 Py_INCREF(Py_None);
560 return Py_None;
563 onoutofrange:
564 PyErr_SetString(PyExc_ValueError, "seek out of range");
565 return NULL;
568 static PyObject *
569 mmap_move_method(mmap_object *self, PyObject *args)
571 unsigned long dest, src, count;
572 CHECK_VALID(NULL);
573 if (!PyArg_ParseTuple(args, "kkk:move", &dest, &src, &count) ||
574 !is_writeable(self)) {
575 return NULL;
576 } else {
577 /* bounds check the values */
578 if (/* end of source after end of data?? */
579 ((src+count) > self->size)
580 /* dest will fit? */
581 || (dest+count > self->size)) {
582 PyErr_SetString(PyExc_ValueError,
583 "source or destination out of range");
584 return NULL;
585 } else {
586 memmove(self->data+dest, self->data+src, count);
587 Py_INCREF(Py_None);
588 return Py_None;
593 static struct PyMethodDef mmap_object_methods[] = {
594 {"close", (PyCFunction) mmap_close_method, METH_NOARGS},
595 {"find", (PyCFunction) mmap_find_method, METH_VARARGS},
596 {"flush", (PyCFunction) mmap_flush_method, METH_VARARGS},
597 {"move", (PyCFunction) mmap_move_method, METH_VARARGS},
598 {"read", (PyCFunction) mmap_read_method, METH_VARARGS},
599 {"read_byte", (PyCFunction) mmap_read_byte_method, METH_NOARGS},
600 {"readline", (PyCFunction) mmap_read_line_method, METH_NOARGS},
601 {"resize", (PyCFunction) mmap_resize_method, METH_VARARGS},
602 {"seek", (PyCFunction) mmap_seek_method, METH_VARARGS},
603 {"size", (PyCFunction) mmap_size_method, METH_NOARGS},
604 {"tell", (PyCFunction) mmap_tell_method, METH_NOARGS},
605 {"write", (PyCFunction) mmap_write_method, METH_VARARGS},
606 {"write_byte", (PyCFunction) mmap_write_byte_method, METH_VARARGS},
607 {NULL, NULL} /* sentinel */
610 /* Functions for treating an mmap'ed file as a buffer */
612 static Py_ssize_t
613 mmap_buffer_getreadbuf(mmap_object *self, Py_ssize_t index, const void **ptr)
615 CHECK_VALID(-1);
616 if (index != 0) {
617 PyErr_SetString(PyExc_SystemError,
618 "Accessing non-existent mmap segment");
619 return -1;
621 *ptr = self->data;
622 return self->size;
625 static Py_ssize_t
626 mmap_buffer_getwritebuf(mmap_object *self, Py_ssize_t index, const void **ptr)
628 CHECK_VALID(-1);
629 if (index != 0) {
630 PyErr_SetString(PyExc_SystemError,
631 "Accessing non-existent mmap segment");
632 return -1;
634 if (!is_writeable(self))
635 return -1;
636 *ptr = self->data;
637 return self->size;
640 static Py_ssize_t
641 mmap_buffer_getsegcount(mmap_object *self, Py_ssize_t *lenp)
643 CHECK_VALID(-1);
644 if (lenp)
645 *lenp = self->size;
646 return 1;
649 static Py_ssize_t
650 mmap_buffer_getcharbuffer(mmap_object *self, Py_ssize_t index, const void **ptr)
652 if (index != 0) {
653 PyErr_SetString(PyExc_SystemError,
654 "accessing non-existent buffer segment");
655 return -1;
657 *ptr = (const char *)self->data;
658 return self->size;
661 static PyObject *
662 mmap_object_getattr(mmap_object *self, char *name)
664 return Py_FindMethod(mmap_object_methods, (PyObject *)self, name);
667 static Py_ssize_t
668 mmap_length(mmap_object *self)
670 CHECK_VALID(-1);
671 return self->size;
674 static PyObject *
675 mmap_item(mmap_object *self, Py_ssize_t i)
677 CHECK_VALID(NULL);
678 if (i < 0 || (size_t)i >= self->size) {
679 PyErr_SetString(PyExc_IndexError, "mmap index out of range");
680 return NULL;
682 return PyString_FromStringAndSize(self->data + i, 1);
685 static PyObject *
686 mmap_slice(mmap_object *self, Py_ssize_t ilow, Py_ssize_t ihigh)
688 CHECK_VALID(NULL);
689 if (ilow < 0)
690 ilow = 0;
691 else if ((size_t)ilow > self->size)
692 ilow = self->size;
693 if (ihigh < 0)
694 ihigh = 0;
695 if (ihigh < ilow)
696 ihigh = ilow;
697 else if ((size_t)ihigh > self->size)
698 ihigh = self->size;
700 return PyString_FromStringAndSize(self->data + ilow, ihigh-ilow);
703 static PyObject *
704 mmap_subscript(mmap_object *self, PyObject *item)
706 CHECK_VALID(NULL);
707 if (PyIndex_Check(item)) {
708 Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError);
709 if (i == -1 && PyErr_Occurred())
710 return NULL;
711 if (i < 0)
712 i += self->size;
713 if (i < 0 || (size_t)i > self->size) {
714 PyErr_SetString(PyExc_IndexError,
715 "mmap index out of range");
716 return NULL;
718 return PyString_FromStringAndSize(self->data + i, 1);
720 else if (PySlice_Check(item)) {
721 Py_ssize_t start, stop, step, slicelen;
723 if (PySlice_GetIndicesEx((PySliceObject *)item, self->size,
724 &start, &stop, &step, &slicelen) < 0) {
725 return NULL;
728 if (slicelen <= 0)
729 return PyString_FromStringAndSize("", 0);
730 else if (step == 1)
731 return PyString_FromStringAndSize(self->data + start,
732 slicelen);
733 else {
734 char *result_buf = (char *)PyMem_Malloc(slicelen);
735 Py_ssize_t cur, i;
736 PyObject *result;
738 if (result_buf == NULL)
739 return PyErr_NoMemory();
740 for (cur = start, i = 0; i < slicelen;
741 cur += step, i++) {
742 result_buf[i] = self->data[cur];
744 result = PyString_FromStringAndSize(result_buf,
745 slicelen);
746 PyMem_Free(result_buf);
747 return result;
750 else {
751 PyErr_SetString(PyExc_TypeError,
752 "mmap indices must be integers");
753 return NULL;
757 static PyObject *
758 mmap_concat(mmap_object *self, PyObject *bb)
760 CHECK_VALID(NULL);
761 PyErr_SetString(PyExc_SystemError,
762 "mmaps don't support concatenation");
763 return NULL;
766 static PyObject *
767 mmap_repeat(mmap_object *self, Py_ssize_t n)
769 CHECK_VALID(NULL);
770 PyErr_SetString(PyExc_SystemError,
771 "mmaps don't support repeat operation");
772 return NULL;
775 static int
776 mmap_ass_slice(mmap_object *self, Py_ssize_t ilow, Py_ssize_t ihigh, PyObject *v)
778 const char *buf;
780 CHECK_VALID(-1);
781 if (ilow < 0)
782 ilow = 0;
783 else if ((size_t)ilow > self->size)
784 ilow = self->size;
785 if (ihigh < 0)
786 ihigh = 0;
787 if (ihigh < ilow)
788 ihigh = ilow;
789 else if ((size_t)ihigh > self->size)
790 ihigh = self->size;
792 if (v == NULL) {
793 PyErr_SetString(PyExc_TypeError,
794 "mmap object doesn't support slice deletion");
795 return -1;
797 if (! (PyString_Check(v)) ) {
798 PyErr_SetString(PyExc_IndexError,
799 "mmap slice assignment must be a string");
800 return -1;
802 if (PyString_Size(v) != (ihigh - ilow)) {
803 PyErr_SetString(PyExc_IndexError,
804 "mmap slice assignment is wrong size");
805 return -1;
807 if (!is_writeable(self))
808 return -1;
809 buf = PyString_AsString(v);
810 memcpy(self->data + ilow, buf, ihigh-ilow);
811 return 0;
814 static int
815 mmap_ass_item(mmap_object *self, Py_ssize_t i, PyObject *v)
817 const char *buf;
819 CHECK_VALID(-1);
820 if (i < 0 || (size_t)i >= self->size) {
821 PyErr_SetString(PyExc_IndexError, "mmap index out of range");
822 return -1;
824 if (v == NULL) {
825 PyErr_SetString(PyExc_TypeError,
826 "mmap object doesn't support item deletion");
827 return -1;
829 if (! (PyString_Check(v) && PyString_Size(v)==1) ) {
830 PyErr_SetString(PyExc_IndexError,
831 "mmap assignment must be single-character string");
832 return -1;
834 if (!is_writeable(self))
835 return -1;
836 buf = PyString_AsString(v);
837 self->data[i] = buf[0];
838 return 0;
841 static int
842 mmap_ass_subscript(mmap_object *self, PyObject *item, PyObject *value)
844 CHECK_VALID(-1);
846 if (PyIndex_Check(item)) {
847 Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError);
848 const char *buf;
850 if (i == -1 && PyErr_Occurred())
851 return -1;
852 if (i < 0)
853 i += self->size;
854 if (i < 0 || (size_t)i > self->size) {
855 PyErr_SetString(PyExc_IndexError,
856 "mmap index out of range");
857 return -1;
859 if (value == NULL) {
860 PyErr_SetString(PyExc_TypeError,
861 "mmap object doesn't support item deletion");
862 return -1;
864 if (!PyString_Check(value) || PyString_Size(value) != 1) {
865 PyErr_SetString(PyExc_IndexError,
866 "mmap assignment must be single-character string");
867 return -1;
869 if (!is_writeable(self))
870 return -1;
871 buf = PyString_AsString(value);
872 self->data[i] = buf[0];
873 return 0;
875 else if (PySlice_Check(item)) {
876 Py_ssize_t start, stop, step, slicelen;
878 if (PySlice_GetIndicesEx((PySliceObject *)item,
879 self->size, &start, &stop,
880 &step, &slicelen) < 0) {
881 return -1;
883 if (value == NULL) {
884 PyErr_SetString(PyExc_TypeError,
885 "mmap object doesn't support slice deletion");
886 return -1;
888 if (!PyString_Check(value)) {
889 PyErr_SetString(PyExc_IndexError,
890 "mmap slice assignment must be a string");
891 return -1;
893 if (PyString_Size(value) != slicelen) {
894 PyErr_SetString(PyExc_IndexError,
895 "mmap slice assignment is wrong size");
896 return -1;
898 if (!is_writeable(self))
899 return -1;
901 if (slicelen == 0)
902 return 0;
903 else if (step == 1) {
904 const char *buf = PyString_AsString(value);
906 if (buf == NULL)
907 return -1;
908 memcpy(self->data + start, buf, slicelen);
909 return 0;
911 else {
912 Py_ssize_t cur, i;
913 const char *buf = PyString_AsString(value);
915 if (buf == NULL)
916 return -1;
917 for (cur = start, i = 0; i < slicelen;
918 cur += step, i++) {
919 self->data[cur] = buf[i];
921 return 0;
924 else {
925 PyErr_SetString(PyExc_TypeError,
926 "mmap indices must be integer");
927 return -1;
931 static PySequenceMethods mmap_as_sequence = {
932 (lenfunc)mmap_length, /*sq_length*/
933 (binaryfunc)mmap_concat, /*sq_concat*/
934 (ssizeargfunc)mmap_repeat, /*sq_repeat*/
935 (ssizeargfunc)mmap_item, /*sq_item*/
936 (ssizessizeargfunc)mmap_slice, /*sq_slice*/
937 (ssizeobjargproc)mmap_ass_item, /*sq_ass_item*/
938 (ssizessizeobjargproc)mmap_ass_slice, /*sq_ass_slice*/
941 static PyMappingMethods mmap_as_mapping = {
942 (lenfunc)mmap_length,
943 (binaryfunc)mmap_subscript,
944 (objobjargproc)mmap_ass_subscript,
947 static PyBufferProcs mmap_as_buffer = {
948 (readbufferproc)mmap_buffer_getreadbuf,
949 (writebufferproc)mmap_buffer_getwritebuf,
950 (segcountproc)mmap_buffer_getsegcount,
951 (charbufferproc)mmap_buffer_getcharbuffer,
954 static PyTypeObject mmap_object_type = {
955 PyVarObject_HEAD_INIT(0, 0) /* patched in module init */
956 "mmap.mmap", /* tp_name */
957 sizeof(mmap_object), /* tp_size */
958 0, /* tp_itemsize */
959 /* methods */
960 (destructor) mmap_object_dealloc, /* tp_dealloc */
961 0, /* tp_print */
962 (getattrfunc) mmap_object_getattr, /* tp_getattr */
963 0, /* tp_setattr */
964 0, /* tp_compare */
965 0, /* tp_repr */
966 0, /* tp_as_number */
967 &mmap_as_sequence, /*tp_as_sequence*/
968 &mmap_as_mapping, /*tp_as_mapping*/
969 0, /*tp_hash*/
970 0, /*tp_call*/
971 0, /*tp_str*/
972 0, /*tp_getattro*/
973 0, /*tp_setattro*/
974 &mmap_as_buffer, /*tp_as_buffer*/
975 Py_TPFLAGS_HAVE_GETCHARBUFFER, /*tp_flags*/
976 0, /*tp_doc*/
980 /* extract the map size from the given PyObject
982 Returns -1 on error, with an appropriate Python exception raised. On
983 success, the map size is returned. */
984 static Py_ssize_t
985 _GetMapSize(PyObject *o, const char* param)
987 if (o == NULL)
988 return 0;
989 if (PyIndex_Check(o)) {
990 Py_ssize_t i = PyNumber_AsSsize_t(o, PyExc_OverflowError);
991 if (i==-1 && PyErr_Occurred())
992 return -1;
993 if (i < 0) {
994 PyErr_Format(PyExc_OverflowError,
995 "memory mapped %s must be positive",
996 param);
997 return -1;
999 return i;
1002 PyErr_SetString(PyExc_TypeError, "map size must be an integral value");
1003 return -1;
1006 #ifdef UNIX
1007 static PyObject *
1008 new_mmap_object(PyObject *self, PyObject *args, PyObject *kwdict)
1010 #ifdef HAVE_FSTAT
1011 struct stat st;
1012 #endif
1013 mmap_object *m_obj;
1014 PyObject *map_size_obj = NULL, *offset_obj = NULL;
1015 Py_ssize_t map_size, offset;
1016 int fd, flags = MAP_SHARED, prot = PROT_WRITE | PROT_READ;
1017 int devzero = -1;
1018 int access = (int)ACCESS_DEFAULT;
1019 static char *keywords[] = {"fileno", "length",
1020 "flags", "prot",
1021 "access", "offset", NULL};
1023 if (!PyArg_ParseTupleAndKeywords(args, kwdict, "iO|iiiO", keywords,
1024 &fd, &map_size_obj, &flags, &prot,
1025 &access, &offset_obj))
1026 return NULL;
1027 map_size = _GetMapSize(map_size_obj, "size");
1028 if (map_size < 0)
1029 return NULL;
1030 offset = _GetMapSize(offset_obj, "offset");
1031 if (offset < 0)
1032 return NULL;
1034 if ((access != (int)ACCESS_DEFAULT) &&
1035 ((flags != MAP_SHARED) || (prot != (PROT_WRITE | PROT_READ))))
1036 return PyErr_Format(PyExc_ValueError,
1037 "mmap can't specify both access and flags, prot.");
1038 switch ((access_mode)access) {
1039 case ACCESS_READ:
1040 flags = MAP_SHARED;
1041 prot = PROT_READ;
1042 break;
1043 case ACCESS_WRITE:
1044 flags = MAP_SHARED;
1045 prot = PROT_READ | PROT_WRITE;
1046 break;
1047 case ACCESS_COPY:
1048 flags = MAP_PRIVATE;
1049 prot = PROT_READ | PROT_WRITE;
1050 break;
1051 case ACCESS_DEFAULT:
1052 /* use the specified or default values of flags and prot */
1053 break;
1054 default:
1055 return PyErr_Format(PyExc_ValueError,
1056 "mmap invalid access parameter.");
1059 #ifdef HAVE_FSTAT
1060 # ifdef __VMS
1061 /* on OpenVMS we must ensure that all bytes are written to the file */
1062 if (fd != -1) {
1063 fsync(fd);
1065 # endif
1066 if (fd != -1 && fstat(fd, &st) == 0 && S_ISREG(st.st_mode)) {
1067 if (map_size == 0) {
1068 map_size = st.st_size;
1069 } else if ((size_t)offset + (size_t)map_size > st.st_size) {
1070 PyErr_SetString(PyExc_ValueError,
1071 "mmap length is greater than file size");
1072 return NULL;
1075 #endif
1076 m_obj = PyObject_New(mmap_object, &mmap_object_type);
1077 if (m_obj == NULL) {return NULL;}
1078 m_obj->data = NULL;
1079 m_obj->size = (size_t) map_size;
1080 m_obj->pos = (size_t) 0;
1081 m_obj->offset = offset;
1082 if (fd == -1) {
1083 m_obj->fd = -1;
1084 /* Assume the caller wants to map anonymous memory.
1085 This is the same behaviour as Windows. mmap.mmap(-1, size)
1086 on both Windows and Unix map anonymous memory.
1088 #ifdef MAP_ANONYMOUS
1089 /* BSD way to map anonymous memory */
1090 flags |= MAP_ANONYMOUS;
1091 #else
1092 /* SVR4 method to map anonymous memory is to open /dev/zero */
1093 fd = devzero = open("/dev/zero", O_RDWR);
1094 if (devzero == -1) {
1095 Py_DECREF(m_obj);
1096 PyErr_SetFromErrno(mmap_module_error);
1097 return NULL;
1099 #endif
1100 } else {
1101 m_obj->fd = dup(fd);
1102 if (m_obj->fd == -1) {
1103 Py_DECREF(m_obj);
1104 PyErr_SetFromErrno(mmap_module_error);
1105 return NULL;
1109 m_obj->data = mmap(NULL, map_size,
1110 prot, flags,
1111 fd, offset);
1113 if (devzero != -1) {
1114 close(devzero);
1117 if (m_obj->data == (char *)-1) {
1118 m_obj->data = NULL;
1119 Py_DECREF(m_obj);
1120 PyErr_SetFromErrno(mmap_module_error);
1121 return NULL;
1123 m_obj->access = (access_mode)access;
1124 return (PyObject *)m_obj;
1126 #endif /* UNIX */
1128 #ifdef MS_WINDOWS
1129 static PyObject *
1130 new_mmap_object(PyObject *self, PyObject *args, PyObject *kwdict)
1132 mmap_object *m_obj;
1133 PyObject *map_size_obj = NULL, *offset_obj = NULL;
1134 Py_ssize_t map_size, offset;
1135 DWORD off_hi; /* upper 32 bits of offset */
1136 DWORD off_lo; /* lower 32 bits of offset */
1137 DWORD size_hi; /* upper 32 bits of size */
1138 DWORD size_lo; /* lower 32 bits of size */
1139 char *tagname = "";
1140 DWORD dwErr = 0;
1141 int fileno;
1142 HANDLE fh = 0;
1143 int access = (access_mode)ACCESS_DEFAULT;
1144 DWORD flProtect, dwDesiredAccess;
1145 static char *keywords[] = { "fileno", "length",
1146 "tagname",
1147 "access", "offset", NULL };
1149 if (!PyArg_ParseTupleAndKeywords(args, kwdict, "iO|ziO", keywords,
1150 &fileno, &map_size_obj,
1151 &tagname, &access, &offset_obj)) {
1152 return NULL;
1155 switch((access_mode)access) {
1156 case ACCESS_READ:
1157 flProtect = PAGE_READONLY;
1158 dwDesiredAccess = FILE_MAP_READ;
1159 break;
1160 case ACCESS_DEFAULT: case ACCESS_WRITE:
1161 flProtect = PAGE_READWRITE;
1162 dwDesiredAccess = FILE_MAP_WRITE;
1163 break;
1164 case ACCESS_COPY:
1165 flProtect = PAGE_WRITECOPY;
1166 dwDesiredAccess = FILE_MAP_COPY;
1167 break;
1168 default:
1169 return PyErr_Format(PyExc_ValueError,
1170 "mmap invalid access parameter.");
1173 map_size = _GetMapSize(map_size_obj, "size");
1174 if (map_size < 0)
1175 return NULL;
1176 offset = _GetMapSize(offset_obj, "offset");
1177 if (offset < 0)
1178 return NULL;
1180 /* assume -1 and 0 both mean invalid filedescriptor
1181 to 'anonymously' map memory.
1182 XXX: fileno == 0 is a valid fd, but was accepted prior to 2.5.
1183 XXX: Should this code be added?
1184 if (fileno == 0)
1185 PyErr_Warn(PyExc_DeprecationWarning,
1186 "don't use 0 for anonymous memory");
1188 if (fileno != -1 && fileno != 0) {
1189 fh = (HANDLE)_get_osfhandle(fileno);
1190 if (fh==(HANDLE)-1) {
1191 PyErr_SetFromErrno(mmap_module_error);
1192 return NULL;
1194 /* Win9x appears to need us seeked to zero */
1195 lseek(fileno, 0, SEEK_SET);
1198 m_obj = PyObject_New(mmap_object, &mmap_object_type);
1199 if (m_obj == NULL)
1200 return NULL;
1201 /* Set every field to an invalid marker, so we can safely
1202 destruct the object in the face of failure */
1203 m_obj->data = NULL;
1204 m_obj->file_handle = INVALID_HANDLE_VALUE;
1205 m_obj->map_handle = INVALID_HANDLE_VALUE;
1206 m_obj->tagname = NULL;
1207 m_obj->offset = offset;
1209 if (fh) {
1210 /* It is necessary to duplicate the handle, so the
1211 Python code can close it on us */
1212 if (!DuplicateHandle(
1213 GetCurrentProcess(), /* source process handle */
1214 fh, /* handle to be duplicated */
1215 GetCurrentProcess(), /* target proc handle */
1216 (LPHANDLE)&m_obj->file_handle, /* result */
1217 0, /* access - ignored due to options value */
1218 FALSE, /* inherited by child processes? */
1219 DUPLICATE_SAME_ACCESS)) { /* options */
1220 dwErr = GetLastError();
1221 Py_DECREF(m_obj);
1222 PyErr_SetFromWindowsErr(dwErr);
1223 return NULL;
1225 if (!map_size) {
1226 DWORD low,high;
1227 low = GetFileSize(fh, &high);
1228 /* low might just happen to have the value INVALID_FILE_SIZE;
1229 so we need to check the last error also. */
1230 if (low == INVALID_FILE_SIZE &&
1231 (dwErr = GetLastError()) != NO_ERROR) {
1232 Py_DECREF(m_obj);
1233 return PyErr_SetFromWindowsErr(dwErr);
1236 #if SIZEOF_SIZE_T > 4
1237 m_obj->size = (((size_t)high)<<32) + low;
1238 #else
1239 if (high)
1240 /* File is too large to map completely */
1241 m_obj->size = (size_t)-1;
1242 else
1243 m_obj->size = low;
1244 #endif
1245 } else {
1246 m_obj->size = map_size;
1249 else {
1250 m_obj->size = map_size;
1253 /* set the initial position */
1254 m_obj->pos = (size_t) 0;
1256 /* set the tag name */
1257 if (tagname != NULL && *tagname != '\0') {
1258 m_obj->tagname = PyMem_Malloc(strlen(tagname)+1);
1259 if (m_obj->tagname == NULL) {
1260 PyErr_NoMemory();
1261 Py_DECREF(m_obj);
1262 return NULL;
1264 strcpy(m_obj->tagname, tagname);
1266 else
1267 m_obj->tagname = NULL;
1269 m_obj->access = (access_mode)access;
1270 /* DWORD is a 4-byte int. If we're on a box where size_t consumes
1271 * more than 4 bytes, we need to break it apart. Else (size_t
1272 * consumes 4 bytes), C doesn't define what happens if we shift
1273 * right by 32, so we need different code.
1275 #if SIZEOF_SIZE_T > 4
1276 size_hi = (DWORD)((offset + m_obj->size) >> 32);
1277 size_lo = (DWORD)((offset + m_obj->size) & 0xFFFFFFFF);
1278 off_hi = (DWORD)(offset >> 32);
1279 off_lo = (DWORD)(offset & 0xFFFFFFFF);
1280 #else
1281 size_hi = 0;
1282 size_lo = (DWORD)(offset + m_obj->size);
1283 off_hi = 0;
1284 off_lo = (DWORD)offset;
1285 #endif
1286 /* For files, it would be sufficient to pass 0 as size.
1287 For anonymous maps, we have to pass the size explicitly. */
1288 m_obj->map_handle = CreateFileMapping(m_obj->file_handle,
1289 NULL,
1290 flProtect,
1291 size_hi,
1292 size_lo,
1293 m_obj->tagname);
1294 if (m_obj->map_handle != NULL) {
1295 m_obj->data = (char *) MapViewOfFile(m_obj->map_handle,
1296 dwDesiredAccess,
1297 off_hi,
1298 off_lo,
1300 if (m_obj->data != NULL)
1301 return (PyObject *)m_obj;
1302 else
1303 dwErr = GetLastError();
1304 } else
1305 dwErr = GetLastError();
1306 Py_DECREF(m_obj);
1307 PyErr_SetFromWindowsErr(dwErr);
1308 return NULL;
1310 #endif /* MS_WINDOWS */
1312 /* List of functions exported by this module */
1313 static struct PyMethodDef mmap_functions[] = {
1314 {"mmap", (PyCFunction) new_mmap_object,
1315 METH_VARARGS|METH_KEYWORDS},
1316 {NULL, NULL} /* Sentinel */
1319 static void
1320 setint(PyObject *d, const char *name, long value)
1322 PyObject *o = PyInt_FromLong(value);
1323 if (o && PyDict_SetItemString(d, name, o) == 0) {
1324 Py_DECREF(o);
1328 PyMODINIT_FUNC
1329 initmmap(void)
1331 PyObject *dict, *module;
1333 /* Patch the object type */
1334 Py_TYPE(&mmap_object_type) = &PyType_Type;
1336 module = Py_InitModule("mmap", mmap_functions);
1337 if (module == NULL)
1338 return;
1339 dict = PyModule_GetDict(module);
1340 if (!dict)
1341 return;
1342 mmap_module_error = PyExc_EnvironmentError;
1343 PyDict_SetItemString(dict, "error", mmap_module_error);
1344 #ifdef PROT_EXEC
1345 setint(dict, "PROT_EXEC", PROT_EXEC);
1346 #endif
1347 #ifdef PROT_READ
1348 setint(dict, "PROT_READ", PROT_READ);
1349 #endif
1350 #ifdef PROT_WRITE
1351 setint(dict, "PROT_WRITE", PROT_WRITE);
1352 #endif
1354 #ifdef MAP_SHARED
1355 setint(dict, "MAP_SHARED", MAP_SHARED);
1356 #endif
1357 #ifdef MAP_PRIVATE
1358 setint(dict, "MAP_PRIVATE", MAP_PRIVATE);
1359 #endif
1360 #ifdef MAP_DENYWRITE
1361 setint(dict, "MAP_DENYWRITE", MAP_DENYWRITE);
1362 #endif
1363 #ifdef MAP_EXECUTABLE
1364 setint(dict, "MAP_EXECUTABLE", MAP_EXECUTABLE);
1365 #endif
1366 #ifdef MAP_ANONYMOUS
1367 setint(dict, "MAP_ANON", MAP_ANONYMOUS);
1368 setint(dict, "MAP_ANONYMOUS", MAP_ANONYMOUS);
1369 #endif
1371 setint(dict, "PAGESIZE", (long)my_getpagesize());
1373 setint(dict, "ALLOCATIONGRANULARITY", (long)my_getallocationgranularity());
1375 setint(dict, "ACCESS_READ", ACCESS_READ);
1376 setint(dict, "ACCESS_WRITE", ACCESS_WRITE);
1377 setint(dict, "ACCESS_COPY", ACCESS_COPY);