Modules/mmapmodule.c

   1 /*
   2  /  Author: Sam Rushing <rushing@nightmare.com>
   3  /  Hacked for Unix by AMK
   4  /  $Id$
   5
   6  / mmapmodule.cpp -- map a view of a file into memory
   7  /
   8  / todo: need permission flags, perhaps a 'chsize' analog
   9  /   not all functions check range yet!!!
  10  /
  11  /
  12  / This version of mmapmodule.c has been changed significantly
  13  / from the original mmapfile.c on which it was based.
  14  / The original version of mmapfile is maintained by Sam at
  15  / ftp://squirl.nightmare.com/pub/python/python-ext.
  16 */
  17
  18 #define PY_SSIZE_T_CLEAN
  19 #include <Python.h>
  20
  21 #ifndef MS_WINDOWS
  22 #define UNIX
  23 #endif
  24
  25 #ifdef MS_WINDOWS
  26 #include <windows.h>
  27 static int
  28 my_getpagesize(void)
  29 {
  30         SYSTEM_INFO si;
  31         GetSystemInfo(&si);
  32         return si.dwPageSize;
  33 }
  34 #endif
  35
  36 #ifdef UNIX
  37 #include <sys/mman.h>
  38 #include <sys/stat.h>
  39
  40 #if defined(HAVE_SYSCONF) && defined(_SC_PAGESIZE)
  41 static int
  42 my_getpagesize(void)
  43 {
  44         return sysconf(_SC_PAGESIZE);
  45 }
  46 #else
  47 #define my_getpagesize getpagesize
  48 #endif
  49
  50 #endif /* UNIX */
  51
  52 #include <string.h>
  53
  54 #ifdef HAVE_SYS_TYPES_H
  55 #include <sys/types.h>
  56 #endif /* HAVE_SYS_TYPES_H */
  57
  58 /* Prefer MAP_ANONYMOUS since MAP_ANON is deprecated according to man page. */
  59 #if !defined(MAP_ANONYMOUS) && defined(MAP_ANON)
  60 #  define MAP_ANONYMOUS MAP_ANON
  61 #endif
  62
  63 static PyObject *mmap_module_error;
  64
  65 typedef enum
  66 {
  67         ACCESS_DEFAULT,
  68         ACCESS_READ,
  69         ACCESS_WRITE,
  70         ACCESS_COPY
  71 } access_mode;
  72
  73 typedef struct {
  74         PyObject_HEAD
  75         char *  data;
  76         size_t  size;
  77         size_t  pos;
  78
  79 #ifdef MS_WINDOWS
  80         HANDLE  map_handle;
  81         HANDLE  file_handle;
  82         char *  tagname;
  83 #endif
  84
  85 #ifdef UNIX
  86         int fd;
  87 #endif
  88
  89         access_mode access;
  90 } mmap_object;
  91
  92
  93 static void
  94 mmap_object_dealloc(mmap_object *m_obj)
  95 {
  96 #ifdef MS_WINDOWS
  97         if (m_obj->data != NULL)
  98                 UnmapViewOfFile (m_obj->data);
  99         if (m_obj->map_handle != INVALID_HANDLE_VALUE)
 100                 CloseHandle (m_obj->map_handle);
 101         if (m_obj->file_handle != INVALID_HANDLE_VALUE)
 102                 CloseHandle (m_obj->file_handle);
 103         if (m_obj->tagname)
 104                 PyMem_Free(m_obj->tagname);
 105 #endif /* MS_WINDOWS */
 106
 107 #ifdef UNIX
 108         if (m_obj->fd >= 0)
 109                 (void) close(m_obj->fd);
 110         if (m_obj->data!=NULL) {
 111                 msync(m_obj->data, m_obj->size, MS_SYNC);
 112                 munmap(m_obj->data, m_obj->size);
 113         }
 114 #endif /* UNIX */
 115
 116         PyObject_Del(m_obj);
 117 }
 118
 119 static PyObject *
 120 mmap_close_method(mmap_object *self, PyObject *unused)
 121 {
 122 #ifdef MS_WINDOWS
 123         /* For each resource we maintain, we need to check
 124            the value is valid, and if so, free the resource
 125            and set the member value to an invalid value so
 126            the dealloc does not attempt to resource clearing
 127            again.
 128            TODO - should we check for errors in the close operations???
 129         */
 130         if (self->data != NULL) {
 131                 UnmapViewOfFile(self->data);
 132                 self->data = NULL;
 133         }
 134         if (self->map_handle != INVALID_HANDLE_VALUE) {
 135                 CloseHandle(self->map_handle);
 136                 self->map_handle = INVALID_HANDLE_VALUE;
 137         }
 138         if (self->file_handle != INVALID_HANDLE_VALUE) {
 139                 CloseHandle(self->file_handle);
 140                 self->file_handle = INVALID_HANDLE_VALUE;
 141         }
 142 #endif /* MS_WINDOWS */
 143
 144 #ifdef UNIX
 145         (void) close(self->fd);
 146         self->fd = -1;
 147         if (self->data != NULL) {
 148                 munmap(self->data, self->size);
 149                 self->data = NULL;
 150         }
 151 #endif
 152
 153         Py_INCREF(Py_None);
 154         return Py_None;
 155 }
 156
 157 #ifdef MS_WINDOWS
 158 #define CHECK_VALID(err)                                                \
 159 do {                                                                    \
 160     if (self->map_handle == INVALID_HANDLE_VALUE) {                                             \
 161         PyErr_SetString(PyExc_ValueError, "mmap closed or invalid");    \
 162         return err;                                                     \
 163     }                                                                   \
 164 } while (0)
 165 #endif /* MS_WINDOWS */
 166
 167 #ifdef UNIX
 168 #define CHECK_VALID(err)                                                \
 169 do {                                                                    \
 170     if (self->data == NULL) {                                           \
 171         PyErr_SetString(PyExc_ValueError, "mmap closed or invalid");    \
 172         return err;                                                     \
 173         }                                                               \
 174 } while (0)
 175 #endif /* UNIX */
 176
 177 static PyObject *
 178 mmap_read_byte_method(mmap_object *self,
 179                       PyObject *unused)
 180 {
 181         CHECK_VALID(NULL);
 182         if (self->pos < self->size) {
 183                 char value = self->data[self->pos];
 184                 self->pos += 1;
 185                 return Py_BuildValue("c", value);
 186         } else {
 187                 PyErr_SetString(PyExc_ValueError, "read byte out of range");
 188                 return NULL;
 189         }
 190 }
 191
 192 static PyObject *
 193 mmap_read_line_method(mmap_object *self,
 194                       PyObject *unused)
 195 {
 196         char *start = self->data+self->pos;
 197         char *eof = self->data+self->size;
 198         char *eol;
 199         PyObject *result;
 200
 201         CHECK_VALID(NULL);
 202
 203         eol = memchr(start, '\n', self->size - self->pos);
 204         if (!eol)
 205                 eol = eof;
 206         else
 207                 ++eol;          /* we're interested in the position after the
 208                                    newline. */
 209         result = PyString_FromStringAndSize(start, (eol - start));
 210         self->pos += (eol - start);
 211         return result;
 212 }
 213
 214 static PyObject *
 215 mmap_read_method(mmap_object *self,
 216                  PyObject *args)
 217 {
 218         Py_ssize_t num_bytes;
 219         PyObject *result;
 220
 221         CHECK_VALID(NULL);
 222         if (!PyArg_ParseTuple(args, "n:read", &num_bytes))
 223                 return(NULL);
 224
 225         /* silently 'adjust' out-of-range requests */
 226         if ((self->pos + num_bytes) > self->size) {
 227                 num_bytes -= (self->pos+num_bytes) - self->size;
 228         }
 229         result = Py_BuildValue("s#", self->data+self->pos, num_bytes);
 230         self->pos += num_bytes;
 231         return result;
 232 }
 233
 234 static PyObject *
 235 mmap_find_method(mmap_object *self,
 236                  PyObject *args)
 237 {
 238         Py_ssize_t start = self->pos;
 239         char *needle;
 240         Py_ssize_t len;
 241
 242         CHECK_VALID(NULL);
 243         if (!PyArg_ParseTuple(args, "s#|n:find", &needle, &len, &start)) {
 244                 return NULL;
 245         } else {
 246                 char *p;
 247                 char *e = self->data + self->size;
 248
 249                 if (start < 0)
 250                         start += self->size;
 251                 if (start < 0)
 252                         start = 0;
 253                 else if ((size_t)start > self->size)
 254                         start = self->size;
 255
 256                 for (p = self->data + start; p + len <= e; ++p) {
 257                         Py_ssize_t i;
 258                         for (i = 0; i < len && needle[i] == p[i]; ++i)
 259                                 /* nothing */;
 260                         if (i == len) {
 261                                 return PyInt_FromSsize_t(p - self->data);
 262                         }
 263                 }
 264                 return PyInt_FromLong(-1);
 265         }
 266 }
 267
 268 static int
 269 is_writeable(mmap_object *self)
 270 {
 271         if (self->access != ACCESS_READ)
 272                 return 1;
 273         PyErr_Format(PyExc_TypeError, "mmap can't modify a readonly memory map.");
 274         return 0;
 275 }
 276
 277 static int
 278 is_resizeable(mmap_object *self)
 279 {
 280         if ((self->access == ACCESS_WRITE) || (self->access == ACCESS_DEFAULT))
 281                 return 1;
 282         PyErr_Format(PyExc_TypeError,
 283                      "mmap can't resize a readonly or copy-on-write memory map.");
 284         return 0;
 285 }
 286
 287
 288 static PyObject *
 289 mmap_write_method(mmap_object *self,
 290                   PyObject *args)
 291 {
 292         Py_ssize_t length;
 293         char *data;
 294
 295         CHECK_VALID(NULL);
 296         if (!PyArg_ParseTuple(args, "s#:write", &data, &length))
 297                 return(NULL);
 298
 299         if (!is_writeable(self))
 300                 return NULL;
 301
 302         if ((self->pos + length) > self->size) {
 303                 PyErr_SetString(PyExc_ValueError, "data out of range");
 304                 return NULL;
 305         }
 306         memcpy(self->data+self->pos, data, length);
 307         self->pos = self->pos+length;
 308         Py_INCREF(Py_None);
 309         return Py_None;
 310 }
 311
 312 static PyObject *
 313 mmap_write_byte_method(mmap_object *self,
 314                        PyObject *args)
 315 {
 316         char value;
 317
 318         CHECK_VALID(NULL);
 319         if (!PyArg_ParseTuple(args, "c:write_byte", &value))
 320                 return(NULL);
 321
 322         if (!is_writeable(self))
 323                 return NULL;
 324         *(self->data+self->pos) = value;
 325         self->pos += 1;
 326         Py_INCREF(Py_None);
 327         return Py_None;
 328 }
 329
 330 static PyObject *
 331 mmap_size_method(mmap_object *self,
 332                  PyObject *unused)
 333 {
 334         CHECK_VALID(NULL);
 335
 336 #ifdef MS_WINDOWS
 337         if (self->file_handle != INVALID_HANDLE_VALUE) {
 338                 DWORD low,high;
 339                 PY_LONG_LONG size;
 340                 low = GetFileSize(self->file_handle, &high);
 341                 if (low == INVALID_FILE_SIZE) {
 342                         /* It might be that the function appears to have failed,
 343                            when indeed its size equals INVALID_FILE_SIZE */
 344                         DWORD error = GetLastError();
 345                         if (error != NO_ERROR)
 346                                 return PyErr_SetFromWindowsErr(error);
 347                 }
 348                 if (!high && low < LONG_MAX)
 349                         return PyInt_FromLong((long)low);
 350                 size = (((PY_LONG_LONG)high)<<32) + low;
 351                 return PyLong_FromLongLong(size);
 352         } else {
 353                 return PyInt_FromSsize_t(self->size);
 354         }
 355 #endif /* MS_WINDOWS */
 356
 357 #ifdef UNIX
 358         {
 359                 struct stat buf;
 360                 if (-1 == fstat(self->fd, &buf)) {
 361                         PyErr_SetFromErrno(mmap_module_error);
 362                         return NULL;
 363                 }
 364                 return PyInt_FromSsize_t(buf.st_size);
 365         }
 366 #endif /* UNIX */
 367 }
 368
 369 /* This assumes that you want the entire file mapped,
 370  / and when recreating the map will make the new file
 371  / have the new size
 372  /
 373  / Is this really necessary?  This could easily be done
 374  / from python by just closing and re-opening with the
 375  / new size?
 376  */
 377
 378 static PyObject *
 379 mmap_resize_method(mmap_object *self,
 380                    PyObject *args)
 381 {
 382         Py_ssize_t new_size;
 383         CHECK_VALID(NULL);
 384         if (!PyArg_ParseTuple(args, "n:resize", &new_size) ||
 385             !is_resizeable(self)) {
 386                 return NULL;
 387 #ifdef MS_WINDOWS
 388         } else {
 389                 DWORD dwErrCode = 0;
 390                 DWORD newSizeLow, newSizeHigh;
 391                 /* First, unmap the file view */
 392                 UnmapViewOfFile(self->data);
 393                 /* Close the mapping object */
 394                 CloseHandle(self->map_handle);
 395                 /* Move to the desired EOF position */
 396 #if SIZEOF_SIZE_T > 4
 397                 newSizeHigh = (DWORD)(new_size >> 32);
 398                 newSizeLow = (DWORD)(new_size & 0xFFFFFFFF);
 399 #else
 400                 newSizeHigh = 0;
 401                 newSizeLow = (DWORD)new_size;
 402 #endif
 403                 SetFilePointer(self->file_handle,
 404                                newSizeLow, &newSizeHigh, FILE_BEGIN);
 405                 /* Change the size of the file */
 406                 SetEndOfFile(self->file_handle);
 407                 /* Create another mapping object and remap the file view */
 408                 self->map_handle = CreateFileMapping(
 409                         self->file_handle,
 410                         NULL,
 411                         PAGE_READWRITE,
 412                         newSizeHigh,
 413                         newSizeLow,
 414                         self->tagname);
 415                 if (self->map_handle != NULL) {
 416                         self->data = (char *) MapViewOfFile(self->map_handle,
 417                                                             FILE_MAP_WRITE,
 418                                                             0,
 419                                                             0,
 420                                                             0);
 421                         if (self->data != NULL) {
 422                                 self->size = new_size;
 423                                 Py_INCREF(Py_None);
 424                                 return Py_None;
 425                         } else {
 426                                 dwErrCode = GetLastError();
 427                         }
 428                 } else {
 429                         dwErrCode = GetLastError();
 430                 }
 431                 PyErr_SetFromWindowsErr(dwErrCode);
 432                 return NULL;
 433 #endif /* MS_WINDOWS */
 434
 435 #ifdef UNIX
 436 #ifndef HAVE_MREMAP
 437         } else {
 438                 PyErr_SetString(PyExc_SystemError,
 439                                 "mmap: resizing not available--no mremap()");
 440                 return NULL;
 441 #else
 442         } else {
 443                 void *newmap;
 444
 445                 if (ftruncate(self->fd, new_size) == -1) {
 446                         PyErr_SetFromErrno(mmap_module_error);
 447                         return NULL;
 448                 }
 449
 450 #ifdef MREMAP_MAYMOVE
 451                 newmap = mremap(self->data, self->size, new_size, MREMAP_MAYMOVE);
 452 #else
 453                 newmap = mremap(self->data, self->size, new_size, 0);
 454 #endif
 455                 if (newmap == (void *)-1)
 456                 {
 457                         PyErr_SetFromErrno(mmap_module_error);
 458                         return NULL;
 459                 }
 460                 self->data = newmap;
 461                 self->size = new_size;
 462                 Py_INCREF(Py_None);
 463                 return Py_None;
 464 #endif /* HAVE_MREMAP */
 465 #endif /* UNIX */
 466         }
 467 }
 468
 469 static PyObject *
 470 mmap_tell_method(mmap_object *self, PyObject *unused)
 471 {
 472         CHECK_VALID(NULL);
 473         return PyInt_FromSize_t(self->pos);
 474 }
 475
 476 static PyObject *
 477 mmap_flush_method(mmap_object *self, PyObject *args)
 478 {
 479         Py_ssize_t offset = 0;
 480         Py_ssize_t size = self->size;
 481         CHECK_VALID(NULL);
 482         if (!PyArg_ParseTuple(args, "|nn:flush", &offset, &size))
 483                 return NULL;
 484         if ((size_t)(offset + size) > self->size) {
 485                 PyErr_SetString(PyExc_ValueError, "flush values out of range");
 486                 return NULL;
 487         } else {
 488 #ifdef MS_WINDOWS
 489                 return PyInt_FromLong((long)
 490                                       FlushViewOfFile(self->data+offset, size));
 491 #endif /* MS_WINDOWS */
 492 #ifdef UNIX
 493                 /* XXX semantics of return value? */
 494                 /* XXX flags for msync? */
 495                 if (-1 == msync(self->data + offset, size,
 496                                 MS_SYNC))
 497                 {
 498                         PyErr_SetFromErrno(mmap_module_error);
 499                         return NULL;
 500                 }
 501                 return PyInt_FromLong(0);
 502 #endif /* UNIX */
 503         }
 504 }
 505
 506 static PyObject *
 507 mmap_seek_method(mmap_object *self, PyObject *args)
 508 {
 509         Py_ssize_t dist;
 510         int how=0;
 511         CHECK_VALID(NULL);
 512         if (!PyArg_ParseTuple(args, "n|i:seek", &dist, &how))
 513                 return NULL;
 514         else {
 515                 size_t where;
 516                 switch (how) {
 517                 case 0: /* relative to start */
 518                         if (dist < 0)
 519                                 goto onoutofrange;
 520                         where = dist;
 521                         break;
 522                 case 1: /* relative to current position */
 523                         if ((Py_ssize_t)self->pos + dist < 0)
 524                                 goto onoutofrange;
 525                         where = self->pos + dist;
 526                         break;
 527                 case 2: /* relative to end */
 528                         if ((Py_ssize_t)self->size + dist < 0)
 529                                 goto onoutofrange;
 530                         where = self->size + dist;
 531                         break;
 532                 default:
 533                         PyErr_SetString(PyExc_ValueError, "unknown seek type");
 534                         return NULL;
 535                 }
 536                 if (where > self->size)
 537                         goto onoutofrange;
 538                 self->pos = where;
 539                 Py_INCREF(Py_None);
 540                 return Py_None;
 541         }
 542
 543   onoutofrange:
 544         PyErr_SetString(PyExc_ValueError, "seek out of range");
 545         return NULL;
 546 }
 547
 548 static PyObject *
 549 mmap_move_method(mmap_object *self, PyObject *args)
 550 {
 551         unsigned long dest, src, count;
 552         CHECK_VALID(NULL);
 553         if (!PyArg_ParseTuple(args, "kkk:move", &dest, &src, &count) ||
 554             !is_writeable(self)) {
 555                 return NULL;
 556         } else {
 557                 /* bounds check the values */
 558                 if (/* end of source after end of data?? */
 559                         ((src+count) > self->size)
 560                         /* dest will fit? */
 561                         || (dest+count > self->size)) {
 562                         PyErr_SetString(PyExc_ValueError,
 563                                         "source or destination out of range");
 564                         return NULL;
 565                 } else {
 566                         memmove(self->data+dest, self->data+src, count);
 567                         Py_INCREF(Py_None);
 568                         return Py_None;
 569                 }
 570         }
 571 }
 572
 573 static struct PyMethodDef mmap_object_methods[] = {
 574         {"close",       (PyCFunction) mmap_close_method,        METH_NOARGS},
 575         {"find",        (PyCFunction) mmap_find_method,         METH_VARARGS},
 576         {"flush",       (PyCFunction) mmap_flush_method,        METH_VARARGS},
 577         {"move",        (PyCFunction) mmap_move_method,         METH_VARARGS},
 578         {"read",        (PyCFunction) mmap_read_method,         METH_VARARGS},
 579         {"read_byte",   (PyCFunction) mmap_read_byte_method,    METH_NOARGS},
 580         {"readline",    (PyCFunction) mmap_read_line_method,    METH_NOARGS},
 581         {"resize",      (PyCFunction) mmap_resize_method,       METH_VARARGS},
 582         {"seek",        (PyCFunction) mmap_seek_method,         METH_VARARGS},
 583         {"size",        (PyCFunction) mmap_size_method,         METH_NOARGS},
 584         {"tell",        (PyCFunction) mmap_tell_method,         METH_NOARGS},
 585         {"write",       (PyCFunction) mmap_write_method,        METH_VARARGS},
 586         {"write_byte",  (PyCFunction) mmap_write_byte_method,   METH_VARARGS},
 587         {NULL,     NULL}       /* sentinel */
 588 };
 589
 590 /* Functions for treating an mmap'ed file as a buffer */
 591
 592 static Py_ssize_t
 593 mmap_buffer_getreadbuf(mmap_object *self, Py_ssize_t index, const void **ptr)
 594 {
 595         CHECK_VALID(-1);
 596         if (index != 0) {
 597                 PyErr_SetString(PyExc_SystemError,
 598                                 "Accessing non-existent mmap segment");
 599                 return -1;
 600         }
 601         *ptr = self->data;
 602         return self->size;
 603 }
 604
 605 static Py_ssize_t
 606 mmap_buffer_getwritebuf(mmap_object *self, Py_ssize_t index, const void **ptr)
 607 {
 608         CHECK_VALID(-1);
 609         if (index != 0) {
 610                 PyErr_SetString(PyExc_SystemError,
 611                                 "Accessing non-existent mmap segment");
 612                 return -1;
 613         }
 614         if (!is_writeable(self))
 615                 return -1;
 616         *ptr = self->data;
 617         return self->size;
 618 }
 619
 620 static Py_ssize_t
 621 mmap_buffer_getsegcount(mmap_object *self, Py_ssize_t *lenp)
 622 {
 623         CHECK_VALID(-1);
 624         if (lenp)
 625                 *lenp = self->size;
 626         return 1;
 627 }
 628
 629 static Py_ssize_t
 630 mmap_buffer_getcharbuffer(mmap_object *self, Py_ssize_t index, const void **ptr)
 631 {
 632         if (index != 0) {
 633                 PyErr_SetString(PyExc_SystemError,
 634                                 "accessing non-existent buffer segment");
 635                 return -1;
 636         }
 637         *ptr = (const char *)self->data;
 638         return self->size;
 639 }
 640
 641 static PyObject *
 642 mmap_object_getattr(mmap_object *self, char *name)
 643 {
 644         return Py_FindMethod(mmap_object_methods, (PyObject *)self, name);
 645 }
 646
 647 static Py_ssize_t
 648 mmap_length(mmap_object *self)
 649 {
 650         CHECK_VALID(-1);
 651         return self->size;
 652 }
 653
 654 static PyObject *
 655 mmap_item(mmap_object *self, Py_ssize_t i)
 656 {
 657         CHECK_VALID(NULL);
 658         if (i < 0 || (size_t)i >= self->size) {
 659                 PyErr_SetString(PyExc_IndexError, "mmap index out of range");
 660                 return NULL;
 661         }
 662         return PyString_FromStringAndSize(self->data + i, 1);
 663 }
 664
 665 static PyObject *
 666 mmap_slice(mmap_object *self, Py_ssize_t ilow, Py_ssize_t ihigh)
 667 {
 668         CHECK_VALID(NULL);
 669         if (ilow < 0)
 670                 ilow = 0;
 671         else if ((size_t)ilow > self->size)
 672                 ilow = self->size;
 673         if (ihigh < 0)
 674                 ihigh = 0;
 675         if (ihigh < ilow)
 676                 ihigh = ilow;
 677         else if ((size_t)ihigh > self->size)
 678                 ihigh = self->size;
 679
 680         return PyString_FromStringAndSize(self->data + ilow, ihigh-ilow);
 681 }
 682
 683 static PyObject *
 684 mmap_subscript(mmap_object *self, PyObject *item)
 685 {
 686         CHECK_VALID(NULL);
 687         if (PyIndex_Check(item)) {
 688                 Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError);
 689                 if (i == -1 && PyErr_Occurred())
 690                         return NULL;
 691                 if (i < 0)
 692                         i += self->size;
 693                 if (i < 0 || i > self->size) {
 694                         PyErr_SetString(PyExc_IndexError,
 695                                 "mmap index out of range");
 696                         return NULL;
 697                 }
 698                 return PyString_FromStringAndSize(self->data + i, 1);
 699         }
 700         else if (PySlice_Check(item)) {
 701                 Py_ssize_t start, stop, step, slicelen;
 702
 703                 if (PySlice_GetIndicesEx((PySliceObject *)item, self->size,
 704                                  &start, &stop, &step, &slicelen) < 0) {
 705                         return NULL;
 706                 }
 707
 708                 if (slicelen <= 0)
 709                         return PyString_FromStringAndSize("", 0);
 710                 else if (step == 1)
 711                         return PyString_FromStringAndSize(self->data + start,
 712                                                           slicelen);
 713                 else {
 714                         char *result_buf = (char *)PyMem_Malloc(slicelen);
 715                         Py_ssize_t cur, i;
 716                         PyObject *result;
 717
 718                         if (result_buf == NULL)
 719                                 return PyErr_NoMemory();
 720                         for (cur = start, i = 0; i < slicelen;
 721                              cur += step, i++) {
 722                                 result_buf[i] = self->data[cur];
 723                         }
 724                         result = PyString_FromStringAndSize(result_buf,
 725                                                             slicelen);
 726                         PyMem_Free(result_buf);
 727                         return result;
 728                 }
 729         }
 730         else {
 731                 PyErr_SetString(PyExc_TypeError,
 732                                 "mmap indices must be integers");
 733                 return NULL;
 734         }
 735 }
 736
 737 static PyObject *
 738 mmap_concat(mmap_object *self, PyObject *bb)
 739 {
 740         CHECK_VALID(NULL);
 741         PyErr_SetString(PyExc_SystemError,
 742                         "mmaps don't support concatenation");
 743         return NULL;
 744 }
 745
 746 static PyObject *
 747 mmap_repeat(mmap_object *self, Py_ssize_t n)
 748 {
 749         CHECK_VALID(NULL);
 750         PyErr_SetString(PyExc_SystemError,
 751                         "mmaps don't support repeat operation");
 752         return NULL;
 753 }
 754
 755 static int
 756 mmap_ass_slice(mmap_object *self, Py_ssize_t ilow, Py_ssize_t ihigh, PyObject *v)
 757 {
 758         const char *buf;
 759
 760         CHECK_VALID(-1);
 761         if (ilow < 0)
 762                 ilow = 0;
 763         else if ((size_t)ilow > self->size)
 764                 ilow = self->size;
 765         if (ihigh < 0)
 766                 ihigh = 0;
 767         if (ihigh < ilow)
 768                 ihigh = ilow;
 769         else if ((size_t)ihigh > self->size)
 770                 ihigh = self->size;
 771
 772         if (v == NULL) {
 773                 PyErr_SetString(PyExc_TypeError,
 774                                 "mmap object doesn't support slice deletion");
 775                 return -1;
 776         }
 777         if (! (PyString_Check(v)) ) {
 778                 PyErr_SetString(PyExc_IndexError,
 779                                 "mmap slice assignment must be a string");
 780                 return -1;
 781         }
 782         if (PyString_Size(v) != (ihigh - ilow)) {
 783                 PyErr_SetString(PyExc_IndexError,
 784                                 "mmap slice assignment is wrong size");
 785                 return -1;
 786         }
 787         if (!is_writeable(self))
 788                 return -1;
 789         buf = PyString_AsString(v);
 790         memcpy(self->data + ilow, buf, ihigh-ilow);
 791         return 0;
 792 }
 793
 794 static int
 795 mmap_ass_item(mmap_object *self, Py_ssize_t i, PyObject *v)
 796 {
 797         const char *buf;
 798
 799         CHECK_VALID(-1);
 800         if (i < 0 || (size_t)i >= self->size) {
 801                 PyErr_SetString(PyExc_IndexError, "mmap index out of range");
 802                 return -1;
 803         }
 804         if (v == NULL) {
 805                 PyErr_SetString(PyExc_TypeError,
 806                                 "mmap object doesn't support item deletion");
 807                 return -1;
 808         }
 809         if (! (PyString_Check(v) && PyString_Size(v)==1) ) {
 810                 PyErr_SetString(PyExc_IndexError,
 811                                 "mmap assignment must be single-character string");
 812                 return -1;
 813         }
 814         if (!is_writeable(self))
 815                 return -1;
 816         buf = PyString_AsString(v);
 817         self->data[i] = buf[0];
 818         return 0;
 819 }
 820
 821 static int
 822 mmap_ass_subscript(mmap_object *self, PyObject *item, PyObject *value)
 823 {
 824         CHECK_VALID(-1);
 825
 826         if (PyIndex_Check(item)) {
 827                 Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError);
 828                 const char *buf;
 829
 830                 if (i == -1 && PyErr_Occurred())
 831                         return -1;
 832                 if (i < 0)
 833                         i += self->size;
 834                 if (i < 0 || i > self->size) {
 835                         PyErr_SetString(PyExc_IndexError,
 836                                 "mmap index out of range");
 837                         return -1;
 838                 }
 839                 if (value == NULL) {
 840                         PyErr_SetString(PyExc_TypeError,
 841                                 "mmap object doesn't support item deletion");
 842                         return -1;
 843                 }
 844                 if (!PyString_Check(value) || PyString_Size(value) != 1) {
 845                         PyErr_SetString(PyExc_IndexError,
 846                           "mmap assignment must be single-character string");
 847                         return -1;
 848                 }
 849                 if (!is_writeable(self))
 850                         return -1;
 851                 buf = PyString_AsString(value);
 852                 self->data[i] = buf[0];
 853                 return 0;
 854         }
 855         else if (PySlice_Check(item)) {
 856                 Py_ssize_t start, stop, step, slicelen;
 857
 858                 if (PySlice_GetIndicesEx((PySliceObject *)item,
 859                                          self->size, &start, &stop,
 860                                          &step, &slicelen) < 0) {
 861                         return -1;
 862                 }
 863                 if (value == NULL) {
 864                         PyErr_SetString(PyExc_TypeError,
 865                                 "mmap object doesn't support slice deletion");
 866                         return -1;
 867                 }
 868                 if (!PyString_Check(value)) {
 869                         PyErr_SetString(PyExc_IndexError,
 870                                 "mmap slice assignment must be a string");
 871                         return -1;
 872                 }
 873                 if (PyString_Size(value) != slicelen) {
 874                         PyErr_SetString(PyExc_IndexError,
 875                                 "mmap slice assignment is wrong size");
 876                         return -1;
 877                 }
 878                 if (!is_writeable(self))
 879                         return -1;
 880
 881                 if (slicelen == 0)
 882                         return 0;
 883                 else if (step == 1) {
 884                         const char *buf = PyString_AsString(value);
 885
 886                         if (buf == NULL)
 887                                 return -1;
 888                         memcpy(self->data + start, buf, slicelen);
 889                         return 0;
 890                 }
 891                 else {
 892                         Py_ssize_t cur, i;
 893                         const char *buf = PyString_AsString(value);
 894
 895                         if (buf == NULL)
 896                                 return -1;
 897                         for (cur = start, i = 0; i < slicelen;
 898                              cur += step, i++) {
 899                                 self->data[cur] = buf[i];
 900                         }
 901                         return 0;
 902                 }
 903         }
 904         else {
 905                 PyErr_SetString(PyExc_TypeError,
 906                                 "mmap indices must be integer");
 907                 return -1;
 908         }
 909 }
 910
 911 static PySequenceMethods mmap_as_sequence = {
 912         (lenfunc)mmap_length,                  /*sq_length*/
 913         (binaryfunc)mmap_concat,               /*sq_concat*/
 914         (ssizeargfunc)mmap_repeat,             /*sq_repeat*/
 915         (ssizeargfunc)mmap_item,                       /*sq_item*/
 916         (ssizessizeargfunc)mmap_slice,         /*sq_slice*/
 917         (ssizeobjargproc)mmap_ass_item,        /*sq_ass_item*/
 918         (ssizessizeobjargproc)mmap_ass_slice,      /*sq_ass_slice*/
 919 };
 920
 921 static PyMappingMethods mmap_as_mapping = {
 922         (lenfunc)mmap_length,
 923         (binaryfunc)mmap_subscript,
 924         (objobjargproc)mmap_ass_subscript,
 925 };
 926
 927 static PyBufferProcs mmap_as_buffer = {
 928         (readbufferproc)mmap_buffer_getreadbuf,
 929         (writebufferproc)mmap_buffer_getwritebuf,
 930         (segcountproc)mmap_buffer_getsegcount,
 931         (charbufferproc)mmap_buffer_getcharbuffer,
 932 };
 933
 934 static PyTypeObject mmap_object_type = {
 935         PyVarObject_HEAD_INIT(0, 0) /* patched in module init */
 936         "mmap.mmap",                            /* tp_name */
 937         sizeof(mmap_object),                    /* tp_size */
 938         0,                                      /* tp_itemsize */
 939         /* methods */
 940         (destructor) mmap_object_dealloc,       /* tp_dealloc */
 941         0,                                      /* tp_print */
 942         (getattrfunc) mmap_object_getattr,      /* tp_getattr */
 943         0,                                      /* tp_setattr */
 944         0,                                      /* tp_compare */
 945         0,                                      /* tp_repr */
 946         0,                                      /* tp_as_number */
 947         &mmap_as_sequence,                      /*tp_as_sequence*/
 948         &mmap_as_mapping,                       /*tp_as_mapping*/
 949         0,                                      /*tp_hash*/
 950         0,                                      /*tp_call*/
 951         0,                                      /*tp_str*/
 952         0,                                      /*tp_getattro*/
 953         0,                                      /*tp_setattro*/
 954         &mmap_as_buffer,                        /*tp_as_buffer*/
 955         Py_TPFLAGS_HAVE_GETCHARBUFFER,          /*tp_flags*/
 956         0,                                      /*tp_doc*/
 957 };
 958
 959
 960 /* extract the map size from the given PyObject
 961
 962    Returns -1 on error, with an appropriate Python exception raised. On
 963    success, the map size is returned. */
 964 static Py_ssize_t
 965 _GetMapSize(PyObject *o)
 966 {
 967         if (PyIndex_Check(o)) {
 968                 Py_ssize_t i = PyNumber_AsSsize_t(o, PyExc_OverflowError);
 969                 if (i==-1 && PyErr_Occurred())
 970                         return -1;
 971                 if (i < 0) {
 972                         PyErr_SetString(PyExc_OverflowError,
 973                                         "memory mapped size must be positive");
 974                         return -1;
 975                 }
 976                 return i;
 977         }
 978
 979         PyErr_SetString(PyExc_TypeError, "map size must be an integral value");
 980         return -1;
 981 }
 982
 983 #ifdef UNIX
 984 static PyObject *
 985 new_mmap_object(PyObject *self, PyObject *args, PyObject *kwdict)
 986 {
 987 #ifdef HAVE_FSTAT
 988         struct stat st;
 989 #endif
 990         mmap_object *m_obj;
 991         PyObject *map_size_obj = NULL;
 992         Py_ssize_t map_size;
 993         int fd, flags = MAP_SHARED, prot = PROT_WRITE | PROT_READ;
 994         int devzero = -1;
 995         int access = (int)ACCESS_DEFAULT;
 996         static char *keywords[] = {"fileno", "length",
 997                                          "flags", "prot",
 998                                          "access", NULL};
 999
1000         if (!PyArg_ParseTupleAndKeywords(args, kwdict, "iO|iii", keywords,
1001                                          &fd, &map_size_obj, &flags, &prot,
1002                                          &access))
1003                 return NULL;
1004         map_size = _GetMapSize(map_size_obj);
1005         if (map_size < 0)
1006                 return NULL;
1007
1008         if ((access != (int)ACCESS_DEFAULT) &&
1009             ((flags != MAP_SHARED) || (prot != (PROT_WRITE | PROT_READ))))
1010                 return PyErr_Format(PyExc_ValueError,
1011                                     "mmap can't specify both access and flags, prot.");
1012         switch ((access_mode)access) {
1013         case ACCESS_READ:
1014                 flags = MAP_SHARED;
1015                 prot = PROT_READ;
1016                 break;
1017         case ACCESS_WRITE:
1018                 flags = MAP_SHARED;
1019                 prot = PROT_READ | PROT_WRITE;
1020                 break;
1021         case ACCESS_COPY:
1022                 flags = MAP_PRIVATE;
1023                 prot = PROT_READ | PROT_WRITE;
1024                 break;
1025         case ACCESS_DEFAULT:
1026                 /* use the specified or default values of flags and prot */
1027                 break;
1028         default:
1029                 return PyErr_Format(PyExc_ValueError,
1030                                     "mmap invalid access parameter.");
1031         }
1032
1033 #ifdef HAVE_FSTAT
1034 #  ifdef __VMS
1035         /* on OpenVMS we must ensure that all bytes are written to the file */
1036         fsync(fd);
1037 #  endif
1038         if (fstat(fd, &st) == 0 && S_ISREG(st.st_mode)) {
1039                 if (map_size == 0) {
1040                         map_size = st.st_size;
1041                 } else if ((size_t)map_size > st.st_size) {
1042                         PyErr_SetString(PyExc_ValueError,
1043                                         "mmap length is greater than file size");
1044                         return NULL;
1045                 }
1046         }
1047 #endif
1048         m_obj = PyObject_New(mmap_object, &mmap_object_type);
1049         if (m_obj == NULL) {return NULL;}
1050         m_obj->data = NULL;
1051         m_obj->size = (size_t) map_size;
1052         m_obj->pos = (size_t) 0;
1053         if (fd == -1) {
1054                 m_obj->fd = -1;
1055                 /* Assume the caller wants to map anonymous memory.
1056                    This is the same behaviour as Windows.  mmap.mmap(-1, size)
1057                    on both Windows and Unix map anonymous memory.
1058                 */
1059 #ifdef MAP_ANONYMOUS
1060                 /* BSD way to map anonymous memory */
1061                 flags |= MAP_ANONYMOUS;
1062 #else
1063                 /* SVR4 method to map anonymous memory is to open /dev/zero */
1064                 fd = devzero = open("/dev/zero", O_RDWR);
1065                 if (devzero == -1) {
1066                         Py_DECREF(m_obj);
1067                         PyErr_SetFromErrno(mmap_module_error);
1068                         return NULL;
1069                 }
1070 #endif
1071         } else {
1072                 m_obj->fd = dup(fd);
1073                 if (m_obj->fd == -1) {
1074                         Py_DECREF(m_obj);
1075                         PyErr_SetFromErrno(mmap_module_error);
1076                         return NULL;
1077                 }
1078         }
1079
1080         m_obj->data = mmap(NULL, map_size,
1081                            prot, flags,
1082                            fd, 0);
1083
1084         if (devzero != -1) {
1085                 close(devzero);
1086         }
1087
1088         if (m_obj->data == (char *)-1) {
1089                 m_obj->data = NULL;
1090                 Py_DECREF(m_obj);
1091                 PyErr_SetFromErrno(mmap_module_error);
1092                 return NULL;
1093         }
1094         m_obj->access = (access_mode)access;
1095         return (PyObject *)m_obj;
1096 }
1097 #endif /* UNIX */
1098
1099 #ifdef MS_WINDOWS
1100 static PyObject *
1101 new_mmap_object(PyObject *self, PyObject *args, PyObject *kwdict)
1102 {
1103         mmap_object *m_obj;
1104         PyObject *map_size_obj = NULL;
1105         Py_ssize_t map_size;
1106         DWORD size_hi;  /* upper 32 bits of m_obj->size */
1107         DWORD size_lo;  /* lower 32 bits of m_obj->size */
1108         char *tagname = "";
1109         DWORD dwErr = 0;
1110         int fileno;
1111         HANDLE fh = 0;
1112         int access = (access_mode)ACCESS_DEFAULT;
1113         DWORD flProtect, dwDesiredAccess;
1114         static char *keywords[] = { "fileno", "length",
1115                                           "tagname",
1116                                           "access", NULL };
1117
1118         if (!PyArg_ParseTupleAndKeywords(args, kwdict, "iO|zi", keywords,
1119                                          &fileno, &map_size_obj,
1120                                          &tagname, &access)) {
1121                 return NULL;
1122         }
1123
1124         switch((access_mode)access) {
1125         case ACCESS_READ:
1126                 flProtect = PAGE_READONLY;
1127                 dwDesiredAccess = FILE_MAP_READ;
1128                 break;
1129         case ACCESS_DEFAULT:  case ACCESS_WRITE:
1130                 flProtect = PAGE_READWRITE;
1131                 dwDesiredAccess = FILE_MAP_WRITE;
1132                 break;
1133         case ACCESS_COPY:
1134                 flProtect = PAGE_WRITECOPY;
1135                 dwDesiredAccess = FILE_MAP_COPY;
1136                 break;
1137         default:
1138                 return PyErr_Format(PyExc_ValueError,
1139                                     "mmap invalid access parameter.");
1140         }
1141
1142         map_size = _GetMapSize(map_size_obj);
1143         if (map_size < 0)
1144                 return NULL;
1145
1146         /* assume -1 and 0 both mean invalid filedescriptor
1147            to 'anonymously' map memory.
1148            XXX: fileno == 0 is a valid fd, but was accepted prior to 2.5.
1149            XXX: Should this code be added?
1150            if (fileno == 0)
1151                 PyErr_Warn(PyExc_DeprecationWarning,
1152                            "don't use 0 for anonymous memory");
1153          */
1154         if (fileno != -1 && fileno != 0) {
1155                 fh = (HANDLE)_get_osfhandle(fileno);
1156                 if (fh==(HANDLE)-1) {
1157                         PyErr_SetFromErrno(mmap_module_error);
1158                         return NULL;
1159                 }
1160                 /* Win9x appears to need us seeked to zero */
1161                 lseek(fileno, 0, SEEK_SET);
1162         }
1163
1164         m_obj = PyObject_New(mmap_object, &mmap_object_type);
1165         if (m_obj == NULL)
1166                 return NULL;
1167         /* Set every field to an invalid marker, so we can safely
1168            destruct the object in the face of failure */
1169         m_obj->data = NULL;
1170         m_obj->file_handle = INVALID_HANDLE_VALUE;
1171         m_obj->map_handle = INVALID_HANDLE_VALUE;
1172         m_obj->tagname = NULL;
1173
1174         if (fh) {
1175                 /* It is necessary to duplicate the handle, so the
1176                    Python code can close it on us */
1177                 if (!DuplicateHandle(
1178                         GetCurrentProcess(), /* source process handle */
1179                         fh, /* handle to be duplicated */
1180                         GetCurrentProcess(), /* target proc handle */
1181                         (LPHANDLE)&m_obj->file_handle, /* result */
1182                         0, /* access - ignored due to options value */
1183                         FALSE, /* inherited by child processes? */
1184                         DUPLICATE_SAME_ACCESS)) { /* options */
1185                         dwErr = GetLastError();
1186                         Py_DECREF(m_obj);
1187                         PyErr_SetFromWindowsErr(dwErr);
1188                         return NULL;
1189                 }
1190                 if (!map_size) {
1191                         DWORD low,high;
1192                         low = GetFileSize(fh, &high);
1193                         /* low might just happen to have the value INVALID_FILE_SIZE;
1194                            so we need to check the last error also. */
1195                         if (low == INVALID_FILE_SIZE &&
1196                             (dwErr = GetLastError()) != NO_ERROR) {
1197                                 Py_DECREF(m_obj);
1198                                 return PyErr_SetFromWindowsErr(dwErr);
1199                         }
1200
1201 #if SIZEOF_SIZE_T > 4
1202                         m_obj->size = (((size_t)high)<<32) + low;
1203 #else
1204                         if (high)
1205                                 /* File is too large to map completely */
1206                                 m_obj->size = (size_t)-1;
1207                         else
1208                                 m_obj->size = low;
1209 #endif
1210                 } else {
1211                         m_obj->size = map_size;
1212                 }
1213         }
1214         else {
1215                 m_obj->size = map_size;
1216         }
1217
1218         /* set the initial position */
1219         m_obj->pos = (size_t) 0;
1220
1221         /* set the tag name */
1222         if (tagname != NULL && *tagname != '\0') {
1223                 m_obj->tagname = PyMem_Malloc(strlen(tagname)+1);
1224                 if (m_obj->tagname == NULL) {
1225                         PyErr_NoMemory();
1226                         Py_DECREF(m_obj);
1227                         return NULL;
1228                 }
1229                 strcpy(m_obj->tagname, tagname);
1230         }
1231         else
1232                 m_obj->tagname = NULL;
1233
1234         m_obj->access = (access_mode)access;
1235         /* DWORD is a 4-byte int.  If we're on a box where size_t consumes
1236          * more than 4 bytes, we need to break it apart.  Else (size_t
1237          * consumes 4 bytes), C doesn't define what happens if we shift
1238          * right by 32, so we need different code.
1239          */
1240 #if SIZEOF_SIZE_T > 4
1241         size_hi = (DWORD)(m_obj->size >> 32);
1242         size_lo = (DWORD)(m_obj->size & 0xFFFFFFFF);
1243 #else
1244         size_hi = 0;
1245         size_lo = (DWORD)m_obj->size;
1246 #endif
1247         m_obj->map_handle = CreateFileMapping(m_obj->file_handle,
1248                                               NULL,
1249                                               flProtect,
1250                                               size_hi,
1251                                               size_lo,
1252                                               m_obj->tagname);
1253         if (m_obj->map_handle != NULL) {
1254                 m_obj->data = (char *) MapViewOfFile(m_obj->map_handle,
1255                                                      dwDesiredAccess,
1256                                                      0,
1257                                                      0,
1258                                                      0);
1259                 if (m_obj->data != NULL)
1260                         return (PyObject *)m_obj;
1261                 else
1262                         dwErr = GetLastError();
1263         } else
1264                 dwErr = GetLastError();
1265         Py_DECREF(m_obj);
1266         PyErr_SetFromWindowsErr(dwErr);
1267         return NULL;
1268 }
1269 #endif /* MS_WINDOWS */
1270
1271 /* List of functions exported by this module */
1272 static struct PyMethodDef mmap_functions[] = {
1273         {"mmap",        (PyCFunction) new_mmap_object,
1274          METH_VARARGS|METH_KEYWORDS},
1275         {NULL,          NULL}        /* Sentinel */
1276 };
1277
1278 static void
1279 setint(PyObject *d, const char *name, long value)
1280 {
1281         PyObject *o = PyInt_FromLong(value);
1282         if (o && PyDict_SetItemString(d, name, o) == 0) {
1283                 Py_DECREF(o);
1284         }
1285 }
1286
1287 PyMODINIT_FUNC
1288         initmmap(void)
1289 {
1290         PyObject *dict, *module;
1291
1292         /* Patch the object type */
1293         Py_Type(&mmap_object_type) = &PyType_Type;
1294
1295         module = Py_InitModule("mmap", mmap_functions);
1296         if (module == NULL)
1297                 return;
1298         dict = PyModule_GetDict(module);
1299         if (!dict)
1300                 return;
1301         mmap_module_error = PyExc_EnvironmentError;
1302         PyDict_SetItemString(dict, "error", mmap_module_error);
1303 #ifdef PROT_EXEC
1304         setint(dict, "PROT_EXEC", PROT_EXEC);
1305 #endif
1306 #ifdef PROT_READ
1307         setint(dict, "PROT_READ", PROT_READ);
1308 #endif
1309 #ifdef PROT_WRITE
1310         setint(dict, "PROT_WRITE", PROT_WRITE);
1311 #endif
1312
1313 #ifdef MAP_SHARED
1314         setint(dict, "MAP_SHARED", MAP_SHARED);
1315 #endif
1316 #ifdef MAP_PRIVATE
1317         setint(dict, "MAP_PRIVATE", MAP_PRIVATE);
1318 #endif
1319 #ifdef MAP_DENYWRITE
1320         setint(dict, "MAP_DENYWRITE", MAP_DENYWRITE);
1321 #endif
1322 #ifdef MAP_EXECUTABLE
1323         setint(dict, "MAP_EXECUTABLE", MAP_EXECUTABLE);
1324 #endif
1325 #ifdef MAP_ANONYMOUS
1326         setint(dict, "MAP_ANON", MAP_ANONYMOUS);
1327         setint(dict, "MAP_ANONYMOUS", MAP_ANONYMOUS);
1328 #endif
1329
1330         setint(dict, "PAGESIZE", (long)my_getpagesize());
1331
1332         setint(dict, "ACCESS_READ", ACCESS_READ);
1333         setint(dict, "ACCESS_WRITE", ACCESS_WRITE);
1334         setint(dict, "ACCESS_COPY", ACCESS_COPY);
1335 }