From 11eb2e4e3d92cd41236f81ded67735ba9c0ce31b Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Fri, 22 May 2015 17:10:34 +0200 Subject: [PATCH] pytdb: Port to Python 3 - Use bytes for all data, text strings for repr() - Use PyLong instead of PyInt on py3 - Use new module initialization - Update tests - Run tests in "make test" Signed-off-by: Petr Viktorin Reviewed-by: Andrew Bartlett Reviewed-by: Stefan Metzmacher --- lib/tdb/pytdb.c | 168 ++++++++++++++++++++++++++--------------- lib/tdb/python/tests/simple.py | 62 +++++++-------- lib/tdb/wscript | 5 +- 3 files changed, 141 insertions(+), 94 deletions(-) diff --git a/lib/tdb/pytdb.c b/lib/tdb/pytdb.c index bf50258572d..534e6131b99 100644 --- a/lib/tdb/pytdb.c +++ b/lib/tdb/pytdb.c @@ -31,13 +31,25 @@ /* Include tdb headers */ #include +#if PY_MAJOR_VERSION >= 3 +#define PyStr_FromString PyUnicode_FromString +#define PyStr_FromFormat PyUnicode_FromFormat +#define PyInt_FromLong PyLong_FromLong +#define PyInt_Check PyLong_Check +#define PyInt_AsLong PyLong_AsLong +#define Py_TPFLAGS_HAVE_ITER 0 +#else +#define PyStr_FromString PyString_FromString +#define PyStr_FromFormat PyString_FromFormat +#endif + typedef struct { PyObject_HEAD TDB_CONTEXT *ctx; bool closed; } PyTdbObject; -staticforward PyTypeObject PyTdb; +static PyTypeObject PyTdb; static void PyErr_SetTDBError(TDB_CONTEXT *tdb) { @@ -45,21 +57,21 @@ static void PyErr_SetTDBError(TDB_CONTEXT *tdb) Py_BuildValue("(i,s)", tdb_error(tdb), tdb_errorstr(tdb))); } -static TDB_DATA PyString_AsTDB_DATA(PyObject *data) +static TDB_DATA PyBytes_AsTDB_DATA(PyObject *data) { TDB_DATA ret; - ret.dptr = (unsigned char *)PyString_AsString(data); - ret.dsize = PyString_Size(data); + ret.dptr = (unsigned char *)PyBytes_AsString(data); + ret.dsize = PyBytes_Size(data); return ret; } -static PyObject *PyString_FromTDB_DATA(TDB_DATA data) +static PyObject *PyBytes_FromTDB_DATA(TDB_DATA data) { if (data.dptr == NULL && data.dsize == 0) { Py_RETURN_NONE; } else { - PyObject *ret = PyString_FromStringAndSize((const char *)data.dptr, - data.dsize); + PyObject *ret = PyBytes_FromStringAndSize((const char *)data.dptr, + data.dsize); free(data.dptr); return ret; } @@ -233,11 +245,11 @@ static PyObject *obj_get(PyTdbObject *self, PyObject *args) if (!PyArg_ParseTuple(args, "O", &py_key)) return NULL; - key = PyString_AsTDB_DATA(py_key); + key = PyBytes_AsTDB_DATA(py_key); if (!key.dptr) return NULL; - return PyString_FromTDB_DATA(tdb_fetch(self->ctx, key)); + return PyBytes_FromTDB_DATA(tdb_fetch(self->ctx, key)); } static PyObject *obj_append(PyTdbObject *self, PyObject *args) @@ -251,10 +263,10 @@ static PyObject *obj_append(PyTdbObject *self, PyObject *args) if (!PyArg_ParseTuple(args, "OO", &py_key, &py_data)) return NULL; - key = PyString_AsTDB_DATA(py_key); + key = PyBytes_AsTDB_DATA(py_key); if (!key.dptr) return NULL; - data = PyString_AsTDB_DATA(py_data); + data = PyBytes_AsTDB_DATA(py_data); if (!data.dptr) return NULL; @@ -267,7 +279,7 @@ static PyObject *obj_firstkey(PyTdbObject *self) { PyErr_TDB_RAISE_IF_CLOSED(self); - return PyString_FromTDB_DATA(tdb_firstkey(self->ctx)); + return PyBytes_FromTDB_DATA(tdb_firstkey(self->ctx)); } static PyObject *obj_nextkey(PyTdbObject *self, PyObject *args) @@ -279,11 +291,11 @@ static PyObject *obj_nextkey(PyTdbObject *self, PyObject *args) if (!PyArg_ParseTuple(args, "O", &py_key)) return NULL; - key = PyString_AsTDB_DATA(py_key); + key = PyBytes_AsTDB_DATA(py_key); if (!key.dptr) return NULL; - return PyString_FromTDB_DATA(tdb_nextkey(self->ctx, key)); + return PyBytes_FromTDB_DATA(tdb_nextkey(self->ctx, key)); } static PyObject *obj_delete(PyTdbObject *self, PyObject *args) @@ -296,7 +308,7 @@ static PyObject *obj_delete(PyTdbObject *self, PyObject *args) if (!PyArg_ParseTuple(args, "O", &py_key)) return NULL; - key = PyString_AsTDB_DATA(py_key); + key = PyBytes_AsTDB_DATA(py_key); if (!key.dptr) return NULL; ret = tdb_delete(self->ctx, key); @@ -314,7 +326,7 @@ static PyObject *obj_has_key(PyTdbObject *self, PyObject *args) if (!PyArg_ParseTuple(args, "O", &py_key)) return NULL; - key = PyString_AsTDB_DATA(py_key); + key = PyBytes_AsTDB_DATA(py_key); if (!key.dptr) return NULL; ret = tdb_exists(self->ctx, key); @@ -337,10 +349,10 @@ static PyObject *obj_store(PyTdbObject *self, PyObject *args) if (!PyArg_ParseTuple(args, "OO|i", &py_key, &py_value, &flag)) return NULL; - key = PyString_AsTDB_DATA(py_key); + key = PyBytes_AsTDB_DATA(py_key); if (!key.dptr) return NULL; - value = PyString_AsTDB_DATA(py_value); + value = PyBytes_AsTDB_DATA(py_value); if (!value.dptr) return NULL; @@ -389,7 +401,7 @@ static PyObject *tdb_iter_next(PyTdbIteratorObject *self) return NULL; current = self->current; self->current = tdb_nextkey(self->iteratee->ctx, self->current); - ret = PyString_FromTDB_DATA(current); + ret = PyBytes_FromTDB_DATA(current); return ret; } @@ -538,7 +550,7 @@ static PyObject *obj_get_flags(PyTdbObject *self, void *closure) static PyObject *obj_get_filename(PyTdbObject *self, void *closure) { PyErr_TDB_RAISE_IF_CLOSED(self); - return PyString_FromString(tdb_name(self->ctx)); + return PyBytes_FromString(tdb_name(self->ctx)); } static PyObject *obj_get_seqnum(PyTdbObject *self, void *closure) @@ -571,9 +583,9 @@ static PyObject *tdb_object_repr(PyTdbObject *self) { PyErr_TDB_RAISE_IF_CLOSED(self); if (tdb_get_flags(self->ctx) & TDB_INTERNAL) { - return PyString_FromString("Tdb()"); + return PyStr_FromString("Tdb()"); } else { - return PyString_FromFormat("Tdb('%s')", tdb_name(self->ctx)); + return PyStr_FromFormat("Tdb('%s')", tdb_name(self->ctx)); } } @@ -581,27 +593,27 @@ static void tdb_object_dealloc(PyTdbObject *self) { if (!self->closed) tdb_close(self->ctx); - self->ob_type->tp_free(self); + Py_TYPE(self)->tp_free(self); } static PyObject *obj_getitem(PyTdbObject *self, PyObject *key) { TDB_DATA tkey, val; PyErr_TDB_RAISE_IF_CLOSED(self); - if (!PyString_Check(key)) { - PyErr_SetString(PyExc_TypeError, "Expected string as key"); + if (!PyBytes_Check(key)) { + PyErr_SetString(PyExc_TypeError, "Expected bytestring as key"); return NULL; } - tkey.dptr = (unsigned char *)PyString_AsString(key); - tkey.dsize = PyString_Size(key); + tkey.dptr = (unsigned char *)PyBytes_AsString(key); + tkey.dsize = PyBytes_Size(key); val = tdb_fetch(self->ctx, tkey); if (val.dptr == NULL) { PyErr_SetString(PyExc_KeyError, "No such TDB entry"); return NULL; } else { - return PyString_FromTDB_DATA(val); + return PyBytes_FromTDB_DATA(val); } } @@ -610,22 +622,22 @@ static int obj_setitem(PyTdbObject *self, PyObject *key, PyObject *value) TDB_DATA tkey, tval; int ret; PyErr_TDB_RAISE_RETURN_MINUS_1_IF_CLOSED(self); - if (!PyString_Check(key)) { - PyErr_SetString(PyExc_TypeError, "Expected string as key"); + if (!PyBytes_Check(key)) { + PyErr_SetString(PyExc_TypeError, "Expected bytestring as key"); return -1; } - tkey = PyString_AsTDB_DATA(key); + tkey = PyBytes_AsTDB_DATA(key); if (value == NULL) { ret = tdb_delete(self->ctx, tkey); } else { - if (!PyString_Check(value)) { + if (!PyBytes_Check(value)) { PyErr_SetString(PyExc_TypeError, "Expected string as value"); return -1; } - tval = PyString_AsTDB_DATA(value); + tval = PyBytes_AsTDB_DATA(value); ret = tdb_store(self->ctx, tkey, tval, TDB_REPLACE); } @@ -662,46 +674,78 @@ static PyMethodDef tdb_methods[] = { { NULL } }; -void inittdb(void); -void inittdb(void) +#define MODULE_DOC "simple key-value database that supports multiple writers." + +#if PY_MAJOR_VERSION >= 3 +static struct PyModuleDef moduledef = { + PyModuleDef_HEAD_INIT, + .m_name = "tdb", + .m_doc = MODULE_DOC, + .m_size = -1, + .m_methods = tdb_methods, +}; +#endif + +PyObject* module_init(void); +PyObject* module_init(void) { PyObject *m; if (PyType_Ready(&PyTdb) < 0) - return; + return NULL; if (PyType_Ready(&PyTdbIterator) < 0) - return; + return NULL; - m = Py_InitModule3("tdb", tdb_methods, - "simple key-value database that supports multiple writers."); +#if PY_MAJOR_VERSION >= 3 + m = PyModule_Create(&moduledef); +#else + m = Py_InitModule3("tdb", tdb_methods, MODULE_DOC); +#endif if (m == NULL) - return; - - PyModule_AddObject(m, "REPLACE", PyInt_FromLong(TDB_REPLACE)); - PyModule_AddObject(m, "INSERT", PyInt_FromLong(TDB_INSERT)); - PyModule_AddObject(m, "MODIFY", PyInt_FromLong(TDB_MODIFY)); - - PyModule_AddObject(m, "DEFAULT", PyInt_FromLong(TDB_DEFAULT)); - PyModule_AddObject(m, "CLEAR_IF_FIRST", PyInt_FromLong(TDB_CLEAR_IF_FIRST)); - PyModule_AddObject(m, "INTERNAL", PyInt_FromLong(TDB_INTERNAL)); - PyModule_AddObject(m, "NOLOCK", PyInt_FromLong(TDB_NOLOCK)); - PyModule_AddObject(m, "NOMMAP", PyInt_FromLong(TDB_NOMMAP)); - PyModule_AddObject(m, "CONVERT", PyInt_FromLong(TDB_CONVERT)); - PyModule_AddObject(m, "BIGENDIAN", PyInt_FromLong(TDB_BIGENDIAN)); - PyModule_AddObject(m, "NOSYNC", PyInt_FromLong(TDB_NOSYNC)); - PyModule_AddObject(m, "SEQNUM", PyInt_FromLong(TDB_SEQNUM)); - PyModule_AddObject(m, "VOLATILE", PyInt_FromLong(TDB_VOLATILE)); - PyModule_AddObject(m, "ALLOW_NESTING", PyInt_FromLong(TDB_ALLOW_NESTING)); - PyModule_AddObject(m, "DISALLOW_NESTING", PyInt_FromLong(TDB_DISALLOW_NESTING)); - PyModule_AddObject(m, "INCOMPATIBLE_HASH", PyInt_FromLong(TDB_INCOMPATIBLE_HASH)); - - PyModule_AddObject(m, "__docformat__", PyString_FromString("restructuredText")); - - PyModule_AddObject(m, "__version__", PyString_FromString(PACKAGE_VERSION)); + return NULL; + + PyModule_AddIntConstant(m, "REPLACE", TDB_REPLACE); + PyModule_AddIntConstant(m, "INSERT", TDB_INSERT); + PyModule_AddIntConstant(m, "MODIFY", TDB_MODIFY); + + PyModule_AddIntConstant(m, "DEFAULT", TDB_DEFAULT); + PyModule_AddIntConstant(m, "CLEAR_IF_FIRST", TDB_CLEAR_IF_FIRST); + PyModule_AddIntConstant(m, "INTERNAL", TDB_INTERNAL); + PyModule_AddIntConstant(m, "NOLOCK", TDB_NOLOCK); + PyModule_AddIntConstant(m, "NOMMAP", TDB_NOMMAP); + PyModule_AddIntConstant(m, "CONVERT", TDB_CONVERT); + PyModule_AddIntConstant(m, "BIGENDIAN", TDB_BIGENDIAN); + PyModule_AddIntConstant(m, "NOSYNC", TDB_NOSYNC); + PyModule_AddIntConstant(m, "SEQNUM", TDB_SEQNUM); + PyModule_AddIntConstant(m, "VOLATILE", TDB_VOLATILE); + PyModule_AddIntConstant(m, "ALLOW_NESTING", TDB_ALLOW_NESTING); + PyModule_AddIntConstant(m, "DISALLOW_NESTING", TDB_DISALLOW_NESTING); + PyModule_AddIntConstant(m, "INCOMPATIBLE_HASH", TDB_INCOMPATIBLE_HASH); + + PyModule_AddStringConstant(m, "__docformat__", "restructuredText"); + + PyModule_AddStringConstant(m, "__version__", PACKAGE_VERSION); Py_INCREF(&PyTdb); PyModule_AddObject(m, "Tdb", (PyObject *)&PyTdb); Py_INCREF(&PyTdbIterator); + + return m; +} + + +#if PY_MAJOR_VERSION >= 3 +PyMODINIT_FUNC PyInit_tdb(void); +PyMODINIT_FUNC PyInit_tdb(void) +{ + return module_init(); +} +#else +void inittdb(void); +void inittdb(void) +{ + module_init(); } +#endif diff --git a/lib/tdb/python/tests/simple.py b/lib/tdb/python/tests/simple.py index 4751f9b55fb..39c319a0a47 100644 --- a/lib/tdb/python/tests/simple.py +++ b/lib/tdb/python/tests/simple.py @@ -75,25 +75,25 @@ class SimpleTdbTests(TestCase): self.tdb.reopen() def test_store(self): - self.tdb.store("bar", "bla") - self.assertEquals("bla", self.tdb.get("bar")) + self.tdb.store(b"bar", b"bla") + self.assertEquals(b"bla", self.tdb.get(b"bar")) def test_getitem(self): - self.tdb["bar"] = "foo" + self.tdb[b"bar"] = b"foo" self.tdb.reopen() - self.assertEquals("foo", self.tdb["bar"]) + self.assertEquals(b"foo", self.tdb[b"bar"]) def test_delete(self): - self.tdb["bar"] = "foo" - del self.tdb["bar"] - self.assertRaises(KeyError, lambda: self.tdb["bar"]) + self.tdb[b"bar"] = b"foo" + del self.tdb[b"bar"] + self.assertRaises(KeyError, lambda: self.tdb[b"bar"]) def test_contains(self): - self.tdb["bla"] = "bloe" - self.assertTrue("bla" in self.tdb) + self.tdb[b"bla"] = b"bloe" + self.assertTrue(b"bla" in self.tdb) def test_keyerror(self): - self.assertRaises(KeyError, lambda: self.tdb["bla"]) + self.assertRaises(KeyError, lambda: self.tdb[b"bla"]) def test_hash_size(self): self.tdb.hash_size @@ -108,51 +108,51 @@ class SimpleTdbTests(TestCase): self.tdb.filename def test_iterator(self): - self.tdb["bla"] = "1" - self.tdb["brainslug"] = "2" + self.tdb[b"bla"] = b"1" + self.tdb[b"brainslug"] = b"2" l = list(self.tdb) l.sort() - self.assertEquals(["bla", "brainslug"], l) + self.assertEquals([b"bla", b"brainslug"], l) def test_transaction_cancel(self): - self.tdb["bloe"] = "2" + self.tdb[b"bloe"] = b"2" self.tdb.transaction_start() - self.tdb["bloe"] = "1" + self.tdb[b"bloe"] = b"1" self.tdb.transaction_cancel() - self.assertEquals("2", self.tdb["bloe"]) + self.assertEquals(b"2", self.tdb[b"bloe"]) def test_transaction_commit(self): - self.tdb["bloe"] = "2" + self.tdb[b"bloe"] = b"2" self.tdb.transaction_start() - self.tdb["bloe"] = "1" + self.tdb[b"bloe"] = b"1" self.tdb.transaction_commit() - self.assertEquals("1", self.tdb["bloe"]) + self.assertEquals(b"1", self.tdb[b"bloe"]) def test_transaction_prepare_commit(self): - self.tdb["bloe"] = "2" + self.tdb[b"bloe"] = b"2" self.tdb.transaction_start() - self.tdb["bloe"] = "1" + self.tdb[b"bloe"] = b"1" self.tdb.transaction_prepare_commit() self.tdb.transaction_commit() - self.assertEquals("1", self.tdb["bloe"]) + self.assertEquals(b"1", self.tdb[b"bloe"]) def test_iterkeys(self): - self.tdb["bloe"] = "2" - self.tdb["bla"] = "25" + self.tdb[b"bloe"] = b"2" + self.tdb[b"bla"] = b"25" i = self.tdb.iterkeys() - self.assertEquals(set(["bloe", "bla"]), set([i.next(), i.next()])) + self.assertEquals(set([b"bloe", b"bla"]), set([next(i), next(i)])) def test_clear(self): - self.tdb["bloe"] = "2" - self.tdb["bla"] = "25" + self.tdb[b"bloe"] = b"2" + self.tdb[b"bla"] = b"25" self.assertEquals(2, len(list(self.tdb))) self.tdb.clear() self.assertEquals(0, len(list(self.tdb))) def test_repack(self): - self.tdb["foo"] = "abc" - self.tdb["bar"] = "def" - del self.tdb["foo"] + self.tdb[b"foo"] = b"abc" + self.tdb[b"bar"] = b"def" + del self.tdb[b"foo"] self.tdb.repack() def test_seqnum(self): @@ -164,7 +164,7 @@ class SimpleTdbTests(TestCase): def test_len(self): self.assertEquals(0, len(list(self.tdb))) - self.tdb["entry"] = "value" + self.tdb[b"entry"] = b"value" self.assertEquals(1, len(list(self.tdb))) def test_add_flags(self): diff --git a/lib/tdb/wscript b/lib/tdb/wscript index c573d36b7e0..acc0d5d1695 100644 --- a/lib/tdb/wscript +++ b/lib/tdb/wscript @@ -225,7 +225,10 @@ def testonly(ctx): print("testsuite returned %d" % ret) if ret != 0: ecode = ret - sys.exit(ecode) + + pyret = samba_utils.RUN_PYTHON_TESTS(['python/tests/simple.py']) + print("python testsuite returned %d" % pyret) + sys.exit(ecode or pyret) # WAF doesn't build the unit tests for this, maybe because they don't link with tdb? # This forces it -- 2.11.4.GIT