From 82d2ef15e3810abf984dc70c800dac3ebca02f03 Mon Sep 17 00:00:00 2001
From: Barry Warsaw <barry@list.org>
Date: Fri, 17 Aug 2012 22:27:32 -0400
Subject: [PATCH]  * Non-unicode values in msgdata broke pending requests. (LP:
 #1031391)

---
 src/mailman/app/tests/test_moderation.py | 12 ++++++++++++
 src/mailman/docs/NEWS.rst                |  1 +
 src/mailman/model/requests.py            | 26 ++++++++++++++++++++------
 3 files changed, 33 insertions(+), 6 deletions(-)

diff --git a/src/mailman/app/tests/test_moderation.py b/src/mailman/app/tests/test_moderation.py
index bc324faea..ef6adf5ed 100644
--- a/src/mailman/app/tests/test_moderation.py
+++ b/src/mailman/app/tests/test_moderation.py
@@ -37,6 +37,7 @@ from mailman.runners.pipeline import PipelineRunner
 from mailman.testing.helpers import (
     make_testable_runner, specialized_message_from_string)
 from mailman.testing.layers import SMTPLayer
+from mailman.utilities.datetime import now
 
 
 
@@ -109,3 +110,14 @@ Message-ID: <alpha>
         handle_message(self._mlist, request_id, Action.hold)
         key, data = requests_db.get_request(request_id)
         self.assertEqual(key, '<alpha>')
+
+    def test_lp_1031391(self):
+        # LP: #1031391 msgdata['received_time'] gets added by the LMTP server.
+        # The value is a datetime.  If this message gets held, it will break
+        # pending requests since they require string keys and values.
+        received_time = now()
+        msgdata = dict(received_time=received_time)
+        request_id = hold_message(self._mlist, self._msg, msgdata)
+        requests_db = IListRequests(self._mlist)
+        key, data = requests_db.get_request(request_id)
+        self.assertEqual(data['received_time'], received_time)
diff --git a/src/mailman/docs/NEWS.rst b/src/mailman/docs/NEWS.rst
index ea79a3084..9d4f69099 100644
--- a/src/mailman/docs/NEWS.rst
+++ b/src/mailman/docs/NEWS.rst
@@ -101,6 +101,7 @@ Bug fixes
  * Fixed a typo when returning the configuration file's header match checks.
    (LP: #953497)
  * List-Post should be NO when posting is not allowed. (LP: #987563)
+ * Non-unicode values in msgdata broke pending requests. (LP: #1031391)
 
 .. _`passlib`: http://packages.python.org/passlib/index.html
 
diff --git a/src/mailman/model/requests.py b/src/mailman/model/requests.py
index a92332e4a..5eb940233 100644
--- a/src/mailman/model/requests.py
+++ b/src/mailman/model/requests.py
@@ -24,6 +24,7 @@ __all__ = [
     ]
 
 
+from cPickle import dumps, loads
 from datetime import timedelta
 from storm.locals import AutoReload, Int, RawStr, Reference, Unicode
 from zope.component import getUtility
@@ -39,7 +40,19 @@ from mailman.interfaces.requests import IListRequests, RequestType
 
 @implementer(IPendable)
 class DataPendable(dict):
-    pass
+    def update(self, mapping):
+        # Keys and values must be strings (unicodes, but bytes values are
+        # accepted for now).  Any other types for keys are a programming
+        # error.  If we find a non-Unicode value, pickle it and encode it in
+        # such a way that it will be properly reconstituted when unpended.
+        clean_mapping = {}
+        for key, value in mapping.items():
+            assert isinstance(key, basestring)
+            if not isinstance(value, unicode):
+                key = '_pck_' + key
+                value = dumps(value).decode('raw-unicode-escape')
+            clean_mapping[key] = value
+        super(DataPendable, self).update(clean_mapping)
 
 
 
@@ -82,10 +95,6 @@ class ListRequests:
         if data is None:
             data_hash = None
         else:
-            # We're abusing the pending database as a way of storing arbitrary
-            # key/value pairs, where both are strings.  This isn't ideal but
-            # it lets us get auxiliary data almost for free.  We may need to
-            # lock this down more later.
             pendable = DataPendable()
             pendable.update(data)
             token = getUtility(IPendings).add(pendable, timedelta(days=5000))
@@ -106,7 +115,12 @@ class ListRequests:
         pendable = getUtility(IPendings).confirm(
             result.data_hash, expunge=False)
         data = dict()
-        data.update(pendable)
+        # Unpickle any non-Unicode values.
+        for key, value in pendable.items():
+            if key.startswith('_pck_'):
+                data[key[5:]] = loads(value.encode('raw-unicode-escape'))
+            else:
+                data[key] = value
         return result.key, data
 
     @dbconnection
-- 
2.11.4.GIT