config/tests/unitMozZipFile.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 from __future__ import absolute_import
   6 import unittest
   7
   8 import shutil
   9 import os
  10 import sys
  11 import random
  12 import copy
  13 from string import letters
  14
  15 """
  16 Test case infrastructure for MozZipFile.
  17
  18 This isn't really a unit test, but a test case generator and runner.
  19 For a given set of files, lengths, and number of writes, we create
  20 a testcase for every combination of the three. There are some
  21 symmetries used to reduce the number of test cases, the first file
  22 written is always the first file, the second is either the first or
  23 the second, the third is one of the first three. That is, if we
  24 had 4 files, but only three writes, the fourth file would never even
  25 get tried.
  26
  27 The content written to the jars is pseudorandom with a fixed seed.
  28 """
  29
  30 if not __file__:
  31     __file__ = sys.argv[0]
  32 sys.path.append(os.path.join(os.path.dirname(__file__), ".."))
  33
  34 from MozZipFile import ZipFile
  35 import zipfile
  36
  37 leafs = ("firstdir/oneleaf", "seconddir/twoleaf", "thirddir/with/sub/threeleaf")
  38 _lengths = map(lambda n: n * 64, [16, 64, 80])
  39 lengths = 3
  40 writes = 5
  41
  42
  43 def givenlength(i):
  44     """Return a length given in the _lengths array to allow manual
  45     tuning of which lengths of zip entries to use.
  46     """
  47     return _lengths[i]
  48
  49
  50 def prod(*iterables):
  51     """'Tensor product of a list of iterables.
  52
  53     This generator returns lists of items, one of each given
  54     iterable. It iterates over all possible combinations.
  55     """
  56     for item in iterables[0]:
  57         if len(iterables) == 1:
  58             yield [item]
  59         else:
  60             for others in prod(*iterables[1:]):
  61                 yield [item] + others
  62
  63
  64 def getid(descs):
  65     "Convert a list of ints to a string."
  66     return reduce(lambda x, y: x + "{0}{1}".format(*tuple(y)), descs, "")
  67
  68
  69 def getContent(length):
  70     "Get pseudo random content of given length."
  71     rv = [None] * length
  72     for i in xrange(length):
  73         rv[i] = random.choice(letters)
  74     return "".join(rv)
  75
  76
  77 def createWriter(sizer, *items):
  78     "Helper method to fill in tests, one set of writes, one for each item"
  79     locitems = copy.deepcopy(items)
  80     for item in locitems:
  81         item["length"] = sizer(item.pop("length", 0))
  82
  83     def helper(self):
  84         mode = "w"
  85         if os.path.isfile(self.f):
  86             mode = "a"
  87         zf = ZipFile(self.f, mode, self.compression)
  88         for item in locitems:
  89             self._write(zf, **item)
  90         zf = None
  91         pass
  92
  93     return helper
  94
  95
  96 def createTester(name, *writes):
  97     """Helper method to fill in tests, calls into a list of write
  98     helper methods.
  99     """
 100     _writes = copy.copy(writes)
 101
 102     def tester(self):
 103         for w in _writes:
 104             getattr(self, w)()
 105         self._verifyZip()
 106         pass
 107
 108     # unit tests get confused if the method name isn't test...
 109     tester.__name__ = name
 110     return tester
 111
 112
 113 class TestExtensiveStored(unittest.TestCase):
 114     """Unit tests for MozZipFile
 115
 116     The testcase are actually populated by code following the class
 117     definition.
 118     """
 119
 120     stage = "mozzipfilestage"
 121     compression = zipfile.ZIP_STORED
 122
 123     def leaf(self, *leafs):
 124         return os.path.join(self.stage, *leafs)
 125
 126     def setUp(self):
 127         if os.path.exists(self.stage):
 128             shutil.rmtree(self.stage)
 129         os.mkdir(self.stage)
 130         self.f = self.leaf("test.jar")
 131         self.ref = {}
 132         self.seed = 0
 133
 134     def tearDown(self):
 135         self.f = None
 136         self.ref = None
 137
 138     def _verifyZip(self):
 139         zf = zipfile.ZipFile(self.f)
 140         badEntry = zf.testzip()
 141         self.failIf(badEntry, badEntry)
 142         zlist = zf.namelist()
 143         zlist.sort()
 144         vlist = self.ref.keys()
 145         vlist.sort()
 146         self.assertEqual(zlist, vlist)
 147         for leaf, content in self.ref.iteritems():
 148             zcontent = zf.read(leaf)
 149             self.assertEqual(content, zcontent)
 150
 151     def _write(self, zf, seed=None, leaf=0, length=0):
 152         if seed is None:
 153             seed = self.seed
 154             self.seed += 1
 155         random.seed(seed)
 156         leaf = leafs[leaf]
 157         content = getContent(length)
 158         self.ref[leaf] = content
 159         zf.writestr(leaf, content)
 160         dir = os.path.dirname(self.leaf("stage", leaf))
 161         if not os.path.isdir(dir):
 162             os.makedirs(dir)
 163         open(self.leaf("stage", leaf), "w").write(content)
 164
 165
 166 # all leafs in all lengths
 167 atomics = list(prod(xrange(len(leafs)), xrange(lengths)))
 168
 169 # populate TestExtensiveStore with testcases
 170 for w in xrange(writes):
 171     # Don't iterate over all files for the the first n passes,
 172     # those are redundant as long as w < lengths.
 173     # There are symmetries in the trailing end, too, but I don't know
 174     # how to reduce those out right now.
 175     nonatomics = [
 176         list(prod(range(min(i, len(leafs))), xrange(lengths))) for i in xrange(1, w + 1)
 177     ] + [atomics]
 178     for descs in prod(*nonatomics):
 179         suffix = getid(descs)
 180         dicts = [dict(leaf=leaf, length=length) for leaf, length in descs]
 181         setattr(
 182             TestExtensiveStored, "_write" + suffix, createWriter(givenlength, *dicts)
 183         )
 184         setattr(
 185             TestExtensiveStored,
 186             "test" + suffix,
 187             createTester("test" + suffix, "_write" + suffix),
 188         )
 189
 190 # now create another round of tests, with two writing passes
 191 # first, write all file combinations into the jar, close it,
 192 # and then write all atomics again.
 193 # This should catch more or less all artifacts generated
 194 # by the final ordering step when closing the jar.
 195 files = [list(prod([i], xrange(lengths))) for i in xrange(len(leafs))]
 196 allfiles = reduce(
 197     lambda l, r: l + r, [list(prod(*files[: (i + 1)])) for i in xrange(len(leafs))]
 198 )
 199
 200 for first in allfiles:
 201     testbasename = "test{0}_".format(getid(first))
 202     test = [None, "_write" + getid(first), None]
 203     for second in atomics:
 204         test[0] = testbasename + getid([second])
 205         test[2] = "_write" + getid([second])
 206         setattr(TestExtensiveStored, test[0], createTester(*test))
 207
 208
 209 class TestExtensiveDeflated(TestExtensiveStored):
 210     "Test all that has been tested with ZIP_STORED with DEFLATED, too."
 211     compression = zipfile.ZIP_DEFLATED
 212
 213
 214 if __name__ == "__main__":
 215     unittest.main()