config/tests/unitMozZipFile.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 import unittest
   6
   7 import shutil
   8 import os
   9 import re
  10 import sys
  11 import random
  12 import copy
  13 from string import letters
  14
  15 '''
  16 Test case infrastructure for MozZipFile.
  17
  18 This isn't really a unit test, but a test case generator and runner.
  19 For a given set of files, lengths, and number of writes, we create
  20 a testcase for every combination of the three. There are some
  21 symmetries used to reduce the number of test cases, the first file
  22 written is always the first file, the second is either the first or
  23 the second, the third is one of the first three. That is, if we
  24 had 4 files, but only three writes, the fourth file would never even
  25 get tried.
  26
  27 The content written to the jars is pseudorandom with a fixed seed.
  28 '''
  29
  30 if not __file__:
  31   __file__ = sys.argv[0]
  32 sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
  33
  34 from MozZipFile import ZipFile
  35 import zipfile
  36
  37 leafs = (
  38   'firstdir/oneleaf',
  39   'seconddir/twoleaf',
  40   'thirddir/with/sub/threeleaf')
  41 _lengths = map(lambda n: n * 64, [16, 64, 80])
  42 lengths = 3
  43 writes = 5
  44
  45 def givenlength(i):
  46   '''Return a length given in the _lengths array to allow manual
  47   tuning of which lengths of zip entries to use.
  48   '''
  49   return _lengths[i]
  50
  51
  52 def prod(*iterables):
  53   ''''Tensor product of a list of iterables.
  54
  55   This generator returns lists of items, one of each given
  56   iterable. It iterates over all possible combinations.
  57   '''
  58   for item in iterables[0]:
  59     if len(iterables) == 1:
  60       yield [item]
  61     else:
  62       for others in prod(*iterables[1:]):
  63         yield [item] + others
  64
  65
  66 def getid(descs):
  67   'Convert a list of ints to a string.'
  68   return reduce(lambda x,y: x+'{0}{1}'.format(*tuple(y)), descs,'')
  69
  70
  71 def getContent(length):
  72   'Get pseudo random content of given length.'
  73   rv = [None] * length
  74   for i in xrange(length):
  75     rv[i] = random.choice(letters)
  76   return ''.join(rv)
  77
  78
  79 def createWriter(sizer, *items):
  80   'Helper method to fill in tests, one set of writes, one for each item'
  81   locitems = copy.deepcopy(items)
  82   for item in locitems:
  83     item['length'] = sizer(item.pop('length', 0))
  84   def helper(self):
  85     mode  = 'w'
  86     if os.path.isfile(self.f):
  87       mode = 'a'
  88     zf = ZipFile(self.f, mode, self.compression)
  89     for item in locitems:
  90       self._write(zf, **item)
  91     zf = None
  92     pass
  93   return helper
  94
  95 def createTester(name, *writes):
  96   '''Helper method to fill in tests, calls into a list of write
  97   helper methods.
  98   '''
  99   _writes = copy.copy(writes)
 100   def tester(self):
 101     for w in _writes:
 102       getattr(self, w)()
 103     self._verifyZip()
 104     pass
 105   # unit tests get confused if the method name isn't test...
 106   tester.__name__ = name
 107   return tester
 108
 109 class TestExtensiveStored(unittest.TestCase):
 110   '''Unit tests for MozZipFile
 111
 112   The testcase are actually populated by code following the class
 113   definition.
 114   '''
 115
 116   stage = "mozzipfilestage"
 117   compression = zipfile.ZIP_STORED
 118
 119   def leaf(self, *leafs):
 120     return os.path.join(self.stage, *leafs)
 121   def setUp(self):
 122     if os.path.exists(self.stage):
 123       shutil.rmtree(self.stage)
 124     os.mkdir(self.stage)
 125     self.f = self.leaf('test.jar')
 126     self.ref = {}
 127     self.seed = 0
 128
 129   def tearDown(self):
 130     self.f = None
 131     self.ref = None
 132
 133   def _verifyZip(self):
 134     zf = zipfile.ZipFile(self.f)
 135     badEntry = zf.testzip()
 136     self.failIf(badEntry, badEntry)
 137     zlist = zf.namelist()
 138     zlist.sort()
 139     vlist = self.ref.keys()
 140     vlist.sort()
 141     self.assertEqual(zlist, vlist)
 142     for leaf, content in self.ref.iteritems():
 143       zcontent = zf.read(leaf)
 144       self.assertEqual(content, zcontent)
 145
 146   def _write(self, zf, seed=None, leaf=0, length=0):
 147     if seed is None:
 148       seed = self.seed
 149       self.seed += 1
 150     random.seed(seed)
 151     leaf = leafs[leaf]
 152     content = getContent(length)
 153     self.ref[leaf] = content
 154     zf.writestr(leaf, content)
 155     dir = os.path.dirname(self.leaf('stage', leaf))
 156     if not os.path.isdir(dir):
 157       os.makedirs(dir)
 158     open(self.leaf('stage', leaf), 'w').write(content)
 159
 160 # all leafs in all lengths
 161 atomics = list(prod(xrange(len(leafs)), xrange(lengths)))
 162
 163 # populate TestExtensiveStore with testcases
 164 for w in xrange(writes):
 165   # Don't iterate over all files for the the first n passes,
 166   # those are redundant as long as w < lengths.
 167   # There are symmetries in the trailing end, too, but I don't know
 168   # how to reduce those out right now.
 169   nonatomics = [list(prod(range(min(i,len(leafs))), xrange(lengths)))
 170                 for i in xrange(1, w+1)] + [atomics]
 171   for descs in prod(*nonatomics):
 172     suffix = getid(descs)
 173     dicts = [dict(leaf=leaf, length=length) for leaf, length in descs]
 174     setattr(TestExtensiveStored, '_write' + suffix,
 175             createWriter(givenlength, *dicts))
 176     setattr(TestExtensiveStored, 'test' + suffix,
 177             createTester('test' + suffix, '_write' + suffix))
 178
 179 # now create another round of tests, with two writing passes
 180 # first, write all file combinations into the jar, close it,
 181 # and then write all atomics again.
 182 # This should catch more or less all artifacts generated
 183 # by the final ordering step when closing the jar.
 184 files = [list(prod([i], xrange(lengths))) for i in xrange(len(leafs))]
 185 allfiles = reduce(lambda l,r:l+r,
 186                   [list(prod(*files[:(i+1)])) for i in xrange(len(leafs))])
 187
 188 for first in allfiles:
 189   testbasename = 'test{0}_'.format(getid(first))
 190   test = [None, '_write' + getid(first), None]
 191   for second in atomics:
 192     test[0] = testbasename + getid([second])
 193     test[2] = '_write' + getid([second])
 194     setattr(TestExtensiveStored, test[0], createTester(*test))
 195
 196 class TestExtensiveDeflated(TestExtensiveStored):
 197   'Test all that has been tested with ZIP_STORED with DEFLATED, too.'
 198   compression = zipfile.ZIP_DEFLATED
 199
 200 if __name__ == '__main__':
 201   unittest.main()