mirror/source_unittest.py

   1 #!/usr/bin/python
   2 # Copyright 2009 Google Inc. Released under the GPL v2
   3
   4 import unittest, cStringIO, httplib, time, os
   5
   6 import common
   7 from autotest_lib.mirror import source
   8 from autotest_lib.client.common_lib.test_utils import mock
   9
  10
  11 class common_source(unittest.TestCase):
  12     """
  13     Common support class for source unit tests.
  14     """
  15     def setUp(self):
  16         self.god = mock.mock_god()
  17         self.db_mock = self.god.create_mock_class(
  18                 source.database.database, 'database')
  19
  20         # Set fixed timezone so parsing time values does not break.
  21         self._old_tz = getattr(os.environ, 'TZ', '')
  22         os.environ['TZ'] = 'America/Los_Angeles'
  23         time.tzset()
  24
  25
  26     def tearDown(self):
  27         self.god.unstub_all()
  28         os.environ['TZ'] = self._old_tz
  29         time.tzset()
  30
  31
  32 class rsync_source_unittest(common_source):
  33     _cmd_template = '/usr/bin/rsync -rltz --no-motd %s %s/%s'
  34     _prefix = 'rsync://rsync.kernel.org/pub/linux/kernel'
  35     _path1 = 'v2.6/patch-2.6.*.bz2'
  36     _path2 = 'v2.6/testing/patch*.bz2'
  37
  38     _output1 = """\
  39 -rw-rw-r--       10727 2003/12/17 19:04:34 patch-2.6.0.bz2
  40 -rw-rw-r--      777959 2004/01/08 23:31:48 patch-2.6.1.bz2
  41 -rw-rw-r--     4851041 2004/12/24 14:38:58 patch-2.6.10.bz2
  42 -rw-r--r--         713 2005/03/08 16:59:09 patch-2.6.11.1.bz2
  43 -rw-r--r--       15141 2005/05/16 11:17:23 patch-2.6.11.10.bz2
  44 -rw-rw-r--       20868 2005/05/26 22:51:21 patch-2.6.11.11.bz2
  45 -rw-rw-r--       23413 2005/06/11 19:57:26 patch-2.6.11.12.bz2
  46 -rw-r--r--        1010 2005/03/12 22:55:52 patch-2.6.11.2.bz2
  47 """
  48     _output2 = """\
  49 -rw-rw-r--    10462721 2009/04/07 15:45:35 patch-2.6.30-rc1.bz2
  50 -rw-rw-r--    10815919 2009/04/14 19:01:40 patch-2.6.30-rc2.bz2
  51 -rw-rw-r--    11032734 2009/04/21 20:28:11 patch-2.6.30-rc3.bz2
  52 """
  53     _output_excluded = """\
  54 -rw-rw-r--    10462721 2009/04/07 15:45:35 patch-2.6.30-rc1.bz2
  55 -rw-rw-r--    11032734 2009/04/21 20:28:11 patch-2.6.30-rc3.bz2
  56 """
  57     _known_files = {
  58         'v2.6/patch-2.6.1.bz2': source.database.item(
  59             'v2.6/patch-2.6.1.bz2', 777959, 1073633508),
  60         'v2.6/patch-2.6.11.10.bz2': source.database.item(
  61             'v2.6/patch-2.6.11.10.bz2', 15141, 1116267443),
  62         'v2.6/testing/patch-2.6.30-rc1.bz2': source.database.item(
  63             'v2.6/testing/patch-2.6.30-rc1.bz2', 10462721, 1239144335),
  64         }
  65     _result = {
  66         'v2.6/patch-2.6.0.bz2': source.database.item(
  67             'v2.6/patch-2.6.0.bz2', 10727, 1071716674),
  68         'v2.6/patch-2.6.10.bz2': source.database.item(
  69             'v2.6/patch-2.6.10.bz2', 4851041, 1103927938),
  70         'v2.6/patch-2.6.11.12.bz2': source.database.item(
  71             'v2.6/patch-2.6.11.12.bz2', 23413, 1118545046),
  72         'v2.6/patch-2.6.11.11.bz2': source.database.item(
  73             'v2.6/patch-2.6.11.11.bz2', 20868, 1117173081),
  74         'v2.6/patch-2.6.11.2.bz2': source.database.item(
  75             'v2.6/patch-2.6.11.2.bz2', 1010, 1110696952),
  76         'v2.6/patch-2.6.11.1.bz2': source.database.item(
  77             'v2.6/patch-2.6.11.1.bz2', 713, 1110329949),
  78         'v2.6/testing/patch-2.6.30-rc3.bz2': source.database.item(
  79             'v2.6/testing/patch-2.6.30-rc3.bz2', 11032734, 1240370891),
  80         'v2.6/testing/patch-2.6.30-rc2.bz2': source.database.item(
  81             'v2.6/testing/patch-2.6.30-rc2.bz2', 10815919, 1239760900),
  82         }
  83
  84     def setUp(self):
  85         super(rsync_source_unittest, self).setUp()
  86
  87         self.god.stub_function(source.utils, 'system_output')
  88
  89
  90     def test_simple(self):
  91         # record
  92         (source.utils.system_output.expect_call(
  93             self._cmd_template % ('', self._prefix, self._path1))
  94             .and_return(self._output1))
  95         (source.utils.system_output.expect_call(
  96             self._cmd_template % ('', self._prefix, self._path2))
  97             .and_return(self._output2))
  98         self.db_mock.get_dictionary.expect_call().and_return(self._known_files)
  99
 100         # playback
 101         s = source.rsync_source(self.db_mock, self._prefix)
 102         s.add_path('v2.6/patch-2.6.*.bz2', 'v2.6')
 103         s.add_path('v2.6/testing/patch*.bz2', 'v2.6/testing')
 104         self.assertEquals(s.get_new_files(), self._result)
 105         self.god.check_playback()
 106
 107
 108     def test_exclusions(self):
 109         # setup
 110         exclude_str = '--exclude "2.6.30-rc2"'
 111         excluded_result = dict(self._result)
 112         del excluded_result['v2.6/testing/patch-2.6.30-rc2.bz2']
 113
 114         # record
 115         (source.utils.system_output.expect_call(
 116             self._cmd_template % (exclude_str, self._prefix, self._path1))
 117             .and_return(self._output1))
 118         (source.utils.system_output.expect_call(
 119             self._cmd_template % (exclude_str, self._prefix, self._path2))
 120             .and_return(self._output_excluded))
 121         self.db_mock.get_dictionary.expect_call().and_return(self._known_files)
 122
 123         # playback
 124         s = source.rsync_source(self.db_mock, self._prefix,
 125                                 excludes=('2.6.30-rc2',))
 126         s.add_path('v2.6/patch-2.6.*.bz2', 'v2.6')
 127         s.add_path('v2.6/testing/patch*.bz2', 'v2.6/testing')
 128         self.assertEquals(s.get_new_files(), excluded_result)
 129         self.god.check_playback()
 130
 131
 132 class url_source_unittest(common_source):
 133     _prefix = 'http://www.kernel.org/pub/linux/kernel/'
 134
 135     _path1 = 'v2.6/'
 136     _full_path1 = '%s%s' % (_prefix, _path1)
 137
 138     _path2 = 'v2.6/testing'
 139     _full_path2 = '%s%s/' % (_prefix, _path2)
 140
 141     _output1 = """\
 142 <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 3.2 Final//EN">
 143 <html>
 144  <head>
 145   <title>Index of /pub/linux/kernel/v2.6</title>
 146  </head>
 147  <body>
 148 <h1>Index of /pub/linux/kernel/v2.6</h1>
 149 <pre><a href="?C=N;O=D">Name</a>                         <a href="?C=M;O=A">Last modified</a>      <a href="?C=S;O=A">Size</a>  <hr><a href="/pub/linux/kernel/">Parent Directory</a>                                  -
 150 <a href="incr/">incr/</a>                        23-Mar-2009 22:13    -
 151 <a href="pre-releases/">pre-releases/</a>                18-Dec-2003 15:50    -
 152 <a href="snapshots/">snapshots/</a>                   25-Apr-2009 00:18    -
 153 <a href="stable-review/">stable-review/</a>               23-Apr-2009 07:51    -
 154 <a href="testing/">testing/</a>                     22-Apr-2009 03:31    -
 155 <a href="ChangeLog-2.6.0">ChangeLog-2.6.0</a>              18-Dec-2003 03:04   12K
 156 <a href="ChangeLog-2.6.1">ChangeLog-2.6.1</a>              09-Jan-2004 07:08  189K
 157 <a href="ChangeLog-2.6.2">ChangeLog-2.6.2</a>              04-Feb-2004 04:06  286K
 158 <a href="patch-2.6.19.6.bz2.sign">patch-2.6.19.6.bz2.sign</a>      03-Mar-2007 01:06  248
 159 <a href="patch-2.6.19.6.gz">patch-2.6.19.6.gz</a>            03-Mar-2007 01:06   68K
 160 <a href="patch-2.6.19.6.gz.sign">patch-2.6.19.6.gz.sign</a>       03-Mar-2007 01:06  248
 161 <a href="patch-2.6.19.6.sign">patch-2.6.19.6.sign</a>          03-Mar-2007 01:06  248
 162 <a href="patch-2.6.19.7.bz2">patch-2.6.19.7.bz2</a>           03-Mar-2007 05:29   62K
 163 <a href="patch-2.6.19.7.bz2.sign">patch-2.6.19.7.bz2.sign</a>      03-Mar-2007 05:29  248
 164 <a href="linux-2.6.28.1.tar.sign">linux-2.6.28.1.tar.sign</a>      18-Jan-2009 18:48  248
 165 <a href="linux-2.6.28.2.tar.bz2">linux-2.6.28.2.tar.bz2</a>       25-Jan-2009 00:47   50M
 166 <a href="linux-2.6.28.2.tar.bz2.sign">linux-2.6.28.2.tar.bz2.sign</a>  25-Jan-2009 00:47  248
 167 <a href="linux-2.6.28.2.tar.gz">linux-2.6.28.2.tar.gz</a>        25-Jan-2009 00:47   64M
 168 <a href="linux-2.6.28.2.tar.gz.sign">linux-2.6.28.2.tar.gz.sign</a>   25-Jan-2009 00:47  248
 169 <a href="linux-2.6.28.2.tar.sign">linux-2.6.28.2.tar.sign</a>      25-Jan-2009 00:47  248
 170 <a href="linux-2.6.28.3.tar.bz2">linux-2.6.28.3.tar.bz2</a>       02-Feb-2009 18:21   50M
 171 <hr></pre>
 172 </body></html>
 173 """
 174     _output2 = """\
 175 <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 3.2 Final//EN">
 176 <html>
 177  <head>
 178   <title>Index of /pub/linux/kernel/v2.6/testing</title>
 179  </head>
 180  <body>
 181 <h1>Index of /pub/linux/kernel/v2.6/testing</h1>
 182 <pre><a href="?C=N;O=D">Name</a>                          <a href="?C=M;O=A">Last modified</a>      <a href="?C=S;O=A">Size</a>  <hr><a href="/pub/linux/kernel/v2.6/">Parent Directory</a>                                   -
 183 <a href="cset/">cset/</a>                         04-Apr-2005 17:12    -
 184 <a href="incr/">incr/</a>                         22-Apr-2009 03:30    -
 185 <a href="old/">old/</a>                          14-Jul-2003 16:06    -
 186 <a href="v2.6.1/">v2.6.1/</a>                       15-Feb-2008 21:47    -
 187 <a href="v2.6.2/">v2.6.2/</a>                       15-Feb-2008 21:47    -
 188 <a href="LATEST-IS-2.6.30-rc3">LATEST-IS-2.6.30-rc3</a>          22-Apr-2009 03:13    0
 189 <a href="linux-2.6.30-rc1.tar.bz2">linux-2.6.30-rc1.tar.bz2</a>      07-Apr-2009 22:43   57M
 190 <a href="linux-2.6.30-rc1.tar.bz2.sign">linux-2.6.30-rc1.tar.bz2.sign</a> 07-Apr-2009 22:43  248
 191 <a href="linux-2.6.30-rc3.tar.gz.sign">linux-2.6.30-rc3.tar.gz.sign</a>  22-Apr-2009 03:25  248
 192 <a href="linux-2.6.30-rc3.tar.sign">linux-2.6.30-rc3.tar.sign</a>     22-Apr-2009 03:25  248
 193 <a href="patch-2.6.30-rc1.bz2">patch-2.6.30-rc1.bz2</a>          07-Apr-2009 22:45   10M
 194 <a href="patch-2.6.30-rc1.bz2.sign">patch-2.6.30-rc1.bz2.sign</a>     07-Apr-2009 22:45  248
 195 <hr></pre>
 196 </body></html>
 197 """
 198     _extracted_links1 = (
 199         (_full_path1 + 'patch-2.6.19.6.gz', '70021',
 200             (2007, 3, 3, 1, 6, 0, 0, 1, 0)),
 201         (_full_path1 + 'patch-2.6.19.7.bz2', '63424',
 202             (2007, 3, 3, 5, 29, 0, 0, 1, 0)),
 203         (_full_path1 + 'linux-2.6.28.2.tar.bz2', '52697313',
 204             (2009, 1, 25, 0, 47, 0, 0, 1, 0)),
 205         (_full_path1 + 'linux-2.6.28.2.tar.gz', '66781113',
 206             (2009, 1, 25, 0, 47, 0, 0, 1, 0)),
 207         (_full_path1 + 'linux-2.6.28.3.tar.bz2', '52703558',
 208             (2009, 2, 2, 18, 21, 0, 0, 1, 0)),
 209         )
 210
 211     _extracted_links2 = (
 212         (_full_path2 + 'patch-2.6.30-rc1.bz2', '10462721',
 213             (2009, 4, 7, 22, 43, 0, 0, 1, 0)),
 214         )
 215
 216     _known_files = {
 217         _full_path1 + 'linux-2.6.28.2.tar.gz': source.database.item(
 218             _full_path1 + 'linux-2.6.28.2.tar.gz', 66781113, 1232873220),
 219         }
 220
 221     _result = {
 222         _full_path1 + 'linux-2.6.28.3.tar.bz2': source.database.item(
 223             _full_path1 + 'linux-2.6.28.3.tar.bz2', 52703558, 1233627660),
 224         _full_path2 + 'patch-2.6.30-rc1.bz2': source.database.item(
 225             _full_path2 + 'patch-2.6.30-rc1.bz2', 10462721, 1239172980),
 226         _full_path1 + 'patch-2.6.19.7.bz2': source.database.item(
 227             _full_path1 + 'patch-2.6.19.7.bz2', 63424, 1172928540),
 228         _full_path1 + 'linux-2.6.28.2.tar.bz2': source.database.item(
 229             _full_path1 + 'linux-2.6.28.2.tar.bz2', 52697313, 1232873220),
 230         _full_path1 + 'patch-2.6.19.6.gz': source.database.item(
 231             _full_path1 + 'patch-2.6.19.6.gz', 70021, 1172912760),
 232         }
 233
 234     def setUp(self):
 235         super(url_source_unittest, self).setUp()
 236
 237         self.god.stub_function(source.urllib2, 'urlopen')
 238         self.addinfourl_mock = self.god.create_mock_class(
 239             source.urllib2.addinfourl, 'addinfourl')
 240         self.mime_mock = self.god.create_mock_class(
 241             httplib.HTTPMessage, 'HTTPMessage')
 242
 243
 244     def test_get_new_files(self):
 245         # record
 246         (source.urllib2.urlopen.expect_call(self._full_path1)
 247             .and_return(cStringIO.StringIO(self._output1)))
 248         for link, size, time in self._extracted_links1:
 249             (source.urllib2.urlopen.expect_call(link)
 250                 .and_return(self.addinfourl_mock))
 251             self.addinfourl_mock.info.expect_call().and_return(self.mime_mock)
 252             self.mime_mock.get.expect_call('content-length').and_return(size)
 253             self.mime_mock.getdate.expect_call('date').and_return(time)
 254
 255         (source.urllib2.urlopen.expect_call(self._full_path2)
 256             .and_return(cStringIO.StringIO(self._output2)))
 257         for link, size, time in self._extracted_links2:
 258             (source.urllib2.urlopen.expect_call(link)
 259                 .and_return(self.addinfourl_mock))
 260             self.addinfourl_mock.info.expect_call().and_return(self.mime_mock)
 261             self.mime_mock.get.expect_call('content-length').and_return(size)
 262             self.mime_mock.getdate.expect_call('date').and_return(time)
 263
 264         self.db_mock.get_dictionary.expect_call().and_return(self._known_files)
 265
 266         # playback
 267         s = source.url_source(self.db_mock, self._prefix)
 268         s.add_url(self._path1, r'.*\.(gz|bz2)$')
 269         s.add_url(self._path2, r'.*patch-[0-9.]+(-rc[0-9]+)?\.bz2$')
 270         self.assertEquals(s.get_new_files(), self._result)
 271         self.god.check_playback()
 272
 273
 274 class directory_source_unittest(common_source):
 275     """
 276     Unit test class for directory_source.
 277     """
 278     def setUp(self):
 279         super(directory_source_unittest, self).setUp()
 280
 281         self.god.stub_function(os, 'listdir')
 282         self._stat_mock = self.god.create_mock_function('stat')
 283
 284
 285     @staticmethod
 286     def _get_stat_result(mode=0644, ino=12345, dev=12345, nlink=1, uid=1000,
 287                          gid=1000, size=10, atime=123, mtime=123, ctime=123):
 288         """
 289         Build an os.stat_result() instance with many default values.
 290
 291         @param mode, ino, dev, nlink, uid, gid, size, atime, mtime, ctime:
 292                 See help(os.stat_result).
 293         """
 294         return os.stat_result((mode, ino, dev, nlink, uid, gid, size, atime,
 295                                mtime, ctime))
 296
 297
 298     def test_get_new_files_invalid_path(self):
 299         """
 300         Test directory_source.get_new_files() on an invalid path.
 301         """
 302         path = '/some/invalid/path'
 303         os.listdir.expect_call(path).and_raises(OSError('Error'))
 304
 305         s = source.directory_source(self.db_mock, path)
 306         self.assertRaises(OSError, s.get_new_files)
 307         self.god.check_playback()
 308
 309
 310     def test_get_new_files_stat_fails(self):
 311         """
 312         Test directory_source.get_new_files() when stat fails.
 313         """
 314         path = '/some/valid/path'
 315         os.listdir.expect_call(path).and_return(['file1', 'file2'])
 316         (self._stat_mock.expect_call(os.path.join(path, 'file1'))
 317                 .and_raises(OSError('Error')))
 318         stat_result = self._get_stat_result(size=1010, mtime=123)
 319         file2_full_path = os.path.join(path, 'file2')
 320         self._stat_mock.expect_call(file2_full_path).and_return(stat_result)
 321
 322         self.db_mock.get_dictionary.expect_call().and_return({})
 323
 324         s = source.directory_source(self.db_mock, path)
 325         expected = {'file2': source.database.item(file2_full_path, 1010, 123)}
 326         self.assertEquals(expected, s.get_new_files(_stat_func=self._stat_mock))
 327         self.god.check_playback()
 328
 329
 330     def test_get_new_files_success(self):
 331         """
 332         Test directory_source.get_new_files() success.
 333         """
 334         path = '/some/valid/path'
 335         file1_full_path = os.path.join(path, 'file1')
 336         file2_full_path = os.path.join(path, 'file2')
 337         file3_full_path = os.path.join(path, 'file3')
 338
 339         os.listdir.expect_call(path).and_return(['file1', 'file2', 'file3'])
 340
 341         stat_result = self._get_stat_result(size=1010, mtime=123)
 342         self._stat_mock.expect_call(file1_full_path).and_return(stat_result)
 343
 344         stat_result = self._get_stat_result(size=1020, mtime=1234)
 345         self._stat_mock.expect_call(file2_full_path).and_return(stat_result)
 346
 347         stat_result = self._get_stat_result(size=1030, mtime=12345)
 348         self._stat_mock.expect_call(file3_full_path).and_return(stat_result)
 349
 350         known_files = {
 351                 'file2': source.database.item(file2_full_path, 1020, 1234),
 352                 }
 353         self.db_mock.get_dictionary.expect_call().and_return(known_files)
 354
 355         s = source.directory_source(self.db_mock, path)
 356         expected = {
 357                 'file1': source.database.item(file1_full_path, 1010, 123),
 358                 'file3': source.database.item(file3_full_path, 1030, 12345),
 359                 }
 360         self.assertEquals(expected, s.get_new_files(_stat_func=self._stat_mock))
 361         self.god.check_playback()
 362
 363
 364 if __name__ == "__main__":
 365     unittest.main()