buildbot/test/test_ec2buildslave.py

   1 # Portions copyright Canonical Ltd. 2009
   2
   3 import os
   4 import sys
   5 import StringIO
   6 import textwrap
   7
   8 from twisted.trial import unittest
   9 from twisted.internet import defer, reactor
  10
  11 from buildbot.process.base import BuildRequest
  12 from buildbot.sourcestamp import SourceStamp
  13 from buildbot.status.builder import SUCCESS
  14 from buildbot.test.runutils import RunMixin
  15
  16
  17 PENDING = 'pending'
  18 RUNNING = 'running'
  19 SHUTTINGDOWN = 'shutting-down'
  20 TERMINATED = 'terminated'
  21
  22
  23 class EC2ResponseError(Exception):
  24     def __init__(self, code):
  25         self.code = code
  26
  27
  28 class Stub:
  29     def __init__(self, **kwargs):
  30         self.__dict__.update(kwargs)
  31
  32
  33 class Instance:
  34
  35     def __init__(self, data, ami, **kwargs):
  36         self.data = data
  37         self.state = PENDING
  38         self.id = ami
  39         self.public_dns_name = 'ec2-012-345-678-901.compute-1.amazonaws.com'
  40         self.__dict__.update(kwargs)
  41         self.output = Stub(name='output', output='example_output')
  42
  43     def update(self):
  44         if self.state == PENDING:
  45             self.data.testcase.connectOneSlave(self.data.slave.slavename)
  46             self.state = RUNNING
  47         elif self.state == SHUTTINGDOWN:
  48             slavename = self.data.slave.slavename
  49             slaves = self.data.testcase.slaves
  50             if slavename in slaves:
  51                 def discard(data):
  52                     pass
  53                 s = slaves.pop(slavename)
  54                 bot = s.getServiceNamed("bot")
  55                 for buildername in self.data.slave.slavebuilders:
  56                     remote = bot.builders[buildername].remote
  57                     if remote is None:
  58                         continue
  59                     broker = remote.broker
  60                     broker.dataReceived = discard # seal its ears
  61                     # and take away its voice
  62                     broker.transport.write = discard
  63                 # also discourage it from reconnecting once the connection
  64                 # goes away
  65                 s.bf.continueTrying = False
  66                 # stop the service for cleanliness
  67                 s.stopService()
  68             self.state = TERMINATED
  69
  70     def get_console_output(self):
  71         return self.output
  72
  73     def use_ip(self, elastic_ip):
  74         if isinstance(elastic_ip, Stub):
  75             elastic_ip = elastic_ip.public_ip
  76         if self.data.addresses[elastic_ip] is not None:
  77             raise ValueError('elastic ip already used')
  78         self.data.addresses[elastic_ip] = self
  79
  80     def stop(self):
  81         self.state = SHUTTINGDOWN
  82
  83 class Image:
  84
  85     def __init__(self, data, ami, owner, location):
  86         self.data = data
  87         self.id = ami
  88         self.owner = owner
  89         self.location = location
  90
  91     def run(self, **kwargs):
  92         return Stub(name='reservation',
  93                     instances=[Instance(self.data, self.id, **kwargs)])
  94
  95     def create(klass, data, ami, owner, location):
  96         assert ami not in data.images
  97         self = klass(data, ami, owner, location)
  98         data.images[ami] = self
  99         return self
 100     create = classmethod(create)
 101
 102
 103 class Connection:
 104
 105     def __init__(self, data):
 106         self.data = data
 107
 108     def get_all_key_pairs(self, keypair_name):
 109         try:
 110             return [self.data.keys[keypair_name]]
 111         except KeyError:
 112             raise EC2ResponseError('InvalidKeyPair.NotFound')
 113
 114     def create_key_pair(self, keypair_name):
 115         return Key.create(keypair_name, self.data.keys)
 116
 117     def get_all_security_groups(self, security_name):
 118         try:
 119             return [self.data.security_groups[security_name]]
 120         except KeyError:
 121             raise EC2ResponseError('InvalidGroup.NotFound')
 122
 123     def create_security_group(self, security_name, description):
 124         assert security_name not in self.data.security_groups
 125         res = Stub(name='security_group', value=security_name,
 126                    description=description)
 127         self.data.security_groups[security_name] = res
 128         return res
 129
 130     def get_all_images(self, owners=None):
 131         # return a list of images.  images have .location and .id.
 132         res = self.data.images.values()
 133         if owners:
 134             res = [image for image in res if image.owner in owners]
 135         return res
 136
 137     def get_image(self, machine_id):
 138         # return image or raise an error
 139         return self.data.images[machine_id]
 140
 141     def get_all_addresses(self, elastic_ips):
 142         res = []
 143         for ip in elastic_ips:
 144             if ip in self.data.addresses:
 145                 res.append(Stub(public_ip=ip))
 146             else:
 147                 raise EC2ResponseError('...bad address...')
 148         return res
 149
 150     def disassociate_address(self, address):
 151         if address not in self.data.addresses:
 152             raise EC2ResponseError('...unknown address...')
 153         self.data.addresses[address] = None
 154
 155
 156 class Key:
 157
 158     # this is what we would need to do if we actually needed a real key.
 159     # We don't right now.
 160     #def __init__(self):
 161     #    self.raw = paramiko.RSAKey.generate(256)
 162     #    f = StringIO.StringIO()
 163     #    self.raw.write_private_key(f)
 164     #    self.material = f.getvalue()
 165
 166     def create(klass, name, keys):
 167         self = klass()
 168         self.name = name
 169         self.keys = keys
 170         assert name not in keys
 171         keys[name] = self
 172         return self
 173     create = classmethod(create)
 174
 175     def delete(self):
 176         del self.keys[self.name]
 177
 178
 179 class Boto:
 180
 181     slave = None # must be set in setUp
 182
 183     def __init__(self, testcase):
 184         self.testcase = testcase
 185         self.keys = {}
 186         Key.create('latent_buildbot_slave', self.keys)
 187         Key.create('buildbot_slave', self.keys)
 188         kk = self.keys.keys()
 189         kk.sort()
 190         assert kk == ['buildbot_slave', 'latent_buildbot_slave']
 191         self.original_keys = dict(self.keys)
 192         self.security_groups = {
 193             'latent_buildbot_slave': Stub(name='security_group',
 194                                           value='latent_buildbot_slave')}
 195         self.addresses = {'127.0.0.1': None}
 196         self.images = {}
 197         Image.create(self, 'ami-12345', 12345667890,
 198                      'test-xx/image.manifest.xml')
 199         Image.create(self, 'ami-AF000', 11111111111,
 200                      'test-f0a/image.manifest.xml')
 201         Image.create(self, 'ami-CE111', 22222222222,
 202                      'test-e1b/image.manifest.xml')
 203         Image.create(self, 'ami-ED222', 22222222222,
 204                      'test-d2c/image.manifest.xml')
 205         Image.create(self, 'ami-FC333', 22222222222,
 206                      'test-c30d/image.manifest.xml')
 207         Image.create(self, 'ami-DB444', 11111111111,
 208                      'test-b4e/image.manifest.xml')
 209         Image.create(self, 'ami-BA555', 11111111111,
 210                      'test-a5f/image.manifest.xml')
 211
 212     def connect_ec2(self, identifier, secret_identifier):
 213         assert identifier == 'publickey', identifier
 214         assert secret_identifier == 'privatekey', secret_identifier
 215         return Connection(self)
 216
 217     exception = Stub(EC2ResponseError=EC2ResponseError)
 218
 219
 220 class Mixin(RunMixin):
 221
 222     def doBuild(self):
 223         br = BuildRequest("forced", SourceStamp(), 'test_builder')
 224         d = br.waitUntilFinished()
 225         self.control.getBuilder('b1').requestBuild(br)
 226         return d
 227
 228     def setUp(self):
 229         self.boto_setUp1()
 230         self.master.loadConfig(self.config)
 231         self.boto_setUp2()
 232         self.boto_setUp3()
 233
 234     def boto_setUp1(self):
 235         # debugging
 236         #import twisted.internet.base
 237         #twisted.internet.base.DelayedCall.debug = True
 238         # debugging
 239         RunMixin.setUp(self)
 240         self.boto = boto = Boto(self)
 241         if 'boto' not in sys.modules:
 242             sys.modules['boto'] = boto
 243             sys.modules['boto.exception'] = boto.exception
 244         if 'buildbot.ec2buildslave' in sys.modules:
 245             sys.modules['buildbot.ec2buildslave'].boto = boto
 246
 247     def boto_setUp2(self):
 248         if sys.modules['boto'] is self.boto:
 249             del sys.modules['boto']
 250             del sys.modules['boto.exception']
 251
 252     def boto_setUp3(self):
 253         self.master.startService()
 254         self.boto.slave = self.bot1 = self.master.botmaster.slaves['bot1']
 255         self.bot1._poll_resolution = 0.1
 256         self.b1 = self.master.botmaster.builders['b1']
 257
 258     def tearDown(self):
 259         try:
 260             import boto
 261             import boto.exception
 262         except ImportError:
 263             pass
 264         else:
 265             sys.modules['buildbot.ec2buildslave'].boto = boto
 266         return RunMixin.tearDown(self)
 267
 268
 269 class BasicConfig(Mixin, unittest.TestCase):
 270     config = textwrap.dedent("""\
 271         from buildbot.process import factory
 272         from buildbot.steps import dummy
 273         from buildbot.ec2buildslave import EC2LatentBuildSlave
 274         s = factory.s
 275
 276         BuildmasterConfig = c = {}
 277         c['slaves'] = [EC2LatentBuildSlave('bot1', 'sekrit', 'm1.large',
 278                                            'ami-12345',
 279                                            identifier='publickey',
 280                                            secret_identifier='privatekey'
 281                                            )]
 282         c['schedulers'] = []
 283         c['slavePortnum'] = 0
 284         c['schedulers'] = []
 285
 286         f1 = factory.BuildFactory([s(dummy.RemoteDummy, timeout=1)])
 287
 288         c['builders'] = [
 289             {'name': 'b1', 'slavenames': ['bot1'],
 290              'builddir': 'b1', 'factory': f1},
 291             ]
 292         """)
 293
 294     def testSequence(self):
 295         # test with secrets in config, a single AMI, and defaults/
 296         self.assertEqual(self.bot1.ami, 'ami-12345')
 297         self.assertEqual(self.bot1.instance_type, 'm1.large')
 298         self.assertEqual(self.bot1.keypair_name, 'latent_buildbot_slave')
 299         self.assertEqual(self.bot1.security_name, 'latent_buildbot_slave')
 300         # this would be appropriate if we were recreating keys.
 301         #self.assertNotEqual(self.boto.keys['latent_buildbot_slave'],
 302         #                    self.boto.original_keys['latent_buildbot_slave'])
 303         self.failUnless(isinstance(self.bot1.get_image(), Image))
 304         self.assertEqual(self.bot1.get_image().id, 'ami-12345')
 305         self.assertIdentical(self.bot1.elastic_ip, None)
 306         self.assertIdentical(self.bot1.instance, None)
 307         # let's start a build...
 308         self.build_deferred = self.doBuild()
 309         # ...and wait for the ec2 slave to show up
 310         d = self.bot1.substantiation_deferred
 311         d.addCallback(self._testSequence_1)
 312         return d
 313     def _testSequence_1(self, res):
 314         # bot 1 is substantiated.
 315         self.assertNotIdentical(self.bot1.slave, None)
 316         self.failUnless(self.bot1.substantiated)
 317         self.failUnless(isinstance(self.bot1.instance, Instance))
 318         self.assertEqual(self.bot1.instance.id, 'ami-12345')
 319         self.assertEqual(self.bot1.instance.state, RUNNING)
 320         self.assertEqual(self.bot1.instance.key_name, 'latent_buildbot_slave')
 321         self.assertEqual(self.bot1.instance.security_groups,
 322                          ['latent_buildbot_slave'])
 323         self.assertEqual(self.bot1.instance.instance_type, 'm1.large')
 324         self.assertEqual(self.bot1.output.output, 'example_output')
 325         # now we'll wait for the build to complete
 326         d = self.build_deferred
 327         del self.build_deferred
 328         d.addCallback(self._testSequence_2)
 329         return d
 330     def _testSequence_2(self, res):
 331         # build was a success!
 332         self.failUnlessEqual(res.getResults(), SUCCESS)
 333         self.failUnlessEqual(res.getSlavename(), "bot1")
 334         # Let's let it shut down.  We'll set the build_wait_timer to fire
 335         # sooner, and wait for it to fire.
 336         self.bot1.build_wait_timer.reset(0)
 337         # we'll stash the instance around to look at it
 338         self.instance = self.bot1.instance
 339         # now we wait.
 340         d = defer.Deferred()
 341         reactor.callLater(0.5, d.callback, None)
 342         d.addCallback(self._testSequence_3)
 343         return d
 344     def _testSequence_3(self, res):
 345         # slave is insubstantiated
 346         self.assertIdentical(self.bot1.slave, None)
 347         self.failIf(self.bot1.substantiated)
 348         self.assertIdentical(self.bot1.instance, None)
 349         self.assertEqual(self.instance.state, TERMINATED)
 350         del self.instance
 351
 352 class ElasticIP(Mixin, unittest.TestCase):
 353     config = textwrap.dedent("""\
 354         from buildbot.process import factory
 355         from buildbot.steps import dummy
 356         from buildbot.ec2buildslave import EC2LatentBuildSlave
 357         s = factory.s
 358
 359         BuildmasterConfig = c = {}
 360         c['slaves'] = [EC2LatentBuildSlave('bot1', 'sekrit', 'm1.large',
 361                                            'ami-12345',
 362                                            identifier='publickey',
 363                                            secret_identifier='privatekey',
 364                                            elastic_ip='127.0.0.1'
 365                                            )]
 366         c['schedulers'] = []
 367         c['slavePortnum'] = 0
 368         c['schedulers'] = []
 369
 370         f1 = factory.BuildFactory([s(dummy.RemoteDummy, timeout=1)])
 371
 372         c['builders'] = [
 373             {'name': 'b1', 'slavenames': ['bot1'],
 374              'builddir': 'b1', 'factory': f1},
 375             ]
 376         """)
 377
 378     def testSequence(self):
 379         self.assertEqual(self.bot1.elastic_ip.public_ip, '127.0.0.1')
 380         self.assertIdentical(self.boto.addresses['127.0.0.1'], None)
 381         # let's start a build...
 382         d = self.doBuild()
 383         d.addCallback(self._testSequence_1)
 384         return d
 385     def _testSequence_1(self, res):
 386         # build was a success!
 387         self.failUnlessEqual(res.getResults(), SUCCESS)
 388         self.failUnlessEqual(res.getSlavename(), "bot1")
 389         # we have our address
 390         self.assertIdentical(self.boto.addresses['127.0.0.1'],
 391                              self.bot1.instance)
 392         # Let's let it shut down.  We'll set the build_wait_timer to fire
 393         # sooner, and wait for it to fire.
 394         self.bot1.build_wait_timer.reset(0)
 395         d = defer.Deferred()
 396         reactor.callLater(0.5, d.callback, None)
 397         d.addCallback(self._testSequence_2)
 398         return d
 399     def _testSequence_2(self, res):
 400         # slave is insubstantiated
 401         self.assertIdentical(self.bot1.slave, None)
 402         self.failIf(self.bot1.substantiated)
 403         self.assertIdentical(self.bot1.instance, None)
 404         # the address is free again
 405         self.assertIdentical(self.boto.addresses['127.0.0.1'], None)
 406
 407
 408 class Initialization(Mixin, unittest.TestCase):
 409
 410     def setUp(self):
 411         self.boto_setUp1()
 412
 413     def tearDown(self):
 414         self.boto_setUp2()
 415         return Mixin.tearDown(self)
 416
 417     def testDefaultSeparateFile(self):
 418         # set up .ec2/aws_id
 419         home = os.environ['HOME']
 420         fake_home = os.path.join(os.getcwd(), 'basedir') # see RunMixin.setUp
 421         os.environ['HOME'] = fake_home
 422         dir = os.path.join(fake_home, '.ec2')
 423         os.mkdir(dir)
 424         f = open(os.path.join(dir, 'aws_id'), 'w')
 425         f.write('publickey\nprivatekey')
 426         f.close()
 427         # The Connection checks the file, so if the secret file is not parsed
 428         # correctly, *this* is where it would fail. This is the real test.
 429         from buildbot.ec2buildslave import EC2LatentBuildSlave
 430         bot1 = EC2LatentBuildSlave('bot1', 'sekrit', 'm1.large',
 431                                    'ami-12345')
 432         # for completeness, we'll show that the connection actually exists.
 433         self.failUnless(isinstance(bot1.conn, Connection))
 434         # clean up.
 435         os.environ['HOME'] = home
 436         self.rmtree(dir)
 437
 438     def testCustomSeparateFile(self):
 439         # set up .ec2/aws_id
 440         file_path = os.path.join(os.getcwd(), 'basedir', 'custom_aws_id')
 441         f = open(file_path, 'w')
 442         f.write('publickey\nprivatekey')
 443         f.close()
 444         # The Connection checks the file, so if the secret file is not parsed
 445         # correctly, *this* is where it would fail. This is the real test.
 446         from buildbot.ec2buildslave import EC2LatentBuildSlave
 447         bot1 = EC2LatentBuildSlave('bot1', 'sekrit', 'm1.large',
 448                                    'ami-12345', aws_id_file_path=file_path)
 449         # for completeness, we'll show that the connection actually exists.
 450         self.failUnless(isinstance(bot1.conn, Connection))
 451
 452     def testNoAMIBroken(self):
 453         # you must specify an AMI, or at least one of valid_ami_owners or
 454         # valid_ami_location_regex
 455         from buildbot.ec2buildslave import EC2LatentBuildSlave
 456         self.assertRaises(ValueError, EC2LatentBuildSlave, 'bot1', 'sekrit',
 457                           'm1.large', identifier='publickey',
 458                           secret_identifier='privatekey')
 459
 460     def testAMIOwnerFilter(self):
 461         # if you only specify an owner, you get the image owned by any of the
 462         # owners that sorts last by the AMI's location.
 463         from buildbot.ec2buildslave import EC2LatentBuildSlave
 464         bot1 = EC2LatentBuildSlave('bot1', 'sekrit', 'm1.large',
 465                                    valid_ami_owners=[11111111111],
 466                                    identifier='publickey',
 467                                    secret_identifier='privatekey'
 468                                    )
 469         self.assertEqual(bot1.get_image().location,
 470                          'test-f0a/image.manifest.xml')
 471         bot1 = EC2LatentBuildSlave('bot1', 'sekrit', 'm1.large',
 472                                    valid_ami_owners=[11111111111,
 473                                                      22222222222],
 474                                    identifier='publickey',
 475                                    secret_identifier='privatekey'
 476                                    )
 477         self.assertEqual(bot1.get_image().location,
 478                          'test-f0a/image.manifest.xml')
 479         bot1 = EC2LatentBuildSlave('bot1', 'sekrit', 'm1.large',
 480                                    valid_ami_owners=[22222222222],
 481                                    identifier='publickey',
 482                                    secret_identifier='privatekey'
 483                                    )
 484         self.assertEqual(bot1.get_image().location,
 485                          'test-e1b/image.manifest.xml')
 486         bot1 = EC2LatentBuildSlave('bot1', 'sekrit', 'm1.large',
 487                                    valid_ami_owners=12345667890,
 488                                    identifier='publickey',
 489                                    secret_identifier='privatekey'
 490                                    )
 491         self.assertEqual(bot1.get_image().location,
 492                          'test-xx/image.manifest.xml')
 493
 494     def testAMISimpleRegexFilter(self):
 495         from buildbot.ec2buildslave import EC2LatentBuildSlave
 496         bot1 = EC2LatentBuildSlave(
 497             'bot1', 'sekrit', 'm1.large',
 498             valid_ami_location_regex=r'test\-[a-z]\w+/image.manifest.xml',
 499             identifier='publickey', secret_identifier='privatekey')
 500         self.assertEqual(bot1.get_image().location,
 501                          'test-xx/image.manifest.xml')
 502         bot1 = EC2LatentBuildSlave(
 503             'bot1', 'sekrit', 'm1.large',
 504             valid_ami_location_regex=r'test\-[a-z]\d+\w/image.manifest.xml',
 505             identifier='publickey', secret_identifier='privatekey')
 506         self.assertEqual(bot1.get_image().location,
 507                          'test-f0a/image.manifest.xml')
 508         bot1 = EC2LatentBuildSlave(
 509             'bot1', 'sekrit', 'm1.large', valid_ami_owners=[22222222222],
 510             valid_ami_location_regex=r'test\-[a-z]\d+\w/image.manifest.xml',
 511             identifier='publickey', secret_identifier='privatekey')
 512         self.assertEqual(bot1.get_image().location,
 513                          'test-e1b/image.manifest.xml')
 514
 515     def testAMIRegexAlphaSortFilter(self):
 516         from buildbot.ec2buildslave import EC2LatentBuildSlave
 517         bot1 = EC2LatentBuildSlave(
 518             'bot1', 'sekrit', 'm1.large',
 519             valid_ami_owners=[11111111111, 22222222222],
 520             valid_ami_location_regex=r'test\-[a-z]\d+([a-z])/image.manifest.xml',
 521             identifier='publickey', secret_identifier='privatekey')
 522         self.assertEqual(bot1.get_image().location,
 523                          'test-a5f/image.manifest.xml')
 524
 525     def testAMIRegexIntSortFilter(self):
 526         from buildbot.ec2buildslave import EC2LatentBuildSlave
 527         bot1 = EC2LatentBuildSlave(
 528             'bot1', 'sekrit', 'm1.large',
 529             valid_ami_owners=[11111111111, 22222222222],
 530             valid_ami_location_regex=r'test\-[a-z](\d+)[a-z]/image.manifest.xml',
 531             identifier='publickey', secret_identifier='privatekey')
 532         self.assertEqual(bot1.get_image().location,
 533                          'test-c30d/image.manifest.xml')
 534
 535     def testNewSecurityGroup(self):
 536         from buildbot.ec2buildslave import EC2LatentBuildSlave
 537         bot1 = EC2LatentBuildSlave(
 538             'bot1', 'sekrit', 'm1.large', 'ami-12345',
 539             identifier='publickey', secret_identifier='privatekey',
 540             security_name='custom_security_name')
 541         self.assertEqual(
 542             self.boto.security_groups['custom_security_name'].value,
 543             'custom_security_name')
 544         self.assertEqual(bot1.security_name, 'custom_security_name')
 545
 546     def testNewKeypairName(self):
 547         from buildbot.ec2buildslave import EC2LatentBuildSlave
 548         bot1 = EC2LatentBuildSlave(
 549             'bot1', 'sekrit', 'm1.large', 'ami-12345',
 550             identifier='publickey', secret_identifier='privatekey',
 551             keypair_name='custom_keypair_name')
 552         self.assertIn('custom_keypair_name', self.boto.keys)
 553         self.assertEqual(bot1.keypair_name, 'custom_keypair_name')