python/google/appengine/tools/devappserver2/server_test.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright 2007 Google Inc.
   4 #
   5 # Licensed under the Apache License, Version 2.0 (the "License");
   6 # you may not use this file except in compliance with the License.
   7 # You may obtain a copy of the License at
   8 #
   9 #     http://www.apache.org/licenses/LICENSE-2.0
  10 #
  11 # Unless required by applicable law or agreed to in writing, software
  12 # distributed under the License is distributed on an "AS IS" BASIS,
  13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14 # See the License for the specific language governing permissions and
  15 # limitations under the License.
  16 #
  17 """Tests for google.apphosting.tools.devappserver2.server."""
  18
  19
  20 import httplib
  21 import logging
  22 import os
  23 import re
  24 import time
  25 import unittest
  26
  27 import google
  28 import mox
  29
  30 from google.appengine.api import appinfo
  31 from google.appengine.api import request_info
  32 from google.appengine.tools.devappserver2 import api_server
  33 from google.appengine.tools.devappserver2 import application_configuration
  34 from google.appengine.tools.devappserver2 import constants
  35 from google.appengine.tools.devappserver2 import dispatcher
  36 from google.appengine.tools.devappserver2 import instance
  37 from google.appengine.tools.devappserver2 import server
  38 from google.appengine.tools.devappserver2 import start_response_utils
  39 from google.appengine.tools.devappserver2 import wsgi_server
  40
  41
  42 class ServerConfigurationStub(object):
  43   def __init__(self,
  44                application_root='/root',
  45                application='app',
  46                server_name='default',
  47                automatic_scaling=appinfo.AutomaticScaling(),
  48                version='version',
  49                runtime='python27',
  50                threadsafe=False,
  51                skip_files='',
  52                inbound_services=['warmup'],
  53                handlers=[appinfo.URLMap(url=r'/python-(.*)',
  54                                         script=r'\1.py')],
  55                normalized_libraries=None,
  56                env_variables=None,
  57                manual_scaling=None,
  58                basic_scaling=None):
  59     self.application_root = application_root
  60     self.application = application
  61     self.server_name = server_name
  62     self.automatic_scaling = automatic_scaling
  63     self.manual_scaling = manual_scaling
  64     self.basic_scaling = basic_scaling
  65     self.major_version = version
  66     self.runtime = runtime
  67     self.threadsafe = threadsafe
  68     self.skip_files = skip_files
  69     self.inbound_services = inbound_services
  70     self.handlers = handlers
  71     self.normalized_libraries = normalized_libraries or []
  72     self.env_variables = env_variables or []
  73     self.version_id = '%s:%s.%s' % (server_name, version, '12345')
  74     self.is_backend = False
  75
  76   def check_for_updates(self):
  77     return set()
  78
  79
  80 class ServerFacade(server.Server):
  81   def __init__(self,
  82                server_configuration=ServerConfigurationStub(),
  83                instance_factory=None,
  84                ready=True):
  85     super(ServerFacade, self).__init__(
  86         server_configuration,
  87         host='fakehost',
  88         balanced_port=0,
  89         api_port=8080,
  90         runtime_stderr_loglevel=1,
  91
  92         python_config=None,
  93         cloud_sql_config=None,
  94         default_version_port=8080,
  95         port_registry=dispatcher.PortRegistry(),
  96         request_data=None,
  97         dispatcher=None,
  98         max_instances=None,
  99         use_mtime_file_watcher=False,
 100         automatic_restarts=True)
 101     if instance_factory is not None:
 102       self._instance_factory = instance_factory
 103     self._ready = ready
 104
 105   @property
 106   def ready(self):
 107     return self._ready
 108
 109   @property
 110   def balanced_port(self):
 111     return self._balanced_port
 112
 113
 114 class AutoScalingServerFacade(server.AutoScalingServer):
 115   def __init__(self,
 116                server_configuration=ServerConfigurationStub(),
 117                balanced_port=0,
 118                instance_factory=None,
 119                max_instances=None,
 120                ready=True):
 121     super(AutoScalingServerFacade, self).__init__(
 122         server_configuration,
 123         host='fakehost',
 124         balanced_port=balanced_port,
 125         api_port=8080,
 126         runtime_stderr_loglevel=1,
 127
 128         python_config=None,
 129         cloud_sql_config=None,
 130         default_version_port=8080,
 131         port_registry=dispatcher.PortRegistry(),
 132         request_data=None,
 133         dispatcher=None,
 134         max_instances=max_instances,
 135         use_mtime_file_watcher=False,
 136         automatic_restarts=True)
 137     if instance_factory is not None:
 138       self._instance_factory = instance_factory
 139     self._ready = ready
 140
 141   @property
 142   def ready(self):
 143     return self._ready
 144
 145   @property
 146   def balanced_port(self):
 147     return self._balanced_port
 148
 149
 150 class ManualScalingServerFacade(server.ManualScalingServer):
 151   def __init__(self,
 152                server_configuration=ServerConfigurationStub(),
 153                balanced_port=0,
 154                instance_factory=None,
 155                ready=True):
 156     super(ManualScalingServerFacade, self).__init__(
 157         server_configuration,
 158         host='fakehost',
 159         balanced_port=balanced_port,
 160         api_port=8080,
 161         runtime_stderr_loglevel=1,
 162
 163         python_config=None,
 164         cloud_sql_config=None,
 165         default_version_port=8080,
 166         port_registry=dispatcher.PortRegistry(),
 167         request_data=None,
 168         dispatcher=None,
 169         max_instances=None,
 170         use_mtime_file_watcher=False,
 171         automatic_restarts=True)
 172     if instance_factory is not None:
 173       self._instance_factory = instance_factory
 174     self._ready = ready
 175
 176   @property
 177   def ready(self):
 178     return self._ready
 179
 180   @property
 181   def balanced_port(self):
 182     return self._balanced_port
 183
 184
 185 class BasicScalingServerFacade(server.BasicScalingServer):
 186   def __init__(self,
 187                host='fakehost',
 188                server_configuration=ServerConfigurationStub(),
 189                balanced_port=0,
 190                instance_factory=None,
 191                ready=True):
 192     super(BasicScalingServerFacade, self).__init__(
 193         server_configuration,
 194         host,
 195         balanced_port=balanced_port,
 196         api_port=8080,
 197         runtime_stderr_loglevel=1,
 198
 199         python_config=None,
 200         cloud_sql_config=None,
 201         default_version_port=8080,
 202         port_registry=dispatcher.PortRegistry(),
 203         request_data=None,
 204         dispatcher=None,
 205         max_instances=None,
 206         use_mtime_file_watcher=False,
 207         automatic_restarts=True)
 208     if instance_factory is not None:
 209       self._instance_factory = instance_factory
 210     self._ready = ready
 211
 212   @property
 213   def ready(self):
 214     return self._ready
 215
 216   @property
 217   def balanced_port(self):
 218     return self._balanced_port
 219
 220
 221 class BuildRequestEnvironTest(unittest.TestCase):
 222
 223   def setUp(self):
 224     api_server.test_setup_stubs()
 225     self.server = ServerFacade()
 226
 227   def test_build_request_environ(self):
 228     expected_environ = {
 229         constants.FAKE_IS_ADMIN_HEADER: '1',
 230         'HTTP_HOST': 'fakehost:8080',
 231         'HTTP_HEADER': 'Value',
 232         'HTTP_OTHER': 'Values',
 233         'CONTENT_LENGTH': '4',
 234         'PATH_INFO': '/foo',
 235         'QUERY_STRING': 'bar=baz',
 236         'REQUEST_METHOD': 'PUT',
 237         'REMOTE_ADDR': '1.2.3.4',
 238         'SERVER_NAME': 'fakehost',
 239         'SERVER_PORT': '8080',
 240         'SERVER_PROTOCOL': 'HTTP/1.1',
 241         'wsgi.version': (1, 0),
 242         'wsgi.url_scheme': 'http',
 243         'wsgi.multithread': True,
 244         'wsgi.multiprocess': True}
 245     environ = self.server.build_request_environ(
 246         'PUT', '/foo?bar=baz', [('Header', 'Value'), ('Other', 'Values')],
 247         'body', '1.2.3.4', 8080)
 248     self.assertEqual('', environ.pop('wsgi.errors').getvalue())
 249     self.assertEqual('body', environ.pop('wsgi.input').getvalue())
 250     self.assertEqual(expected_environ, environ)
 251
 252   def test_build_request_environ_fake_is_logged_in(self):
 253     expected_environ = {
 254         constants.FAKE_IS_ADMIN_HEADER: '1',
 255         constants.FAKE_LOGGED_IN_HEADER: '1',
 256         'HTTP_HOST': 'fakehost:8080',
 257         'HTTP_HEADER': 'Value',
 258         'HTTP_OTHER': 'Values',
 259         'CONTENT_LENGTH': '4',
 260         'PATH_INFO': '/foo',
 261         'QUERY_STRING': 'bar=baz',
 262         'REQUEST_METHOD': 'PUT',
 263         'REMOTE_ADDR': '1.2.3.4',
 264         'SERVER_NAME': 'fakehost',
 265         'SERVER_PORT': '8080',
 266         'SERVER_PROTOCOL': 'HTTP/1.1',
 267         'wsgi.version': (1, 0),
 268         'wsgi.url_scheme': 'http',
 269         'wsgi.multithread': True,
 270         'wsgi.multiprocess': True}
 271     environ = self.server.build_request_environ(
 272         'PUT', '/foo?bar=baz', [('Header', 'Value'), ('Other', 'Values')],
 273         'body', '1.2.3.4', 8080, fake_login=True)
 274     self.assertEqual('', environ.pop('wsgi.errors').getvalue())
 275     self.assertEqual('body', environ.pop('wsgi.input').getvalue())
 276     self.assertEqual(expected_environ, environ)
 277
 278   def test_build_request_environ_unicode_body(self):
 279     expected_environ = {
 280         constants.FAKE_IS_ADMIN_HEADER: '1',
 281         'HTTP_HOST': 'fakehost',
 282         'HTTP_HEADER': 'Value',
 283         'HTTP_OTHER': 'Values',
 284         'CONTENT_LENGTH': '4',
 285         'PATH_INFO': '/foo',
 286         'QUERY_STRING': 'bar=baz',
 287         'REQUEST_METHOD': 'PUT',
 288         'REMOTE_ADDR': '1.2.3.4',
 289         'SERVER_NAME': 'fakehost',
 290         'SERVER_PORT': '80',
 291         'SERVER_PROTOCOL': 'HTTP/1.1',
 292         'wsgi.version': (1, 0),
 293         'wsgi.url_scheme': 'http',
 294         'wsgi.multithread': True,
 295         'wsgi.multiprocess': True}
 296     environ = self.server.build_request_environ(
 297         'PUT', '/foo?bar=baz', [('Header', 'Value'), ('Other', 'Values')],
 298         u'body', '1.2.3.4', 80)
 299     self.assertEqual('', environ.pop('wsgi.errors').getvalue())
 300     self.assertEqual('body', environ.pop('wsgi.input').getvalue())
 301     self.assertEqual(expected_environ, environ)
 302
 303
 304 class TestServerCreateUrlHandlers(unittest.TestCase):
 305   """Tests for server.Server._create_url_handlers."""
 306
 307   def setUp(self):
 308     self.server_configuration = ServerConfigurationStub()
 309     self.instance_factory = instance.InstanceFactory(None, 1)
 310     self.servr = ServerFacade(instance_factory=self.instance_factory,
 311                               server_configuration=self.server_configuration)
 312     self.instance_factory.START_URL_MAP = appinfo.URLMap(
 313         url='/_ah/start',
 314         script='start_handler',
 315         login='admin')
 316     self.instance_factory.WARMUP_URL_MAP = appinfo.URLMap(
 317         url='/_ah/warmup',
 318         script='warmup_handler',
 319         login='admin')
 320
 321   def test_match_all(self):
 322     self.server_configuration.handlers = [appinfo.URLMap(url=r'.*',
 323                                                          script=r'foo.py')]
 324     handlers = self.servr._create_url_handlers()
 325     self.assertEqual(6, len(handlers))
 326
 327   def test_match_start_only(self):
 328     self.server_configuration.handlers = [appinfo.URLMap(url=r'/_ah/start',
 329                                                          script=r'foo.py')]
 330     handlers = self.servr._create_url_handlers()
 331     self.assertEqual(7, len(handlers))
 332     self.assertEqual(self.instance_factory.WARMUP_URL_MAP, handlers[0].url_map)
 333
 334   def test_match_warmup_only(self):
 335     self.server_configuration.handlers = [appinfo.URLMap(url=r'/_ah/warmup',
 336                                                          script=r'foo.py')]
 337     handlers = self.servr._create_url_handlers()
 338     self.assertEqual(7, len(handlers))
 339     self.assertEqual(self.instance_factory.START_URL_MAP, handlers[0].url_map)
 340
 341   def test_match_neither_warmup_nor_start(self):
 342     self.server_configuration.handlers = [appinfo.URLMap(url=r'/',
 343                                                          script=r'foo.py')]
 344     handlers = self.servr._create_url_handlers()
 345     self.assertEqual(8, len(handlers))
 346     self.assertEqual(self.instance_factory.WARMUP_URL_MAP, handlers[0].url_map)
 347     self.assertEqual(self.instance_factory.START_URL_MAP, handlers[1].url_map)
 348
 349   def test_match_static_only(self):
 350     self.server_configuration.handlers = [
 351         appinfo.URLMap(url=r'/_ah/start', static_dir='foo'),
 352         appinfo.URLMap(url=r'/_ah/warmup', static_files='foo', upload='foo')]
 353     handlers = self.servr._create_url_handlers()
 354     self.assertEqual(9, len(handlers))
 355     self.assertEqual(self.instance_factory.WARMUP_URL_MAP, handlers[0].url_map)
 356     self.assertEqual(self.instance_factory.START_URL_MAP, handlers[1].url_map)
 357
 358   def test_match_start_only_no_inbound_warmup(self):
 359     self.server_configuration.inbound_services = None
 360     self.server_configuration.handlers = [appinfo.URLMap(url=r'/_ah/start',
 361                                                          script=r'foo.py')]
 362     handlers = self.servr._create_url_handlers()
 363     self.assertEqual(6, len(handlers))
 364
 365   def test_match_warmup_only_no_inbound_warmup(self):
 366     self.server_configuration.inbound_services = None
 367     self.server_configuration.handlers = [appinfo.URLMap(url=r'/_ah/warmup',
 368                                                          script=r'foo.py')]
 369     handlers = self.servr._create_url_handlers()
 370     self.assertEqual(7, len(handlers))
 371     self.assertEqual(self.instance_factory.START_URL_MAP, handlers[0].url_map)
 372
 373   def test_match_neither_warmup_nor_start_no_inbound_warmup(self):
 374     self.server_configuration.inbound_services = None
 375     self.server_configuration.handlers = [appinfo.URLMap(url=r'/',
 376                                                          script=r'foo.py')]
 377     handlers = self.servr._create_url_handlers()
 378     self.assertEqual(7, len(handlers))
 379     self.assertEqual(self.instance_factory.START_URL_MAP, handlers[0].url_map)
 380
 381
 382 class TestServerGetRuntimeConfig(unittest.TestCase):
 383   """Tests for server.Server._get_runtime_config."""
 384
 385   def setUp(self):
 386     self.server_configuration = ServerConfigurationStub()
 387     self.instance_factory = instance.InstanceFactory(None, 1)
 388     self.servr = ServerFacade(instance_factory=self.instance_factory,
 389                               server_configuration=self.server_configuration)
 390
 391   def test_static_files_regex(self):
 392     self.server_configuration.handlers = [
 393         appinfo.URLMap(url=r'/static', static_dir='static'),
 394         appinfo.URLMap(url=r'/app_read_static', static_dir='app_read_static',
 395                        application_readable=True),
 396         appinfo.URLMap(url=r'/static_images/*.png',
 397                        static_files=r'static_images/\\1',
 398                        upload=r'static_images/*.png'),
 399         appinfo.URLMap(url=r'/app_readable_static_images/*.png',
 400                        static_files=r'app_readable_static_images/\\1',
 401                        upload=r'app_readable_static_images/*.png',
 402                        application_readable=True),
 403         ]
 404     config = self.servr._get_runtime_config()
 405     self.assertEqual(r'^(static%s.*)|(static_images/*.png)$' %
 406                      re.escape(os.path.sep),
 407                      config.static_files)
 408
 409
 410 class TestServerShutdownInstance(unittest.TestCase):
 411   """Tests for server.Server._shutdown_instance."""
 412
 413   def setUp(self):
 414     api_server.test_setup_stubs()
 415     self.mox = mox.Mox()
 416     self.server_configuration = ServerConfigurationStub()
 417     self.instance_factory = instance.InstanceFactory(None, 1)
 418     self.servr = ServerFacade(instance_factory=self.instance_factory,
 419                               server_configuration=self.server_configuration)
 420     self.mox.StubOutWithMock(logging, 'exception')
 421     self.mox.StubOutWithMock(self.servr, '_handle_request')
 422     self.mox.StubOutWithMock(self.servr._quit_event, 'wait')
 423     self.mox.StubOutWithMock(server.Server, 'build_request_environ')
 424     self.inst = self.mox.CreateMock(instance.Instance)
 425     self.time = 0
 426     self.mox.stubs.Set(time, 'time', lambda: self.time)
 427
 428   def tearDown(self):
 429     self.mox.UnsetStubs()
 430
 431   def test_shutdown_instance(self):
 432
 433     def advance_time(*unused_args, **unused_kwargs):
 434       self.time += 10
 435
 436     environ = object()
 437     self.servr.build_request_environ(
 438         'GET', '/_ah/stop', [], '', '0.1.0.3', 9000, fake_login=True).AndReturn(
 439             environ)
 440     self.servr._handle_request(
 441         environ,
 442         start_response_utils.null_start_response,
 443         inst=self.inst,
 444         request_type=instance.SHUTDOWN_REQUEST).WithSideEffects(advance_time)
 445     self.servr._quit_event.wait(20)
 446     self.inst.quit(force=True)
 447     self.mox.ReplayAll()
 448     self.servr._shutdown_instance(self.inst, 9000)
 449     self.mox.VerifyAll()
 450
 451
 452 class TestAutoScalingServerWarmup(unittest.TestCase):
 453   """Tests for server.AutoScalingServer._warmup."""
 454
 455   def setUp(self):
 456     api_server.test_setup_stubs()
 457     self.mox = mox.Mox()
 458     self.mox.StubOutWithMock(server.Server, 'build_request_environ')
 459
 460   def tearDown(self):
 461     self.mox.UnsetStubs()
 462
 463   def test_warmup(self):
 464     s = AutoScalingServerFacade(balanced_port=8080)
 465     self.mox.StubOutWithMock(s, '_handle_request')
 466     self.mox.StubOutWithMock(s._condition, 'notify')
 467
 468     inst = self.mox.CreateMock(instance.Instance)
 469
 470     environ = object()
 471     s.build_request_environ('GET', '/_ah/warmup', [], '', '0.1.0.3', 8080,
 472                             fake_login=True).AndReturn(environ)
 473     s._handle_request(environ,
 474                       mox.IgnoreArg(),
 475                       inst=inst,
 476                       request_type=instance.READY_REQUEST)
 477     s._condition.notify(1)
 478
 479     self.mox.ReplayAll()
 480     s._warmup(inst)
 481     self.mox.VerifyAll()
 482
 483
 484 class TestAutoScalingServerAddInstance(unittest.TestCase):
 485   """Tests for server.AutoScalingServer._add_instance."""
 486
 487   def setUp(self):
 488     api_server.test_setup_stubs()
 489     self.mox = mox.Mox()
 490     self.factory = self.mox.CreateMock(instance.InstanceFactory)
 491     self.factory.max_concurrent_requests = 10
 492
 493   def tearDown(self):
 494     self.mox.UnsetStubs()
 495
 496   def test_permit_warmup(self):
 497     s = AutoScalingServerFacade(instance_factory=self.factory)
 498     self.mox.StubOutWithMock(s, '_async_warmup')
 499     self.mox.StubOutWithMock(s._condition, 'notify')
 500
 501     inst = self.mox.CreateMock(instance.Instance)
 502     self.factory.new_instance(mox.Regex('[a-f0-9]{36}'),
 503                               expect_ready_request=True).AndReturn(inst)
 504     inst.start().AndReturn(True)
 505     s._async_warmup(inst)
 506
 507     self.mox.ReplayAll()
 508     self.assertEqual(inst, s._add_instance(permit_warmup=True))
 509     self.mox.VerifyAll()
 510
 511     self.assertEqual(1, len(s._instances))
 512
 513   def test_no_permit_warmup(self):
 514     s = AutoScalingServerFacade(instance_factory=self.factory)
 515     self.mox.StubOutWithMock(s._condition, 'notify')
 516
 517     inst = self.mox.CreateMock(instance.Instance)
 518     self.factory.new_instance(mox.Regex('[a-f0-9]{36}'),
 519                               expect_ready_request=False).AndReturn(inst)
 520     inst.start().AndReturn(True)
 521     s._condition.notify(10)
 522
 523     self.mox.ReplayAll()
 524     self.assertEqual(inst, s._add_instance(permit_warmup=False))
 525     self.mox.VerifyAll()
 526
 527     self.assertIn(inst, s._instances)
 528
 529   def test_failed_to_start(self):
 530     s = AutoScalingServerFacade(instance_factory=self.factory)
 531     self.mox.StubOutWithMock(s, '_async_warmup')
 532     self.mox.StubOutWithMock(s._condition, 'notify')
 533
 534     inst = self.mox.CreateMock(instance.Instance)
 535     self.factory.new_instance(mox.Regex('[a-f0-9]{36}'),
 536                               expect_ready_request=True).AndReturn(inst)
 537     inst.start().AndReturn(False)
 538
 539     self.mox.ReplayAll()
 540     self.assertIsNone(s._add_instance(permit_warmup=True))
 541     self.mox.VerifyAll()
 542
 543     self.assertEqual(1, len(s._instances))
 544
 545   def test_max_instances(self):
 546     s = AutoScalingServerFacade(instance_factory=self.factory,
 547                                 max_instances=1)
 548     self.mox.StubOutWithMock(s._condition, 'notify')
 549
 550     inst = self.mox.CreateMock(instance.Instance)
 551     self.factory.new_instance(mox.Regex('[a-f0-9]{36}'),
 552                               expect_ready_request=False).AndReturn(inst)
 553     inst.start().AndReturn(True)
 554     s._condition.notify(10)
 555
 556     self.mox.ReplayAll()
 557     self.assertEqual(inst, s._add_instance(permit_warmup=False))
 558     self.assertEqual(None, s._add_instance(permit_warmup=False))
 559     self.mox.VerifyAll()
 560
 561     self.assertEqual(1, len(s._instances))
 562
 563
 564 class TestAutoScalingInstancePoolHandleScriptRequest(unittest.TestCase):
 565   """Tests for server.AutoScalingServer.handle."""
 566
 567   def setUp(self):
 568     api_server.test_setup_stubs()
 569     self.mox = mox.Mox()
 570
 571     self.inst = self.mox.CreateMock(instance.Instance)
 572     self.environ = {}
 573     self.start_response = object()
 574     self.response = [object()]
 575     self.url_map = object()
 576     self.match = object()
 577     self.request_id = object()
 578     self.auto_server = AutoScalingServerFacade(
 579         instance_factory=instance.InstanceFactory(object(), 10))
 580     self.mox.StubOutWithMock(self.auto_server, '_choose_instance')
 581     self.mox.StubOutWithMock(self.auto_server, '_add_instance')
 582     self.mox.stubs.Set(time, 'time', lambda: 0.0)
 583
 584   def tearDown(self):
 585     self.mox.UnsetStubs()
 586
 587   def test_handle_script_request(self):
 588     self.auto_server._choose_instance(0.1).AndReturn(self.inst)
 589     self.inst.handle(self.environ,
 590                      self.start_response,
 591                      self.url_map,
 592                      self.match,
 593                      self.request_id,
 594                      instance.NORMAL_REQUEST).AndReturn(self.response)
 595
 596     self.mox.ReplayAll()
 597     self.assertEqual(
 598         self.response,
 599         self.auto_server._handle_script_request(self.environ,
 600                                                 self.start_response,
 601                                                 self.url_map,
 602                                                 self.match,
 603                                                 self.request_id))
 604     self.mox.VerifyAll()
 605     self.assertEqual([(mox.IgnoreArg(), 1)],
 606                      list(self.auto_server._outstanding_request_history))
 607
 608   def test_handle_cannot_accept_request(self):
 609     self.auto_server._choose_instance(0.1).AndReturn(self.inst)
 610     self.auto_server._choose_instance(0.1).AndReturn(self.inst)
 611     self.inst.handle(
 612         self.environ, self.start_response, self.url_map, self.match,
 613         self.request_id, instance.NORMAL_REQUEST).AndRaise(
 614             instance.CannotAcceptRequests)
 615     self.inst.handle(
 616         self.environ, self.start_response, self.url_map, self.match,
 617         self.request_id, instance.NORMAL_REQUEST).AndReturn(
 618             self.response)
 619
 620     self.mox.ReplayAll()
 621     self.assertEqual(
 622         self.response,
 623         self.auto_server._handle_script_request(self.environ,
 624                                                 self.start_response,
 625                                                 self.url_map,
 626                                                 self.match,
 627                                                 self.request_id))
 628     self.mox.VerifyAll()
 629     self.assertEqual([(mox.IgnoreArg(), 1)],
 630                      list(self.auto_server._outstanding_request_history))
 631
 632   def test_handle_new_instance(self):
 633     self.auto_server._choose_instance(0.1).AndReturn(None)
 634     self.auto_server._add_instance(permit_warmup=False).AndReturn(self.inst)
 635
 636     self.inst.handle(
 637         self.environ, self.start_response, self.url_map, self.match,
 638         self.request_id, instance.NORMAL_REQUEST).AndReturn(
 639             self.response)
 640
 641     self.mox.ReplayAll()
 642     self.assertEqual(
 643         self.response,
 644         self.auto_server._handle_script_request(self.environ,
 645                                                 self.start_response,
 646                                                 self.url_map,
 647                                                 self.match,
 648                                                 self.request_id))
 649     self.mox.VerifyAll()
 650
 651   def test_handle_new_instance_none_returned(self):
 652     self.auto_server._choose_instance(0.1).AndReturn(None)
 653     self.auto_server._add_instance(permit_warmup=False).AndReturn(None)
 654     self.auto_server._choose_instance(0.2).AndReturn(self.inst)
 655
 656     self.inst.handle(
 657         self.environ, self.start_response, self.url_map, self.match,
 658         self.request_id, instance.NORMAL_REQUEST).AndReturn(
 659             self.response)
 660
 661     self.mox.ReplayAll()
 662     self.assertEqual(
 663         self.response,
 664         self.auto_server._handle_script_request(self.environ,
 665                                                 self.start_response,
 666                                                 self.url_map,
 667                                                 self.match,
 668                                                 self.request_id))
 669     self.mox.VerifyAll()
 670
 671
 672 class TestAutoScalingInstancePoolTrimRequestTimesAndOutstanding(
 673     unittest.TestCase):
 674   """Tests for AutoScalingServer._trim_outstanding_request_history."""
 675
 676   def setUp(self):
 677     api_server.test_setup_stubs()
 678
 679   def test_trim_outstanding_request_history(self):
 680     servr = AutoScalingServerFacade(
 681         instance_factory=instance.InstanceFactory(object(), 10))
 682     servr._outstanding_request_history.append((0, 100))
 683     servr._outstanding_request_history.append((1.0, 101))
 684     servr._outstanding_request_history.append((1.2, 102))
 685     servr._outstanding_request_history.append((2.5, 103))
 686
 687     now = time.time()
 688     servr._outstanding_request_history.append((now, 42))
 689     servr._outstanding_request_history.append((now + 1, 43))
 690     servr._outstanding_request_history.append((now + 3, 44))
 691     servr._outstanding_request_history.append((now + 4, 45))
 692
 693     servr._trim_outstanding_request_history()
 694     self.assertEqual([(now, 42), (now + 1, 43), (now + 3, 44), (now + 4, 45)],
 695                      list(servr._outstanding_request_history))
 696
 697
 698 class TestAutoScalingInstancePoolGetNumRequiredInstances(unittest.TestCase):
 699   """Tests for AutoScalingServer._outstanding_request_history."""
 700
 701   def setUp(self):
 702     api_server.test_setup_stubs()
 703     self.servr = AutoScalingServerFacade(
 704         instance_factory=instance.InstanceFactory(object(), 5))
 705
 706   def test_get_num_required_instances(self):
 707     now = time.time()
 708     self.servr._outstanding_request_history.append((now, 42))
 709     self.servr._outstanding_request_history.append((now + 1, 43))
 710     self.servr._outstanding_request_history.append((now + 3, 44))
 711     self.servr._outstanding_request_history.append((now + 4, 45))
 712     self.assertEqual(9, self.servr._get_num_required_instances())
 713
 714   def test_no_requests(self):
 715     self.assertEqual(0, self.servr._get_num_required_instances())
 716
 717
 718 class TestAutoScalingInstancePoolSplitInstances(unittest.TestCase):
 719   """Tests for server.AutoScalingServer._split_instances."""
 720
 721   class Instance(object):
 722     def __init__(self, num_outstanding_requests, can_accept_requests=True):
 723       self.num_outstanding_requests = num_outstanding_requests
 724       self.can_accept_requests = can_accept_requests
 725
 726     def __repr__(self):
 727       return str(self.num_outstanding_requests)
 728
 729   def setUp(self):
 730     api_server.test_setup_stubs()
 731
 732     self.mox = mox.Mox()
 733     self.servr = AutoScalingServerFacade(
 734         instance_factory=instance.InstanceFactory(object(), 10))
 735     self.mox.StubOutWithMock(self.servr, '_get_num_required_instances')
 736
 737   def tearDown(self):
 738     self.mox.UnsetStubs()
 739
 740   def test_split_instances(self):
 741     instance1 = self.Instance(1)
 742     instance2 = self.Instance(2, can_accept_requests=False)
 743     instance3 = self.Instance(3)
 744     instance4 = self.Instance(4)
 745     instance5 = self.Instance(5)
 746     instance6 = self.Instance(6)
 747     instance7 = self.Instance(7)
 748     instance8 = self.Instance(8, can_accept_requests=False)
 749     instance9 = self.Instance(9)
 750     instance10 = self.Instance(10)
 751
 752     self.servr._get_num_required_instances().AndReturn(5)
 753     self.servr._instances = set([instance1, instance2, instance3, instance4,
 754                                  instance5, instance6, instance7, instance8,
 755                                  instance9, instance10])
 756
 757     self.mox.ReplayAll()
 758     self.assertEqual(
 759         (set([instance10, instance9, instance7,
 760               instance6, instance5]),
 761          set([instance1, instance2, instance3, instance4, instance8])),
 762         self.servr._split_instances())
 763     self.mox.VerifyAll()
 764
 765   def test_split_instances_no_instances(self):
 766     self.servr._get_num_required_instances().AndReturn(5)
 767     self.servr._instances = set([])
 768
 769     self.mox.ReplayAll()
 770     self.assertEqual((set([]), set([])),
 771                      self.servr._split_instances())
 772     self.mox.VerifyAll()
 773
 774   def test_split_instances_no_instances_not_enough_accepting_requests(self):
 775     instance1 = self.Instance(1)
 776     instance2 = self.Instance(1, can_accept_requests=False)
 777     instance3 = self.Instance(2, can_accept_requests=False)
 778
 779     self.servr._get_num_required_instances().AndReturn(5)
 780     self.servr._instances = set([instance1, instance2, instance3])
 781
 782     self.mox.ReplayAll()
 783     self.assertEqual((set([instance1]), set([instance2, instance3])),
 784                      self.servr._split_instances())
 785     self.mox.VerifyAll()
 786
 787   def test_split_instances_no_required_instances(self):
 788     instance1 = self.Instance(1)
 789     instance2 = self.Instance(2, can_accept_requests=False)
 790     instance3 = self.Instance(3, can_accept_requests=False)
 791     instance4 = self.Instance(4)
 792     instance5 = self.Instance(5)
 793     instance6 = self.Instance(6)
 794     instance7 = self.Instance(7)
 795     instance8 = self.Instance(8)
 796
 797     self.servr._get_num_required_instances().AndReturn(0)
 798     self.servr._instances = set([instance1, instance2, instance3, instance4,
 799                                  instance5, instance6, instance7, instance8])
 800
 801     self.mox.ReplayAll()
 802     self.assertEqual(
 803         (set(),
 804          set([instance8, instance7, instance6, instance5, instance4,
 805               instance3, instance2, instance1])),
 806         self.servr._split_instances())
 807     self.mox.VerifyAll()
 808
 809
 810 class TestAutoScalingInstancePoolChooseInstances(unittest.TestCase):
 811   """Tests for server.AutoScalingServer._choose_instance."""
 812
 813   class Instance(object):
 814     def __init__(self, num_outstanding_requests, can_accept_requests=True):
 815       self.num_outstanding_requests = num_outstanding_requests
 816       self.remaining_request_capacity = 10 - num_outstanding_requests
 817       self.can_accept_requests = can_accept_requests
 818
 819   def setUp(self):
 820     api_server.test_setup_stubs()
 821
 822     self.mox = mox.Mox()
 823     self.servr = AutoScalingServerFacade(
 824         instance_factory=instance.InstanceFactory(object(), 10))
 825     self.mox.StubOutWithMock(self.servr, '_split_instances')
 826     self.mox.StubOutWithMock(self.servr._condition, 'wait')
 827     self.time = 10
 828     self.mox.stubs.Set(time, 'time', lambda: self.time)
 829
 830   def advance_time(self, *unused_args):
 831     self.time += 10
 832
 833   def tearDown(self):
 834     self.mox.UnsetStubs()
 835
 836   def test_choose_instance_required_available(self):
 837     instance1 = self.Instance(1)
 838     instance2 = self.Instance(2)
 839     instance3 = self.Instance(3)
 840     instance4 = self.Instance(4)
 841
 842     self.servr._split_instances().AndReturn((set([instance3, instance4]),
 843                                              set([instance1, instance2])))
 844
 845     self.mox.ReplayAll()
 846     self.assertEqual(instance3,  # Least busy required instance.
 847                      self.servr._choose_instance(15))
 848     self.mox.VerifyAll()
 849
 850   def test_choose_instance_no_instances(self):
 851     self.servr._split_instances().AndReturn((set([]), set([])))
 852     self.servr._condition.wait(5).WithSideEffects(self.advance_time)
 853
 854     self.mox.ReplayAll()
 855     self.assertEqual(None, self.servr._choose_instance(15))
 856     self.mox.VerifyAll()
 857
 858   def test_choose_instance_no_instance_that_can_accept_requests(self):
 859     instance1 = self.Instance(1, can_accept_requests=False)
 860     self.servr._split_instances().AndReturn((set([]), set([instance1])))
 861     self.servr._condition.wait(5).WithSideEffects(self.advance_time)
 862
 863     self.mox.ReplayAll()
 864     self.assertEqual(None, self.servr._choose_instance(15))
 865     self.mox.VerifyAll()
 866
 867   def test_choose_instance_required_full(self):
 868     instance1 = self.Instance(1)
 869     instance2 = self.Instance(2)
 870     instance3 = self.Instance(10)
 871     instance4 = self.Instance(10)
 872
 873     self.servr._split_instances().AndReturn((set([instance3, instance4]),
 874                                              set([instance1, instance2])))
 875
 876     self.mox.ReplayAll()
 877     self.assertEqual(instance2,  # Busyest non-required instance.
 878                      self.servr._choose_instance(15))
 879     self.mox.VerifyAll()
 880
 881   def test_choose_instance_must_wait(self):
 882     instance1 = self.Instance(10)
 883     instance2 = self.Instance(10)
 884
 885     self.servr._split_instances().AndReturn((set([instance1]),
 886                                              set([instance2])))
 887     self.servr._condition.wait(5).WithSideEffects(self.advance_time)
 888
 889     self.mox.ReplayAll()
 890     self.assertIsNone(self.servr._choose_instance(15))
 891     self.mox.VerifyAll()
 892
 893
 894 class TestAutoScalingInstancePoolAdjustInstances(unittest.TestCase):
 895   """Tests for server.AutoScalingServer._adjust_instances."""
 896
 897   class Instance(object):
 898     def __init__(self, num_outstanding_requests):
 899       self.num_outstanding_requests = num_outstanding_requests
 900
 901     def quit(self):
 902       pass
 903
 904   def setUp(self):
 905     api_server.test_setup_stubs()
 906     self.mox = mox.Mox()
 907     self.servr = AutoScalingServerFacade(
 908         server_configuration=ServerConfigurationStub(
 909             automatic_scaling=appinfo.AutomaticScaling(
 910                 min_pending_latency='0.1s',
 911                 max_pending_latency='1.0s',
 912                 min_idle_instances=1,
 913                 max_idle_instances=2)),
 914         instance_factory=instance.InstanceFactory(object(), 10))
 915
 916     self.mox.StubOutWithMock(self.servr, '_split_instances')
 917     self.mox.StubOutWithMock(self.servr, '_add_instance')
 918
 919   def tearDown(self):
 920     self.mox.UnsetStubs()
 921
 922   def test_adjust_instances_create_new(self):
 923     instance1 = self.Instance(0)
 924     instance2 = self.Instance(2)
 925     instance3 = self.Instance(3)
 926     instance4 = self.Instance(4)
 927
 928     self.servr._instances = set([instance1, instance2, instance3, instance4])
 929     self.servr._split_instances().AndReturn(
 930         (set([instance1, instance2, instance3, instance4]),
 931          set([])))
 932     self.servr._add_instance(permit_warmup=True)
 933
 934     self.mox.ReplayAll()
 935     self.servr._adjust_instances()
 936     self.mox.VerifyAll()
 937
 938   def test_adjust_instances_quit_idle(self):
 939     instance1 = self.Instance(0)
 940     instance2 = self.Instance(2)
 941     instance3 = self.Instance(3)
 942     instance4 = self.Instance(4)
 943
 944     self.mox.StubOutWithMock(instance1, 'quit')
 945
 946     self.servr._instances = set([instance1, instance2, instance3, instance4])
 947     self.servr._split_instances().AndReturn(
 948         (set([]),
 949          set([instance1, instance2, instance3, instance4])))
 950     instance1.quit()
 951
 952     self.mox.ReplayAll()
 953     self.servr._adjust_instances()
 954     self.mox.VerifyAll()
 955
 956   def test_adjust_instances_quit_idle_with_race(self):
 957     instance1 = self.Instance(0)
 958     instance2 = self.Instance(2)
 959     instance3 = self.Instance(3)
 960     instance4 = self.Instance(4)
 961
 962     self.mox.StubOutWithMock(instance1, 'quit')
 963
 964     self.servr._instances = set([instance1, instance2, instance3, instance4])
 965     self.servr._split_instances().AndReturn(
 966         (set([]),
 967          set([instance1, instance2, instance3, instance4])))
 968     instance1.quit().AndRaise(instance.CannotQuitServingInstance)
 969
 970     self.mox.ReplayAll()
 971     self.servr._adjust_instances()
 972     self.mox.VerifyAll()
 973
 974
 975 class TestAutoScalingInstancePoolHandleChanges(unittest.TestCase):
 976   """Tests for server.AutoScalingServer._handle_changes."""
 977
 978   def setUp(self):
 979     api_server.test_setup_stubs()
 980
 981     self.mox = mox.Mox()
 982     self.instance_factory = instance.InstanceFactory(object(), 10)
 983     self.servr = AutoScalingServerFacade(
 984         instance_factory=self.instance_factory)
 985     self.mox.StubOutWithMock(self.instance_factory, 'files_changed')
 986     self.mox.StubOutWithMock(self.instance_factory, 'configuration_changed')
 987     self.mox.StubOutWithMock(self.servr, '_maybe_restart_instances')
 988     self.mox.StubOutWithMock(self.servr, '_create_url_handlers')
 989     self.mox.StubOutWithMock(self.servr._server_configuration,
 990                              'check_for_updates')
 991     self.mox.StubOutWithMock(self.servr._watcher, 'has_changes')
 992
 993   def tearDown(self):
 994     self.mox.UnsetStubs()
 995
 996   def test_no_changes(self):
 997     self.servr._server_configuration.check_for_updates().AndReturn(frozenset())
 998     self.servr._watcher.has_changes().AndReturn(False)
 999     self.servr._maybe_restart_instances(config_changed=False,
1000                                         file_changed=False)
1001     self.mox.ReplayAll()
1002     self.servr._handle_changes()
1003     self.mox.VerifyAll()
1004
1005   def test_irrelevant_config_change(self):
1006     self.servr._server_configuration.check_for_updates().AndReturn(frozenset())
1007     self.servr._watcher.has_changes().AndReturn(False)
1008     self.servr._maybe_restart_instances(config_changed=False,
1009                                         file_changed=False)
1010
1011     self.mox.ReplayAll()
1012     self.servr._handle_changes()
1013     self.mox.VerifyAll()
1014
1015   def test_restart_config_change(self):
1016     conf_change = frozenset([application_configuration.ENV_VARIABLES_CHANGED])
1017     self.servr._server_configuration.check_for_updates().AndReturn(conf_change)
1018     self.servr._watcher.has_changes().AndReturn(False)
1019     self.instance_factory.configuration_changed(conf_change)
1020     self.servr._maybe_restart_instances(config_changed=True, file_changed=False)
1021
1022     self.mox.ReplayAll()
1023     self.servr._handle_changes()
1024     self.mox.VerifyAll()
1025
1026   def test_handler_change(self):
1027     conf_change = frozenset([application_configuration.HANDLERS_CHANGED])
1028     self.servr._server_configuration.check_for_updates().AndReturn(conf_change)
1029     self.servr._watcher.has_changes().AndReturn(False)
1030     self.servr._create_url_handlers()
1031     self.instance_factory.configuration_changed(conf_change)
1032     self.servr._maybe_restart_instances(config_changed=True, file_changed=False)
1033
1034     self.mox.ReplayAll()
1035     self.servr._handle_changes()
1036     self.mox.VerifyAll()
1037
1038   def test_file_change(self):
1039     self.servr._server_configuration.check_for_updates().AndReturn(frozenset())
1040     self.servr._watcher.has_changes().AndReturn(True)
1041     self.instance_factory.files_changed()
1042     self.servr._maybe_restart_instances(config_changed=False, file_changed=True)
1043
1044     self.mox.ReplayAll()
1045     self.servr._handle_changes()
1046     self.mox.VerifyAll()
1047
1048
1049 class TestAutoScalingInstancePoolMaybeRestartInstances(unittest.TestCase):
1050   """Tests for server.AutoScalingServer._maybe_restart_instances."""
1051
1052   def setUp(self):
1053     api_server.test_setup_stubs()
1054
1055     self.mox = mox.Mox()
1056     self.instance_factory = instance.InstanceFactory(object(), 10)
1057     self.instance_factory.FILE_CHANGE_INSTANCE_RESTART_POLICY = instance.ALWAYS
1058     self.servr = AutoScalingServerFacade(instance_factory=self.instance_factory)
1059
1060     self.inst1 = self.mox.CreateMock(instance.Instance)
1061     self.inst2 = self.mox.CreateMock(instance.Instance)
1062     self.inst3 = self.mox.CreateMock(instance.Instance)
1063     self.inst1.total_requests = 2
1064     self.inst2.total_requests = 0
1065     self.inst3.total_requests = 4
1066     self.servr._instances.add(self.inst1)
1067     self.servr._instances.add(self.inst2)
1068     self.servr._instances.add(self.inst3)
1069
1070   def tearDown(self):
1071     self.mox.UnsetStubs()
1072
1073   def test_no_changes(self):
1074     self.mox.ReplayAll()
1075     self.servr._maybe_restart_instances(config_changed=False,
1076                                         file_changed=False)
1077     self.mox.VerifyAll()
1078
1079   def test_config_change(self):
1080     self.inst1.quit(allow_async=True).InAnyOrder()
1081     self.inst2.quit(allow_async=True).InAnyOrder()
1082     self.inst3.quit(allow_async=True).InAnyOrder()
1083
1084     self.mox.ReplayAll()
1085     self.servr._maybe_restart_instances(config_changed=True,
1086                                         file_changed=False)
1087     self.mox.VerifyAll()
1088
1089   def test_file_change_restart_always(self):
1090     self.instance_factory.FILE_CHANGE_INSTANCE_RESTART_POLICY = instance.ALWAYS
1091     self.inst1.quit(allow_async=True).InAnyOrder()
1092     self.inst2.quit(allow_async=True).InAnyOrder()
1093     self.inst3.quit(allow_async=True).InAnyOrder()
1094
1095     self.mox.ReplayAll()
1096     self.servr._maybe_restart_instances(config_changed=False,
1097                                         file_changed=True)
1098     self.mox.VerifyAll()
1099     self.assertSequenceEqual(set(), self.servr._instances)
1100
1101   def test_file_change_restart_after_first_request(self):
1102     self.instance_factory.FILE_CHANGE_INSTANCE_RESTART_POLICY = (
1103         instance.AFTER_FIRST_REQUEST)
1104     self.inst1.quit(allow_async=True).InAnyOrder()
1105     self.inst3.quit(allow_async=True).InAnyOrder()
1106
1107     self.mox.ReplayAll()
1108     self.servr._maybe_restart_instances(config_changed=False,
1109                                         file_changed=True)
1110     self.mox.VerifyAll()
1111     self.assertSequenceEqual(set([self.inst2]), self.servr._instances)
1112
1113   def test_file_change_restart_never(self):
1114     self.instance_factory.FILE_CHANGE_INSTANCE_RESTART_POLICY = instance.NEVER
1115
1116     self.mox.ReplayAll()
1117     self.servr._maybe_restart_instances(config_changed=False,
1118                                         file_changed=True)
1119     self.mox.VerifyAll()
1120     self.assertSequenceEqual(set([self.inst1, self.inst2, self.inst3]),
1121                              self.servr._instances)
1122
1123
1124 class TestAutoScalingInstancePoolLoopAdjustingInstances(unittest.TestCase):
1125   """Tests for server.AutoScalingServer._adjust_instances."""
1126
1127   def setUp(self):
1128     api_server.test_setup_stubs()
1129
1130     self.mox = mox.Mox()
1131     self.servr = AutoScalingServerFacade(
1132         instance_factory=instance.InstanceFactory(object(), 10))
1133
1134   def tearDown(self):
1135     self.mox.UnsetStubs()
1136
1137   def test_loop_and_quit(self):
1138     self.mox.StubOutWithMock(self.servr, '_adjust_instances')
1139     self.mox.StubOutWithMock(self.servr, '_handle_changes')
1140
1141     inst1 = self.mox.CreateMock(instance.Instance)
1142     inst2 = self.mox.CreateMock(instance.Instance)
1143     inst3 = self.mox.CreateMock(instance.Instance)
1144     self.servr._instances.add(inst1)
1145     self.servr._instances.add(inst2)
1146     self.servr._instances.add(inst3)
1147
1148     self.servr._handle_changes()
1149
1150     def do_quit(*unused_args):
1151       self.servr._quit_event.set()
1152
1153     self.servr._adjust_instances().WithSideEffects(do_quit)
1154
1155     self.mox.ReplayAll()
1156     self.servr._loop_adjusting_instances()
1157     self.mox.VerifyAll()
1158
1159
1160 class TestAutoScalingInstancePoolAutomaticScaling(unittest.TestCase):
1161
1162   def setUp(self):
1163     api_server.test_setup_stubs()
1164
1165   def _create_server(self, automatic_scaling):
1166     return AutoScalingServerFacade(
1167         server_configuration=ServerConfigurationStub(
1168             automatic_scaling=automatic_scaling),
1169         instance_factory=instance.InstanceFactory(object(), 10))
1170
1171   def test_unset_automatic_settings(self):
1172     settings = appinfo.AutomaticScaling()
1173     pool = self._create_server(settings)
1174     self.assertEqual(0.1, pool._min_pending_latency)
1175     self.assertEqual(0.5, pool._max_pending_latency)
1176     self.assertEqual(1, pool._min_idle_instances)
1177     self.assertEqual(1000, pool._max_idle_instances)
1178
1179   def test_automatic_automatic_settings(self):
1180     settings = appinfo.AutomaticScaling(
1181         min_pending_latency='automatic',
1182         max_pending_latency='automatic',
1183         min_idle_instances='automatic',
1184         max_idle_instances='automatic')
1185     pool = self._create_server(settings)
1186     self.assertEqual(0.1, pool._min_pending_latency)
1187     self.assertEqual(0.5, pool._max_pending_latency)
1188     self.assertEqual(1, pool._min_idle_instances)
1189     self.assertEqual(1000, pool._max_idle_instances)
1190
1191   def test_explicit_automatic_settings(self):
1192     settings = appinfo.AutomaticScaling(
1193         min_pending_latency='1234ms',
1194         max_pending_latency='5.67s',
1195         min_idle_instances='3',
1196         max_idle_instances='20')
1197     pool = self._create_server(settings)
1198     self.assertEqual(1.234, pool._min_pending_latency)
1199     self.assertEqual(5.67, pool._max_pending_latency)
1200     self.assertEqual(3, pool._min_idle_instances)
1201     self.assertEqual(20, pool._max_idle_instances)
1202
1203
1204 class TestManualScalingServerStart(unittest.TestCase):
1205   """Tests for server.ManualScalingServer._start_instance."""
1206
1207   def setUp(self):
1208     api_server.test_setup_stubs()
1209     self.mox = mox.Mox()
1210     self.mox.StubOutWithMock(server.Server, 'build_request_environ')
1211
1212   def tearDown(self):
1213     self.mox.UnsetStubs()
1214
1215   def test_instance_start_success(self):
1216     s = ManualScalingServerFacade(balanced_port=8080)
1217     self.mox.StubOutWithMock(s, '_handle_request')
1218     self.mox.StubOutWithMock(s._condition, 'notify')
1219
1220     wsgi_servr = self.mox.CreateMock(wsgi_server.WsgiServer)
1221     wsgi_servr.port = 12345
1222     inst = self.mox.CreateMock(instance.Instance)
1223     inst.instance_id = 0
1224     inst.start().AndReturn(True)
1225
1226     environ = object()
1227     s.build_request_environ('GET', '/_ah/start', [], '', '0.1.0.3', 12345,
1228                             fake_login=True).AndReturn(environ)
1229     s._handle_request(environ,
1230                       mox.IgnoreArg(),
1231                       inst=inst,
1232                       request_type=instance.READY_REQUEST)
1233     s._condition.notify(1)
1234
1235     self.mox.ReplayAll()
1236     s._start_instance(wsgi_servr, inst)
1237     self.mox.VerifyAll()
1238
1239   def test_instance_start_failure(self):
1240     s = ManualScalingServerFacade(balanced_port=8080)
1241     self.mox.StubOutWithMock(s, '_handle_request')
1242     self.mox.StubOutWithMock(s._condition, 'notify')
1243
1244     wsgi_servr = self.mox.CreateMock(wsgi_server.WsgiServer)
1245     wsgi_servr.port = 12345
1246     inst = self.mox.CreateMock(instance.Instance)
1247     inst.instance_id = 0
1248     inst.start().AndReturn(False)
1249
1250     self.mox.ReplayAll()
1251     s._start_instance(wsgi_servr, inst)
1252     self.mox.VerifyAll()
1253
1254
1255 class TestManualScalingServerAddInstance(unittest.TestCase):
1256   """Tests for server.ManualScalingServer._add_instance."""
1257
1258   class WsgiServer(object):
1259     def __init__(self, port):
1260       self.port = port
1261
1262   def setUp(self):
1263     api_server.test_setup_stubs()
1264     self.mox = mox.Mox()
1265     self.factory = self.mox.CreateMock(instance.InstanceFactory)
1266     self.factory.max_concurrent_requests = 10
1267
1268   def tearDown(self):
1269     self.mox.UnsetStubs()
1270
1271   def test_add_while_started(self):
1272     servr = ManualScalingServerFacade(instance_factory=self.factory)
1273
1274     inst = self.mox.CreateMock(instance.Instance)
1275     self.mox.StubOutWithMock(server._THREAD_POOL, 'submit')
1276     self.mox.StubOutWithMock(wsgi_server.WsgiServer, 'start')
1277     self.mox.StubOutWithMock(wsgi_server.WsgiServer, 'port')
1278     wsgi_server.WsgiServer.port = 12345
1279     self.factory.new_instance(0, expect_ready_request=True).AndReturn(inst)
1280     wsgi_server.WsgiServer.start()
1281     server._THREAD_POOL.submit(servr._start_instance,
1282                                mox.IsA(wsgi_server.WsgiServer), inst)
1283
1284     self.mox.ReplayAll()
1285     servr._add_instance()
1286     self.mox.VerifyAll()
1287     self.assertIn(inst, servr._instances)
1288     self.assertEqual((servr, inst), servr._port_registry.get(12345))
1289
1290   def test_add_while_stopped(self):
1291     servr = ManualScalingServerFacade(instance_factory=self.factory)
1292     servr._suspended = True
1293
1294     inst = self.mox.CreateMock(instance.Instance)
1295     self.mox.StubOutWithMock(wsgi_server.WsgiServer, 'start')
1296     self.mox.StubOutWithMock(wsgi_server.WsgiServer, 'port')
1297     wsgi_server.WsgiServer.port = 12345
1298     self.mox.StubOutWithMock(server._THREAD_POOL, 'submit')
1299     self.factory.new_instance(0, expect_ready_request=True).AndReturn(inst)
1300     wsgi_server.WsgiServer.start()
1301
1302     self.mox.ReplayAll()
1303     servr._add_instance()
1304     self.mox.VerifyAll()
1305
1306     self.assertIn(inst, servr._instances)
1307     self.assertEqual((servr, inst), servr._port_registry.get(12345))
1308
1309
1310 class TestManualScalingInstancePoolHandleScriptRequest(unittest.TestCase):
1311   """Tests for server.ManualScalingServer.handle."""
1312
1313   def setUp(self):
1314     api_server.test_setup_stubs()
1315     self.mox = mox.Mox()
1316
1317     self.inst = self.mox.CreateMock(instance.Instance)
1318     self.inst.instance_id = 0
1319     self.environ = {}
1320     self.start_response = object()
1321     self.response = [object()]
1322     self.url_map = object()
1323     self.match = object()
1324     self.request_id = object()
1325     self.manual_server = ManualScalingServerFacade(
1326         instance_factory=instance.InstanceFactory(object(), 10))
1327     self.mox.StubOutWithMock(self.manual_server, '_choose_instance')
1328     self.mox.StubOutWithMock(self.manual_server, '_add_instance')
1329     self.mox.StubOutWithMock(self.manual_server._condition, 'notify')
1330     self.mox.stubs.Set(time, 'time', lambda: 0.0)
1331
1332   def tearDown(self):
1333     self.mox.UnsetStubs()
1334
1335   def test_handle_script_request(self):
1336     self.manual_server._choose_instance(10.0).AndReturn(self.inst)
1337     self.inst.handle(self.environ,
1338                      self.start_response,
1339                      self.url_map,
1340                      self.match,
1341                      self.request_id,
1342                      instance.NORMAL_REQUEST).AndReturn(self.response)
1343     self.manual_server._condition.notify()
1344
1345     self.mox.ReplayAll()
1346     self.assertEqual(
1347         self.response,
1348         self.manual_server._handle_script_request(self.environ,
1349                                                   self.start_response,
1350                                                   self.url_map,
1351                                                   self.match,
1352                                                   self.request_id))
1353     self.mox.VerifyAll()
1354
1355   def test_handle_cannot_accept_request(self):
1356     self.manual_server._choose_instance(10.0).AndReturn(self.inst)
1357     self.manual_server._choose_instance(10.0).AndReturn(self.inst)
1358     self.inst.handle(
1359         self.environ, self.start_response, self.url_map, self.match,
1360         self.request_id, instance.NORMAL_REQUEST).AndRaise(
1361             instance.CannotAcceptRequests)
1362     self.manual_server._condition.notify()
1363     self.inst.handle(
1364         self.environ, self.start_response, self.url_map, self.match,
1365         self.request_id, instance.NORMAL_REQUEST).AndReturn(
1366             self.response)
1367     self.manual_server._condition.notify()
1368
1369     self.mox.ReplayAll()
1370     self.assertEqual(
1371         self.response,
1372         self.manual_server._handle_script_request(self.environ,
1373                                                   self.start_response,
1374                                                   self.url_map,
1375                                                   self.match,
1376                                                   self.request_id))
1377     self.mox.VerifyAll()
1378
1379   def test_handle_must_wait(self):
1380     self.manual_server._choose_instance(10.0).AndReturn(None)
1381     self.manual_server._choose_instance(10.0).AndReturn(self.inst)
1382
1383     self.inst.handle(
1384         self.environ, self.start_response, self.url_map, self.match,
1385         self.request_id, instance.NORMAL_REQUEST).AndReturn(
1386             self.response)
1387     self.manual_server._condition.notify()
1388
1389     self.mox.ReplayAll()
1390     self.assertEqual(
1391         self.response,
1392         self.manual_server._handle_script_request(self.environ,
1393                                                   self.start_response,
1394                                                   self.url_map,
1395                                                   self.match,
1396                                                   self.request_id))
1397     self.mox.VerifyAll()
1398
1399   def test_handle_timeout(self):
1400     self.time = 0.0
1401
1402     def advance_time(*unused_args):
1403       self.time += 11
1404
1405     self.mox.stubs.Set(time, 'time', lambda: self.time)
1406     self.mox.StubOutWithMock(self.manual_server, '_error_response')
1407
1408     self.manual_server._choose_instance(10.0).WithSideEffects(advance_time)
1409     self.manual_server._error_response(self.environ, self.start_response,
1410                                        503).AndReturn(self.response)
1411     self.mox.ReplayAll()
1412     self.assertEqual(
1413         self.response,
1414         self.manual_server._handle_script_request(self.environ,
1415                                                   self.start_response,
1416                                                   self.url_map,
1417                                                   self.match,
1418                                                   self.request_id))
1419     self.mox.VerifyAll()
1420
1421
1422 class TestManualScalingInstancePoolChooseInstances(unittest.TestCase):
1423   """Tests for server.ManualScalingServer._choose_instance."""
1424
1425   class Instance(object):
1426     def __init__(self, can_accept_requests):
1427       self.can_accept_requests = can_accept_requests
1428
1429   def setUp(self):
1430     self.mox = mox.Mox()
1431     api_server.test_setup_stubs()
1432     self.servr = ManualScalingServerFacade(
1433         instance_factory=instance.InstanceFactory(object(), 10))
1434     self.mox.StubOutWithMock(self.servr._condition, 'wait')
1435     self.time = 0
1436     self.mox.stubs.Set(time, 'time', lambda: self.time)
1437
1438   def advance_time(self, *unused_args):
1439     self.time += 10
1440
1441   def tearDown(self):
1442     self.mox.UnsetStubs()
1443
1444   def test_choose_instance_first_can_accept(self):
1445     instance1 = self.Instance(True)
1446     instance2 = self.Instance(True)
1447     self.servr._instances = [instance1, instance2]
1448     self.mox.ReplayAll()
1449     self.assertEqual(instance1, self.servr._choose_instance(1))
1450     self.mox.VerifyAll()
1451
1452   def test_choose_instance_first_cannot_accept(self):
1453     instance1 = self.Instance(False)
1454     instance2 = self.Instance(True)
1455     self.servr._instances = [instance1, instance2]
1456     self.mox.ReplayAll()
1457     self.assertEqual(instance2, self.servr._choose_instance(1))
1458     self.mox.VerifyAll()
1459
1460   def test_choose_instance_none_can_accept(self):
1461     instance1 = self.Instance(False)
1462     instance2 = self.Instance(False)
1463     self.servr._instances = [instance1, instance2]
1464     self.servr._condition.wait(5).WithSideEffects(self.advance_time)
1465     self.mox.ReplayAll()
1466     self.assertEqual(None, self.servr._choose_instance(5))
1467     self.mox.VerifyAll()
1468
1469   def test_choose_instance_no_instances(self):
1470     self.servr._condition.wait(5).WithSideEffects(self.advance_time)
1471     self.mox.ReplayAll()
1472     self.assertEqual(None, self.servr._choose_instance(5))
1473     self.mox.VerifyAll()
1474
1475
1476 class TestManualScalingInstancePoolSetNumInstances(unittest.TestCase):
1477   """Tests for server.ManualScalingServer.set_num_instances."""
1478
1479   def setUp(self):
1480     self.mox = mox.Mox()
1481     api_server.test_setup_stubs()
1482     self.server = ManualScalingServerFacade(
1483         instance_factory=instance.InstanceFactory(object(), 10))
1484     self._instance = self.mox.CreateMock(instance.Instance)
1485     self._wsgi_server = self.mox.CreateMock(wsgi_server.WsgiServer)
1486     self._wsgi_server.port = 8080
1487     self.server._instances = [self._instance]
1488     self.server._wsgi_servers = [self._wsgi_server]
1489     self.mox.StubOutWithMock(server._THREAD_POOL, 'submit')
1490     self.mox.StubOutWithMock(self.server, '_add_instance')
1491     self.mox.StubOutWithMock(self.server, '_shutdown_instance')
1492
1493   def tearDown(self):
1494     self.mox.UnsetStubs()
1495
1496   def test_no_op(self):
1497     self.mox.ReplayAll()
1498     self.assertEqual(1, self.server.get_num_instances())
1499     self.server.set_num_instances(1)
1500     self.mox.VerifyAll()
1501
1502   def test_add_an_instance(self):
1503     self.server._add_instance()
1504     self.mox.ReplayAll()
1505     self.assertEqual(1, self.server.get_num_instances())
1506     self.server.set_num_instances(2)
1507     self.mox.VerifyAll()
1508
1509   def test_remove_an_instance(self):
1510     server._THREAD_POOL.submit(self.server._quit_instance,
1511                                self._instance,
1512                                self._wsgi_server)
1513     self._instance.quit(expect_shutdown=True)
1514     self._wsgi_server.quit()
1515     self.server._shutdown_instance(self._instance, 8080)
1516     self.mox.ReplayAll()
1517     self.assertEqual(1, self.server.get_num_instances())
1518     self.server.set_num_instances(0)
1519     self.server._quit_instance(self._instance,
1520                                self._wsgi_server)
1521     self.mox.VerifyAll()
1522
1523
1524 class TestManualScalingInstancePoolSuspendAndResume(unittest.TestCase):
1525   """Tests for server.ManualScalingServer.suspend and resume."""
1526
1527   def setUp(self):
1528     self.mox = mox.Mox()
1529     api_server.test_setup_stubs()
1530     self.factory = self.mox.CreateMock(instance.InstanceFactory)
1531     self.server = ManualScalingServerFacade(
1532         instance_factory=self.factory)
1533     self._instance = self.mox.CreateMock(instance.Instance)
1534     self._wsgi_server = wsgi_server.WsgiServer(('localhost', 0), None)
1535     self.server._instances = [self._instance]
1536     self.server._wsgi_servers = [self._wsgi_server]
1537     self.mox.StubOutWithMock(server._THREAD_POOL, 'submit')
1538     self.mox.StubOutWithMock(self.server, '_shutdown_instance')
1539     self._wsgi_server.start()
1540
1541   def tearDown(self):
1542     self._wsgi_server.quit()
1543     self.mox.UnsetStubs()
1544
1545   def test_already_suspended(self):
1546     self.server._suspended = True
1547     self.assertRaises(request_info.ServerAlreadyStoppedError,
1548                       self.server.suspend)
1549
1550   def test_already_resumed(self):
1551     self.assertRaises(request_info.ServerAlreadyStartedError,
1552                       self.server.resume)
1553
1554   def test_suspend_instance(self):
1555     server._THREAD_POOL.submit(self.server._suspend_instance, self._instance,
1556                                self._wsgi_server.port)
1557     self._instance.quit(expect_shutdown=True)
1558     port = object()
1559     self.server._shutdown_instance(self._instance, port)
1560     self.mox.ReplayAll()
1561     self.server.suspend()
1562     self.server._suspend_instance(self._instance, port)
1563     self.mox.VerifyAll()
1564     self.assertEqual(404, self._wsgi_server._error)
1565     self.assertEqual(None, self._wsgi_server._app)
1566     self.assertTrue(self.server._suspended)
1567
1568   def test_resume(self):
1569     self.server._suspended = True
1570     self.server._instances = [object()]
1571     self.factory.new_instance(0, expect_ready_request=True).AndReturn(
1572         self._instance)
1573     server._THREAD_POOL.submit(self.server._start_instance, self._wsgi_server,
1574                                self._instance)
1575     self.mox.ReplayAll()
1576     self.server.resume()
1577     self.mox.VerifyAll()
1578     self.assertEqual(self.server._handle_request,
1579                      self._wsgi_server._app.func)
1580     self.assertEqual({'inst': self._instance},
1581                      self._wsgi_server._app.keywords)
1582     self.assertFalse(self.server._suspended)
1583
1584   def test_restart(self):
1585     self._new_instance = self.mox.CreateMock(instance.Instance)
1586     self.factory.new_instance(0, expect_ready_request=True).AndReturn(
1587         self._new_instance)
1588     server._THREAD_POOL.submit(self.server._suspend_instance, self._instance,
1589                                self._wsgi_server.port)
1590     server._THREAD_POOL.submit(self.server._start_instance, self._wsgi_server,
1591                                self._new_instance)
1592     self._instance.quit(expect_shutdown=True)
1593     port = object()
1594     self.server._shutdown_instance(self._instance, port)
1595     self.mox.ReplayAll()
1596     self.server.restart()
1597     self.server._suspend_instance(self._instance, port)
1598     self.mox.VerifyAll()
1599     self.assertEqual(self.server._handle_request,
1600                      self._wsgi_server._app.func)
1601     self.assertEqual({'inst': self._new_instance},
1602                      self._wsgi_server._app.keywords)
1603     self.assertFalse(self.server._suspended)
1604
1605
1606 class TestManualScalingInstancePoolHandleChanges(unittest.TestCase):
1607   """Tests for server.ManualScalingServer._handle_changes."""
1608
1609   def setUp(self):
1610     api_server.test_setup_stubs()
1611
1612     self.mox = mox.Mox()
1613     self.instance_factory = instance.InstanceFactory(object(), 10)
1614     self.servr = ManualScalingServerFacade(
1615         instance_factory=self.instance_factory)
1616     self.mox.StubOutWithMock(self.instance_factory, 'files_changed')
1617     self.mox.StubOutWithMock(self.instance_factory, 'configuration_changed')
1618     self.mox.StubOutWithMock(self.servr, 'restart')
1619     self.mox.StubOutWithMock(self.servr, '_create_url_handlers')
1620     self.mox.StubOutWithMock(self.servr._server_configuration,
1621                              'check_for_updates')
1622     self.mox.StubOutWithMock(self.servr._watcher, 'has_changes')
1623
1624   def tearDown(self):
1625     self.mox.UnsetStubs()
1626
1627   def test_no_changes(self):
1628     self.servr._server_configuration.check_for_updates().AndReturn(frozenset())
1629     self.servr._watcher.has_changes().AndReturn(False)
1630
1631     self.mox.ReplayAll()
1632     self.servr._handle_changes()
1633     self.mox.VerifyAll()
1634
1635   def test_irrelevant_config_change(self):
1636     self.servr._server_configuration.check_for_updates().AndReturn(frozenset())
1637     self.servr._watcher.has_changes().AndReturn(False)
1638
1639     self.mox.ReplayAll()
1640     self.servr._handle_changes()
1641     self.mox.VerifyAll()
1642
1643   def test_restart_config_change(self):
1644     conf_change = frozenset([application_configuration.ENV_VARIABLES_CHANGED])
1645     self.servr._server_configuration.check_for_updates().AndReturn(conf_change)
1646     self.servr._watcher.has_changes().AndReturn(False)
1647     self.instance_factory.configuration_changed(conf_change)
1648     self.servr.restart()
1649
1650     self.mox.ReplayAll()
1651     self.servr._handle_changes()
1652     self.mox.VerifyAll()
1653
1654   def test_handler_change(self):
1655     conf_change = frozenset([application_configuration.HANDLERS_CHANGED])
1656     self.servr._server_configuration.check_for_updates().AndReturn(conf_change)
1657     self.servr._watcher.has_changes().AndReturn(False)
1658     self.servr._create_url_handlers()
1659     self.instance_factory.configuration_changed(conf_change)
1660
1661     self.servr.restart()
1662
1663     self.mox.ReplayAll()
1664     self.servr._handle_changes()
1665     self.mox.VerifyAll()
1666
1667   def test_file_change(self):
1668     self.servr._server_configuration.check_for_updates().AndReturn(frozenset())
1669     self.servr._watcher.has_changes().AndReturn(True)
1670     self.instance_factory.files_changed()
1671     self.servr.restart()
1672
1673     self.mox.ReplayAll()
1674     self.servr._handle_changes()
1675     self.mox.VerifyAll()
1676
1677   def test_restart_config_change_suspended(self):
1678     self.servr._suspended = True
1679     conf_change = frozenset([application_configuration.ENV_VARIABLES_CHANGED])
1680     self.servr._server_configuration.check_for_updates().AndReturn(conf_change)
1681     self.servr._watcher.has_changes().AndReturn(False)
1682     self.instance_factory.configuration_changed(conf_change)
1683
1684     self.mox.ReplayAll()
1685     self.servr._handle_changes()
1686     self.mox.VerifyAll()
1687
1688   def test_handler_change_suspended(self):
1689     self.servr._suspended = True
1690     conf_change = frozenset([application_configuration.HANDLERS_CHANGED])
1691     self.servr._server_configuration.check_for_updates().AndReturn(conf_change)
1692     self.servr._watcher.has_changes().AndReturn(False)
1693     self.servr._create_url_handlers()
1694     self.instance_factory.configuration_changed(conf_change)
1695
1696     self.mox.ReplayAll()
1697     self.servr._handle_changes()
1698     self.mox.VerifyAll()
1699
1700   def test_file_change_suspended(self):
1701     self.servr._suspended = True
1702     self.servr._server_configuration.check_for_updates().AndReturn(frozenset())
1703     self.servr._watcher.has_changes().AndReturn(True)
1704     self.instance_factory.files_changed()
1705
1706     self.mox.ReplayAll()
1707     self.servr._handle_changes()
1708     self.mox.VerifyAll()
1709
1710
1711 class TestBasicScalingServerStart(unittest.TestCase):
1712   """Tests for server.BasicScalingServer._start_instance."""
1713
1714   def setUp(self):
1715     api_server.test_setup_stubs()
1716     self.mox = mox.Mox()
1717     self.mox.StubOutWithMock(server.Server, 'build_request_environ')
1718
1719   def tearDown(self):
1720     self.mox.UnsetStubs()
1721
1722   def test_instance_start_success(self):
1723     s = BasicScalingServerFacade(balanced_port=8080)
1724     self.mox.StubOutWithMock(s, '_handle_request')
1725     self.mox.StubOutWithMock(s._condition, 'notify')
1726
1727     wsgi_servr = self.mox.CreateMock(wsgi_server.WsgiServer)
1728     wsgi_servr.port = 12345
1729     s._wsgi_servers[0] = wsgi_servr
1730     inst = self.mox.CreateMock(instance.Instance)
1731     inst.instance_id = 0
1732     s._instances[0] = inst
1733     inst.start().AndReturn(True)
1734
1735     environ = object()
1736     s.build_request_environ('GET', '/_ah/start', [], '', '0.1.0.3', 12345,
1737                             fake_login=True).AndReturn(environ)
1738     s._handle_request(environ,
1739                       mox.IgnoreArg(),
1740                       inst=inst,
1741                       request_type=instance.READY_REQUEST)
1742     s._condition.notify(1)
1743
1744     self.mox.ReplayAll()
1745     s._start_instance(0)
1746     self.mox.VerifyAll()
1747
1748   def test_instance_start_failure(self):
1749     s = BasicScalingServerFacade(balanced_port=8080)
1750     self.mox.StubOutWithMock(s, '_handle_request')
1751     self.mox.StubOutWithMock(s._condition, 'notify')
1752
1753     wsgi_servr = self.mox.CreateMock(wsgi_server.WsgiServer)
1754     wsgi_servr.port = 12345
1755     s._wsgi_servers[0] = wsgi_servr
1756     inst = self.mox.CreateMock(instance.Instance)
1757     inst.instance_id = 0
1758     s._instances[0] = inst
1759     inst.start().AndReturn(False)
1760
1761     self.mox.ReplayAll()
1762     s._start_instance(0)
1763     self.mox.VerifyAll()
1764
1765   def test_start_any_instance_success(self):
1766     s = BasicScalingServerFacade(balanced_port=8080)
1767     s._instance_running = [True, False, False, True]
1768     inst = object()
1769     s._instances = [None, inst, None, None]
1770     self.mox.StubOutWithMock(server._THREAD_POOL, 'submit')
1771     server._THREAD_POOL.submit(s._start_instance, 1)
1772     self.mox.ReplayAll()
1773     self.assertEqual(inst, s._start_any_instance())
1774     self.mox.VerifyAll()
1775     self.assertEqual([True, True, False, True], s._instance_running)
1776
1777   def test_start_any_instance_all_already_running(self):
1778     s = BasicScalingServerFacade(balanced_port=8080)
1779     s._instance_running = [True, True, True, True]
1780     self.mox.StubOutWithMock(server._THREAD_POOL, 'submit')
1781     self.mox.ReplayAll()
1782     self.assertIsNone(s._start_any_instance())
1783     self.mox.VerifyAll()
1784     self.assertEqual([True, True, True, True], s._instance_running)
1785
1786
1787 class TestBasicScalingInstancePoolHandleScriptRequest(unittest.TestCase):
1788   """Tests for server.BasicScalingServer.handle."""
1789
1790   def setUp(self):
1791     api_server.test_setup_stubs()
1792     self.mox = mox.Mox()
1793
1794     self.inst = self.mox.CreateMock(instance.Instance)
1795     self.inst.instance_id = 0
1796     self.environ = {}
1797     self.start_response = object()
1798     self.response = [object()]
1799     self.url_map = object()
1800     self.match = object()
1801     self.request_id = object()
1802     self.basic_server = BasicScalingServerFacade(
1803         instance_factory=instance.InstanceFactory(object(), 10))
1804     self.mox.StubOutWithMock(self.basic_server, '_choose_instance')
1805     self.mox.StubOutWithMock(self.basic_server, '_start_any_instance')
1806     self.mox.StubOutWithMock(self.basic_server, '_start_instance')
1807     self.mox.StubOutWithMock(self.basic_server._condition, 'wait')
1808     self.mox.StubOutWithMock(self.basic_server._condition, 'notify')
1809     self.time = 10
1810     self.mox.stubs.Set(time, 'time', lambda: self.time)
1811
1812   def advance_time(self, *unused_args):
1813     self.time += 11
1814
1815   def tearDown(self):
1816     self.mox.UnsetStubs()
1817
1818   def test_handle_script_request(self):
1819     self.basic_server._choose_instance(20).AndReturn(self.inst)
1820     self.inst.handle(self.environ,
1821                      self.start_response,
1822                      self.url_map,
1823                      self.match,
1824                      self.request_id,
1825                      instance.NORMAL_REQUEST).AndReturn(self.response)
1826     self.basic_server._condition.notify()
1827
1828     self.mox.ReplayAll()
1829     self.assertEqual(
1830         self.response,
1831         self.basic_server._handle_script_request(self.environ,
1832                                                  self.start_response,
1833                                                  self.url_map,
1834                                                  self.match,
1835                                                  self.request_id))
1836     self.mox.VerifyAll()
1837
1838   def test_handle_cannot_accept_request(self):
1839     self.basic_server._choose_instance(20).AndReturn(self.inst)
1840     self.inst.handle(
1841         self.environ, self.start_response, self.url_map, self.match,
1842         self.request_id, instance.NORMAL_REQUEST).AndRaise(
1843             instance.CannotAcceptRequests)
1844     self.basic_server._condition.notify()
1845     self.basic_server._choose_instance(20).AndReturn(self.inst)
1846     self.inst.handle(
1847         self.environ, self.start_response, self.url_map, self.match,
1848         self.request_id, instance.NORMAL_REQUEST).AndReturn(
1849             self.response)
1850     self.basic_server._condition.notify()
1851
1852     self.mox.ReplayAll()
1853     self.assertEqual(
1854         self.response,
1855         self.basic_server._handle_script_request(self.environ,
1856                                                  self.start_response,
1857                                                  self.url_map,
1858                                                  self.match,
1859                                                  self.request_id))
1860     self.mox.VerifyAll()
1861
1862   def test_handle_timeout(self):
1863     self.mox.StubOutWithMock(self.basic_server, '_error_response')
1864
1865     self.basic_server._choose_instance(20).WithSideEffects(self.advance_time)
1866     self.basic_server._error_response(self.environ, self.start_response,
1867                                       503).AndReturn(self.response)
1868
1869     self.mox.ReplayAll()
1870     self.assertEqual(
1871         self.response,
1872         self.basic_server._handle_script_request(self.environ,
1873                                                  self.start_response,
1874                                                  self.url_map,
1875                                                  self.match,
1876                                                  self.request_id))
1877     self.mox.VerifyAll()
1878
1879   def test_handle_instance(self):
1880     self.inst.instance_id = 0
1881     self.inst.has_quit = False
1882
1883     self.inst.handle(
1884         self.environ, self.start_response, self.url_map, self.match,
1885         self.request_id, instance.NORMAL_REQUEST).AndReturn(
1886             self.response)
1887     self.basic_server._condition.notify()
1888
1889     self.mox.ReplayAll()
1890     self.assertEqual(
1891         self.response,
1892         self.basic_server._handle_script_request(self.environ,
1893                                                  self.start_response,
1894                                                  self.url_map,
1895                                                  self.match,
1896                                                  self.request_id,
1897                                                  inst=self.inst))
1898     self.mox.VerifyAll()
1899
1900   def test_handle_instance_start_the_instance(self):
1901     self.inst.instance_id = 0
1902     self.inst.has_quit = False
1903
1904     self.inst.handle(
1905         self.environ, self.start_response, self.url_map, self.match,
1906         self.request_id, instance.NORMAL_REQUEST).AndRaise(
1907             instance.CannotAcceptRequests)
1908     self.basic_server._start_instance(0).AndReturn(True)
1909     self.inst.handle(
1910         self.environ, self.start_response, self.url_map, self.match,
1911         self.request_id, instance.NORMAL_REQUEST).AndReturn(
1912             self.response)
1913     self.basic_server._condition.notify()
1914
1915     self.mox.ReplayAll()
1916     self.assertEqual(
1917         self.response,
1918         self.basic_server._handle_script_request(self.environ,
1919                                                  self.start_response,
1920                                                  self.url_map,
1921                                                  self.match,
1922                                                  self.request_id,
1923                                                  inst=self.inst))
1924     self.mox.VerifyAll()
1925
1926   def test_handle_instance_already_running(self):
1927     self.inst.instance_id = 0
1928     self.inst.has_quit = False
1929
1930     self.basic_server._instance_running[0] = True
1931     self.inst.handle(
1932         self.environ, self.start_response, self.url_map, self.match,
1933         self.request_id, instance.NORMAL_REQUEST).AndRaise(
1934             instance.CannotAcceptRequests)
1935     self.inst.wait(20)
1936     self.inst.handle(
1937         self.environ, self.start_response, self.url_map, self.match,
1938         self.request_id, instance.NORMAL_REQUEST).AndReturn(
1939             self.response)
1940     self.basic_server._condition.notify()
1941
1942     self.mox.ReplayAll()
1943     self.assertEqual(
1944         self.response,
1945         self.basic_server._handle_script_request(self.environ,
1946                                                  self.start_response,
1947                                                  self.url_map,
1948                                                  self.match,
1949                                                  self.request_id,
1950                                                  inst=self.inst))
1951     self.mox.VerifyAll()
1952
1953   def test_handle_instance_timeout(self):
1954     self.mox.StubOutWithMock(self.basic_server, '_error_response')
1955
1956     self.inst.instance_id = 0
1957     self.inst.has_quit = False
1958
1959     self.basic_server._instance_running[0] = True
1960     self.inst.handle(
1961         self.environ, self.start_response, self.url_map, self.match,
1962         self.request_id, instance.NORMAL_REQUEST).AndRaise(
1963             instance.CannotAcceptRequests)
1964     self.inst.wait(20).WithSideEffects(self.advance_time)
1965     self.basic_server._error_response(self.environ, self.start_response,
1966                                       503).AndReturn(self.response)
1967     self.basic_server._condition.notify()
1968
1969     self.mox.ReplayAll()
1970     self.assertEqual(
1971         self.response,
1972         self.basic_server._handle_script_request(self.environ,
1973                                                  self.start_response,
1974                                                  self.url_map,
1975                                                  self.match,
1976                                                  self.request_id,
1977                                                  inst=self.inst))
1978     self.mox.VerifyAll()
1979
1980
1981 class TestBasicScalingInstancePoolChooseInstances(unittest.TestCase):
1982   """Tests for server.BasicScalingServer._choose_instance."""
1983
1984   class Instance(object):
1985     def __init__(self, can_accept_requests):
1986       self.can_accept_requests = can_accept_requests
1987
1988   def setUp(self):
1989     api_server.test_setup_stubs()
1990     self.mox = mox.Mox()
1991     self.servr = BasicScalingServerFacade(
1992         instance_factory=instance.InstanceFactory(object(), 10))
1993     self.mox.stubs.Set(time, 'time', lambda: self.time)
1994     self.mox.StubOutWithMock(self.servr._condition, 'wait')
1995     self.mox.StubOutWithMock(self.servr, '_start_any_instance')
1996     self.time = 0
1997
1998   def tearDown(self):
1999     self.mox.UnsetStubs()
2000
2001   def advance_time(self, *unused_args):
2002     self.time += 10
2003
2004   def test_choose_instance_first_can_accept(self):
2005     instance1 = self.Instance(True)
2006     instance2 = self.Instance(True)
2007     self.servr._instances = [instance1, instance2]
2008     self.mox.ReplayAll()
2009     self.assertEqual(instance1, self.servr._choose_instance(1))
2010     self.mox.VerifyAll()
2011
2012   def test_choose_instance_first_cannot_accept(self):
2013     instance1 = self.Instance(False)
2014     instance2 = self.Instance(True)
2015     self.servr._instances = [instance1, instance2]
2016     self.mox.ReplayAll()
2017     self.assertEqual(instance2, self.servr._choose_instance(1))
2018     self.mox.VerifyAll()
2019
2020   def test_choose_instance_none_can_accept(self):
2021     instance1 = self.Instance(False)
2022     instance2 = self.Instance(False)
2023     self.servr._instance_running = [True, True]
2024     self.servr._instances = [instance1, instance2]
2025     self.servr._start_any_instance().AndReturn(None)
2026     self.servr._condition.wait(1).WithSideEffects(self.advance_time)
2027     self.mox.ReplayAll()
2028     self.assertEqual(None, self.servr._choose_instance(1))
2029     self.mox.VerifyAll()
2030
2031   def test_choose_instance_start_an_instance(self):
2032     instance1 = self.Instance(False)
2033     instance2 = self.Instance(False)
2034     mock_instance = self.mox.CreateMock(instance.Instance)
2035     self.servr._instances = [instance1, instance2]
2036     self.servr._instance_running = [True, False]
2037     self.servr._start_any_instance().AndReturn(mock_instance)
2038     mock_instance.wait(1)
2039     self.mox.ReplayAll()
2040     self.assertEqual(mock_instance, self.servr._choose_instance(1))
2041     self.mox.VerifyAll()
2042
2043   def test_choose_instance_no_instances(self):
2044     self.servr._start_any_instance().AndReturn(None)
2045     self.servr._condition.wait(1).WithSideEffects(self.advance_time)
2046     self.mox.ReplayAll()
2047     self.assertEqual(None, self.servr._choose_instance(1))
2048     self.mox.VerifyAll()
2049
2050
2051 class TestBasicScalingInstancePoolInstanceManagement(unittest.TestCase):
2052
2053   def setUp(self):
2054     api_server.test_setup_stubs()
2055     self.mox = mox.Mox()
2056     self.factory = self.mox.CreateMock(instance.InstanceFactory)
2057     self.factory.max_concurrent_requests = 10
2058     self.mox.StubOutWithMock(server._THREAD_POOL, 'submit')
2059     self.server = BasicScalingServerFacade(instance_factory=self.factory,
2060                                            host='localhost')
2061     self.wsgi_server = self.server._wsgi_servers[0]
2062     self.wsgi_server.start()
2063
2064   def tearDown(self):
2065     self.wsgi_server.quit()
2066     self.mox.UnsetStubs()
2067
2068   def test_restart(self):
2069     old_instances = [self.mox.CreateMock(instance.Instance),
2070                      self.mox.CreateMock(instance.Instance)]
2071     self.server._instances = old_instances[:]
2072     self.server._instance_running = [True, False]
2073     new_instance = self.mox.CreateMock(instance.Instance)
2074     self.factory.new_instance(0, expect_ready_request=True).AndReturn(
2075         new_instance)
2076     server._THREAD_POOL.submit(self.server._start_instance, 0)
2077     old_instances[0].quit(expect_shutdown=True)
2078     server._THREAD_POOL.submit(self.server._shutdown_instance, old_instances[0],
2079                                self.wsgi_server.port)
2080
2081     self.mox.ReplayAll()
2082     self.server.restart()
2083     self.mox.VerifyAll()
2084     self.assertEqual([True, False], self.server._instance_running)
2085     self.assertEqual(new_instance, self.server._instances[0])
2086     self.assertEqual(self.server._handle_request,
2087                      self.server._wsgi_servers[0]._app.func)
2088     self.assertEqual({'inst': new_instance},
2089                      self.server._wsgi_servers[0]._app.keywords)
2090
2091   def test_shutdown_idle_instances(self):
2092     s = BasicScalingServerFacade(instance_factory=self.factory)
2093     old_instances = [self.mox.CreateMock(instance.Instance),
2094                      self.mox.CreateMock(instance.Instance),
2095                      self.mox.CreateMock(instance.Instance)]
2096     self.server._instances = old_instances[:]
2097     old_instances[0].idle_seconds = (self.server._instance_idle_timeout + 1)
2098     old_instances[1].idle_seconds = 0
2099     old_instances[2].idle_seconds = (self.server._instance_idle_timeout + 1)
2100     self.server._instance_running = [True, True, False]
2101     new_instance = self.mox.CreateMock(instance.Instance)
2102     self.factory.new_instance(0, expect_ready_request=True).AndReturn(
2103         new_instance)
2104     old_instances[0].quit(expect_shutdown=True)
2105     server._THREAD_POOL.submit(self.server._shutdown_instance, old_instances[0],
2106                                self.wsgi_server.port)
2107
2108     self.mox.ReplayAll()
2109     self.server._shutdown_idle_instances()
2110     self.mox.VerifyAll()
2111     self.assertEqual([False, True, False], self.server._instance_running)
2112     self.assertEqual(new_instance, self.server._instances[0])
2113     self.assertEqual(self.server._handle_request,
2114                      self.server._wsgi_servers[0]._app.func)
2115     self.assertEqual({'inst': new_instance},
2116                      self.server._wsgi_servers[0]._app.keywords)
2117
2118
2119 class TestBasicScalingInstancePoolHandleChanges(unittest.TestCase):
2120   """Tests for server.BasicScalingServer._handle_changes."""
2121
2122   def setUp(self):
2123     api_server.test_setup_stubs()
2124
2125     self.mox = mox.Mox()
2126     self.instance_factory = instance.InstanceFactory(object(), 10)
2127     self.servr = BasicScalingServerFacade(
2128         instance_factory=self.instance_factory)
2129     self.mox.StubOutWithMock(self.instance_factory, 'files_changed')
2130     self.mox.StubOutWithMock(self.instance_factory, 'configuration_changed')
2131     self.mox.StubOutWithMock(self.servr, 'restart')
2132     self.mox.StubOutWithMock(self.servr, '_create_url_handlers')
2133     self.mox.StubOutWithMock(self.servr._server_configuration,
2134                              'check_for_updates')
2135     self.mox.StubOutWithMock(self.servr._watcher, 'has_changes')
2136
2137   def tearDown(self):
2138     self.mox.UnsetStubs()
2139
2140   def test_no_changes(self):
2141     self.servr._server_configuration.check_for_updates().AndReturn(frozenset())
2142     self.servr._watcher.has_changes().AndReturn(False)
2143
2144     self.mox.ReplayAll()
2145     self.servr._handle_changes()
2146     self.mox.VerifyAll()
2147
2148   def test_irrelevant_config_change(self):
2149     self.servr._server_configuration.check_for_updates().AndReturn(frozenset())
2150     self.servr._watcher.has_changes().AndReturn(False)
2151
2152     self.mox.ReplayAll()
2153     self.servr._handle_changes()
2154     self.mox.VerifyAll()
2155
2156   def test_restart_config_change(self):
2157     conf_change = frozenset([application_configuration.ENV_VARIABLES_CHANGED])
2158     self.servr._server_configuration.check_for_updates().AndReturn(conf_change)
2159     self.servr._watcher.has_changes().AndReturn(False)
2160     self.instance_factory.configuration_changed(conf_change)
2161     self.servr.restart()
2162
2163     self.mox.ReplayAll()
2164     self.servr._handle_changes()
2165     self.mox.VerifyAll()
2166
2167   def test_handler_change(self):
2168     conf_change = frozenset([application_configuration.HANDLERS_CHANGED])
2169     self.servr._server_configuration.check_for_updates().AndReturn(conf_change)
2170     self.servr._watcher.has_changes().AndReturn(False)
2171     self.servr._create_url_handlers()
2172     self.instance_factory.configuration_changed(conf_change)
2173     self.servr.restart()
2174
2175     self.mox.ReplayAll()
2176     self.servr._handle_changes()
2177     self.mox.VerifyAll()
2178
2179   def test_file_change(self):
2180     self.servr._server_configuration.check_for_updates().AndReturn(frozenset())
2181     self.servr._watcher.has_changes().AndReturn(True)
2182     self.instance_factory.files_changed().AndReturn(True)
2183     self.servr.restart()
2184
2185     self.mox.ReplayAll()
2186     self.servr._handle_changes()
2187     self.mox.VerifyAll()
2188
2189
2190 class TestInteractiveCommandServer(unittest.TestCase):
2191   def setUp(self):
2192     api_server.test_setup_stubs()
2193
2194     self.mox = mox.Mox()
2195     self.inst = self.mox.CreateMock(instance.Instance)
2196     self.inst.instance_id = 0
2197     self.environ = object()
2198     self.start_response = object()
2199     self.response = [object()]
2200     self.url_map = object()
2201     self.match = object()
2202     self.request_id = object()
2203
2204     self.servr = server.InteractiveCommandServer(
2205         ServerConfigurationStub(),
2206         'fakehost',
2207         balanced_port=8000,
2208         api_port=9000,
2209         runtime_stderr_loglevel=1,
2210
2211         python_config=None,
2212         cloud_sql_config=None,
2213         default_version_port=8080,
2214         port_registry=dispatcher.PortRegistry(),
2215         request_data=None,
2216         dispatcher=None,
2217         use_mtime_file_watcher=False)
2218     self.mox.StubOutWithMock(self.servr._instance_factory, 'new_instance')
2219     self.mox.StubOutWithMock(self.servr, '_handle_request')
2220     self.mox.StubOutWithMock(self.servr, 'build_request_environ')
2221
2222   def test_send_interactive_command(self):
2223     def good_response(unused_environ, start_response, request_type):
2224       start_response('200 OK', [])
2225       return ['10\n']
2226
2227     environ = object()
2228     self.servr.build_request_environ(
2229         'POST', '/', [], 'print 5+5', '192.0.2.0', 8000).AndReturn(environ)
2230     self.servr._handle_request(
2231         environ,
2232         mox.IgnoreArg(),
2233         request_type=instance.INTERACTIVE_REQUEST).WithSideEffects(
2234             good_response)
2235
2236     self.mox.ReplayAll()
2237     self.assertEqual('10\n', self.servr.send_interactive_command('print 5+5'))
2238     self.mox.VerifyAll()
2239
2240   def test_send_interactive_command_handle_request_exception(self):
2241     environ = object()
2242     self.servr.build_request_environ(
2243         'POST', '/', [], 'print 5+5', '192.0.2.0', 8000).AndReturn(environ)
2244     self.servr._handle_request(
2245         environ,
2246         mox.IgnoreArg(),
2247         request_type=instance.INTERACTIVE_REQUEST).AndRaise(Exception('error'))
2248
2249     self.mox.ReplayAll()
2250     self.assertRaisesRegexp(server.InteractiveCommandError,
2251                             'error',
2252                             self.servr.send_interactive_command,
2253                             'print 5+5')
2254     self.mox.VerifyAll()
2255
2256   def test_send_interactive_command_handle_request_failure(self):
2257     def good_response(unused_environ, start_response, request_type):
2258       start_response('503 Service Unavailable', [])
2259       return ['Instance was restarted while executing command']
2260
2261     environ = object()
2262     self.servr.build_request_environ(
2263         'POST', '/', [], 'print 5+5', '192.0.2.0', 8000).AndReturn(environ)
2264     self.servr._handle_request(
2265         environ,
2266         mox.IgnoreArg(),
2267         request_type=instance.INTERACTIVE_REQUEST).WithSideEffects(
2268             good_response)
2269
2270     self.mox.ReplayAll()
2271     self.assertRaisesRegexp(server.InteractiveCommandError,
2272                             'Instance was restarted while executing command',
2273                             self.servr.send_interactive_command,
2274                             'print 5+5')
2275     self.mox.VerifyAll()
2276
2277   def test_handle_script_request(self):
2278     self.servr._instance_factory.new_instance(
2279         mox.IgnoreArg(),
2280         expect_ready_request=False).AndReturn(self.inst)
2281     self.inst.start()
2282     self.inst.handle(self.environ,
2283                      self.start_response,
2284                      self.url_map,
2285                      self.match,
2286                      self.request_id,
2287                      instance.INTERACTIVE_REQUEST).AndReturn(['10\n'])
2288
2289     self.mox.ReplayAll()
2290     self.assertEqual(
2291         ['10\n'],
2292         self.servr._handle_script_request(self.environ,
2293                                           self.start_response,
2294                                           self.url_map,
2295                                           self.match,
2296                                           self.request_id))
2297     self.mox.VerifyAll()
2298
2299   def test_handle_script_request_busy(self):
2300     self.servr._instance_factory.new_instance(
2301         mox.IgnoreArg(),
2302         expect_ready_request=False).AndReturn(self.inst)
2303     self.inst.start()
2304     self.inst.handle(
2305         self.environ,
2306         self.start_response,
2307         self.url_map,
2308         self.match,
2309         self.request_id,
2310         instance.INTERACTIVE_REQUEST).AndRaise(instance.CannotAcceptRequests())
2311     self.inst.wait(mox.IgnoreArg())
2312     self.inst.handle(self.environ,
2313                      self.start_response,
2314                      self.url_map,
2315                      self.match,
2316                      self.request_id,
2317                      instance.INTERACTIVE_REQUEST).AndReturn(['10\n'])
2318
2319     self.mox.ReplayAll()
2320     self.assertEqual(
2321         ['10\n'],
2322         self.servr._handle_script_request(self.environ,
2323                                           self.start_response,
2324                                           self.url_map,
2325                                           self.match,
2326                                           self.request_id))
2327     self.mox.VerifyAll()
2328
2329   def test_handle_script_request_timeout(self):
2330     self.servr._MAX_REQUEST_WAIT_TIME = 0
2331     start_response = start_response_utils.CapturingStartResponse()
2332
2333     self.mox.ReplayAll()
2334     self.assertEqual(
2335         ['The command timed-out while waiting for another one to complete'],
2336         self.servr._handle_script_request(self.environ,
2337                                           start_response,
2338                                           self.url_map,
2339                                           self.match,
2340                                           self.request_id))
2341     self.mox.VerifyAll()
2342     self.assertEqual('503 Service Unavailable',
2343                      start_response.status)
2344
2345   def test_handle_script_request_restart(self):
2346     def restart_and_raise(*args):
2347       self.servr._inst = None
2348       raise httplib.BadStatusLine('line')
2349
2350     start_response = start_response_utils.CapturingStartResponse()
2351     self.servr._instance_factory.new_instance(
2352         mox.IgnoreArg(),
2353         expect_ready_request=False).AndReturn(self.inst)
2354     self.inst.start()
2355     self.inst.handle(
2356         self.environ,
2357         start_response,
2358         self.url_map,
2359         self.match,
2360         self.request_id,
2361         instance.INTERACTIVE_REQUEST).WithSideEffects(restart_and_raise)
2362
2363     self.mox.ReplayAll()
2364     self.assertEqual(
2365         ['Instance was restarted while executing command'],
2366         self.servr._handle_script_request(self.environ,
2367                                           start_response,
2368                                           self.url_map,
2369                                           self.match,
2370                                           self.request_id))
2371     self.mox.VerifyAll()
2372     self.assertEqual('503 Service Unavailable',
2373                      start_response.status)
2374
2375   def test_handle_script_request_unexpected_instance_exception(self):
2376     self.servr._instance_factory.new_instance(
2377         mox.IgnoreArg(),
2378         expect_ready_request=False).AndReturn(self.inst)
2379     self.inst.start()
2380     self.inst.handle(
2381         self.environ,
2382         self.start_response,
2383         self.url_map,
2384         self.match,
2385         self.request_id,
2386         instance.INTERACTIVE_REQUEST).AndRaise(httplib.BadStatusLine('line'))
2387
2388     self.mox.ReplayAll()
2389     self.assertRaises(
2390         httplib.BadStatusLine,
2391         self.servr._handle_script_request,
2392         self.environ,
2393         self.start_response,
2394         self.url_map,
2395         self.match,
2396         self.request_id)
2397     self.mox.VerifyAll()
2398
2399 if __name__ == '__main__':
2400   unittest.main()