python/google/appengine/tools/devappserver2/instance_test.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright 2007 Google Inc.
   4 #
   5 # Licensed under the Apache License, Version 2.0 (the "License");
   6 # you may not use this file except in compliance with the License.
   7 # You may obtain a copy of the License at
   8 #
   9 #     http://www.apache.org/licenses/LICENSE-2.0
  10 #
  11 # Unless required by applicable law or agreed to in writing, software
  12 # distributed under the License is distributed on an "AS IS" BASIS,
  13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14 # See the License for the specific language governing permissions and
  15 # limitations under the License.
  16 #
  17 """Tests for google.apphosting.tools.devappserver2.instance."""
  18
  19
  20 import time
  21 import unittest
  22
  23 import google
  24 import mox
  25
  26 from google.appengine.tools.devappserver2 import instance
  27 from google.appengine.tools.devappserver2 import wsgi_request_info
  28
  29
  30 class TestInstance(unittest.TestCase):
  31   """Tests for instance.Instance."""
  32
  33   def setUp(self):
  34     self.mox = mox.Mox()
  35     self.proxy = self.mox.CreateMock(instance.RuntimeProxy)
  36     self.environ = object()
  37     self.start_response = object()
  38     self.url_map = object()
  39     self.match = object()
  40     self.request_id = object()
  41     self.response = [object()]
  42     self.request_data = self.mox.CreateMock(wsgi_request_info.WSGIRequestInfo)
  43
  44   def tearDown(self):
  45     self.mox.UnsetStubs()
  46
  47   def test_new_instance(self):
  48     inst = instance.Instance(
  49         self.request_data, 'name', self.proxy, max_concurrent_requests=5,
  50         expect_ready_request=True)
  51     self.assertEqual(0, inst.total_requests)
  52     self.assertEqual(5, inst.remaining_request_capacity)
  53     self.assertEqual(0, inst.num_outstanding_requests)
  54     self.assertFalse(inst.can_accept_requests)
  55     self.assertTrue(inst.handling_ready_request)
  56     self.assertAlmostEqual(0, inst.idle_seconds, places=2)
  57     self.assertEqual(0, inst.get_latency_60s())
  58     self.assertEqual(0, inst.get_qps_60s())
  59     self.assertEqual('name', inst.instance_id)
  60
  61   def test_handle(self):
  62     inst = instance.Instance(self.request_data, 'name', self.proxy,
  63                              max_concurrent_requests=5)
  64     self.mox.StubOutWithMock(inst._condition, 'notify')
  65
  66     self.proxy.start()
  67     self.environ = {}
  68     self.request_data.set_request_instance(self.request_id, inst)
  69     self.proxy.handle(self.environ,
  70                       self.start_response,
  71                       self.url_map,
  72                       self.match,
  73                       self.request_id,
  74                       instance.NORMAL_REQUEST).AndReturn(self.response)
  75     inst._condition.notify()
  76     self.mox.ReplayAll()
  77     now = time.time()
  78     inst._request_history.append((now - 100, now - 80))
  79     inst.start()
  80     self.assertTrue(inst.can_accept_requests)
  81     self.assertEqual(
  82         self.response,
  83         list(inst.handle(self.environ,
  84                          self.start_response,
  85                          self.url_map,
  86                          self.match,
  87                          self.request_id,
  88                          instance.NORMAL_REQUEST)))
  89     self.mox.VerifyAll()
  90
  91     self.assertEqual(1, len(inst._request_history))
  92     self.assertEqual(1, inst.total_requests)
  93     self.assertEqual(5, inst.remaining_request_capacity)
  94     self.assertEqual(0, inst.num_outstanding_requests)
  95     self.assertTrue(0 < inst.get_qps_60s())
  96
  97   def test_handle_ready_request(self):
  98     inst = instance.Instance(self.request_data, 'name', self.proxy,
  99                              max_concurrent_requests=5,
 100                              expect_ready_request=True)
 101     self.mox.StubOutWithMock(inst._condition, 'notify')
 102
 103     self.proxy.start()
 104     self.environ = {}
 105     self.request_data.set_request_instance(self.request_id, inst)
 106     self.proxy.handle(self.environ,
 107                       self.start_response,
 108                       self.url_map,
 109                       self.match,
 110                       self.request_id,
 111                       instance.READY_REQUEST).AndReturn(self.response)
 112     inst._condition.notify(5)
 113     self.mox.ReplayAll()
 114     inst.start()
 115     self.assertFalse(inst.can_accept_requests)
 116     self.assertRaises(instance.CannotAcceptRequests,
 117                       inst.handle,
 118                       self.environ,
 119                       self.start_response,
 120                       self.url_map,
 121                       self.match,
 122                       self.request_id,
 123                       instance.NORMAL_REQUEST)
 124     self.assertEqual(
 125         self.response,
 126         list(inst.handle(self.environ,
 127                          self.start_response,
 128                          self.url_map,
 129                          self.match,
 130                          self.request_id,
 131                          instance.READY_REQUEST)))
 132     self.mox.VerifyAll()
 133
 134     self.assertEqual(1, inst.total_requests)
 135     self.assertEqual(5, inst.remaining_request_capacity)
 136     self.assertEqual(0, inst.num_outstanding_requests)
 137     self.assertTrue(0 < inst.get_qps_60s())
 138     self.assertFalse(inst.handling_ready_request)
 139
 140   def test_handle_background_request(self):
 141     inst = instance.Instance(self.request_data, 'name', self.proxy,
 142                              max_concurrent_requests=5,
 143                              max_background_threads=2)
 144     inst._num_running_background_threads = 1
 145     self.mox.StubOutWithMock(inst._condition, 'notify')
 146
 147     self.proxy.start()
 148     self.environ = {}
 149     self.request_data.set_request_instance(self.request_id, inst)
 150     self.proxy.handle(self.environ,
 151                       self.start_response,
 152                       self.url_map,
 153                       self.match,
 154                       self.request_id,
 155                       instance.BACKGROUND_REQUEST).AndReturn(self.response)
 156     self.mox.ReplayAll()
 157     inst.start()
 158     self.assertTrue(inst.can_accept_requests)
 159     self.assertEqual(1, inst.remaining_background_thread_capacity)
 160     self.assertEqual(
 161         self.response,
 162         list(inst.handle(self.environ,
 163                          self.start_response,
 164                          self.url_map,
 165                          self.match,
 166                          self.request_id,
 167                          instance.BACKGROUND_REQUEST)))
 168     self.mox.VerifyAll()
 169
 170     self.assertEqual(1, inst.total_requests)
 171     self.assertEqual(5, inst.remaining_request_capacity)
 172     self.assertEqual(0, inst.num_outstanding_requests)
 173     self.assertTrue(0 < inst.get_qps_60s())
 174     self.assertEqual(2, inst.remaining_background_thread_capacity)
 175     self.assertFalse(inst.handling_ready_request)
 176
 177   def test_handle_shutdown_request(self):
 178     inst = instance.Instance(self.request_data, 'name', self.proxy,
 179                              max_concurrent_requests=1)
 180     inst._num_outstanding_requests = 0
 181     self.mox.StubOutWithMock(inst._condition, 'notify_all')
 182
 183     self.proxy.start()
 184     self.environ = {}
 185     self.request_data.set_request_instance(self.request_id, inst)
 186     self.proxy.handle(self.environ,
 187                       self.start_response,
 188                       self.url_map,
 189                       self.match,
 190                       self.request_id,
 191                       instance.SHUTDOWN_REQUEST).AndReturn(self.response)
 192     self.proxy.quit()
 193     inst._condition.notify_all()
 194     self.mox.ReplayAll()
 195     inst.start()
 196     self.assertTrue(inst.can_accept_requests)
 197     self.assertFalse(inst.has_quit)
 198     inst.quit(expect_shutdown=True)
 199     self.assertFalse(inst.can_accept_requests)
 200     self.assertTrue(inst.has_quit)
 201     self.assertEqual(
 202         self.response,
 203         list(inst.handle(self.environ,
 204                          self.start_response,
 205                          self.url_map,
 206                          self.match,
 207                          self.request_id,
 208                          instance.SHUTDOWN_REQUEST)))
 209     self.mox.VerifyAll()
 210
 211     self.assertEqual(1, inst.total_requests)
 212     self.assertEqual(1, inst.remaining_request_capacity)
 213     self.assertEqual(0, inst.num_outstanding_requests)
 214     self.assertTrue(0 < inst.get_qps_60s())
 215     self.assertFalse(inst._quitting)
 216     self.assertTrue(inst._quit)
 217
 218   def test_handle_shutdown_request_running_request(self):
 219     inst = instance.Instance(self.request_data, 'name', self.proxy,
 220                              max_concurrent_requests=1)
 221     inst._num_outstanding_requests = 1
 222     self.mox.StubOutWithMock(inst._condition, 'notify_all')
 223
 224     self.proxy.start()
 225     self.environ = {}
 226     self.request_data.set_request_instance(self.request_id, inst)
 227     self.proxy.handle(self.environ,
 228                       self.start_response,
 229                       self.url_map,
 230                       self.match,
 231                       self.request_id,
 232                       instance.SHUTDOWN_REQUEST).AndReturn(self.response)
 233     self.mox.ReplayAll()
 234     inst.start()
 235     self.assertTrue(inst.can_accept_requests)
 236     self.assertFalse(inst.has_quit)
 237     inst.quit(expect_shutdown=True)
 238     self.assertFalse(inst.can_accept_requests)
 239     self.assertTrue(inst.has_quit)
 240     self.assertEqual(
 241         self.response,
 242         list(inst.handle(self.environ,
 243                          self.start_response,
 244                          self.url_map,
 245                          self.match,
 246                          self.request_id,
 247                          instance.SHUTDOWN_REQUEST)))
 248     self.mox.VerifyAll()
 249
 250     self.assertEqual(1, inst.total_requests)
 251     self.assertEqual(0, inst.remaining_request_capacity)
 252     self.assertEqual(1, inst.num_outstanding_requests)
 253     self.assertEqual(0, inst.idle_seconds)
 254     self.assertTrue(0 < inst.get_qps_60s())
 255     self.assertTrue(inst._quitting)
 256     self.assertFalse(inst._quit)
 257
 258   def test_handle_before_start(self):
 259     inst = instance.Instance(self.request_data, 'name', self.proxy,
 260                              max_concurrent_requests=5)
 261     self.mox.StubOutWithMock(inst._condition, 'notify')
 262
 263     self.assertRaises(instance.CannotAcceptRequests,
 264                       inst.handle,
 265                       self.environ,
 266                       self.start_response,
 267                       self.url_map,
 268                       self.match,
 269                       self.request_id,
 270                       instance.NORMAL_REQUEST)
 271
 272   def test_handle_after_quit(self):
 273     inst = instance.Instance(self.request_data, 'name', self.proxy,
 274                              max_concurrent_requests=5)
 275     self.mox.StubOutWithMock(inst._condition, 'notify')
 276     self.mox.StubOutWithMock(inst._condition, 'notify_all')
 277
 278     self.proxy.start()
 279     self.proxy.quit()
 280     inst._condition.notify_all()
 281
 282     self.mox.ReplayAll()
 283     inst.start()
 284     inst.quit()
 285
 286     self.assertRaises(instance.CannotAcceptRequests,
 287                       inst.handle,
 288                       self.environ,
 289                       self.start_response,
 290                       self.url_map,
 291                       self.match,
 292                       self.request_id,
 293                       instance.NORMAL_REQUEST)
 294     self.mox.VerifyAll()
 295
 296   def test_handle_while_quitting(self):
 297     inst = instance.Instance(self.request_data, 'name', self.proxy,
 298                              max_concurrent_requests=5)
 299     self.mox.StubOutWithMock(inst._condition, 'notify')
 300     inst._num_outstanding_requests = 1
 301
 302     self.proxy.start()
 303     self.mox.ReplayAll()
 304     inst.start()
 305     inst.quit(allow_async=True)
 306     self.mox.VerifyAll()
 307
 308     self.assertRaises(instance.CannotAcceptRequests,
 309                       inst.handle,
 310                       self.environ,
 311                       self.start_response,
 312                       self.url_map,
 313                       self.match,
 314                       self.request_id,
 315                       instance.NORMAL_REQUEST)
 316
 317   def test_handle_no_capacity(self):
 318     inst = instance.Instance(self.request_data, 'name', self.proxy,
 319                              max_concurrent_requests=1)
 320     self.mox.StubOutWithMock(inst._condition, 'notify')
 321     inst._num_outstanding_requests = 1
 322     self.proxy.start()
 323     self.assertRaises(instance.CannotAcceptRequests,
 324                       inst.handle,
 325                       self.environ,
 326                       self.start_response,
 327                       self.url_map,
 328                       self.match,
 329                       self.request_id,
 330                       instance.NORMAL_REQUEST)
 331
 332   def test_reserve_background_thread_success(self):
 333     inst = instance.Instance(self.request_data, 'name', self.proxy,
 334                              max_concurrent_requests=5,
 335                              max_background_threads=2)
 336     inst._started = True
 337
 338     self.mox.ReplayAll()
 339     self.assertEqual(2, inst.remaining_background_thread_capacity)
 340     inst.reserve_background_thread()
 341     self.assertEqual(1, inst.remaining_background_thread_capacity)
 342     self.mox.VerifyAll()
 343
 344   def test_reserve_background_thread_quitting(self):
 345     inst = instance.Instance(self.request_data, 'name', self.proxy,
 346                              max_concurrent_requests=5,
 347                              max_background_threads=2)
 348     inst._started = True
 349     inst._quitting = True
 350
 351     self.mox.ReplayAll()
 352     self.assertEqual(2, inst.remaining_background_thread_capacity)
 353     inst.reserve_background_thread()
 354     self.assertEqual(1, inst.remaining_background_thread_capacity)
 355     self.mox.VerifyAll()
 356
 357   def test_reserve_background_thread_no_capacity(self):
 358     inst = instance.Instance(self.request_data, 'name', self.proxy,
 359                              max_concurrent_requests=5,
 360                              max_background_threads=0)
 361     inst._started = True
 362     self.mox.ReplayAll()
 363     self.assertEqual(0, inst.remaining_background_thread_capacity)
 364     self.assertRaises(instance.CannotAcceptRequests,
 365                       inst.reserve_background_thread)
 366     self.mox.VerifyAll()
 367     self.assertEqual(0, inst.remaining_background_thread_capacity)
 368
 369   def test_reserve_background_thread_not_started(self):
 370     inst = instance.Instance(self.request_data, 'name', self.proxy,
 371                              max_concurrent_requests=5,
 372                              max_background_threads=1)
 373     self.mox.ReplayAll()
 374     self.assertEqual(1, inst.remaining_background_thread_capacity)
 375     self.assertRaises(instance.CannotAcceptRequests,
 376                       inst.reserve_background_thread)
 377     self.mox.VerifyAll()
 378     self.assertEqual(1, inst.remaining_background_thread_capacity)
 379
 380   def test_reserve_background_thread_quit(self):
 381     inst = instance.Instance(self.request_data, 'name', self.proxy,
 382                              max_concurrent_requests=5,
 383                              max_background_threads=1)
 384     inst._started = True
 385     inst._quit = True
 386     self.mox.ReplayAll()
 387     self.assertEqual(1, inst.remaining_background_thread_capacity)
 388     self.assertRaises(instance.CannotAcceptRequests,
 389                       inst.reserve_background_thread)
 390     self.mox.VerifyAll()
 391     self.assertEqual(1, inst.remaining_background_thread_capacity)
 392
 393   def test_reserve_background_thread_not_ready(self):
 394     inst = instance.Instance(self.request_data, 'name', self.proxy,
 395                              max_concurrent_requests=5,
 396                              max_background_threads=2,
 397                              expect_ready_request=True)
 398     inst._started = True
 399     self.mox.ReplayAll()
 400     self.assertEqual(2, inst.remaining_background_thread_capacity)
 401     inst.reserve_background_thread()
 402     self.mox.VerifyAll()
 403     self.assertEqual(1, inst.remaining_background_thread_capacity)
 404
 405   def test_wait_with_capacity(self):
 406     inst = instance.Instance(self.request_data, 'name', self.proxy,
 407                              max_concurrent_requests=1)
 408     inst._started = True
 409     self.mox.StubOutWithMock(inst._condition, 'wait')
 410     self.mox.stubs.Set(time, 'time', lambda: 0)
 411     self.mox.ReplayAll()
 412     self.assertTrue(inst.wait(1))
 413     self.mox.VerifyAll()
 414
 415   def test_wait_waiting_for_can_accept(self):
 416     inst = instance.Instance(self.request_data, 'name', self.proxy,
 417                              max_concurrent_requests=1,
 418                              expect_ready_request=True)
 419     inst._started = True
 420     self.mox.StubOutWithMock(inst._condition, 'wait')
 421     self.time = 0
 422     self.mox.stubs.Set(time, 'time', lambda: self.time)
 423
 424     def advance_time(*unused_args):
 425       self.time += 10
 426
 427     inst._condition.wait(1).WithSideEffects(advance_time)
 428     self.mox.ReplayAll()
 429     self.assertFalse(inst.wait(1))
 430     self.mox.VerifyAll()
 431
 432   def test_wait_timed_out_with_capacity(self):
 433     inst = instance.Instance(self.request_data, 'name', self.proxy,
 434                              max_concurrent_requests=1)
 435     inst._started = True
 436     self.mox.StubOutWithMock(inst._condition, 'wait')
 437     self.mox.ReplayAll()
 438     self.assertTrue(inst.wait(0))
 439     self.mox.VerifyAll()
 440
 441   def test_wait_without_capacity(self):
 442     inst = instance.Instance(self.request_data, 'name', self.proxy,
 443                              max_concurrent_requests=0)
 444     inst._started = True
 445     self.mox.StubOutWithMock(inst._condition, 'wait')
 446     self.time = 0
 447     self.mox.stubs.Set(time, 'time', lambda: self.time)
 448
 449     def advance_time(*unused_args):
 450       self.time += 10
 451
 452     inst._condition.wait(1).WithSideEffects(advance_time)
 453     self.mox.ReplayAll()
 454     self.assertFalse(inst.wait(1))
 455     self.mox.VerifyAll()
 456
 457   def test_wait_timed_out_without_capacity(self):
 458     inst = instance.Instance(self.request_data, 'name', self.proxy,
 459                              max_concurrent_requests=0)
 460     inst._started = True
 461     self.mox.StubOutWithMock(inst._condition, 'wait')
 462     self.mox.ReplayAll()
 463     self.assertFalse(inst.wait(0))
 464     self.mox.VerifyAll()
 465
 466   def test_wait_quit_while_starting(self):
 467     inst = instance.Instance(self.request_data, 'name', self.proxy,
 468                              max_concurrent_requests=5)
 469     self.mox.StubOutWithMock(inst._condition, 'notify_all')
 470     self.proxy.start().WithSideEffects(inst.quit)
 471
 472     self.proxy.quit()
 473
 474     self.mox.ReplayAll()
 475     inst.start()
 476     self.mox.VerifyAll()
 477     self.assertFalse(inst.can_accept_requests)
 478
 479   def test_wait_quit_while_waiting(self):
 480     self.mox.stubs.Set(time, 'time', lambda: 0)
 481     inst = instance.Instance(self.request_data, 'name', self.proxy,
 482                              max_concurrent_requests=0)
 483     self.mox.StubOutWithMock(inst._condition, 'wait')
 484     inst._condition.wait(1).WithSideEffects(lambda *unused_args: inst.quit())
 485     self.mox.ReplayAll()
 486     self.assertFalse(inst.wait(1))
 487     self.mox.VerifyAll()
 488
 489   def test_health(self):
 490     inst = instance.Instance(self.request_data, 'name', self.proxy,
 491                              max_concurrent_requests=5)
 492     self.mox.StubOutWithMock(inst._condition, 'notify_all')
 493     self.proxy.start()
 494
 495     self.proxy.quit()
 496     inst._condition.notify_all()
 497
 498     self.mox.ReplayAll()
 499     inst.start()
 500     self.assertTrue(inst.can_accept_requests)
 501     inst.set_health(False)
 502     self.assertFalse(inst.can_accept_requests)
 503     inst.set_health(True)
 504     self.assertTrue(inst.can_accept_requests)
 505
 506   def test_quit(self):
 507     inst = instance.Instance(self.request_data, 'name', self.proxy,
 508                              max_concurrent_requests=5)
 509     self.mox.StubOutWithMock(inst._condition, 'notify_all')
 510     self.proxy.start()
 511
 512     self.proxy.quit()
 513     inst._condition.notify_all()
 514
 515     self.mox.ReplayAll()
 516     inst.start()
 517     self.assertTrue(inst.can_accept_requests)
 518     inst.quit()
 519     self.mox.VerifyAll()
 520     self.assertFalse(inst.can_accept_requests)
 521
 522   def test_quit_with_request(self):
 523     inst = instance.Instance(self.request_data, 'name', self.proxy,
 524                              max_concurrent_requests=5)
 525     self.mox.StubOutWithMock(inst._condition, 'notify_all')
 526     self.proxy.start()
 527
 528     self.mox.ReplayAll()
 529     inst.start()
 530     self.mox.VerifyAll()
 531     inst._num_outstanding_requests = 1
 532     self.assertRaises(instance.CannotQuitServingInstance,
 533                       inst.quit)
 534
 535   def test_quit_with_request_force(self):
 536     inst = instance.Instance(self.request_data, 'name', self.proxy,
 537                              max_concurrent_requests=5)
 538     self.mox.StubOutWithMock(inst._condition, 'notify_all')
 539
 540     inst._num_outstanding_requests = 1
 541     self.proxy.start()
 542     self.proxy.quit()
 543     inst._condition.notify_all()
 544
 545     self.mox.ReplayAll()
 546     inst.start()
 547     inst.quit(force=True)
 548     self.mox.VerifyAll()
 549
 550   def test_quit_with_request_force_and_allow_async(self):
 551     inst = instance.Instance(self.request_data, 'name', self.proxy,
 552                              max_concurrent_requests=5)
 553     self.mox.StubOutWithMock(inst._condition, 'notify_all')
 554
 555     inst._num_outstanding_requests = 1
 556     self.proxy.start()
 557     self.proxy.quit()
 558     inst._condition.notify_all()
 559
 560     self.mox.ReplayAll()
 561     inst.start()
 562     inst.quit(force=True, allow_async=True)
 563     self.mox.VerifyAll()
 564
 565   def test_quit_with_request_allow_async(self):
 566     inst = instance.Instance(self.request_data, 'name', self.proxy,
 567                              max_concurrent_requests=5)
 568     self.mox.StubOutWithMock(inst._condition, 'notify_all')
 569
 570     inst._num_outstanding_requests = 1
 571     self.proxy.start()
 572
 573     self.mox.ReplayAll()
 574     inst.start()
 575     inst.quit(allow_async=True)
 576     self.mox.VerifyAll()
 577     self.assertTrue(inst._quitting)
 578
 579   def test_quit_shutdown(self):
 580     inst = instance.Instance(self.request_data, 'name', self.proxy,
 581                              max_concurrent_requests=5)
 582     self.mox.StubOutWithMock(inst._condition, 'notify_all')
 583
 584     inst._num_outstanding_requests = 1
 585     self.proxy.start()
 586
 587     self.mox.ReplayAll()
 588     inst.start()
 589     inst.quit(expect_shutdown=True)
 590     self.mox.VerifyAll()
 591     self.assertTrue(inst._expecting_shutdown_request)
 592     self.assertFalse(inst._quitting)
 593
 594   def test_get_latency_60s(self):
 595     inst = instance.Instance(self.request_data, 'name', self.proxy,
 596                              max_concurrent_requests=5)
 597     now = time.time()
 598     inst._request_history = [(now, now+1), (now+2, now+4)]
 599     self.assertEqual(1.5, inst.get_latency_60s())
 600
 601   def test_get_qps_60s(self):
 602     inst = instance.Instance(self.request_data, 'name', self.proxy,
 603                              max_concurrent_requests=5)
 604     now = time.time()
 605     inst._request_history = [(now, now+1)] * 120
 606     self.assertEqual(2.0, inst.get_qps_60s())
 607
 608   def test__trim_request_history_to_60s(self):
 609     inst = instance.Instance(self.request_data, 'name', self.proxy,
 610                              max_concurrent_requests=5)
 611     inst._request_history.append((0, 100))
 612     inst._request_history.append((1.0, 101))
 613     inst._request_history.append((1.2, 102))
 614     inst._request_history.append((2.5, 103))
 615
 616     now = time.time()
 617     inst._request_history.append((now, 42))
 618     inst._request_history.append((now + 1, 43))
 619     inst._request_history.append((now + 3, 44))
 620     inst._request_history.append((now + 4, 45))
 621
 622     inst._trim_request_history_to_60s()
 623     self.assertEqual([(now, 42), (now + 1, 43), (now + 3, 44), (now + 4, 45)],
 624                      list(inst._request_history))
 625
 626
 627 if __name__ == '__main__':
 628   unittest.main()