Lib/test/test_gc.py

   1 import unittest
   2 from test.test_support import verbose, run_unittest
   3 import sys
   4 import gc
   5 import weakref
   6
   7 ### Support code
   8 ###############################################################################
   9
  10 # Bug 1055820 has several tests of longstanding bugs involving weakrefs and
  11 # cyclic gc.
  12
  13 # An instance of C1055820 has a self-loop, so becomes cyclic trash when
  14 # unreachable.
  15 class C1055820(object):
  16     def __init__(self, i):
  17         self.i = i
  18         self.loop = self
  19
  20 class GC_Detector(object):
  21     # Create an instance I.  Then gc hasn't happened again so long as
  22     # I.gc_happened is false.
  23
  24     def __init__(self):
  25         self.gc_happened = False
  26
  27         def it_happened(ignored):
  28             self.gc_happened = True
  29
  30         # Create a piece of cyclic trash that triggers it_happened when
  31         # gc collects it.
  32         self.wr = weakref.ref(C1055820(666), it_happened)
  33
  34
  35 ### Tests
  36 ###############################################################################
  37
  38 class GCTests(unittest.TestCase):
  39     def test_list(self):
  40         l = []
  41         l.append(l)
  42         gc.collect()
  43         del l
  44         self.assertEqual(gc.collect(), 1)
  45
  46     def test_dict(self):
  47         d = {}
  48         d[1] = d
  49         gc.collect()
  50         del d
  51         self.assertEqual(gc.collect(), 1)
  52
  53     def test_tuple(self):
  54         # since tuples are immutable we close the loop with a list
  55         l = []
  56         t = (l,)
  57         l.append(t)
  58         gc.collect()
  59         del t
  60         del l
  61         self.assertEqual(gc.collect(), 2)
  62
  63     def test_class(self):
  64         class A:
  65             pass
  66         A.a = A
  67         gc.collect()
  68         del A
  69         self.assertNotEqual(gc.collect(), 0)
  70
  71     def test_newstyleclass(self):
  72         class A(object):
  73             pass
  74         gc.collect()
  75         del A
  76         self.assertNotEqual(gc.collect(), 0)
  77
  78     def test_instance(self):
  79         class A:
  80             pass
  81         a = A()
  82         a.a = a
  83         gc.collect()
  84         del a
  85         self.assertNotEqual(gc.collect(), 0)
  86
  87     def test_newinstance(self):
  88         class A(object):
  89             pass
  90         a = A()
  91         a.a = a
  92         gc.collect()
  93         del a
  94         self.assertNotEqual(gc.collect(), 0)
  95         class B(list):
  96             pass
  97         class C(B, A):
  98             pass
  99         a = C()
 100         a.a = a
 101         gc.collect()
 102         del a
 103         self.assertNotEqual(gc.collect(), 0)
 104         del B, C
 105         self.assertNotEqual(gc.collect(), 0)
 106         A.a = A()
 107         del A
 108         self.assertNotEqual(gc.collect(), 0)
 109         self.assertEqual(gc.collect(), 0)
 110
 111     def test_method(self):
 112         # Tricky: self.__init__ is a bound method, it references the instance.
 113         class A:
 114             def __init__(self):
 115                 self.init = self.__init__
 116         a = A()
 117         gc.collect()
 118         del a
 119         self.assertNotEqual(gc.collect(), 0)
 120
 121     def test_finalizer(self):
 122         # A() is uncollectable if it is part of a cycle, make sure it shows up
 123         # in gc.garbage.
 124         class A:
 125             def __del__(self): pass
 126         class B:
 127             pass
 128         a = A()
 129         a.a = a
 130         id_a = id(a)
 131         b = B()
 132         b.b = b
 133         gc.collect()
 134         del a
 135         del b
 136         self.assertNotEqual(gc.collect(), 0)
 137         for obj in gc.garbage:
 138             if id(obj) == id_a:
 139                 del obj.a
 140                 break
 141         else:
 142             self.fail("didn't find obj in garbage (finalizer)")
 143         gc.garbage.remove(obj)
 144
 145     def test_finalizer_newclass(self):
 146         # A() is uncollectable if it is part of a cycle, make sure it shows up
 147         # in gc.garbage.
 148         class A(object):
 149             def __del__(self): pass
 150         class B(object):
 151             pass
 152         a = A()
 153         a.a = a
 154         id_a = id(a)
 155         b = B()
 156         b.b = b
 157         gc.collect()
 158         del a
 159         del b
 160         self.assertNotEqual(gc.collect(), 0)
 161         for obj in gc.garbage:
 162             if id(obj) == id_a:
 163                 del obj.a
 164                 break
 165         else:
 166             self.fail("didn't find obj in garbage (finalizer)")
 167         gc.garbage.remove(obj)
 168
 169     def test_function(self):
 170         # Tricky: f -> d -> f, code should call d.clear() after the exec to
 171         # break the cycle.
 172         d = {}
 173         exec("def f(): pass\n") in d
 174         gc.collect()
 175         del d
 176         self.assertEqual(gc.collect(), 2)
 177
 178     def test_frame(self):
 179         def f():
 180             frame = sys._getframe()
 181         gc.collect()
 182         f()
 183         self.assertEqual(gc.collect(), 1)
 184
 185     def test_saveall(self):
 186         # Verify that cyclic garbage like lists show up in gc.garbage if the
 187         # SAVEALL option is enabled.
 188
 189         # First make sure we don't save away other stuff that just happens to
 190         # be waiting for collection.
 191         gc.collect()
 192         # if this fails, someone else created immortal trash
 193         self.assertEqual(gc.garbage, [])
 194
 195         L = []
 196         L.append(L)
 197         id_L = id(L)
 198
 199         debug = gc.get_debug()
 200         gc.set_debug(debug | gc.DEBUG_SAVEALL)
 201         del L
 202         gc.collect()
 203         gc.set_debug(debug)
 204
 205         self.assertEqual(len(gc.garbage), 1)
 206         obj = gc.garbage.pop()
 207         self.assertEqual(id(obj), id_L)
 208
 209     def test_del(self):
 210         # __del__ methods can trigger collection, make this to happen
 211         thresholds = gc.get_threshold()
 212         gc.enable()
 213         gc.set_threshold(1)
 214
 215         class A:
 216             def __del__(self):
 217                 dir(self)
 218         a = A()
 219         del a
 220
 221         gc.disable()
 222         gc.set_threshold(*thresholds)
 223
 224     def test_del_newclass(self):
 225         # __del__ methods can trigger collection, make this to happen
 226         thresholds = gc.get_threshold()
 227         gc.enable()
 228         gc.set_threshold(1)
 229
 230         class A(object):
 231             def __del__(self):
 232                 dir(self)
 233         a = A()
 234         del a
 235
 236         gc.disable()
 237         gc.set_threshold(*thresholds)
 238
 239     # The following two tests are fragile:
 240     # They precisely count the number of allocations,
 241     # which is highly implementation-dependent.
 242     # For example:
 243     # - disposed tuples are not freed, but reused
 244     # - the call to assertEqual somehow avoids building its args tuple
 245     def test_get_count(self):
 246         # Avoid future allocation of method object
 247         assertEqual = self._baseAssertEqual
 248         gc.collect()
 249         assertEqual(gc.get_count(), (0, 0, 0))
 250         a = dict()
 251         # since gc.collect(), we created two objects:
 252         # the dict, and the tuple returned by get_count()
 253         assertEqual(gc.get_count(), (2, 0, 0))
 254
 255     def test_collect_generations(self):
 256         # Avoid future allocation of method object
 257         assertEqual = self.assertEqual
 258         gc.collect()
 259         a = dict()
 260         gc.collect(0)
 261         assertEqual(gc.get_count(), (0, 1, 0))
 262         gc.collect(1)
 263         assertEqual(gc.get_count(), (0, 0, 1))
 264         gc.collect(2)
 265         assertEqual(gc.get_count(), (0, 0, 0))
 266
 267     def test_trashcan(self):
 268         class Ouch:
 269             n = 0
 270             def __del__(self):
 271                 Ouch.n = Ouch.n + 1
 272                 if Ouch.n % 17 == 0:
 273                     gc.collect()
 274
 275         # "trashcan" is a hack to prevent stack overflow when deallocating
 276         # very deeply nested tuples etc.  It works in part by abusing the
 277         # type pointer and refcount fields, and that can yield horrible
 278         # problems when gc tries to traverse the structures.
 279         # If this test fails (as it does in 2.0, 2.1 and 2.2), it will
 280         # most likely die via segfault.
 281
 282         # Note:  In 2.3 the possibility for compiling without cyclic gc was
 283         # removed, and that in turn allows the trashcan mechanism to work
 284         # via much simpler means (e.g., it never abuses the type pointer or
 285         # refcount fields anymore).  Since it's much less likely to cause a
 286         # problem now, the various constants in this expensive (we force a lot
 287         # of full collections) test are cut back from the 2.2 version.
 288         gc.enable()
 289         N = 150
 290         for count in range(2):
 291             t = []
 292             for i in range(N):
 293                 t = [t, Ouch()]
 294             u = []
 295             for i in range(N):
 296                 u = [u, Ouch()]
 297             v = {}
 298             for i in range(N):
 299                 v = {1: v, 2: Ouch()}
 300         gc.disable()
 301
 302     def test_boom(self):
 303         class Boom:
 304             def __getattr__(self, someattribute):
 305                 del self.attr
 306                 raise AttributeError
 307
 308         a = Boom()
 309         b = Boom()
 310         a.attr = b
 311         b.attr = a
 312
 313         gc.collect()
 314         garbagelen = len(gc.garbage)
 315         del a, b
 316         # a<->b are in a trash cycle now.  Collection will invoke
 317         # Boom.__getattr__ (to see whether a and b have __del__ methods), and
 318         # __getattr__ deletes the internal "attr" attributes as a side effect.
 319         # That causes the trash cycle to get reclaimed via refcounts falling to
 320         # 0, thus mutating the trash graph as a side effect of merely asking
 321         # whether __del__ exists.  This used to (before 2.3b1) crash Python.
 322         # Now __getattr__ isn't called.
 323         self.assertEqual(gc.collect(), 4)
 324         self.assertEqual(len(gc.garbage), garbagelen)
 325
 326     def test_boom2(self):
 327         class Boom2:
 328             def __init__(self):
 329                 self.x = 0
 330
 331             def __getattr__(self, someattribute):
 332                 self.x += 1
 333                 if self.x > 1:
 334                     del self.attr
 335                 raise AttributeError
 336
 337         a = Boom2()
 338         b = Boom2()
 339         a.attr = b
 340         b.attr = a
 341
 342         gc.collect()
 343         garbagelen = len(gc.garbage)
 344         del a, b
 345         # Much like test_boom(), except that __getattr__ doesn't break the
 346         # cycle until the second time gc checks for __del__.  As of 2.3b1,
 347         # there isn't a second time, so this simply cleans up the trash cycle.
 348         # We expect a, b, a.__dict__ and b.__dict__ (4 objects) to get
 349         # reclaimed this way.
 350         self.assertEqual(gc.collect(), 4)
 351         self.assertEqual(len(gc.garbage), garbagelen)
 352
 353     def test_boom_new(self):
 354         # boom__new and boom2_new are exactly like boom and boom2, except use
 355         # new-style classes.
 356
 357         class Boom_New(object):
 358             def __getattr__(self, someattribute):
 359                 del self.attr
 360                 raise AttributeError
 361
 362         a = Boom_New()
 363         b = Boom_New()
 364         a.attr = b
 365         b.attr = a
 366
 367         gc.collect()
 368         garbagelen = len(gc.garbage)
 369         del a, b
 370         self.assertEqual(gc.collect(), 4)
 371         self.assertEqual(len(gc.garbage), garbagelen)
 372
 373     def test_boom2_new(self):
 374         class Boom2_New(object):
 375             def __init__(self):
 376                 self.x = 0
 377
 378             def __getattr__(self, someattribute):
 379                 self.x += 1
 380                 if self.x > 1:
 381                     del self.attr
 382                 raise AttributeError
 383
 384         a = Boom2_New()
 385         b = Boom2_New()
 386         a.attr = b
 387         b.attr = a
 388
 389         gc.collect()
 390         garbagelen = len(gc.garbage)
 391         del a, b
 392         self.assertEqual(gc.collect(), 4)
 393         self.assertEqual(len(gc.garbage), garbagelen)
 394
 395     def test_get_referents(self):
 396         alist = [1, 3, 5]
 397         got = gc.get_referents(alist)
 398         got.sort()
 399         self.assertEqual(got, alist)
 400
 401         atuple = tuple(alist)
 402         got = gc.get_referents(atuple)
 403         got.sort()
 404         self.assertEqual(got, alist)
 405
 406         adict = {1: 3, 5: 7}
 407         expected = [1, 3, 5, 7]
 408         got = gc.get_referents(adict)
 409         got.sort()
 410         self.assertEqual(got, expected)
 411
 412         got = gc.get_referents([1, 2], {3: 4}, (0, 0, 0))
 413         got.sort()
 414         self.assertEqual(got, [0, 0] + range(5))
 415
 416         self.assertEqual(gc.get_referents(1, 'a', 4j), [])
 417
 418     def test_is_tracked(self):
 419         # Atomic built-in types are not tracked, user-defined objects and
 420         # mutable containers are.
 421         # NOTE: types with special optimizations (e.g. tuple) have tests
 422         # in their own test files instead.
 423         self.assertFalse(gc.is_tracked(None))
 424         self.assertFalse(gc.is_tracked(1))
 425         self.assertFalse(gc.is_tracked(1.0))
 426         self.assertFalse(gc.is_tracked(1.0 + 5.0j))
 427         self.assertFalse(gc.is_tracked(True))
 428         self.assertFalse(gc.is_tracked(False))
 429         self.assertFalse(gc.is_tracked("a"))
 430         self.assertFalse(gc.is_tracked(u"a"))
 431         self.assertFalse(gc.is_tracked(bytearray("a")))
 432         self.assertFalse(gc.is_tracked(type))
 433         self.assertFalse(gc.is_tracked(int))
 434         self.assertFalse(gc.is_tracked(object))
 435         self.assertFalse(gc.is_tracked(object()))
 436
 437         class OldStyle:
 438             pass
 439         class NewStyle(object):
 440             pass
 441         self.assertTrue(gc.is_tracked(gc))
 442         self.assertTrue(gc.is_tracked(OldStyle))
 443         self.assertTrue(gc.is_tracked(OldStyle()))
 444         self.assertTrue(gc.is_tracked(NewStyle))
 445         self.assertTrue(gc.is_tracked(NewStyle()))
 446         self.assertTrue(gc.is_tracked([]))
 447         self.assertTrue(gc.is_tracked(set()))
 448
 449     def test_bug1055820b(self):
 450         # Corresponds to temp2b.py in the bug report.
 451
 452         ouch = []
 453         def callback(ignored):
 454             ouch[:] = [wr() for wr in WRs]
 455
 456         Cs = [C1055820(i) for i in range(2)]
 457         WRs = [weakref.ref(c, callback) for c in Cs]
 458         c = None
 459
 460         gc.collect()
 461         self.assertEqual(len(ouch), 0)
 462         # Make the two instances trash, and collect again.  The bug was that
 463         # the callback materialized a strong reference to an instance, but gc
 464         # cleared the instance's dict anyway.
 465         Cs = None
 466         gc.collect()
 467         self.assertEqual(len(ouch), 2)  # else the callbacks didn't run
 468         for x in ouch:
 469             # If the callback resurrected one of these guys, the instance
 470             # would be damaged, with an empty __dict__.
 471             self.assertEqual(x, None)
 472
 473 class GCTogglingTests(unittest.TestCase):
 474     def setUp(self):
 475         gc.enable()
 476
 477     def tearDown(self):
 478         gc.disable()
 479
 480     def test_bug1055820c(self):
 481         # Corresponds to temp2c.py in the bug report.  This is pretty
 482         # elaborate.
 483
 484         c0 = C1055820(0)
 485         # Move c0 into generation 2.
 486         gc.collect()
 487
 488         c1 = C1055820(1)
 489         c1.keep_c0_alive = c0
 490         del c0.loop # now only c1 keeps c0 alive
 491
 492         c2 = C1055820(2)
 493         c2wr = weakref.ref(c2) # no callback!
 494
 495         ouch = []
 496         def callback(ignored):
 497             ouch[:] = [c2wr()]
 498
 499         # The callback gets associated with a wr on an object in generation 2.
 500         c0wr = weakref.ref(c0, callback)
 501
 502         c0 = c1 = c2 = None
 503
 504         # What we've set up:  c0, c1, and c2 are all trash now.  c0 is in
 505         # generation 2.  The only thing keeping it alive is that c1 points to
 506         # it. c1 and c2 are in generation 0, and are in self-loops.  There's a
 507         # global weakref to c2 (c2wr), but that weakref has no callback.
 508         # There's also a global weakref to c0 (c0wr), and that does have a
 509         # callback, and that callback references c2 via c2wr().
 510         #
 511         #               c0 has a wr with callback, which references c2wr
 512         #               ^
 513         #               |
 514         #               |     Generation 2 above dots
 515         #. . . . . . . .|. . . . . . . . . . . . . . . . . . . . . . . .
 516         #               |     Generation 0 below dots
 517         #               |
 518         #               |
 519         #            ^->c1   ^->c2 has a wr but no callback
 520         #            |  |    |  |
 521         #            <--v    <--v
 522         #
 523         # So this is the nightmare:  when generation 0 gets collected, we see
 524         # that c2 has a callback-free weakref, and c1 doesn't even have a
 525         # weakref.  Collecting generation 0 doesn't see c0 at all, and c0 is
 526         # the only object that has a weakref with a callback.  gc clears c1
 527         # and c2.  Clearing c1 has the side effect of dropping the refcount on
 528         # c0 to 0, so c0 goes away (despite that it's in an older generation)
 529         # and c0's wr callback triggers.  That in turn materializes a reference
 530         # to c2 via c2wr(), but c2 gets cleared anyway by gc.
 531
 532         # We want to let gc happen "naturally", to preserve the distinction
 533         # between generations.
 534         junk = []
 535         i = 0
 536         detector = GC_Detector()
 537         while not detector.gc_happened:
 538             i += 1
 539             if i > 10000:
 540                 self.fail("gc didn't happen after 10000 iterations")
 541             self.assertEqual(len(ouch), 0)
 542             junk.append([])  # this will eventually trigger gc
 543
 544         self.assertEqual(len(ouch), 1)  # else the callback wasn't invoked
 545         for x in ouch:
 546             # If the callback resurrected c2, the instance would be damaged,
 547             # with an empty __dict__.
 548             self.assertEqual(x, None)
 549
 550     def test_bug1055820d(self):
 551         # Corresponds to temp2d.py in the bug report.  This is very much like
 552         # test_bug1055820c, but uses a __del__ method instead of a weakref
 553         # callback to sneak in a resurrection of cyclic trash.
 554
 555         ouch = []
 556         class D(C1055820):
 557             def __del__(self):
 558                 ouch[:] = [c2wr()]
 559
 560         d0 = D(0)
 561         # Move all the above into generation 2.
 562         gc.collect()
 563
 564         c1 = C1055820(1)
 565         c1.keep_d0_alive = d0
 566         del d0.loop # now only c1 keeps d0 alive
 567
 568         c2 = C1055820(2)
 569         c2wr = weakref.ref(c2) # no callback!
 570
 571         d0 = c1 = c2 = None
 572
 573         # What we've set up:  d0, c1, and c2 are all trash now.  d0 is in
 574         # generation 2.  The only thing keeping it alive is that c1 points to
 575         # it.  c1 and c2 are in generation 0, and are in self-loops.  There's
 576         # a global weakref to c2 (c2wr), but that weakref has no callback.
 577         # There are no other weakrefs.
 578         #
 579         #               d0 has a __del__ method that references c2wr
 580         #               ^
 581         #               |
 582         #               |     Generation 2 above dots
 583         #. . . . . . . .|. . . . . . . . . . . . . . . . . . . . . . . .
 584         #               |     Generation 0 below dots
 585         #               |
 586         #               |
 587         #            ^->c1   ^->c2 has a wr but no callback
 588         #            |  |    |  |
 589         #            <--v    <--v
 590         #
 591         # So this is the nightmare:  when generation 0 gets collected, we see
 592         # that c2 has a callback-free weakref, and c1 doesn't even have a
 593         # weakref.  Collecting generation 0 doesn't see d0 at all.  gc clears
 594         # c1 and c2.  Clearing c1 has the side effect of dropping the refcount
 595         # on d0 to 0, so d0 goes away (despite that it's in an older
 596         # generation) and d0's __del__ triggers.  That in turn materializes
 597         # a reference to c2 via c2wr(), but c2 gets cleared anyway by gc.
 598
 599         # We want to let gc happen "naturally", to preserve the distinction
 600         # between generations.
 601         detector = GC_Detector()
 602         junk = []
 603         i = 0
 604         while not detector.gc_happened:
 605             i += 1
 606             if i > 10000:
 607                 self.fail("gc didn't happen after 10000 iterations")
 608             self.assertEqual(len(ouch), 0)
 609             junk.append([])  # this will eventually trigger gc
 610
 611         self.assertEqual(len(ouch), 1)  # else __del__ wasn't invoked
 612         for x in ouch:
 613             # If __del__ resurrected c2, the instance would be damaged, with an
 614             # empty __dict__.
 615             self.assertEqual(x, None)
 616
 617 def test_main():
 618     enabled = gc.isenabled()
 619     gc.disable()
 620     assert not gc.isenabled()
 621     debug = gc.get_debug()
 622     gc.set_debug(debug & ~gc.DEBUG_LEAK) # this test is supposed to leak
 623
 624     try:
 625         gc.collect() # Delete 2nd generation garbage
 626         run_unittest(GCTests, GCTogglingTests)
 627     finally:
 628         gc.set_debug(debug)
 629         # test gc.enable() even if GC is disabled by default
 630         if verbose:
 631             print "restoring automatic collection"
 632         # make sure to always test gc.enable()
 633         gc.enable()
 634         assert gc.isenabled()
 635         if not enabled:
 636             gc.disable()
 637
 638 if __name__ == "__main__":
 639     test_main()