scheduler/gc_stats.py

   1 # Compute and gather statistics about garbage collection in this process.
   2 # This module depends on the CPython gc module and garbage collection behavior.
   3
   4 import gc, logging, pprint
   5
   6
   7 verbose = False
   8
   9
  10 # A mapping from type objects to a count of instances of those types in the
  11 # garbage collectors all objects list on the previous call to
  12 # _log_garbage_collector_stats().
  13 _previous_obj_type_map = {}
  14
  15
  16 # A set of object ids for everything in the all objects list on the
  17 # previous call to _log_garbage_collector_stats().
  18 _previous_obj_ids = set()
  19
  20
  21 def _log_garbage_collector_stats(minimum_count=10):
  22     """
  23     Log statistics about how many of what type of Python object exist in this
  24     process.
  25
  26     @param minimum_count: The minimum number of instances of a type for it
  27             to be considered worthy of logging.
  28     """
  29     global _previous_obj_type_map
  30     global _previous_obj_ids
  31
  32     # We get all objects -before- creating any new objects within this function.
  33     # to avoid having our own local instances in the list.
  34     all_objects = gc.get_objects()
  35     obj = None
  36     new_objects = []
  37     try:
  38         obj_type_map = {}
  39         object_ids = set()
  40         for obj in all_objects:
  41             obj_type = type(obj)
  42             obj_type_map.setdefault(obj_type, 0)
  43             obj_type_map[obj_type] += 1
  44             object_ids.add(id(obj))
  45         whats_new_big_str = ''
  46         if verbose and _previous_obj_ids:
  47             new_object_ids = object_ids - _previous_obj_ids
  48             for obj in all_objects:
  49                 if id(obj) in new_object_ids:
  50                     new_objects.append(obj)
  51             whats_new_big_str = pprint.pformat(new_objects, indent=1)
  52     finally:
  53         # Never keep references to stuff returned by gc.get_objects() around
  54         # or it'll just make the future cyclic gc runs more difficult.
  55         del all_objects
  56         del obj
  57         del new_objects
  58
  59
  60     delta = {}
  61     for obj_type, count in obj_type_map.iteritems():
  62         if obj_type not in _previous_obj_type_map:
  63             delta[obj_type] = count
  64         elif _previous_obj_type_map[obj_type] != count:
  65             delta[obj_type] = count - _previous_obj_type_map[obj_type]
  66
  67     sorted_stats = reversed(sorted(
  68             (count, obj_type) for obj_type, count in obj_type_map.iteritems()))
  69     sorted_delta = reversed(sorted(
  70             (count, obj_type) for obj_type, count in delta.iteritems()))
  71
  72     logging.debug('Garbage collector object type counts:')
  73     for count, obj_type in sorted_stats:
  74         if count >= minimum_count:
  75             logging.debug('  %d\t%s', count, obj_type)
  76
  77     logging.info('Change in object counts since previous GC stats:')
  78     for change, obj_type in sorted_delta:
  79         if obj_type_map[obj_type] > minimum_count:
  80             logging.info('  %+d\t%s\tto %d', change, obj_type,
  81                          obj_type_map[obj_type])
  82
  83     if verbose and whats_new_big_str:
  84         logging.debug('Pretty printed representation of the new objects:')
  85         logging.debug(whats_new_big_str)
  86
  87     _previous_obj_type_map = obj_type_map
  88     if verbose:
  89         _previous_obj_ids = object_ids