mygpo/db/couchdb/management/commands/compact-couchdb.py

   1 import sys
   2 from datetime import datetime
   3 from time import sleep
   4 from urlparse import urlparse
   5 from optparse import make_option
   6
   7 from couchdbkit import Database
   8 from restkit import BasicAuth
   9 from restkit.errors import RequestError
  10 from django.core.management.base import BaseCommand
  11 from django.conf import settings
  12
  13 from mygpo.decorators import repeat_on_conflict
  14 from mygpo.utils import progress
  15
  16
  17
  18 class Command(BaseCommand):
  19     """
  20     Compacts the database and all views, and measures the required time
  21     """
  22
  23     option_list = BaseCommand.option_list + (
  24         make_option('--dry-run', action='store_true', dest='dryrun', default=False, help="Don't compact anything."),
  25     )
  26
  27
  28     def handle(self, *args, **options):
  29         db_urls = set(db[1] for db in settings.COUCHDB_DATABASES)
  30
  31         filters = []
  32
  33         couchdb_admins = getattr(settings, 'COUCHDB_ADMINS', ())
  34         if couchdb_admins:
  35             username, passwd = couchdb_admins[0]
  36             filters.append(BasicAuth(username, passwd))
  37
  38         for db_url in db_urls:
  39             db = Database(db_url, filters=filters)
  40             for view_hash, name, compact, is_compacting, get_size in self.get_compacters(db):
  41
  42                 if options.get('dryrun'):
  43                     duration, size_before, size_after = 0, 0, 0
  44                 else:
  45                     duration, size_before, size_after = self.compact_wait(compact, is_compacting, get_size)
  46
  47                 print '%-40s %17s %10s %10s %7s' % (name, duration, self.prettySize(size_before), self.prettySize(size_after), view_hash[:5])
  48
  49
  50     def get_compacters(self, db):
  51         """ Returns tuples containing compaction tasks """
  52
  53         compact_db       = lambda: db.compact()
  54         db_is_compacting = lambda: db.info()['compact_running']
  55         get_db_size      = lambda: db.info()['disk_size']
  56
  57         yield ('', db.dbname, compact_db, db_is_compacting, get_db_size)
  58
  59         for view_hash, design_doc in self.get_design_docs(db):
  60             compact_view       = lambda: db.compact('%s' % design_doc)
  61             view_is_compacting = lambda: db.res.get('/_design/%s/_info' % design_doc).json_body['view_index']['compact_running']
  62             get_view_size      = lambda: db.res.get('/_design/%s/_info' % design_doc).json_body['view_index']['disk_size']
  63             yield (view_hash, design_doc, compact_view, view_is_compacting, get_view_size)
  64
  65
  66     @staticmethod
  67     def get_all_design_docs(db):
  68         """ Returns all design documents in the database """
  69
  70         prefix = '_design/'
  71         prefix_len = len(prefix)
  72         return (ddoc['key'][prefix_len:] for ddoc in db.view('_all_docs', startkey='_design/', endkey='_design0'))
  73
  74
  75     def get_design_docs(self, db):
  76         """
  77         Return one design doc for each index file
  78         """
  79         ddocs = {}
  80         for ddoc in self.get_all_design_docs(db):
  81             sig = db.res.get('/_design/%s/_info' % ddoc).json_body['view_index']['signature']
  82             ddocs[sig] = ddoc
  83
  84         return ddocs.items()
  85
  86
  87     @staticmethod
  88     def compact_wait(compact, is_compacting, get_size, sleep_time=300, inc_factor = 1):
  89         """ Compacts the view and waits for the compaction to finish
  90
  91         Reports elapsed time and the view size, before and after the compaction """
  92
  93         start = datetime.utcnow()
  94         size_before = get_size()
  95
  96         while True:
  97             try:
  98                 compact()
  99                 break
 100             except RequestError as e:
 101                 print >> sys.stderr, e
 102                 sleep(100)
 103
 104         while True:
 105             try:
 106                 is_comp = is_compacting()
 107                 if is_comp:
 108                     size_before = get_size()
 109                     sleep(sleep_time)
 110                     sleep_time *= inc_factor
 111                 else:
 112                     break
 113
 114             except RequestError as e:
 115                 print >> sys.stderr, e
 116                 sleep(100)
 117
 118         end = datetime.utcnow()
 119         size_after = get_size()
 120
 121         return end - start, size_before, size_after
 122
 123
 124     @staticmethod
 125     def prettySize(size):
 126         # http://snippets.dzone.com/posts/show/5434
 127         suffixes = [("B",2**10), ("K",2**20), ("M",2**30), ("G",2**40), ("T",2**50)]
 128         for suf, lim in suffixes:
 129             if size > lim:
 130                 continue
 131             else:
 132                 return round(size/float(lim/2**10),2).__str__()+suf