mygpo/db/couchdb/management/commands/compact-couchdb.py

   1 import sys
   2 from datetime import datetime
   3 from time import sleep
   4 from urlparse import urlparse
   5 from optparse import make_option
   6
   7 from couchdbkit import Database
   8 from restkit import BasicAuth
   9 from restkit.errors import RequestError
  10 from django.core.management.base import BaseCommand
  11 from django.conf import settings
  12
  13 from mygpo.decorators import repeat_on_conflict
  14 from mygpo.core.models import SanitizingRule
  15 from mygpo.utils import progress
  16
  17
  18
  19 class Command(BaseCommand):
  20     """
  21     Compacts the database and all views, and measures the required time
  22     """
  23
  24     option_list = BaseCommand.option_list + (
  25         make_option('--dry-run', action='store_true', dest='dryrun', default=False, help="Don't compact anything."),
  26     )
  27
  28
  29     def handle(self, *args, **options):
  30         db_urls = set(db[1] for db in settings.COUCHDB_DATABASES)
  31
  32         filters = []
  33
  34         couchdb_admins = getattr(settings, 'COUCHDB_ADMINS', ())
  35         if couchdb_admins:
  36             username, passwd = couchdb_admins[0]
  37             filters.append(BasicAuth(username, passwd))
  38
  39         for db_url in db_urls:
  40             db = Database(db_url, filters=filters)
  41             for view_hash, name, compact, is_compacting, get_size in self.get_compacters(db):
  42
  43                 if options.get('dryrun'):
  44                     duration, size_before, size_after = 0, 0, 0
  45                 else:
  46                     duration, size_before, size_after = self.compact_wait(compact, is_compacting, get_size)
  47
  48                 print '%-40s %17s %10s %10s %7s' % (name, duration, self.prettySize(size_before), self.prettySize(size_after), view_hash[:5])
  49
  50
  51     def get_compacters(self, db):
  52         """ Returns tuples containing compaction tasks """
  53
  54         compact_db       = lambda: db.compact()
  55         db_is_compacting = lambda: db.info()['compact_running']
  56         get_db_size      = lambda: db.info()['disk_size']
  57
  58         yield ('', db.dbname, compact_db, db_is_compacting, get_db_size)
  59
  60         for view_hash, design_doc in self.get_design_docs(db):
  61             compact_view       = lambda: db.compact('%s' % design_doc)
  62             view_is_compacting = lambda: db.res.get('/_design/%s/_info' % design_doc).json_body['view_index']['compact_running']
  63             get_view_size      = lambda: db.res.get('/_design/%s/_info' % design_doc).json_body['view_index']['disk_size']
  64             yield (view_hash, design_doc, compact_view, view_is_compacting, get_view_size)
  65
  66
  67     @staticmethod
  68     def get_all_design_docs(db):
  69         """ Returns all design documents in the database """
  70
  71         prefix = '_design/'
  72         prefix_len = len(prefix)
  73         return (ddoc['key'][prefix_len:] for ddoc in db.view('_all_docs', startkey='_design/', endkey='_design0'))
  74
  75
  76     def get_design_docs(self, db):
  77         """
  78         Return one design doc for each index file
  79         """
  80         ddocs = {}
  81         for ddoc in self.get_all_design_docs(db):
  82             sig = db.res.get('/_design/%s/_info' % ddoc).json_body['view_index']['signature']
  83             ddocs[sig] = ddoc
  84
  85         return ddocs.items()
  86
  87
  88     @staticmethod
  89     def compact_wait(compact, is_compacting, get_size, sleep_time=300, inc_factor = 1):
  90         """ Compacts the view and waits for the compaction to finish
  91
  92         Reports elapsed time and the view size, before and after the compaction """
  93
  94         start = datetime.utcnow()
  95         size_before = get_size()
  96
  97         while True:
  98             try:
  99                 compact()
 100                 break
 101             except RequestError as e:
 102                 print >> sys.stderr, e
 103                 sleep(100)
 104
 105         while True:
 106             try:
 107                 is_comp = is_compacting()
 108                 if is_comp:
 109                     size_before = get_size()
 110                     sleep(sleep_time)
 111                     sleep_time *= inc_factor
 112                 else:
 113                     break
 114
 115             except RequestError as e:
 116                 print >> sys.stderr, e
 117                 sleep(100)
 118
 119         end = datetime.utcnow()
 120         size_after = get_size()
 121
 122         return end - start, size_before, size_after
 123
 124
 125     @staticmethod
 126     def prettySize(size):
 127         # http://snippets.dzone.com/posts/show/5434
 128         suffixes = [("B",2**10), ("K",2**20), ("M",2**30), ("G",2**40), ("T",2**50)]
 129         for suf, lim in suffixes:
 130             if size > lim:
 131                 continue
 132             else:
 133                 return round(size/float(lim/2**10),2).__str__()+suf