mygpo/db/couchdb/utils.py

   1 #
   2 # This file is part of my.gpodder.org.
   3 #
   4 # my.gpodder.org is free software: you can redistribute it and/or modify it
   5 # under the terms of the GNU Affero General Public License as published by
   6 # the Free Software Foundation, either version 3 of the License, or (at your
   7 # option) any later version.
   8 #
   9 # my.gpodder.org is distributed in the hope that it will be useful, but
  10 # WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
  11 # or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public
  12 # License for more details.
  13 #
  14 # You should have received a copy of the GNU Affero General Public License
  15 # along with my.gpodder.org. If not, see <http://www.gnu.org/licenses/>.
  16 #
  17
  18 import os.path
  19 import operator
  20 import string
  21 import functools
  22
  23 from couchdbkit.loaders import FileSystemDocsLoader
  24 from couchdbkit.ext.django import loading
  25
  26 from django.conf import settings
  27
  28 from mygpo.db.couchdb import get_main_database
  29
  30 import logging
  31 logger = logging.getLogger(__name__)
  32
  33
  34 def multi_request_view(cls, view, wrap=True, auto_advance=True,
  35         *args, **kwargs):
  36     """
  37     splits up a view request into several requests, which reduces
  38     the server load of the number of returned objects is large.
  39
  40     NOTE: As such a split request is obviously not atomical anymore, results
  41     might skip some elements of contain some twice
  42
  43     If auto_advance is False the method will always request the same range.
  44     This can be useful when the view contain unprocessed items and the caller
  45     processes the items, thus removing them from the view before the next
  46     request.
  47     """
  48
  49     per_page = kwargs.get('limit', 1000)
  50     kwargs['limit'] = per_page + 1
  51     db = get_main_database()
  52     wrapper = kwargs.pop('wrapper', False) or cls.wrap
  53     cont = True
  54
  55     while cont:
  56
  57         resp = db.view(view, *args, **kwargs)
  58         cont = False
  59
  60         for n, obj in enumerate(resp.iterator()):
  61
  62             key = obj['key']
  63
  64             if wrap:
  65                 doc = wrapper(obj['doc']) if wrapper else obj['doc']
  66                 docid = doc._id if wrapper else obj['id']
  67             else:
  68                 docid = obj.get('id', None)
  69                 doc = obj
  70
  71             if n == per_page:
  72                 if auto_advance:
  73                     kwargs['startkey'] = key
  74                     if docid is not None:
  75                         kwargs['startkey_docid'] = docid
  76                     if 'skip' in kwargs:
  77                         del kwargs['skip']
  78
  79                 # we reached the end of the page, load next one
  80                 cont = True
  81                 break
  82
  83             yield doc
  84
  85
  86
  87 def is_couchdb_id(id_str):
  88     f = functools.partial(operator.contains, string.hexdigits)
  89     return len(id_str) == 32 and all(map(f, id_str))
  90
  91
  92 def sync_design_docs():
  93     """ synchronize the design docs for all databases """
  94
  95     base_dir = settings.BASE_DIR
  96
  97     for part, label in settings.COUCHDB_DDOC_MAPPING.items():
  98             path = os.path.join(base_dir, '..', 'couchdb', part, '_design')
  99
 100             logger.info('syncing ddocs for "%s" from "%s"', label, path)
 101
 102             db = loading.get_db(label)
 103             loader = FileSystemDocsLoader(path)
 104             loader.sync(db, verbose=True)
 105
 106
 107 def view_cleanup():
 108     """ do a view-cleanup for all databases """
 109
 110     logger.info('Doing view cleanup for all databases')
 111     for label in settings.COUCHDB_DDOC_MAPPING.values():
 112         logger.info('Doing view cleanup for database "%s"', label)
 113         db = loading.get_db(label)
 114         res = db.view_cleanup()
 115
 116         if res.get('ok', False):
 117             log = logger.info
 118         else:
 119             log = logger.warn
 120
 121         log('Result of view cleanup for database "%s": %s', label, res)