mygpo/db/couchdb/utils.py

   1 #
   2 # This file is part of my.gpodder.org.
   3 #
   4 # my.gpodder.org is free software: you can redistribute it and/or modify it
   5 # under the terms of the GNU Affero General Public License as published by
   6 # the Free Software Foundation, either version 3 of the License, or (at your
   7 # option) any later version.
   8 #
   9 # my.gpodder.org is distributed in the hope that it will be useful, but
  10 # WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
  11 # or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public
  12 # License for more details.
  13 #
  14 # You should have received a copy of the GNU Affero General Public License
  15 # along with my.gpodder.org. If not, see <http://www.gnu.org/licenses/>.
  16 #
  17
  18 import operator
  19 import string
  20 import functools
  21
  22 from mygpo.db.couchdb import get_main_database
  23
  24
  25
  26 def multi_request_view(cls, view, wrap=True, auto_advance=True,
  27         *args, **kwargs):
  28     """
  29     splits up a view request into several requests, which reduces
  30     the server load of the number of returned objects is large.
  31
  32     NOTE: As such a split request is obviously not atomical anymore, results
  33     might skip some elements of contain some twice
  34
  35     If auto_advance is False the method will always request the same range.
  36     This can be useful when the view contain unprocessed items and the caller
  37     processes the items, thus removing them from the view before the next
  38     request.
  39     """
  40
  41     per_page = kwargs.get('limit', 1000)
  42     kwargs['limit'] = per_page + 1
  43     db = get_main_database()
  44     wrapper = kwargs.pop('wrapper', False) or cls.wrap
  45     cont = True
  46
  47     while cont:
  48
  49         resp = db.view(view, *args, **kwargs)
  50         cont = False
  51
  52         for n, obj in enumerate(resp.iterator()):
  53
  54             key = obj['key']
  55
  56             if wrap:
  57                 doc = wrapper(obj['doc']) if wrapper else obj['doc']
  58                 docid = doc._id if wrapper else obj['id']
  59             else:
  60                 docid = obj.get('id', None)
  61                 doc = obj
  62
  63             if n == per_page:
  64                 if auto_advance:
  65                     kwargs['startkey'] = key
  66                     if docid is not None:
  67                         kwargs['startkey_docid'] = docid
  68                     if 'skip' in kwargs:
  69                         del kwargs['skip']
  70
  71                 # we reached the end of the page, load next one
  72                 cont = True
  73                 break
  74
  75             yield doc
  76
  77
  78
  79 def is_couchdb_id(id_str):
  80     f = functools.partial(operator.contains, string.hexdigits)
  81     return len(id_str) == 32 and all(map(f, id_str))