From dff09a0ece9fe28ba491e3bb61a34b317f57014a Mon Sep 17 00:00:00 2001 From: =?utf8?q?Stefan=20K=C3=B6gl?= Date: Mon, 27 Aug 2012 18:09:22 +0200 Subject: [PATCH] reduce queries to Database URL --- mygpo/admin/clients.py | 4 +-- mygpo/api/advanced/__init__.py | 4 +-- mygpo/api/backend.py | 9 +++--- mygpo/core/models.py | 33 +++++++++++----------- mygpo/core/slugs.py | 12 ++++---- mygpo/couchdb.py | 8 ++++++ mygpo/data/podcast.py | 4 +-- .../management/commands/update-episode-toplist.py | 4 --- .../management/commands/update-toplist.py | 4 +-- mygpo/directory/search.py | 3 +- mygpo/directory/tags.py | 4 +-- mygpo/maintenance/management/changescmd.py | 3 +- .../management/commands/assign-episode-slugs.py | 3 +- .../maintenance/management/commands/dump-sample.py | 3 +- .../management/commands/merge-episode-states.py | 4 +-- .../management/commands/sync-design-docs.py | 4 +-- mygpo/maintenance/management/podcastcmd.py | 3 +- mygpo/users/models.py | 8 +++--- mygpo/utils.py | 4 ++- mygpo/web/heatmap.py | 4 +-- 20 files changed, 67 insertions(+), 58 deletions(-) diff --git a/mygpo/admin/clients.py b/mygpo/admin/clients.py index 7b254198..867d7583 100644 --- a/mygpo/admin/clients.py +++ b/mygpo/admin/clients.py @@ -13,12 +13,12 @@ class UserAgentStats(object): def __init__(self): self._useragents = None - self.db = User.get_db() def get_entries(self): if self._useragents is None: - res = self.db.view('clients/by_ua_string', + res = User.view('clients/by_ua_string', + wrap_doc = False, group_level = 1, stale = 'update_after', ) diff --git a/mygpo/api/advanced/__init__.py b/mygpo/api/advanced/__init__.py index e36ae686..73203a05 100644 --- a/mygpo/api/advanced/__init__.py +++ b/mygpo/api/advanced/__init__.py @@ -34,7 +34,7 @@ from mygpo.api.httpresponse import JsonResponse from mygpo.api.sanitizing import sanitize_url, sanitize_urls from mygpo.api.advanced.directory import episode_data, podcast_data from mygpo.api.backend import get_device, get_favorites, BulkSubscribe -from mygpo.couchdb import BulkException +from mygpo.couchdb import BulkException, get_main_database from mygpo.log import log from mygpo.utils import parse_time, format_time, parse_bool, get_to_dict, get_timestamp from mygpo.decorators import allowed_methods, repeat_on_conflict @@ -331,7 +331,7 @@ def update_episodes(user, actions, now, ua_string): fun = partial(update_episode_actions, action_list=action_list) obj_funs.append( (episode_state, fun) ) - db = EpisodeUserState.get_db() + db = get_main_database() bulk_save_retry(db, obj_funs) return update_urls diff --git a/mygpo/api/backend.py b/mygpo/api/backend.py index ec8f977e..268ee282 100644 --- a/mygpo/api/backend.py +++ b/mygpo/api/backend.py @@ -24,11 +24,10 @@ from django.core.cache import cache from mygpo.data.mimetype import get_type, CONTENT_TYPES from mygpo.core.models import Podcast, Episode -from mygpo.users.models import EpisodeUserState, Device, DeviceDoesNotExist, \ - PodcastUserState +from mygpo.users.models import EpisodeUserState, Device, DeviceDoesNotExist from mygpo.decorators import repeat_on_conflict from mygpo.json import json -from mygpo.couchdb import bulk_save_retry +from mygpo.couchdb import bulk_save_retry, get_main_database def get_random_picks(languages=None): @@ -59,7 +58,7 @@ def get_podcast_count_for_language(): counts = defaultdict(int) - db = Podcast.get_db() + db = get_main_database() r = db.view('podcasts/by_language', reduce = True, group_level = 1, @@ -138,7 +137,7 @@ class BulkSubscribe(object): def execute(self): """ Executes all added actions in bulk """ - db = PodcastUserState.get_db() + db = get_main_database() obj_funs = map(self._get_obj_fun, self.actions) bulk_save_retry(db, obj_funs) diff --git a/mygpo/core/models.py b/mygpo/core/models.py index 3534959e..e76df94e 100644 --- a/mygpo/core/models.py +++ b/mygpo/core/models.py @@ -346,11 +346,11 @@ class Podcast(Document, SlugMixin, OldIdMixin): @classmethod def for_slug(cls, slug): - db = cls.get_db() - r = db.view('podcasts/by_slug', + r = cls.view('podcasts/by_slug', startkey = [slug, None], endkey = [slug, {}], include_docs = True, + wrap_doc = False, ) if not r: @@ -378,10 +378,10 @@ class Podcast(Document, SlugMixin, OldIdMixin): @classmethod def get_multi(cls, ids): - db = Podcast.get_db() - r = db.view('podcasts/by_id', - keys=ids, - include_docs=True, + r = cls.view('podcasts/by_id', + keys = ids, + include_docs = True, + wrap_doc = False ) for res in r: @@ -438,15 +438,14 @@ class Podcast(Document, SlugMixin, OldIdMixin): be restricted to this language. chunk_size determines how many podcasts will be fetched at once """ - db = cls.get_db() - while True: rnd = random() - res = db.view('podcasts/random', + res = cls.view('podcasts/random', startkey = [language, rnd], include_docs = True, limit = chunk_size, stale = 'ok', + wrap_doc = False, ) if not res: @@ -463,10 +462,10 @@ class Podcast(Document, SlugMixin, OldIdMixin): @classmethod def by_last_update(cls): - db = cls.get_db() - res = db.view('podcasts/by_last_update', + res = cls.view('podcasts/by_last_update', include_docs = True, stale = 'update_after', + wrap_doc = False, ) for r in res: @@ -483,14 +482,14 @@ class Podcast(Document, SlugMixin, OldIdMixin): @classmethod def for_language(cls, language, **kwargs): - db = cls.get_db() - res = db.view('podcasts/by_language', + res = cls.view('podcasts/by_language', startkey = [language, None], endkey = [language, {}], include_docs = True, reduce = False, stale = 'update_after', + wrap_doc = False, **kwargs ) @@ -823,11 +822,11 @@ class Podcast(Document, SlugMixin, OldIdMixin): """ Returns the latest episode actions for the podcast's episodes """ from mygpo.users.models import EpisodeUserState - db = EpisodeUserState.get_db() - res = db.view('episode_states/by_user_podcast', - startkey= [user_id, self.get_id(), None], - endkey = [user_id, self.get_id(), {}] + res = EpisodeUserState.view('episode_states/by_user_podcast', + startkey = [user_id, self.get_id(), None], + endkey = [user_id, self.get_id(), {}], + wrap_doc = False, ) for r in res: diff --git a/mygpo/core/slugs.py b/mygpo/core/slugs.py index 0c7c42cf..35f59628 100644 --- a/mygpo/core/slugs.py +++ b/mygpo/core/slugs.py @@ -84,10 +84,10 @@ class PodcastGroupSlug(SlugGenerator): def _get_existing_slugs(self): from mygpo.core.models import Podcast - db = Podcast.get_db() - res = db.view('podcasts/by_slug', + res = Podcast.view('podcasts/by_slug', startkey = [self.base_slug, None], - endkey = [self.base_slug + 'ZZZZZ', None] + endkey = [self.base_slug + 'ZZZZZ', None], + wrap_doc = False, ) return [r['key'][0] for r in res] @@ -142,10 +142,10 @@ class EpisodeSlug(SlugGenerator): """ Episode slugs have to be unique within the Podcast """ from mygpo.core.models import Episode - db = Episode.get_db() - res = db.view('episodes/by_slug', + res = Episode.view('episodes/by_slug', startkey = [self.podcast_id, self.base_slug], - endkey = [self.podcast_id, self.base_slug + 'ZZZZZ'] + endkey = [self.podcast_id, self.base_slug + 'ZZZZZ'], + wrap_doc = False, ) return [r['key'][1] for r in res] diff --git a/mygpo/couchdb.py b/mygpo/couchdb.py index a26dddbf..878f7aeb 100644 --- a/mygpo/couchdb.py +++ b/mygpo/couchdb.py @@ -1,6 +1,8 @@ from operator import itemgetter from collections import namedtuple +from django.conf import settings + from couchdbkit import * class BulkException(Exception): @@ -71,3 +73,9 @@ def bulk_save_retry(db, obj_funs, reload_f=__default_reload): if errors: raise BulkException(errors) + + + +def get_main_database(): + db_url = settings.COUCHDB_DATABASES[0][1] + return Database(db_url) diff --git a/mygpo/data/podcast.py b/mygpo/data/podcast.py index 1b0c810c..6c3248f3 100644 --- a/mygpo/data/podcast.py +++ b/mygpo/data/podcast.py @@ -16,7 +16,7 @@ # from mygpo.core.models import Podcast -from mygpo.users.models import PodcastUserState +from mygpo.couchdb import get_main_database try: from collections import Counter @@ -32,7 +32,7 @@ def calc_similar_podcasts(podcast, num=20): Probably an expensive operation """ - db = PodcastUserState.get_db() + db = get_main_database() res = db.view('subscriptions/by_podcast', startkey = [podcast.get_id(), None, None], diff --git a/mygpo/directory/management/commands/update-episode-toplist.py b/mygpo/directory/management/commands/update-episode-toplist.py index 7ac9649e..ea916095 100644 --- a/mygpo/directory/management/commands/update-episode-toplist.py +++ b/mygpo/directory/management/commands/update-episode-toplist.py @@ -42,7 +42,3 @@ class Command(ChangesCommand): def get_query_params(self): return dict(include_docs=True, filter='episode_states/has_play_events') - - - def get_db(self): - return EpisodeUserState.get_db() diff --git a/mygpo/directory/management/commands/update-toplist.py b/mygpo/directory/management/commands/update-toplist.py index bfcdf6e2..96f2ff8e 100644 --- a/mygpo/directory/management/commands/update-toplist.py +++ b/mygpo/directory/management/commands/update-toplist.py @@ -4,7 +4,7 @@ from optparse import make_option from django.core.management.base import BaseCommand from mygpo.core.models import Podcast, SubscriberData -from mygpo.users.models import PodcastUserState +from mygpo.couchdb import get_main_database from mygpo.utils import progress from mygpo.decorators import repeat_on_conflict @@ -52,7 +52,7 @@ class Command(BaseCommand): @staticmethod def get_subscriber_count(podcast): - db = PodcastUserState.get_db() + db = get_main_database() subscriber_sum = 0 for podcast_id in podcast.get_ids(): diff --git a/mygpo/directory/search.py b/mygpo/directory/search.py index 4faabc8a..f7121345 100644 --- a/mygpo/directory/search.py +++ b/mygpo/directory/search.py @@ -1,5 +1,6 @@ from mygpo.core.models import Podcast, PodcastGroup from mygpo.utils import is_url +from mygpo.couchdb import get_main_database from mygpo.data.feeddownloader import update_podcasts from mygpo.api.sanitizing import sanitize_url @@ -29,7 +30,7 @@ def search_podcasts(q, limit=20, skip=0): return [podcast], 1 - db = Podcast.get_db() + db = get_main_database() #FIXME current couchdbkit can't parse responses for multi-query searches q = q.replace(',', '') diff --git a/mygpo/directory/tags.py b/mygpo/directory/tags.py index 8e0f2dce..eac7dcdf 100644 --- a/mygpo/directory/tags.py +++ b/mygpo/directory/tags.py @@ -3,7 +3,7 @@ from operator import itemgetter from mygpo.core.models import Podcast from mygpo.decorators import query_if_required -from mygpo.directory.models import Category +from mygpo.couchdb import get_main_database from mygpo.utils import multi_request_view from mygpo.counter import Counter @@ -135,7 +135,7 @@ class TagCloud(object): return self._entries is None def _query(self): - db = Category.get_db() + db = get_main_database() res = db.view('categories/by_weight', descending = True, skip = self.skip, diff --git a/mygpo/maintenance/management/changescmd.py b/mygpo/maintenance/management/changescmd.py index d591ba81..63aaa20a 100755 --- a/mygpo/maintenance/management/changescmd.py +++ b/mygpo/maintenance/management/changescmd.py @@ -8,6 +8,7 @@ from couchdbkit.exceptions import ResourceNotFound from couchdbkit import Consumer from mygpo.utils import progress +from mygpo.couchdb import get_main_database from mygpo.maintenance.models import CommandStatus, CommandRunStatus try: @@ -131,4 +132,4 @@ class ChangesCommand(BaseCommand): @abstractmethod def get_db(self): - raise NotImplemented + return get_main_database() diff --git a/mygpo/maintenance/management/commands/assign-episode-slugs.py b/mygpo/maintenance/management/commands/assign-episode-slugs.py index e7d5fba4..c7cb434c 100755 --- a/mygpo/maintenance/management/commands/assign-episode-slugs.py +++ b/mygpo/maintenance/management/commands/assign-episode-slugs.py @@ -10,6 +10,7 @@ from mygpo.core.models import Podcast, PodcastGroup, Episode from mygpo.core.slugs import EpisodeSlug, EpisodesMissingSlugs from mygpo.decorators import repeat_on_conflict from mygpo.utils import progress +from mygpo.couchdb import get_main_database from mygpo.maintenance.models import CommandStatus, CommandRunStatus try: @@ -34,7 +35,7 @@ class Command(BaseCommand): def handle(self, *args, **options): - db = Episode.get_db() + db = get_main_database() status = self.get_cmd_status() since = self.get_since(status, options) objects = self.get_objects(db, since) diff --git a/mygpo/maintenance/management/commands/dump-sample.py b/mygpo/maintenance/management/commands/dump-sample.py index 1f397324..a5b6c56b 100755 --- a/mygpo/maintenance/management/commands/dump-sample.py +++ b/mygpo/maintenance/management/commands/dump-sample.py @@ -8,6 +8,7 @@ from django.core.management.base import BaseCommand from mygpo.json import json from mygpo.core.models import Podcast +from mygpo.couchdb import get_main_database from mygpo.users.models import PodcastUserState, EpisodeUserState, \ Suggestions, User from mygpo.directory.models import Category @@ -74,7 +75,7 @@ class Command(BaseCommand): docs.add(e_state._id) - db = Podcast.get_db() + db = get_main_database() docs = sorted(docs) self.dump(docs, db) diff --git a/mygpo/maintenance/management/commands/merge-episode-states.py b/mygpo/maintenance/management/commands/merge-episode-states.py index b8422737..3f430cda 100755 --- a/mygpo/maintenance/management/commands/merge-episode-states.py +++ b/mygpo/maintenance/management/commands/merge-episode-states.py @@ -8,7 +8,7 @@ from mygpo.utils import progress, multi_request_view from mygpo.users.models import EpisodeUserState from mygpo.counter import Counter from mygpo.maintenance.merge import merge_episode_states -from mygpo.couchdb import bulk_save_retry +from mygpo.couchdb import bulk_save_retry, get_main_database class Command(BaseCommand): @@ -25,7 +25,7 @@ class Command(BaseCommand): total = EpisodeUserState.view('episode_states/by_user_episode', limit=0, ).total_rows - db = EpisodeUserState.get_db() + db = get_main_database() actions = Counter() actions['merged'] = 0 diff --git a/mygpo/maintenance/management/commands/sync-design-docs.py b/mygpo/maintenance/management/commands/sync-design-docs.py index 98ceccb1..996c3ac6 100755 --- a/mygpo/maintenance/management/commands/sync-design-docs.py +++ b/mygpo/maintenance/management/commands/sync-design-docs.py @@ -4,7 +4,7 @@ from django.conf import settings from django.core.management.base import BaseCommand from couchdbkit.loaders import FileSystemDocsLoader -from mygpo.core.models import Podcast +from mygpo.couchdb import get_main_database @@ -14,7 +14,7 @@ class Command(BaseCommand): def handle(self, *args, **options): path = os.path.join(settings.BASE_DIR, '..', 'couchdb', '_design') - db = Podcast.get_db() + db = get_main_database() loader = FileSystemDocsLoader(path) loader.sync(db, verbose=True) diff --git a/mygpo/maintenance/management/podcastcmd.py b/mygpo/maintenance/management/podcastcmd.py index 2df67af6..03f68a78 100644 --- a/mygpo/maintenance/management/podcastcmd.py +++ b/mygpo/maintenance/management/podcastcmd.py @@ -5,6 +5,7 @@ from django.core.management.base import BaseCommand from mygpo.core.models import Podcast, PodcastGroup from mygpo.directory.toplist import PodcastToplist +from mygpo.couchdb import get_main_database class PodcastCommand(BaseCommand): @@ -51,7 +52,7 @@ class PodcastCommand(BaseCommand): def get_podcast_with_new_episodes(self): - db = Podcast.get_db() + db = get_main_database() res = db.view('episodes/need_update', group_level = 1, reduce = True, diff --git a/mygpo/users/models.py b/mygpo/users/models.py index 1951f13f..52dc1450 100644 --- a/mygpo/users/models.py +++ b/mygpo/users/models.py @@ -15,6 +15,7 @@ from django_couchdb_utils.registration.models import User as BaseUser from mygpo.core.proxy import proxy_object, DocumentABCMeta from mygpo.core.models import Podcast, Episode from mygpo.utils import linearize, get_to_dict, iterate_together +from mygpo.couchdb import get_main_database from mygpo.decorators import repeat_on_conflict from mygpo.users.ratings import RatingMixin from mygpo.users.sync import SyncedDevicesMixin @@ -136,7 +137,7 @@ class EpisodeAction(DocumentSchema): startkey = [user_id, podcast_id, device_id, since_str] endkey = [user_id, podcast_id, device_id, until_str] - db = EpisodeUserState.get_db() + db = get_main_database() res = db.view(view, startkey = startkey, endkey = endkey @@ -191,8 +192,7 @@ class Chapter(Document): @classmethod def for_episode(cls, episode_id): - db = cls.get_db() - r = db.view('chapters/by_episode', + r = cls.view('chapters/by_episode', startkey = [episode_id, None], endkey = [episode_id, {}], wrap_doc = False, @@ -897,7 +897,7 @@ class History(object): def __init__(self, user, device): self.user = user self.device = device - self._db = EpisodeUserState.get_db() + self._db = get_main_database() if device: self._view = 'history/by_device' diff --git a/mygpo/utils.py b/mygpo/utils.py index cf235257..c0a13dba 100644 --- a/mygpo/utils.py +++ b/mygpo/utils.py @@ -25,6 +25,8 @@ import hashlib from django.core.cache import cache +from mygpo.couchdb import get_main_database + def daterange(from_date, to_date=None, leap=timedelta(days=1)): """ @@ -264,7 +266,7 @@ def multi_request_view(cls, view, wrap=True, auto_advance=True, per_page = kwargs.get('limit', 1000) kwargs['limit'] = per_page + 1 - db = cls.get_db() + db = get_main_database() wrapper = kwargs.pop('wrapper', cls.wrap) cont = True diff --git a/mygpo/web/heatmap.py b/mygpo/web/heatmap.py index 912fd7f8..4284d303 100644 --- a/mygpo/web/heatmap.py +++ b/mygpo/web/heatmap.py @@ -17,7 +17,7 @@ from functools import wraps -from mygpo.users.models import EpisodeUserState +from mygpo.couchdb import get_main_database class EpisodeHeatmap(object): @@ -50,7 +50,7 @@ class EpisodeHeatmap(object): def _query(self): """ Queries the database and stores the heatmap and its borders """ - db = EpisodeUserState.get_db() + db = get_main_database() group_level = len(filter(None, [self.podcast_id, self.episode_id, self.user_id])) -- 2.11.4.GIT