From 55630534e5df4402ccf147dcbb7071161b2301b2 Mon Sep 17 00:00:00 2001 From: =?utf8?q?Stefan=20K=C3=B6gl?= Date: Fri, 21 Sep 2012 19:01:29 +0200 Subject: [PATCH] move more queries into db module --- mygpo/admin/clients.py | 9 ++------ mygpo/db/couchdb/episode.py | 16 +++++++++++++ mygpo/db/couchdb/episode_state.py | 16 +++++++++++++ mygpo/db/couchdb/user.py | 27 ++++++++++++++++++++++ .../management/commands/cleanup-unused-users.py | 14 +++-------- .../management/commands/merge-episode-states.py | 25 ++++---------------- mygpo/users/models.py | 13 ----------- mygpo/web/views/episode.py | 4 ++-- 8 files changed, 71 insertions(+), 53 deletions(-) diff --git a/mygpo/admin/clients.py b/mygpo/admin/clients.py index 867d7583..455b9a30 100644 --- a/mygpo/admin/clients.py +++ b/mygpo/admin/clients.py @@ -3,6 +3,7 @@ from collections import namedtuple from mygpo.users.models import User from mygpo.counter import Counter +from mygpo.db.couchdb.user import user_agent_stats Client = namedtuple('Client', 'client client_version lib lib_version os os_version') @@ -17,13 +18,7 @@ class UserAgentStats(object): def get_entries(self): if self._useragents is None: - res = User.view('clients/by_ua_string', - wrap_doc = False, - group_level = 1, - stale = 'update_after', - ) - - self._useragents = Counter(dict((r['key'], r['value']) for r in res)) + self._useragents = user_agent_stats() return self._useragents diff --git a/mygpo/db/couchdb/episode.py b/mygpo/db/couchdb/episode.py index 042bd257..cef5175f 100644 --- a/mygpo/db/couchdb/episode.py +++ b/mygpo/db/couchdb/episode.py @@ -1,6 +1,7 @@ from datetime import datetime from mygpo.core.models import Podcast, Episode, MergedIdException +from mygpo.users.models import Chapter from mygpo.cache import cache_result from mygpo.utils import is_couchdb_id from mygpo.db.couchdb.podcast import podcast_for_url, podcast_for_slug_id @@ -202,3 +203,18 @@ def favorite_episodes_for_user(user): ) return list(favorites) + +def chapters_for_episode(episode_id): + db = get_main_database() + r = db.view('chapters/by_episode', + startkey = [episode_id, None], + endkey = [episode_id, {}], + ) + + return map(_wrap_chapter, r) + + +def _wrap_chapter(res): + user = res['key'][1] + chapter = Chapter.wrap(res['value']) + return (user, chapter) diff --git a/mygpo/db/couchdb/episode_state.py b/mygpo/db/couchdb/episode_state.py index 34901904..8b9aaa7f 100644 --- a/mygpo/db/couchdb/episode_state.py +++ b/mygpo/db/couchdb/episode_state.py @@ -230,6 +230,22 @@ def episode_states_count(): return r.total_rows +def get_nth_episode_state(n): + first = EpisodeUserState.view('episode_states/by_user_episode', + skip = n, + include_docs = True, + limit = 1, + ) + return first.one() if first else None + + +def get_duplicate_episode_states(user, episode): + states = EpisodeUserState.view('episode_states/by_user_episode', + key = [user, episode], + include_docs = True, + ) + return = list(states) + def _wrap_listener_count(res): date = parser.parse(res['key'][1]).date() diff --git a/mygpo/db/couchdb/user.py b/mygpo/db/couchdb/user.py index 93b6da45..4fe351bc 100644 --- a/mygpo/db/couchdb/user.py +++ b/mygpo/db/couchdb/user.py @@ -1,4 +1,5 @@ from mygpo.cache import cache_result +from mygpo.counter import Counter @cache_result(timeout=60) @@ -103,3 +104,29 @@ def suggestions_for_user(user): s = Suggestions() s.user = user._id return s + + +@cache_result(timeout=60*60) +def user_agent_stats(): + res = User.view('clients/by_ua_string', + wrap_doc = False, + group_level = 1, + stale = 'update_after', + ) + + return Counter(dict((r['key'], r['value']) for r in res)) + + +def deleted_users(): + users = User.view('users/deleted', + include_docs = True, + reduce = False, + ) + return list(users) + + +def deleted_user_count(): + total = User.view('users/deleted', + reduce = True, + ) + return list(total)[0]['value'] if total else 0 diff --git a/mygpo/maintenance/management/commands/cleanup-unused-users.py b/mygpo/maintenance/management/commands/cleanup-unused-users.py index 3c5cc491..8f59c855 100644 --- a/mygpo/maintenance/management/commands/cleanup-unused-users.py +++ b/mygpo/maintenance/management/commands/cleanup-unused-users.py @@ -1,24 +1,16 @@ from django.core.management.base import BaseCommand from mygpo.users.models import User - from mygpo.utils import progress +from mygpo.db.couchdb.user import deleted_users, deleted_user_count class Command(BaseCommand): def handle(self, *args, **options): - users = User.view('users/deleted', - include_docs = True, - reduce = False, - ) - - total = User.view('users/deleted', - reduce = True, - ) - - total = list(total)[0]['value'] if total else 0 + users = deleted_users() + total = deleted_user_count() for n, user in enumerate(users): diff --git a/mygpo/maintenance/management/commands/merge-episode-states.py b/mygpo/maintenance/management/commands/merge-episode-states.py index 6cbf7400..66d066d2 100755 --- a/mygpo/maintenance/management/commands/merge-episode-states.py +++ b/mygpo/maintenance/management/commands/merge-episode-states.py @@ -9,6 +9,8 @@ from mygpo.users.models import EpisodeUserState from mygpo.counter import Counter from mygpo.maintenance.merge import merge_episode_states from mygpo.couch import bulk_save_retry, get_main_database +from mygpo.db.couchdb.episode_state import episode_states_count, \ + get_nth_episode_state, get_duplicate_episode_states class Command(BaseCommand): @@ -22,9 +24,7 @@ class Command(BaseCommand): def handle(self, *args, **options): skip = options.get('skip') - total = EpisodeUserState.view('episode_states/by_user_episode', - limit=0, - ).total_rows + total = episode_states_count() db = get_main_database() actions = Counter() @@ -33,23 +33,8 @@ class Command(BaseCommand): for n in count(skip): - first = EpisodeUserState.view('episode_states/by_user_episode', - skip = n, - include_docs = True, - limit = 1, - ) - first = list(first) - if not first: - break - - first = first[0] - - - states = EpisodeUserState.view('episode_states/by_user_episode', - key = [first.user, first.episode], - include_docs = True, - ) - states = list(states) + first = get_nth_episode_state(n) + states = get_duplicate_episode_states(first.user, first.episode) l1 = len(states) # we don't want to delete this one diff --git a/mygpo/users/models.py b/mygpo/users/models.py index 18a38111..bbaee58f 100644 --- a/mygpo/users/models.py +++ b/mygpo/users/models.py @@ -142,19 +142,6 @@ class Chapter(Document): label = StringProperty() advertisement = BooleanProperty() - @classmethod - def for_episode(cls, episode_id): - db = get_main_database() - r = db.view('chapters/by_episode', - startkey = [episode_id, None], - endkey = [episode_id, {}], - ) - - for res in r: - user = res['key'][1] - chapter = Chapter.wrap(res['value']) - yield (user, chapter) - def __repr__(self): return '<%s %s (%d-%d)>' % (self.__class__.__name__, self.label, diff --git a/mygpo/web/views/episode.py b/mygpo/web/views/episode.py index c64a1bbc..a09b4b2d 100644 --- a/mygpo/web/views/episode.py +++ b/mygpo/web/views/episode.py @@ -42,7 +42,7 @@ from mygpo.utils import parse_time from mygpo.web.heatmap import EpisodeHeatmap from mygpo.web.utils import get_episode_link_target, fetch_episode_data from mygpo.db.couchdb.episode import episode_for_slug_id, episode_for_oldid, \ - favorite_episodes_for_user + favorite_episodes_for_user, chapters_for_episode from mygpo.db.couchdb.podcast import podcast_by_id, podcast_for_url, \ podcasts_to_dict from mygpo.db.couchdb.episode_state import episode_state_for_user_episode @@ -85,7 +85,7 @@ def episode(request, episode): chapters = [] - for user, chapter in Chapter.for_episode(episode._id): + for user, chapter in chapters_for_episode(episode._id): chapter.is_own = request.user.is_authenticated() and \ user == request.user._id chapters.append(chapter) -- 2.11.4.GIT