From ebd42c452fca541b30db1e571c7e04108536445c Mon Sep 17 00:00:00 2001 From: =?utf8?q?Stefan=20K=C3=B6gl?= Date: Fri, 21 Sep 2012 18:33:31 +0200 Subject: [PATCH] move all directory db queries into separate module --- mygpo/api/advanced/directory.py | 4 +- mygpo/db/couchdb/directory.py | 111 +++++++++++++++++++++ .../commands/category-merge-spellings.py | 5 +- .../management/commands/update-categories.py | 5 +- mygpo/directory/models.py | 20 ---- mygpo/directory/tags.py | 84 +--------------- mygpo/directory/toplist.py | 23 +---- mygpo/directory/views.py | 4 +- .../maintenance/management/commands/dump-sample.py | 6 +- mygpo/web/views/__init__.py | 6 +- mygpo/web/views/podcast.py | 6 +- 11 files changed, 134 insertions(+), 140 deletions(-) create mode 100644 mygpo/db/couchdb/directory.py diff --git a/mygpo/api/advanced/directory.py b/mygpo/api/advanced/directory.py index 17f4e1b8..b855d66f 100644 --- a/mygpo/api/advanced/directory.py +++ b/mygpo/api/advanced/directory.py @@ -28,9 +28,9 @@ from mygpo.directory.tags import Topics from mygpo.web.utils import get_episode_link_target, get_podcast_link_target from mygpo.api.httpresponse import JsonResponse from mygpo.api.sanitizing import sanitize_url -from mygpo.directory.models import Category from mygpo.db.couchdb.episode import episode_for_podcast_url from mygpo.db.couchdb.podcast import podcast_by_id, podcast_for_url +from mygpo.db.couchdb.directory import category_for_tag @csrf_exempt @@ -46,7 +46,7 @@ def top_tags(request, count): @cache_page(60 * 60 * 24) def tag_podcasts(request, tag, count): count = parse_range(count, 1, 100, 100) - category = Category.for_tag(tag) + category = category_for_tag(tag) if not category: return JsonResponse([]) diff --git a/mygpo/db/couchdb/directory.py b/mygpo/db/couchdb/directory.py new file mode 100644 index 00000000..a6547f96 --- /dev/null +++ b/mygpo/db/couchdb/directory.py @@ -0,0 +1,111 @@ +from mygpo.directory.models import Category +from mygpo.couch import get_main_database +from mygpo.cache import cache_result + + +@cache_result(timeout=60*60) +def category_for_tag(tag): + r = Category.view('categories/by_tags', + key = tag, + include_docs = True, + stale = 'update_after', + ) + return r.first() if r else None + + +@cache_result(timeout=60*60) +def top_categories(count, wrap=True): + if wrap: + src = Category + else: + src = get_main_database() + + r = src.view('categories/by_weight', + descending = True, + limit = count, + include_docs = True, + stale = 'update_after', + ) + return list(r) + + + +def tags_for_podcast(podcast): + """ all tags for the podcast, in decreasing order of importance """ + + res = Podcast.view('tags/by_podcast', + startkey = [podcast.get_id(), None], + endkey = [podcast.get_id(), {}], + reduce = True, + group = True, + group_level = 2, + stale = 'update_after', + ) + + tags = Counter(dict((x['key'][1], x['value']) for x in res)) + + res = Podcast.view('usertags/by_podcast', + startkey = [podcast.get_id(), None], + endkey = [podcast.get_id(), {}], + reduce = True, + group = True, + group_level = 2, + ) + + tags.update(Counter(dict( (x['key'][1], x['value']) for x in res))) + + get_tag = itemgetter(0) + return map(get_tag, tags.most_common()) + + +def tags_for_user(user, podcast_id=None): + """ mapping of all podcasts tagged by the user with a list of tags """ + + res = Podcast.view('tags/by_user', + startkey = [user._id, podcast_id], + endkey = [user._id, podcast_id or {}] + ) + + tags = defaultdict(list) + for r in res: + tags[r['key'][1]].append(r['value']) + return tags + + +def all_tags(): + """ Returns all tags + + Some tags might be returned twice """ + res = multi_request_view(Podcast, 'podcasts/by_tag', + wrap = False, + reduce = True, + group = True, + group_level = 1 + ) + + for r in res: + yield r['key'][0] + + res = multi_request_view(Podcast, 'usertags/podcasts', + wrap = False, + reduce = True, + group = True, + group_level = 1 + ) + + for r in res: + yield r['key'][0] + + +@cache_result(timeout=60*60) +def toplist(res_cls, view, key, limit, **view_args): + r = res_cls.view(view, + startkey = key + [{}], + endkey = key + [None], + include_docs = True, + descending = True, + limit = limit, + stale = 'update_after', + **view_args + ) + return list(r) diff --git a/mygpo/directory/management/commands/category-merge-spellings.py b/mygpo/directory/management/commands/category-merge-spellings.py index 035a3689..8d5a1e37 100644 --- a/mygpo/directory/management/commands/category-merge-spellings.py +++ b/mygpo/directory/management/commands/category-merge-spellings.py @@ -3,6 +3,7 @@ from datetime import datetime from django.core.management.base import BaseCommand from mygpo.directory.models import Category +from mygpo.db.couchdb.directory import category_for_tag class Command(BaseCommand): @@ -23,7 +24,7 @@ Usage: spellings = args[1:] print "Adding new spellings for %s ..." % cat_name - category = Category.for_tag(cat_name) + category = category_for_tag(cat_name) if not category: print " creating new category %s" % cat_name @@ -31,7 +32,7 @@ Usage: category.label = cat_name for spelling in spellings: - new_cat = Category.for_tag(spelling) + new_cat = category_for_tag(spelling) if spelling == cat_name or (spelling in category.spellings): print " skipped %s: already in category" % spelling diff --git a/mygpo/directory/management/commands/update-categories.py b/mygpo/directory/management/commands/update-categories.py index 9fc7a8ec..bfc4f956 100755 --- a/mygpo/directory/management/commands/update-categories.py +++ b/mygpo/directory/management/commands/update-categories.py @@ -8,6 +8,7 @@ from mygpo.directory.models import Category, CategoryEntry from mygpo.directory.tags import Tag from mygpo import utils from mygpo.db.couchdb.podcast import podcasts_by_id +from mygpo.db.couchdb.directory import category_for_tag, all_tags class Command(BaseCommand): @@ -19,7 +20,7 @@ class Command(BaseCommand): excluded_tags = settings.DIRECTORY_EXCLUDED_TAGS - tags = args or Tag.all() + tags = args or all_tags() for n, tag in enumerate(tags): @@ -40,7 +41,7 @@ class Command(BaseCommand): e.weight = float(weight * podcast.subscriber_count()) podcasts.append(e) - category = Category.for_tag(label) + category = category_for_tag(label) if not category: if not label or label in excluded_tags: diff --git a/mygpo/directory/models.py b/mygpo/directory/models.py index 27b6179a..c167da22 100644 --- a/mygpo/directory/models.py +++ b/mygpo/directory/models.py @@ -29,26 +29,6 @@ class Category(Document): spellings = StringListProperty() podcasts = SchemaListProperty(CategoryEntry) - @classmethod - @cache_result(timeout=60*60) - def for_tag(cls, tag): - r = cls.view('categories/by_tags', - key = tag, - include_docs = True, - stale = 'update_after', - ) - return r.first() if r else None - - @classmethod - @cache_result(timeout=60*60) - def top_categories(cls, count): - return cls.view('categories/by_weight', - descending = True, - limit = count, - include_docs = True, - stale = 'update_after', - ) - def merge_podcasts(self, podcasts): """ diff --git a/mygpo/directory/tags.py b/mygpo/directory/tags.py index 937394dc..5422a3d6 100644 --- a/mygpo/directory/tags.py +++ b/mygpo/directory/tags.py @@ -9,6 +9,7 @@ from mygpo.counter import Counter from mygpo.core.proxy import proxy_object from mygpo.directory.models import Category from mygpo.db.couchdb.podcast import podcasts_for_tag +from mygpo.db.couchdb.directory import top_categories class Tag(object): @@ -17,79 +18,6 @@ class Tag(object): self.tag = tag - @classmethod - def for_podcast(cls, podcast): - """ all tags for the podcast, in decreasing order of importance """ - - res = Podcast.view('tags/by_podcast', - startkey = [podcast.get_id(), None], - endkey = [podcast.get_id(), {}], - reduce = True, - group = True, - group_level = 2, - stale = 'update_after', - ) - - tags = Counter(dict((x['key'][1], x['value']) for x in res)) - - res = Podcast.view('usertags/by_podcast', - startkey = [podcast.get_id(), None], - endkey = [podcast.get_id(), {}], - reduce = True, - group = True, - group_level = 2, - ) - - tags.update(Counter(dict( (x['key'][1], x['value']) for x in res))) - - get_tag = itemgetter(0) - return map(get_tag, tags.most_common()) - - - - @classmethod - def for_user(cls, user, podcast_id=None): - """ mapping of all podcasts tagged by the user with a list of tags """ - - res = Podcast.view('tags/by_user', - startkey = [user._id, podcast_id], - endkey = [user._id, podcast_id or {}] - ) - - tags = defaultdict(list) - for r in res: - tags[r['key'][1]].append(r['value']) - return tags - - - @classmethod - def all(cls): - """ Returns all tags - - Some tags might be returned twice """ - res = multi_request_view(Podcast, 'podcasts/by_tag', - wrap = False, - reduce = True, - group = True, - group_level = 1 - ) - - for r in res: - yield r['key'][0] - - res = multi_request_view(Podcast, 'usertags/podcasts', - wrap = False, - reduce = True, - group = True, - group_level = 1 - ) - - for r in res: - yield r['key'][0] - - - - def get_podcasts(self): """ Returns the podcasts with the current tag. @@ -117,15 +45,7 @@ class Topics(object): def _query(self): - db = get_main_database() - res = db.view('categories/by_weight', - descending = True, - limit = self.total, - stale = 'update_after', - include_docs = True, - ) - - self._entries = list(res) + self._entries = top_categories(self.total, wrap=False) @property diff --git a/mygpo/directory/toplist.py b/mygpo/directory/toplist.py index 25d9a9d7..11e10b4f 100644 --- a/mygpo/directory/toplist.py +++ b/mygpo/directory/toplist.py @@ -7,6 +7,7 @@ from django.core.cache import cache from mygpo.core.models import Episode, Podcast, PodcastGroup from mygpo.data.mimetype import get_type, CONTENT_TYPES from mygpo.utils import daterange +from mygpo.db.couchdb.directory import toplist CACHE_SECONDS = 60*60 @@ -36,27 +37,7 @@ class Toplist(object): def _cache_or_query(self, limit, key): - cache_str = '{view}-{limit}-{key}'.format( - view=self.view, - limit=limit, - key='-'.join(key) - ) - - res = cache.get(cache_str) - if not res: - r = self.cls.view(self.view, - startkey = key + [{}], - endkey = key + [None], - include_docs = True, - descending = True, - limit = limit, - stale = 'update_after', - **self.view_args - ) - res = list(r) - cache.set(cache_str, res, CACHE_SECONDS) - - return res + return toplist(self.cls, self.view, key, limit, self.view_args) def _sort(self, results): diff --git a/mygpo/directory/views.py b/mygpo/directory/views.py index 8ac83a7b..6ac3bc31 100644 --- a/mygpo/directory/views.py +++ b/mygpo/directory/views.py @@ -12,7 +12,6 @@ from django.views.generic.base import View from mygpo.core.models import Podcast from mygpo.core.proxy import proxy_object -from mygpo.directory.models import Category from mygpo.directory.toplist import PodcastToplist, EpisodeToplist, \ TrendingPodcasts from mygpo.directory.search import search_podcasts @@ -22,6 +21,7 @@ from mygpo.share.models import PodcastList from mygpo.users.models import User from mygpo.db.couchdb.podcast import get_podcast_languages, podcasts_by_id, \ random_podcasts, podcasts_to_dict +from mygpo.db.couchdb.directory import category_for_tag @vary_on_cookie @@ -87,7 +87,7 @@ class Directory(View): @cache_control(private=True) @vary_on_cookie def category(request, category, page_size=20): - category = Category.for_tag(category) + category = category_for_tag(category) if not category: return HttpResponseNotFound() diff --git a/mygpo/maintenance/management/commands/dump-sample.py b/mygpo/maintenance/management/commands/dump-sample.py index bb2908eb..9c380a38 100755 --- a/mygpo/maintenance/management/commands/dump-sample.py +++ b/mygpo/maintenance/management/commands/dump-sample.py @@ -11,7 +11,6 @@ from mygpo.core.models import Podcast from mygpo.couch import get_main_database from mygpo.users.models import PodcastUserState, EpisodeUserState, \ Suggestions, User -from mygpo.directory.models import Category from mygpo.utils import progress from mygpo.json import json from mygpo.db.couchdb.episode import episodes_for_podcast @@ -19,6 +18,7 @@ from mygpo.db.couchdb.podcast import podcast_by_id from mygpo.db.couchdb.podcast_state import podcast_states_for_user from mygpo.db.couchdb.podcast_state import episode_state_for_user_episode from mygpo.db.couchdb.user import suggestions_for_user +from mygpo.db.couchdb.directory import category_for_tag class Command(BaseCommand): @@ -58,7 +58,7 @@ class Command(BaseCommand): # Categories for tag in p_state.tags: - c = Category.for_tag(tag) + c = category_for_tag(tag) if c: docs.add(c._id) # Podcast @@ -68,7 +68,7 @@ class Command(BaseCommand): # Categories for s in podcast.tags: for tag in podcast.tags[s]: - c = Category.for_tag(tag) + c = category_for_tag(tag) if c: docs.add(c._id) # Episodes diff --git a/mygpo/web/views/__init__.py b/mygpo/web/views/__init__.py index d68de6e4..6682c25b 100644 --- a/mygpo/web/views/__init__.py +++ b/mygpo/web/views/__init__.py @@ -37,7 +37,6 @@ from mygpo.decorators import repeat_on_conflict from mygpo.core import models from mygpo.core.models import Podcast from mygpo.core.podcasts import PodcastSet -from mygpo.directory.tags import Tag from mygpo.directory.toplist import PodcastToplist from mygpo.users.models import Suggestions, History, HistoryEntry, DeviceDoesNotExist from mygpo.users.models import PodcastUserState, User @@ -48,6 +47,7 @@ from mygpo.db.couchdb.episode import favorite_episodes_for_user from mygpo.db.couchdb.podcast import podcast_by_id, \ podcast_for_oldid, random_podcasts from mygpo.db.couchdb.user import suggestions_for_user +from mygpo.db.couchdb.directory import tags_for_user @vary_on_cookie @@ -102,7 +102,7 @@ def dashboard(request, episode_count=10): if not request.user.get_token('userpage_token'): checklist.append('userpage') - if Tag.for_user(request.user): + if tags_for_user(request.user): checklist.append('tags') if PodcastList.for_user(request.user._id): @@ -227,7 +227,7 @@ def mytags(request): tags_podcast = {} tags_tag = defaultdict(list) - for podcast_id, taglist in Tag.for_user(request.user).items(): + for podcast_id, taglist in tags_for_user(request.user).items(): podcast = podcast_by_id(podcast_id) tags_podcast[podcast] = taglist diff --git a/mygpo/web/views/podcast.py b/mygpo/web/views/podcast.py index b3ba1022..5b4880ce 100644 --- a/mygpo/web/views/podcast.py +++ b/mygpo/web/views/podcast.py @@ -17,7 +17,6 @@ from mygpo.core.proxy import proxy_object from mygpo.api.sanitizing import sanitize_url from mygpo.users.models import HistoryEntry, DeviceDoesNotExist from mygpo.web.forms import PrivacyForm, SyncForm -from mygpo.directory.tags import Tag from mygpo.decorators import allowed_methods, repeat_on_conflict from mygpo.utils import daterange from mygpo.web.utils import get_podcast_link_target @@ -27,6 +26,7 @@ from mygpo.db.couchdb.podcast import podcast_for_slug, podcast_for_slug_id, \ podcast_for_oldid, podcast_for_url from mygpo.db.couchdb.podcast_state import podcast_state_for_user_podcast from mygpo.db.couchdb.episode_state import get_podcasts_episode_states +from mygpo.db.couchdb.directory import tags_for_user, tags_for_podcast MAX_TAGS_ON_PAGE=50 @@ -123,12 +123,12 @@ def show(request, podcast): def get_tags(podcast, user): tags = {} - for t in Tag.for_podcast(podcast): + for t in tags_for_podcast(podcast): tag_str = t.lower() tags[tag_str] = False if not user.is_anonymous(): - users_tags = Tag.for_user(user, podcast.get_id()) + users_tags = tags_for_user(user, podcast.get_id()) for t in users_tags.get(podcast.get_id(), []): tag_str = t.lower() tags[tag_str] = True -- 2.11.4.GIT