From 146bb53f7765dcdb51f80245def6d2924bb6d211 Mon Sep 17 00:00:00 2001 From: =?utf8?q?Stefan=20K=C3=B6gl?= Date: Sun, 28 Sep 2014 15:37:03 +0200 Subject: [PATCH] [Categories] use categories from Django ORM --- mygpo/api/advanced/directory.py | 18 ++-- mygpo/categories/admin.py | 6 +- mygpo/categories/migrations/0001_initial.py | 49 ++++++++- mygpo/categories/models.py | 28 +++-- mygpo/db/couchdb/directory.py | 73 ------------- .../commands/category-merge-spellings.py | 114 ++++++++++----------- mygpo/directory/tags.py | 80 ++++++--------- mygpo/directory/templates/category.html | 6 +- mygpo/directory/templates/directory.html | 25 ++--- mygpo/directory/views.py | 32 +++--- mygpo/maintenance/migrate.py | 5 +- mygpo/podcasts/models.py | 5 + mygpo/web/templatetags/podcasts.py | 2 +- 13 files changed, 209 insertions(+), 234 deletions(-) delete mode 100644 mygpo/db/couchdb/directory.py rewrite mygpo/directory/management/commands/category-merge-spellings.py (67%) diff --git a/mygpo/api/advanced/directory.py b/mygpo/api/advanced/directory.py index 94536f97..ef202f23 100644 --- a/mygpo/api/advanced/directory.py +++ b/mygpo/api/advanced/directory.py @@ -29,8 +29,8 @@ from mygpo.web.utils import get_episode_link_target, get_podcast_link_target from mygpo.web.logo import get_logo_url from mygpo.subscriptions.models import SubscribedPodcast from mygpo.decorators import cors_origin +from mygpo.categories.models import Category from mygpo.api.httpresponse import JsonResponse -from mygpo.db.couchdb.directory import category_for_tag @csrf_exempt @@ -48,13 +48,17 @@ def top_tags(request, count): @cors_origin() def tag_podcasts(request, tag, count): count = parse_range(count, 1, 100, 100) - category = category_for_tag(tag) - if not category: + try: + category = Category.objects.get(tags__tag=tag) + + except Category.DoesNotExist: return JsonResponse([]) domain = RequestSite(request).domain - query = category.get_podcasts(0, count) - resp = map(lambda p: podcast_data(p, domain), query) + entries = category.entries.all()\ + .prefetch_related('podcast', 'podcast__slugs', + 'podcast__urls')[:count] + resp = [podcast_data(entry.podcast, domain) for entry in entries] return JsonResponse(resp) @@ -150,6 +154,6 @@ def episode_data(episode, domain, podcast=None): def category_data(category): return dict( - tag = category.label, - usage = category.get_weight() + tag = category.title, + usage = category.num_entries, ) diff --git a/mygpo/categories/admin.py b/mygpo/categories/admin.py index 7d7479a9..1c3e0d04 100644 --- a/mygpo/categories/admin.py +++ b/mygpo/categories/admin.py @@ -18,16 +18,12 @@ class CategoryAdmin(admin.ModelAdmin): model = Category - list_display = ('title', 'num_podcasts', 'tag_list') + list_display = ('title', 'num_entries', 'tag_list') inlines = [ CategoryEntryInline, CategoryTagInline, ] - def num_podcasts(self, category): - """ number of entries in a category """ - return category.entries.count() - def tag_list(self, category): return ', '.join(t.tag for t in category.tags.all()[:10]) diff --git a/mygpo/categories/migrations/0001_initial.py b/mygpo/categories/migrations/0001_initial.py index 1a5b99ab..8c414ae7 100644 --- a/mygpo/categories/migrations/0001_initial.py +++ b/mygpo/categories/migrations/0001_initial.py @@ -2,10 +2,13 @@ from __future__ import unicode_literals from django.db import models, migrations +import datetime class Migration(migrations.Migration): + replaces = [(b'categories', '0001_initial'), (b'categories', '0002_auto_20140927_1501'), (b'categories', '0003_category_num_entries'), (b'categories', '0004_auto_20140927_1540')] + dependencies = [ ('podcasts', '0029_episode_index_toplist'), ] @@ -39,7 +42,7 @@ class Migration(migrations.Migration): fields=[ ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), ('tag', models.SlugField(unique=True)), - ('category', models.ForeignKey(to='categories.Category')), + ('category', models.ForeignKey(related_name=b'tags', to='categories.Category')), ], options={ }, @@ -49,4 +52,48 @@ class Migration(migrations.Migration): name='categoryentry', unique_together=set([('category', 'podcast')]), ), + migrations.AlterModelOptions( + name='category', + options={'verbose_name': 'Category', 'verbose_name_plural': 'Categories'}, + ), + migrations.AddField( + model_name='category', + name='created', + field=models.DateTimeField(default=datetime.datetime(2014, 9, 28, 13, 26, 28, 914038), auto_now_add=True), + preserve_default=False, + ), + migrations.AddField( + model_name='category', + name='modified', + field=models.DateTimeField(default=datetime.datetime(2014, 9, 28, 13, 26, 28, 914095), auto_now=True), + preserve_default=False, + ), + migrations.AlterField( + model_name='categoryentry', + name='category', + field=models.ForeignKey(related_name=b'entries', to='categories.Category'), + ), + migrations.AlterField( + model_name='categoryentry', + name='modified', + field=models.DateTimeField(auto_now=True), + ), + migrations.AlterIndexTogether( + name='category', + index_together=set([('modified',)]), + ), + migrations.AlterIndexTogether( + name='categoryentry', + index_together=set([('category', 'modified')]), + ), + migrations.AddField( + model_name='category', + name='num_entries', + field=models.IntegerField(default=0), + preserve_default=False, + ), + migrations.AlterIndexTogether( + name='category', + index_together=set([('modified', 'num_entries')]), + ), ] diff --git a/mygpo/categories/models.py b/mygpo/categories/models.py index cd215294..7e373e4c 100644 --- a/mygpo/categories/models.py +++ b/mygpo/categories/models.py @@ -1,20 +1,35 @@ from django.db import models +from mygpo.core.models import UpdateInfoModel from mygpo.podcasts.models import Podcast -class Category(models.Model): +class Category(UpdateInfoModel): """ A category of podcasts """ title = models.CharField(max_length=1000, null=False, blank=False, unique=True) + num_entries = models.IntegerField() + class Meta: verbose_name = 'Category' verbose_name_plural = 'Categories' + index_together = [ + ('modified', 'num_entries'), + ] + + def save(self, *args, **kwargs): + self.num_entries = self.entries.count() + super(Category, self).save(*args, **kwargs) + + @property + def podcasts(self): + return self.entries.prefetch_related('podcast', 'podcast__slugs') + -class CategoryEntry(models.Model): +class CategoryEntry(UpdateInfoModel,): """ A podcast in a category """ category = models.ForeignKey(Category, related_name='entries', @@ -23,16 +38,15 @@ class CategoryEntry(models.Model): podcast = models.ForeignKey(Podcast, on_delete=models.CASCADE) - # this could be used from UpdateInfoModel, except for the index on modified - created = models.DateTimeField(auto_now_add=True) - modified = models.DateTimeField(auto_now=True, db_index=True) - - class Meta: unique_together = [ ('category', 'podcast'), ] + index_together = [ + ('category', 'modified'), + ] + class CategoryTag(models.Model): diff --git a/mygpo/db/couchdb/directory.py b/mygpo/db/couchdb/directory.py deleted file mode 100644 index 0b3de6d2..00000000 --- a/mygpo/db/couchdb/directory.py +++ /dev/null @@ -1,73 +0,0 @@ -from mygpo.directory.models import Category -from mygpo.db.couchdb import get_categories_database, get_single_result -from mygpo.cache import cache_result -from mygpo.db import QueryParameterMissing - - -def category_for_tag_uncached(tag): - - if not tag: - raise QueryParameterMissing('tag') - - db = get_categories_database() - cat = get_single_result(db, 'categories/by_tags', - key = tag, - include_docs = True, - stale = 'update_after', - schema = Category - ) - - return cat - - -category_for_tag = cache_result(timeout=60*60)(category_for_tag_uncached) - - -@cache_result(timeout=60*60) -def top_categories(offset, count, with_podcasts=False): - - if offset is None: - raise QueryParameterMissing('offset') - - if not count: - raise QueryParameterMissing('count') - - db = get_categories_database() - - if with_podcasts: - r = db.view('categories/by_update', - descending = True, - skip = offset, - limit = count, - include_docs = True, - stale = 'update_after', - schema = Category, - ) - - else: - r = db.view('categories/by_update', - descending = True, - skip = offset, - limit = count, - stale = 'update_after', - wrapper = _category_wrapper, - ) - - categories = list(r) - - for cat in categories: - cat.set_db(db) - - return categories - - -def _category_wrapper(r): - c = Category() - c.label = r['value'][0] - c._weight = r['value'][1] - return c - - -def save_category(category): - db = get_categories_database() - db.save_doc(category) diff --git a/mygpo/directory/management/commands/category-merge-spellings.py b/mygpo/directory/management/commands/category-merge-spellings.py dissimilarity index 67% index dc1bdf6f..dcc1a188 100644 --- a/mygpo/directory/management/commands/category-merge-spellings.py +++ b/mygpo/directory/management/commands/category-merge-spellings.py @@ -1,58 +1,56 @@ -from datetime import datetime - -from django.core.management.base import BaseCommand - -from mygpo.directory.models import Category -from mygpo.db.couchdb.directory import category_for_tag, save_category - - -class Command(BaseCommand): - - def handle(self, *args, **options): - - if len(args) < 2: - print """ -Merges multiple categories into one by listing them as alternative spellings - -Usage: - ./manage.py category-merge-spellings [ ...] -""" - return - - start_time = datetime.utcnow() - cat_name = args[0] - spellings = args[1:] - - print "Adding new spellings for %s ..." % cat_name - category = category_for_tag(cat_name) - - if not category: - print " creating new category %s" % cat_name - category = Category() - category.label = cat_name - - for spelling in spellings: - new_cat = category_for_tag(spelling) - - if spelling == cat_name or (spelling in category.spellings): - print " skipped %s: already in category" % spelling - continue - - if not new_cat: - #merged category doesn't yet exist - category.spellings.append(spelling) - - elif new_cat and category._id == new_cat._id: - print " set %s as new label" % cat_name - category.spellings = list(set([x for x in category.spellings + [category.label] if x != cat_name])) - category.label = cat_name - - else: - print " add spelling %s" % spelling - category.spellings = list(set(category.spellings + [new_cat.label] + new_cat.spellings)) - category.merge_podcasts(new_cat.podcasts) - new_cat.delete() - - category.updated = start_time - - save_category(category) +from datetime import datetime + +from django.core.management.base import BaseCommand +from django.utils.text import slugify + +from mygpo.directory.models import Category, CategoryTag + + +class Command(BaseCommand): + + def handle(self, *args, **options): + + if len(args) < 2: + print """ +Merges multiple categories into one by listing them as alternative spellings + +Usage: + ./manage.py category-merge-spellings [ ...] +""" + return + + start_time = datetime.utcnow() + cat_name = args[0] + spellings = args[1:] + + print "Adding new spellings for %s ..." % cat_name + category, created = Category.objects.get_or_create( + tags__tag=slugify(cat_name), + defaults={ + 'title': cat_name, + } + ) + + for spelling in spellings: + + tag, created = CategoryTag.objects.get_or_create( + tag=spelling, + defaults={ + 'category': category, + } + ) + + if created: + # we just created a new tag-assignedment -- nothing else to do + continue + + oldcategory = tag.category + + for entry in oldcategory.entries: + # todo: this might cause a constraint violation if the + # podcast is already a entry of the new category + entry.category = category + entry.save() + + tag.category = category + tag.save() diff --git a/mygpo/directory/tags.py b/mygpo/directory/tags.py index 9400a907..ca074f75 100644 --- a/mygpo/directory/tags.py +++ b/mygpo/directory/tags.py @@ -4,11 +4,11 @@ from datetime import datetime from random import choice from itertools import chain +from django.utils.text import slugify + from mygpo.decorators import query_if_required, repeat_on_conflict from mygpo.core.proxy import proxy_object -from mygpo.directory.models import Category -from mygpo.db.couchdb.directory import top_categories, save_category, \ - category_for_tag_uncached +from mygpo.categories.models import Category, CategoryEntry class Topics(object): @@ -20,36 +20,28 @@ class Topics(object): self._categories = None self._tagcloud = None - def _needs_query(self): return self._categories is None - def _query(self): - self._categories = [] - if self.num_cat > 0: - self._categories = top_categories(0, self.num_cat, True) - - self._tagcloud = [] - if self.total-self.num_cat > 0: - self._tagcloud = top_categories(self.num_cat, self.total-self.num_cat, False) - + categories = list(Category.objects.filter(num_entries__gt=0) + .order_by('-modified')[:self.total]) + self._categories = categories[:self.num_cat] + self._tagcloud = sorted(categories[self.num_cat:], + key=lambda x: x.title.lower()) @property @query_if_required() def tagcloud(self): - self._tagcloud.sort(key = lambda x: x.label.lower()) return self._tagcloud - @query_if_required() - def max_weight(self): - return max([e.get_weight() for e in self.tagcloud] + [0]) + def max_entries(self): + return max([e.num_entries for e in self.tagcloud] + [0]) @query_if_required() - def min_weight(self): - return min([e.get_weight() for e in self.tagcloud]) - + def min_entries(self): + return min([e.num_entries for e in self.tagcloud] + [0]) @property @query_if_required() @@ -57,14 +49,6 @@ class Topics(object): return self._categories - def _prepare_category(self, resp): - category = Category.wrap(resp['doc']) - category = proxy_object(category) - category.podcasts = category.get_podcasts(0, self.podcasts_per_cat) - return category - - - @repeat_on_conflict() def update_category(podcast): all_tags = list(chain.from_iterable(s for s in podcast.tags.values())) @@ -74,23 +58,23 @@ def update_category(podcast): random_tag = choice(all_tags) - category = category_for_tag_uncached(random_tag) - if not category: - category = Category(label=random_tag) - - category.updated = datetime.utcnow() - - category.podcasts = category.podcasts[:999] - - # we don't need to CategoryEntry wrapper anymore - if any(isinstance(x, dict) for x in category.podcasts): - category.podcasts = filter(lambda x: isinstance(x, dict), category.podcasts) - category.podcasts = [e['podcast'] for e in category.podcasts] - - if podcast.get_id() in category.podcasts: - category.podcasts.remove(podcast.get_id()) - - category.podcasts.insert(0, podcast.get_id()) - category.label = category.label.strip() - - save_category(category) + category, created = Category.objects.get_or_create( + tags__tag=slugify(random_tag.strip()), + defaults={ + 'label': random_tag, + } + ) + + if not created: + # update modified timestamp + category.save() + + # add podcast to the category as newest entry + entry, created = CategoryEntry.objects.get_or_create( + category=category, + podcast=podcast, + ) + + if not created: + # update modified timestamp + entry.save() diff --git a/mygpo/directory/templates/category.html b/mygpo/directory/templates/category.html index 0e4ae256..d91b84ed 100644 --- a/mygpo/directory/templates/category.html +++ b/mygpo/directory/templates/category.html @@ -27,10 +27,10 @@ {% for entry in entries %} - {{ entry|podcast_logo}} - {% podcast_group_link entry %} + {{ entry.podcast|podcast_logo}} + {% podcast_group_link entry.podcast %} - {{ entry.description|truncatewords:15 }} + {{ entry.podcast.description|truncatewords:15 }} {% endfor %} diff --git a/mygpo/directory/templates/directory.html b/mygpo/directory/templates/directory.html index 189102f9..22e29a7d 100644 --- a/mygpo/directory/templates/directory.html +++ b/mygpo/directory/templates/directory.html @@ -5,7 +5,6 @@ {% load charts %} {% load math %} {% load utils %} -{% load cache %} {% load menu %} {% block mainmenu %}{{ "/directory/"|main_menu }}{% endblock %} @@ -19,7 +18,6 @@ {% block content %} - {% cache 60 topics %} {% for c in topics.categories %} {% if forloop.counter0|divisibleby:"2" %} @@ -31,14 +29,14 @@ {% url "user" c.username as user-lists-url %}

{% blocktrans with c.title as listtitle and c.username as username %}{{ listtitle }} by {{ username }}{% endblocktrans %}

{% else %} -

{{ c.label }}

+

{{ c.title }}

{% endif %} - {% for podcast in c.get_podcasts %} + {% for entry in c.podcasts|slice:":10" %} - - + + {% endfor %} @@ -47,7 +45,7 @@ {% if c.cls == "PodcastList" %} {% trans "more..." %} {% else %} - {% trans "more..." %} + {% trans "more..." %} {% endif %} @@ -59,25 +57,21 @@ {% endif %} {% endfor %} - {% endcache %}
- {% cache 60 tagcloud %}
- {% for tag in topics.tagcloud %} - - {{ tag.label }} + {% for category in topics.tagcloud %} + + {{ category.title }} {% endfor %}
- {% endcache %} {% endblock %} {% block sidebar %} - {% cache 3600 podcastlist %} {% for podcastlist in podcastlists %} {% if podcastlist and podcastlist.user.username %}
@@ -115,9 +109,7 @@
{% endif %} {% endfor %} - {% endcache %} - {% cache 3600 random_podcast_box %}

{% trans "Random" %}

@@ -138,7 +130,6 @@

- {% endcache %} {% endblock %} diff --git a/mygpo/directory/views.py b/mygpo/directory/views.py index dccdcb31..2edbba90 100644 --- a/mygpo/directory/views.py +++ b/mygpo/directory/views.py @@ -30,10 +30,10 @@ from mygpo.web.utils import process_lang_params, get_language_names, \ get_page_list, get_podcast_link_target, sanitize_language_codes from mygpo.directory.tags import Topics from mygpo.users.settings import FLATTR_TOKEN +from mygpo.categories.models import Category from mygpo.podcastlists.models import PodcastList from mygpo.data.feeddownloader import PodcastUpdater, NoEpisodesException from mygpo.data.tasks import update_podcasts -from mygpo.db.couchdb.directory import category_for_tag class ToplistView(TemplateView): @@ -143,25 +143,31 @@ class Directory(View): @cache_control(private=True) @vary_on_cookie def category(request, category, page_size=20): - category = category_for_tag(category) - if not category: + try: + category = Category.objects.get(tags__tag=category) + except Category.DoesNotExist: return HttpResponseNotFound() - # Make sure page request is an int. If not, deliver first page. - try: - page = int(request.GET.get('page', '1')) - except ValueError: - page = 1 + podcasts = category.entries.all()\ + .prefetch_related('podcast', 'podcast__slugs') - entries = category.get_podcasts( (page-1) * page_size, page*page_size ) - podcasts = filter(None, entries) - num_pages = int(ceil(len(category.podcasts) / page_size)) + paginator = Paginator(podcasts, page_size) - page_list = get_page_list(1, num_pages, page, 15) + page = request.GET.get('page') + try: + podcasts = paginator.page(page) + except PageNotAnInteger: + # If page is not an integer, deliver first page. + podcasts = paginator.page(1) + except EmptyPage: + # If page is out of range (e.g. 9999), deliver last page of results. + podcasts = paginator.page(paginator.num_pages) + + page_list = get_page_list(1, paginator.num_pages, podcasts.number, 15) return render(request, 'category.html', { 'entries': podcasts, - 'category': category.label, + 'category': category.title, 'page_list': page_list, }) diff --git a/mygpo/maintenance/migrate.py b/mygpo/maintenance/migrate.py index 6b9a0c4d..7e58e761 100644 --- a/mygpo/maintenance/migrate.py +++ b/mygpo/maintenance/migrate.py @@ -5,6 +5,7 @@ from datetime import datetime from django.contrib.contenttypes.models import ContentType from django.contrib.auth.models import User +from django.utils.text import slugify from django.db import reset_queries from mygpo.podcasts.models import Tag @@ -143,7 +144,7 @@ def migrate_category(cat): for spelling in cat.spellings + [cat.label]: s, c = CategoryTag.objects.get_or_create( - tag=to_maxlength(CategoryTag, 'tag', spelling.strip()), + tag=slugify(to_maxlength(CategoryTag, 'tag', spelling.strip())), defaults={ 'category': category, } @@ -164,6 +165,8 @@ def migrate_category(cat): entry, c = CategoryEntry.objects.get_or_create(category=category, podcast=podcast) + category.save() + from couchdbkit import Database db = Database('http://127.0.0.1:5984/mygpo_userdata_copy') diff --git a/mygpo/podcasts/models.py b/mygpo/podcasts/models.py index be847a65..47a86e0c 100644 --- a/mygpo/podcasts/models.py +++ b/mygpo/podcasts/models.py @@ -531,6 +531,11 @@ class Podcast(UUIDModel, TitleModel, DescriptionModel, LinkModel, if self.title: return self.title + if not self.url: + logger.warn('Podcast with ID {podcast_id} does not have a URL' + .format(podcast_id=self.id.hex)) + return _('Unknown Podcast') + return _('Unknown Podcast from {domain}'.format( domain=utils.get_domain(self.url))) diff --git a/mygpo/web/templatetags/podcasts.py b/mygpo/web/templatetags/podcasts.py index 38d2b54d..b402172c 100644 --- a/mygpo/web/templatetags/podcasts.py +++ b/mygpo/web/templatetags/podcasts.py @@ -143,7 +143,7 @@ def podcast_group_link(podcast, title=None): def podcast_link(podcast, title=None): """ Returns the link for a single Podcast """ - title = title or getattr(podcast, 'display_title', None) or podcast.title + title = title or podcast.display_title title = strip_tags(title) -- 2.11.4.GIT
{% podcast_group_link podcast %}{% podcast_group_link entry.podcast %}