From e8fe4fde046144236df679bb6519e8dadf06aa93 Mon Sep 17 00:00:00 2001 From: =?utf8?q?Stefan=20K=C3=B6gl?= Date: Sun, 3 Dec 2017 18:47:09 +0100 Subject: [PATCH] Collect results when updating podcasts --- htdocs/media/screen.css | 19 ++++++++ mygpo/data/admin.py | 18 ++++++++ mygpo/data/feeddownloader.py | 47 ++++++++++++++------ mygpo/data/migrations/0002_result_podcast_null.py | 21 +++++++++ .../0003_podcastupdateresult_podcast_url.py | 21 +++++++++ mygpo/data/models.py | 34 ++++++++++++++- mygpo/podcasts/models.py | 4 ++ mygpo/publisher/templates/publisher/podcast.html | 50 +++++++++++++++++++--- mygpo/publisher/views.py | 7 +++ 9 files changed, 200 insertions(+), 21 deletions(-) create mode 100644 mygpo/data/admin.py create mode 100644 mygpo/data/migrations/0002_result_podcast_null.py create mode 100644 mygpo/data/migrations/0003_podcastupdateresult_podcast_url.py diff --git a/htdocs/media/screen.css b/htdocs/media/screen.css index 6a195093..b2d39619 100644 --- a/htdocs/media/screen.css +++ b/htdocs/media/screen.css @@ -1,3 +1,9 @@ +:root { + --color-status-success: 90; + --color-status-error: 0; + --color-status-neutral: 247; +} + /* Landscape phones and down */ @media (min-width: 980px) @@ -628,3 +634,16 @@ div.podcasts div.podcast:hover div.actions button.btn .hosting { text-align: center; } + +.status-success { + background-color: hsla(var(--color-status-success), 100%, 75%, 1); +} + + +.status-error { + background-color: hsla(var(--color-status-error), 100%, 75%, 1); +} + +.status-neutral { + background-color: hsla(var(--color-status-neutral), 16%, 85%, 1); +} diff --git a/mygpo/data/admin.py b/mygpo/data/admin.py new file mode 100644 index 00000000..cc94ec44 --- /dev/null +++ b/mygpo/data/admin.py @@ -0,0 +1,18 @@ +from django.contrib import admin + +from . import models + + +@admin.register(models.PodcastUpdateResult) +class PodcastUpdateResultAdmin(admin.ModelAdmin): + model = models.PodcastUpdateResult + + list_display = ['title', 'start', 'duration', 'successful', + 'episodes_added'] + + readonly_fields = ['id', 'podcast_url', 'podcast', 'start', 'duration', + 'successful', 'error_message', 'podcast_created', + 'episodes_added'] + + def title(self, res): + return res.podcast or res.podcast_url diff --git a/mygpo/data/feeddownloader.py b/mygpo/data/feeddownloader.py index 42b5898a..19a5a448 100755 --- a/mygpo/data/feeddownloader.py +++ b/mygpo/data/feeddownloader.py @@ -28,6 +28,8 @@ from mygpo.pubsub.models import SubscriptionError from mygpo.directory.tags import update_category from mygpo.search import get_index_fields +from . import models + import logging logger = logging.getLogger(__name__) @@ -77,22 +79,31 @@ class PodcastUpdater(object): def update_podcast(self): """ Update the podcast """ - parsed = self.parse_feed() - if not parsed: - return + with models.PodcastUpdateResult(podcast_url=self.podcast_url) as res: - podcast = Podcast.objects.get_or_create_for_url(self.podcast_url) + parsed = self.parse_feed() + if not parsed: + res.podcast_created = False + res.error_message = '"{}" could not be parsed'.format( + self.podcast_url) + return - episode_updater = MultiEpisodeUpdater(podcast) - episode_updater.update_episodes(parsed.get('episodes', [])) + podcast, created = Podcast.objects.get_or_create_for_url( + self.podcast_url) + res.podcast = podcast + res.podcast_created = created - podcast.refresh_from_db() - podcast.episode_count = Episode.objects.filter(podcast=podcast).count() - podcast.save() + res.episodes_added = 0 + episode_updater = MultiEpisodeUpdater(podcast, res) + episode_updater.update_episodes(parsed.get('episodes', [])) + + podcast.refresh_from_db() + podcast.episode_count = episode_updater.count_episodes() + podcast.save() - episode_updater.order_episodes() + episode_updater.order_episodes() - self._update_podcast(podcast, parsed, episode_updater) + self._update_podcast(podcast, parsed, episode_updater) return podcast @@ -123,7 +134,7 @@ class PodcastUpdater(object): # if we fail to parse the URL, we don't even create the # podcast object try: - p = Podcast.objects.get(urls__url=podcast_url) + p = Podcast.objects.get(urls__url=self.podcast_url) # if it exists already, we mark it as outdated self._mark_outdated(p, 'error while fetching feed: {}'.format( str(nee))) @@ -354,8 +365,9 @@ class PodcastUpdater(object): class MultiEpisodeUpdater(object): - def __init__(self, podcast): + def __init__(self, podcast, update_result): self.podcast = podcast + self.update_result = update_result self.updated_episodes = [] self.max_episode_order = None @@ -378,7 +390,11 @@ class MultiEpisodeUpdater(object): logger.info('Updating episode %d / %d', n, len(parsed_episodes)) - episode = Episode.objects.get_or_create_for_url(self.podcast, url) + episode, created = Episode.objects.get_or_create_for_url( + self.podcast, url) + + if created: + self.update_result.episodes_added += 1 updater = EpisodeUpdater(episode, self.podcast) updater.update_episode(parsed) @@ -435,6 +451,9 @@ class MultiEpisodeUpdater(object): return f['urls'][0] return None + def count_episodes(self): + return Episode.objects.filter(podcast=self.podcast).count() + def get_update_interval(self, episodes): """ calculates the avg interval between new episodes """ diff --git a/mygpo/data/migrations/0002_result_podcast_null.py b/mygpo/data/migrations/0002_result_podcast_null.py new file mode 100644 index 00000000..4149dfcf --- /dev/null +++ b/mygpo/data/migrations/0002_result_podcast_null.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.5 on 2017-12-03 17:22 +from __future__ import unicode_literals + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('data', '0001_initial'), + ] + + operations = [ + migrations.AlterField( + model_name='podcastupdateresult', + name='podcast', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, to='podcasts.Podcast'), + ), + ] diff --git a/mygpo/data/migrations/0003_podcastupdateresult_podcast_url.py b/mygpo/data/migrations/0003_podcastupdateresult_podcast_url.py new file mode 100644 index 00000000..89b0d97c --- /dev/null +++ b/mygpo/data/migrations/0003_podcastupdateresult_podcast_url.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.5 on 2017-12-03 17:28 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('data', '0002_result_podcast_null'), + ] + + operations = [ + migrations.AddField( + model_name='podcastupdateresult', + name='podcast_url', + field=models.URLField(default='unknown', max_length=2048), + preserve_default=False, + ), + ] diff --git a/mygpo/data/models.py b/mygpo/data/models.py index 7958f2b5..02b0bc11 100644 --- a/mygpo/data/models.py +++ b/mygpo/data/models.py @@ -1,3 +1,5 @@ +import uuid + from datetime import datetime from django.db import models @@ -11,8 +13,11 @@ class PodcastUpdateResult(UUIDModel): Once an instance is stored, the update is assumed to be finished. """ + # URL of the podcast to be updated + podcast_url = models.URLField(max_length=2048) + # The podcast that was updated - podcast = models.ForeignKey(Podcast, on_delete=models.CASCADE) + podcast = models.ForeignKey(Podcast, on_delete=models.CASCADE, null=True) # The timestamp at which the updated started to be executed start = models.DateTimeField(default=datetime.utcnow) @@ -42,3 +47,30 @@ class PodcastUpdateResult(UUIDModel): models.Index(fields=['podcast', 'start']) ] + def __str__(self): + return 'Update Result for "{}" @ {:%Y-%m-%d %H:%M}'.format( + self.podcast, self.start) + + # Use as context manager + + def __enter__(self): + self.id = uuid.uuid4() + self.start = datetime.utcnow() + return self + + def __exit__(self, exc_type, exc_value, traceback): + self.duration = datetime.utcnow() - self.start + + success = (exc_type, exc_value, traceback) == (None, None, None) + self.successful = success + + if not success: + self.error_message = str(exc_value) + + if self.podcast_created is None: + self.podcast_created = False + + if self.episodes_added is None: + self.episodes_added = 0 + + self.save() diff --git a/mygpo/podcasts/models.py b/mygpo/podcasts/models.py index 54fe96c4..6b5d1f96 100644 --- a/mygpo/podcasts/models.py +++ b/mygpo/podcasts/models.py @@ -696,6 +696,10 @@ class Podcast(UUIDModel, TitleModel, DescriptionModel, LinkModel, return _('Unknown Podcast from {domain}'.format( domain=utils.get_domain(self.url))) + @property + def next_update(self): + return self.last_update + timedelta(hours=self.update_interval) + class EpisodeQuerySet(MergedUUIDQuerySet): """ QuerySet for Episodes """ diff --git a/mygpo/publisher/templates/publisher/podcast.html b/mygpo/publisher/templates/publisher/podcast.html index 93c03a38..314ab388 100644 --- a/mygpo/publisher/templates/publisher/podcast.html +++ b/mygpo/publisher/templates/publisher/podcast.html @@ -4,6 +4,7 @@ {% load podcasts %} {% load charts %} {% load pcharts %} +{% load time %} {% load static %} {% load menu %} {% load utils %} @@ -56,12 +57,49 @@

{% trans "The podcast information is regularly retrieved from the podcast feed" %}

{{ podcast.url }}
-

{% trans "Timing" %}

- +

{% trans "Updates" %}

+ + {% trans "Update interval:" %} {{ podcast.update_interval|hours_to_str }} + + + + + + + + + + + + + + + + + + {% for result in update_results %} + + + + + + + {% empty %} + + + + + + + {% endfor %} + +
StartDurationStatusEpisodes Added
{{ podcast.next_update|naturaltime }}Next
{{ result.start|naturaltime }}{{ result.duration.total_seconds|format_duration }} + {% if result.successful %} + {% trans "Successful" %} + {% else %} + {% trans "Error" %} {{ result.error_message }} + {% endif %} + {{ result.episodes_added }}
{{ podcast.last_update|naturaltime }}
{% csrf_token %} diff --git a/mygpo/publisher/views.py b/mygpo/publisher/views.py index 9e62e98b..5038cdf8 100644 --- a/mygpo/publisher/views.py +++ b/mygpo/publisher/views.py @@ -31,6 +31,7 @@ from mygpo.web.utils import get_podcast_link_target, normalize_twitter, \ get_episode_link_target from django.contrib.sites.requests import RequestSite from mygpo.data.tasks import update_podcasts +from mygpo.data.models import PodcastUpdateResult from mygpo.decorators import requires_token, allowed_methods from mygpo.pubsub.models import HubSubscription @@ -92,6 +93,11 @@ def podcast(request, podcast): except HubSubscription.DoesNotExist: pubsubscription = None + MAX_UPDATE_RESULTS=10 + + update_results = PodcastUpdateResult.objects.filter(podcast=podcast) + update_results = update_results[:MAX_UPDATE_RESULTS] + site = RequestSite(request) feedurl_quoted = urllib.parse.quote(podcast.url.encode('ascii')) @@ -105,6 +111,7 @@ def podcast(request, podcast): 'update_token': update_token, 'feedurl_quoted': feedurl_quoted, 'pubsubscription': pubsubscription, + 'update_results': update_results, }) -- 2.11.4.GIT