From a75a187e9ee4c8520fc85d45d2b1e8e1082c0002 Mon Sep 17 00:00:00 2001 From: Stefan Koegl Date: Sun, 19 Sep 2010 08:57:32 +0300 Subject: [PATCH] refactoring in publisher and Podcast model code --- mygpo/api/models/__init__.py | 18 +++- mygpo/data/models.py | 2 +- mygpo/publisher/templates/publisher/episodes.html | 4 +- mygpo/publisher/utils.py | 117 ++++++++-------------- mygpo/publisher/views.py | 8 +- mygpo/web/templates/podcast.html | 4 +- mygpo/web/views/podcast.py | 7 +- 7 files changed, 66 insertions(+), 94 deletions(-) diff --git a/mygpo/api/models/__init__.py b/mygpo/api/models/__init__.py index 9f40f98e..3a9c2f89 100644 --- a/mygpo/api/models/__init__.py +++ b/mygpo/api/models/__init__.py @@ -81,6 +81,11 @@ class Podcast(models.Model): from mygpo.data.models import Listener return Listener.objects.filter(podcast=self).values('user').distinct().count() + def listener_count_timespan(self, start, end): + return EpisodeAction.objects.filter(episode__podcast=self, + timestamp__range=(start, end), + action='play').values('user_id').distinct().count() + def logo_shortname(self): return hashlib.sha1(self.logo_url).hexdigest() @@ -141,6 +146,9 @@ class Podcast(models.Model): from mygpo.data.models import RelatedPodcast return [r.rel_podcast for r in RelatedPodcast.objects.filter(ref_podcast=self)] + def get_episodes(self): + return Episode.objects.filter(podcast=self) + def __unicode__(self): return self.title if self.title != '' else self.url @@ -294,13 +302,14 @@ class Episode(models.Model): def number(self): m = re.search('\D*(\d+)\D+', self.title) - return m.group(1) + return m.group(1) if m else '' def shortname(self): s = self.title s = s.replace(self.podcast.title, '') s = s.replace(self.number(), '') - s = re.search('\W*(.+)', s).group(1) + m = re.search('\W*(.+)', s) + s = m.group(1) if m else s s = s.strip() return s @@ -308,6 +317,11 @@ class Episode(models.Model): from mygpo.data.models import Listener return Listener.objects.filter(episode=self).values('user').distinct().count() + def listener_count_timespan(self, start, end): + return EpisodeAction.objects.filter(episode=self, + timestamp__range=(start, end), + action='play').values('user_id').distinct().count() + def __unicode__(self): return '%s (%s)' % (self.shortname(), self.podcast) diff --git a/mygpo/data/models.py b/mygpo/data/models.py index b52f65f8..c047f4c6 100755 --- a/mygpo/data/models.py +++ b/mygpo/data/models.py @@ -23,7 +23,7 @@ class PodcastTagManager(models.Manager): """ if not tags: tags = PodcastTag.objects.all() - tags = tags.values('tag').annotate(count=Count('id')) + tags = tags.values('tag').annotate(count=models.Count('id')) tags = sorted(tags, key=lambda x: x['count'], reverse=True) return tags diff --git a/mygpo/publisher/templates/publisher/episodes.html b/mygpo/publisher/templates/publisher/episodes.html index 7adc590c..7793efce 100644 --- a/mygpo/publisher/templates/publisher/episodes.html +++ b/mygpo/publisher/templates/publisher/episodes.html @@ -50,8 +50,8 @@ {{ episode.title|default:"Unknown Episode"|striptags }} {{ episode.timestamp|default:""|naturalday }} - {% if episode.listeners %} - {{ episode.listeners|vertical_bar:max_listeners }} + {% if episode.listener_count %} + {{ episode.listener_count|vertical_bar:max_listeners }} {% endif %} diff --git a/mygpo/publisher/utils.py b/mygpo/publisher/utils.py index 295181eb..fbac2be6 100644 --- a/mygpo/publisher/utils.py +++ b/mygpo/publisher/utils.py @@ -22,55 +22,44 @@ from mygpo.data.models import HistoricPodcastData from mygpo.web.utils import flatten_intervals from mygpo.publisher.models import PodcastPublisher from mygpo.api.constants import DEVICE_TYPES -from django.db.models import Avg +from django.db.models import Avg, Count from django.contrib.auth.models import User -def listener_data(podcasts): - day = timedelta(1) +def listener_data(podcasts, start_date=date(2010, 1, 1), leap=timedelta(days=1)): + episode_actions = EpisodeAction.objects.filter( + episode__podcast__in=podcasts, + timestamp__gte=start_date, + action='play').order_by('timestamp').values('timestamp') - # get start date - d = date(2010, 1, 1) - episode_actions = EpisodeAction.objects.filter(episode__podcast__in=podcasts, timestamp__gte=d, action='play').order_by('timestamp').values('timestamp') if len(episode_actions) == 0: return [] start = episode_actions[0]['timestamp'] # pre-calculate episode list, make it index-able by release-date - episodes = {} - for episode in Episode.objects.filter(podcast__in=podcasts): - if episode.timestamp: - episodes[episode.timestamp.date()] = episode + episodes = Episode.objects.filter(podcast__in=podcasts) + episodes = filter(lambda e: e.timestamp, episodes) + episodes = dict([(e.timestamp.date(), e) for e in episodes]) days = [] - for d in daterange(start): - next = d + timedelta(days=1) - listener_sum = 0 + for d in daterange(start, leap=leap): + next = d + leap - # this is faster than .filter(episode__podcast__in=podcasts) - for p in podcasts: - listeners = EpisodeAction.objects.filter(episode__podcast=p, timestamp__gte=d, timestamp__lt=next, action='play').values('user_id').distinct().count() - listener_sum += listeners + get_listeners = lambda p: p.listener_count_timespan(d, next) + listeners = map(get_listeners, podcasts) + listener_sum = sum(listeners) - if d.date() in episodes: - episode = episodes[d.date()] - else: - episode = None + episode = episodes[d.date()] if d.date() in episodes else None - days.append({ - 'date': d, - 'listeners': listener_sum, - 'episode': episode}) + days.append(dict(date=d, listeners=listener_sum, episode=episode)) return days -def episode_listener_data(episode): - d = date(2010, 1, 1) - leap = timedelta(days=1) - - episodes = EpisodeAction.objects.filter(episode=episode, timestamp__gte=d).order_by('timestamp').values('timestamp') +def episode_listener_data(episode, start_date=date(2010, 1, 1), leap=timedelta(days=1)): + episodes = EpisodeAction.objects.filter(episode=episode, + timestamp__gte=start_date).order_by('timestamp').values('timestamp') if len(episodes) == 0: return [] @@ -79,37 +68,27 @@ def episode_listener_data(episode): intervals = [] for d in daterange(start, leap=leap): next = d + leap - listeners = EpisodeAction.objects.filter(episode=episode, timestamp__gte=d, timestamp__lt=next).values('user_id').distinct().count() - e = episode if episode.timestamp and episode.timestamp >= d and episode.timestamp <= next else None - intervals.append({ - 'date': d, - 'listeners': listeners, - 'episode': e}) + + listeners = episode.listener_count_timespan(d, next) + released_episode = episode if episode.timestamp and episode.timestamp >= d and episode.timestamp <= next else None + intervals.append(dict(date=d, listeners=listeners, episode=released_episode)) return intervals def subscriber_data(podcasts): - data = {} - #this is fater than a subquery - records = [] - for p in podcasts: - records.extend(HistoricPodcastData.objects.filter(podcast=p).order_by('date')) + records = HistoricPodcastData.objects.filter(podcast__in=podcasts).order_by('date') - for r in records: - if r.date.day == 1: - s = r.date.strftime('%y-%m') - val = data.get(s, 0) - data[s] = val + r.subscriber_count + include_record = lambda r: r.date.day == 1 + records = filter(include_record, records) - list = [] - for k, v in data.iteritems(): - list.append({'x': k, 'y': v}) + create_entry = lambda r: dict(x=r.date.strftime('%y-%m'), y=r.subscriber_count) + data = map(create_entry, records) - list.sort(key=lambda x: x['x']) + data.sort(key=lambda x: x['x']) - return list + return data def check_publisher_permission(user, podcast): @@ -121,30 +100,15 @@ def check_publisher_permission(user, podcast): return False -def episode_list(podcast): - episodes = Episode.objects.filter(podcast=podcast).order_by('-timestamp') - for e in episodes: - listeners = EpisodeAction.objects.filter(episode=e, action='play').values('user').distinct() - e.listeners = listeners.count() - - return episodes - def device_stats(podcasts): - res = {} - for type in DEVICE_TYPES: - c = 0 - - # this is faster than a subquery - for p in podcasts: - c += EpisodeAction.objects.filter(episode__podcast=p, device__type=type[0]).values('user_id').distinct().count() - if c > 0: - res[type[1]] = c + l = EpisodeAction.objects.filter(episode__podcast__in=podcasts).values('device__type').annotate(count=Count('id')) + l = filter(lambda x: int(x['count']) > 0, l) + l = map(lambda x: (x['device__type'], x['count']), l) + return dict(l) - return res - -def episode_heatmap(episode, max_part_num=50, min_part_length=10): +def episode_heatmap(episode, max_part_num=30, min_part_length=10): """ Generates "Heatmap Data" for the given episode @@ -157,10 +121,7 @@ def episode_heatmap(episode, max_part_num=50, min_part_length=10): episode_actions = EpisodeAction.objects.filter(episode=episode, action='play') - if episode.duration: - duration = episode.duration - else: - duration = episode_actions.aggregate(duration=Avg('total'))['duration'] + duration = episode.duration or episode_actions.aggregate(duration=Avg('total'))['duration'] if not duration: return [0], 0 @@ -173,8 +134,7 @@ def episode_heatmap(episode, max_part_num=50, min_part_length=10): user_ids = [x['user'] for x in episode_actions.values('user').distinct()] for user_id in user_ids: - user = User.objects.get(id=user_id) - actions = episode_actions.filter(user=user, playmark__isnull=False, started__isnull=False) + actions = episode_actions.filter(user__id=user_id, playmark__isnull=False, started__isnull=False) if actions.exists(): played_parts = flatten_intervals(actions) user_heatmap = played_parts_to_heatmap(played_parts, part_length, part_num) @@ -209,7 +169,8 @@ def played_parts_to_heatmap(played_parts, part_length, part_count): return parts if current_part['start'] <= (part + part_length) and current_part['end'] >= part: - parts[i] = 1 + parts[i] += 1 + return parts diff --git a/mygpo/publisher/views.py b/mygpo/publisher/views.py index 3b43fae1..55b9d635 100644 --- a/mygpo/publisher/views.py +++ b/mygpo/publisher/views.py @@ -6,7 +6,7 @@ from django.contrib.auth.decorators import login_required from mygpo.publisher.models import PodcastPublisher from mygpo.publisher.auth import require_publisher, is_publisher from mygpo.publisher.forms import SearchPodcastForm, EpisodeForm, PodcastForm -from mygpo.publisher.utils import listener_data, episode_listener_data, check_publisher_permission, episode_list, subscriber_data, device_stats, episode_heatmap +from mygpo.publisher.utils import listener_data, episode_listener_data, check_publisher_permission, subscriber_data, device_stats, episode_heatmap from django.contrib.sites.models import Site from mygpo.data.feeddownloader import update_podcasts from mygpo.decorators import requires_token, allowed_methods @@ -131,8 +131,8 @@ def episodes(request, id): if not check_publisher_permission(request.user, p): return HttpResponseForbidden() - episodes = episode_list(p) - max_listeners = max([x.listeners for x in episodes]) if len(episodes) else 0 + episodes = p.get_episodes() + max_listeners = max([x.listener_count() for x in episodes]) if len(episodes) else 0 return render_to_response('publisher/episodes.html', { 'podcast': p, @@ -164,7 +164,7 @@ def episode(request, id): 'episode': e, 'form': form, 'timeline_data': timeline_data, - 'heatmap_data': heatmap_data if any([x > 1 for x in heatmap_data]) else None, + 'heatmap_data': heatmap_data if any([x > 0 for x in heatmap_data]) else None, 'heatmap_part_length': part_length, }, context_instance=RequestContext(request)) diff --git a/mygpo/web/templates/podcast.html b/mygpo/web/templates/podcast.html index ca9ec03d..4af0967a 100755 --- a/mygpo/web/templates/podcast.html +++ b/mygpo/web/templates/podcast.html @@ -136,8 +136,8 @@ {{ episode.timestamp|default:""|date:"Y-m-d" }} - {% if episode.listeners %} - {{ episode.listeners|vertical_bar:max_listeners }} + {% if episode.listener_count %} + {{ episode.listener_count|vertical_bar:max_listeners }} {% endif %} diff --git a/mygpo/web/views/podcast.py b/mygpo/web/views/podcast.py index 47f65b36..1077e5bb 100644 --- a/mygpo/web/views/podcast.py +++ b/mygpo/web/views/podcast.py @@ -22,7 +22,7 @@ MAX_TAGS_ON_PAGE=50 def show(request, pid): podcast = get_object_or_404(Podcast, pk=pid) episodes = episode_list(podcast, request.user) - max_listeners = max([x.listeners for x in episodes]) if len(episodes) else 0 + max_listeners = max([x.listener_count() for x in episodes]) if len(episodes) else 0 related_podcasts = [x for x in podcast.group.podcasts() if x != podcast] if podcast.group else [] tags = get_tags(podcast, request.user) @@ -137,11 +137,8 @@ def episode_list(podcast, user): action. The attribute is unsert if there is no episode-action for the episode. """ - episodes = Episode.objects.filter(podcast=podcast).order_by('-timestamp') + episodes = podcast.get_episodes().order_by('-timestamp') for e in episodes: - listeners = Listener.objects.filter(episode=e).values('user').distinct() - e.listeners = listeners.count() - if user.is_authenticated(): actions = EpisodeAction.objects.filter(episode=e, user=user).order_by('-timestamp') if actions.count() > 0: -- 2.11.4.GIT