From 3f3f9cc929cf1bbf45a0d5bbcc71acf57e158b12 Mon Sep 17 00:00:00 2001 From: =?utf8?q?Stefan=20K=C3=B6gl?= Date: Sun, 26 Oct 2014 11:36:29 +0100 Subject: [PATCH] [History] store episode history in Django ORM --- mygpo/administration/tests.py | 2 - mygpo/api/advanced/__init__.py | 166 ++++++++++++++--------------------------- mygpo/api/advanced/updates.py | 5 +- mygpo/core/tasks.py | 14 ++-- mygpo/history/models.py | 2 +- mygpo/maintenance/tests.py | 4 - mygpo/web/views/episode.py | 24 +++--- 7 files changed, 79 insertions(+), 138 deletions(-) diff --git a/mygpo/administration/tests.py b/mygpo/administration/tests.py index b7a24488..ec1dd07c 100644 --- a/mygpo/administration/tests.py +++ b/mygpo/administration/tests.py @@ -65,7 +65,6 @@ class SimpleTest(TestCase): user = user, action = EpisodeHistoryEntry.PLAY, timestamp = datetime.utcnow(), - created = datetime.utcnow(), ) action3 = EpisodeHistoryEntry.objects.create( @@ -73,7 +72,6 @@ class SimpleTest(TestCase): user = user, action = EpisodeHistoryEntry.PLAY, timestamp = datetime.utcnow(), - created = datetime.utcnow(), ) # we need that for later diff --git a/mygpo/api/advanced/__init__.py b/mygpo/api/advanced/__init__.py index e9dd2b3d..81691806 100644 --- a/mygpo/api/advanced/__init__.py +++ b/mygpo/api/advanced/__init__.py @@ -41,6 +41,7 @@ from mygpo.api.backend import get_device from mygpo.utils import format_time, parse_bool, get_timestamp, \ parse_request_body, normalize_feed_url from mygpo.decorators import allowed_methods, cors_origin +from mygpo.history.models import EpisodeHistoryEntry from mygpo.core.tasks import auto_flattr_episode from mygpo.users.models import (EpisodeAction, Client, InvalidEpisodeActionAttributes, ) @@ -48,9 +49,6 @@ from mygpo.users.settings import FLATTR_AUTO from mygpo.favorites.models import FavoriteEpisode from mygpo.core.json import JSONDecodeError from mygpo.api.basic_auth import require_valid_user, check_username -from mygpo.db.couchdb import bulk_save_retry, get_userdata_database -from mygpo.db.couchdb.episode_state import episode_state_for_ref_urls, \ - get_episode_actions import logging @@ -148,7 +146,7 @@ def episodes(request, username, version=1): device = None changes = get_episode_changes(request.user, podcast, device, since, - now_, aggregated, version) + now, aggregated, version) return JsonResponse(changes) @@ -165,25 +163,22 @@ def convert_position(action): def get_episode_changes(user, podcast, device, since, until, aggregated, version): - devices = {client.id.hex: client.uid for client in user.client_set.all()} + history = EpisodeHistoryEntry.objects.filter(user=user, + timestamp__lt=until) + + if since: + history = history.filter(timestamp__gte=since) - args = {} if podcast is not None: - args['podcast_id'] = podcast.get_id() + history = history.filter(episode__podcast=podcast) if device is not None: - args['device_id'] = device.id.hex - - actions, until = get_episode_actions(user.profile.uuid.hex, since, until, **args) + history = history.filter(client=device) if version == 1: - actions = imap(convert_position, actions) - - clean_data = partial(clean_episode_action_data, - user=user, devices=devices) + history = imap(convert_position, history) - actions = map(clean_data, actions) - actions = filter(None, actions) + actions = [episode_action_json(a, user) for a in history] if aggregated: actions = dict( (a['episode'], a) for a in actions ).values() @@ -191,57 +186,29 @@ def get_episode_changes(user, podcast, device, since, until, aggregated, version return {'actions': actions, 'timestamp': until} +def episode_action_json(history, user): + action = { + 'podcast': history.podcast_ref_url or history.episode.podcast.url, + 'episode': history.episode_ref_url or history.episode.url, + 'action': history.action, + 'timestamp': history.timestamp.isoformat(), + } -def clean_episode_action_data(action, user, devices): - - if None in (action.get('podcast', None), action.get('episode', None)): - return None - - if 'device_id' in action: - device_id = action['device_id'] - device_uid = devices.get(device_id) - if device_uid: - action['device'] = device_uid - - del action['device_id'] - - # remove superfluous keys - for x in action.keys(): - if x not in EPISODE_ACTION_KEYS: - del action[x] - - # set missing keys to None - for x in EPISODE_ACTION_KEYS: - if x not in action: - action[x] = None + if history.client: + action['device'] = history.client.uid - if action['action'] != 'play': - if 'position' in action: - del action['position'] - - if 'total' in action: - del action['total'] - - if 'started' in action: - del action['started'] - - if 'playmark' in action: - del action['playmark'] - - else: - action['position'] = action.get('position', False) or 0 + if history.action == EpisodeHistoryEntry.PLAY: + action['started'] = history.started + action['position'] = history.stopped # TODO: check "playmark" + action['total'] = history.total return action - - - def update_episodes(user, actions, now, ua_string): update_urls = [] - - grouped_actions = defaultdict(list) + auto_flattr = user.profile.get_wksetting(FLATTR_AUTO) # group all actions by their episode for action in actions: @@ -256,73 +223,56 @@ def update_episodes(user, actions, now, ua_string): if episode_url == '': continue - act = parse_episode_action(action, user, update_urls, now, ua_string) - grouped_actions[ (podcast_url, episode_url) ].append(act) - - - auto_flattr_episodes = [] - - # Prepare the updates for each episode state - obj_funs = [] - - for (p_url, e_url), action_list in grouped_actions.iteritems(): - episode_state = episode_state_for_ref_urls(user, p_url, e_url) - - if any(a['action'] == 'play' for a in actions): - auto_flattr_episodes.append(episode_state.episode) - - fun = partial(update_episode_actions, action_list=action_list) - obj_funs.append( (episode_state, fun) ) - - udb = get_userdata_database() - bulk_save_retry(obj_funs, udb) - - if user.profile.get_wksetting(FLATTR_AUTO): - for episode_id in auto_flattr_episodes: - auto_flattr_episode.delay(user, episode_id) + podcast = Podcast.objects.get_or_create_for_url(podcast_url) + episode = Episode.objects.get_or_create_for_url(podcast, episode_url) + + # parse_episode_action returns a EpisodeHistoryEntry obj + history = parse_episode_action(action, user, update_urls, now, + ua_string) + + # we could save ``history`` directly, but we check for duplicates first + EpisodeHistoryEntry.objects.get_or_create( + user = user, + client = history.client, + episode = episode, + action = history.action, + timestamp = history.timestamp, + defaults = { + 'started': history.started, + 'stopped': history.stopped, + 'total': history.total, + } + ) + + if history.action == EpisodeHistoryEntry.PLAY and auto_flattr: + auto_flattr_episode.delay(user, episode.id) return update_urls -def update_episode_actions(episode_state, action_list): - """ Adds actions to the episode state and saves if necessary """ - - len1 = len(episode_state.actions) - episode_state.add_actions(action_list) - - if len(episode_state.actions) == len1: - return None - - return episode_state - - - def parse_episode_action(action, user, update_urls, now, ua_string): action_str = action.get('action', None) if not valid_episodeaction(action_str): raise Exception('invalid action %s' % action_str) - new_action = EpisodeAction() + history = EpisodeHistoryEntry() - new_action.action = action['action'] + history.action = action['action'] if action.get('device', False): - device = get_device(user, action['device'], ua_string) - new_action.device = device.id.hex + client = get_device(user, action['device'], ua_string) + history.client = client if action.get('timestamp', False): - new_action.timestamp = dateutil.parser.parse(action['timestamp']) + history.timestamp = dateutil.parser.parse(action['timestamp']) else: - new_action.timestamp = now - new_action.timestamp = new_action.timestamp.replace(microsecond=0) - - new_action.upload_timestamp = get_timestamp(now) + history.timestamp = now - new_action.started = action.get('started', None) - new_action.playmark = action.get('position', None) - new_action.total = action.get('total', None) + history.started = action.get('started', None) + history.stopped = action.get('position', None) + history.total = action.get('total', None) - return new_action + return history @csrf_exempt diff --git a/mygpo/api/advanced/updates.py b/mygpo/api/advanced/updates.py index 9af4a4a4..a4c7284a 100644 --- a/mygpo/api/advanced/updates.py +++ b/mygpo/api/advanced/updates.py @@ -27,7 +27,7 @@ from django.views.generic.base import View from mygpo.podcasts.models import Episode from mygpo.api.httpresponse import JsonResponse -from mygpo.api.advanced import clean_episode_action_data +from mygpo.api.advanced import episode_action_json from mygpo.api.advanced.directory import episode_data, podcast_data from mygpo.utils import parse_bool, get_timestamp from mygpo.subscriptions import get_subscription_history, subscription_diff @@ -143,8 +143,9 @@ class DeviceUpdates(View): t['status'] = episode_status.status # include latest action (bug 1419) + # TODO if include_actions and episode_status.action: - t['action'] = clean_episode_action_data(episode_status.action, user, devices) + t['action'] = episode_action_json(episode_status.action, user) return t diff --git a/mygpo/core/tasks.py b/mygpo/core/tasks.py index 763e7635..3bde5011 100644 --- a/mygpo/core/tasks.py +++ b/mygpo/core/tasks.py @@ -10,8 +10,7 @@ from mygpo.data.feeddownloader import PodcastUpdater from mygpo.utils import get_timestamp from mygpo.users.models import EpisodeAction from mygpo.flattr import Flattr -from mygpo.db.couchdb.episode_state import episode_state_for_user_episode, \ - add_episode_actions +from mygpo.history.models import EpisodeHistoryEntry @celery.task(max_retries=5, default_retry_delay=60) @@ -71,11 +70,12 @@ def auto_flattr_episode(user, episode_id): return False episode = Episode.objects.get(id=episode_id) - state = episode_state_for_user_episode(user, episode) - action = EpisodeAction() - action.action = 'flattr' - action.upload_timestamp = get_timestamp(datetime.utcnow()) - add_episode_actions(state, [action]) + EpisodeHistoryEntry.objects.create( + episode = episode, + action = EpisodeHistoryEntry.FLATTR, + timestamp = datetime.utcnow(), + user = user, + ) return True diff --git a/mygpo/history/models.py b/mygpo/history/models.py index 4211bee4..d9ff20a8 100644 --- a/mygpo/history/models.py +++ b/mygpo/history/models.py @@ -71,7 +71,7 @@ class EpisodeHistoryEntry(models.Model): timestamp = models.DateTimeField() # the timestamp at which the event was created (provided by the server) - created = models.DateTimeField() + created = models.DateTimeField(auto_now_add=True) # the episode which was involved in the event episode = models.ForeignKey(Episode, db_index=True, null=True, diff --git a/mygpo/maintenance/tests.py b/mygpo/maintenance/tests.py index fab79665..8c92aac8 100644 --- a/mygpo/maintenance/tests.py +++ b/mygpo/maintenance/tests.py @@ -95,7 +95,6 @@ class MergeTests(TransactionTestCase): action1 = EpisodeHistoryEntry.objects.create( timestamp=datetime.utcnow(), - created=datetime.utcnow(), episode=self.episode1, user=self.user, client=None, @@ -106,7 +105,6 @@ class MergeTests(TransactionTestCase): action2 = EpisodeHistoryEntry.objects.create( timestamp=datetime.utcnow(), - created=datetime.utcnow(), episode=self.episode2, user=self.user, client=None, @@ -197,7 +195,6 @@ class MergeGroupTests(TransactionTestCase): action1 = EpisodeHistoryEntry.objects.create( timestamp=datetime.utcnow(), - created=datetime.utcnow(), episode=self.episode1, user=self.user, client=None, @@ -208,7 +205,6 @@ class MergeGroupTests(TransactionTestCase): action2 = EpisodeHistoryEntry.objects.create( timestamp=datetime.utcnow(), - created=datetime.utcnow(), episode=self.episode2, user=self.user, client=None, diff --git a/mygpo/web/views/episode.py b/mygpo/web/views/episode.py index 54f6655d..0ef66a1e 100644 --- a/mygpo/web/views/episode.py +++ b/mygpo/web/views/episode.py @@ -42,8 +42,6 @@ from mygpo.publisher.utils import check_publisher_permission from mygpo.web.utils import get_episode_link_target, check_restrictions from mygpo.history.models import EpisodeHistoryEntry from mygpo.favorites.models import FavoriteEpisode -from mygpo.db.couchdb.episode_state import episode_state_for_user_episode, \ - add_episode_actions from mygpo.userfeeds.feeds import FavoriteFeed @@ -198,14 +196,17 @@ def add_action(request, episode): else: timestamp = datetime.utcnow() - action = EpisodeAction() - action.timestamp = timestamp - action.upload_timestamp = get_timestamp(datetime.utcnow()) - action.device = client.id.hex if client else None - action.action = action_str + history = EpisodeHistoryEntry( + timestamp = timestamp, + action = action_str, + user = user, + episode = episode, + ) + + if client: + history.client = client - state = episode_state_for_user_episode(user, episode) - add_episode_actions(state, [action]) + history.save() podcast = episode.podcast return HttpResponseRedirect(get_episode_link_target(episode, podcast)) @@ -225,11 +226,6 @@ def flattr_episode(request, episode): success, msg = task.get() if success: - action = EpisodeAction() - action.action = 'flattr' - action.upload_timestamp = get_timestamp(datetime.utcnow()) - state = episode_state_for_user_episode(request.user, episode) - add_episode_actions(state, [action]) messages.success(request, _("Flattr\'d")) else: -- 2.11.4.GIT