Merge branch 'upload-timestamps'
[mygpo.git] / mygpo / users / models.py
blobbcbb3e7a3ef09d58c351ecd7672335ccc5878b55
1 import re
2 import uuid, collections
3 from datetime import datetime
4 import dateutil.parser
5 from itertools import imap
6 from operator import itemgetter
7 import random
8 import string
10 from couchdbkit.ext.django.schema import *
12 from django.core.cache import cache
14 from django_couchdb_utils.registration.models import User as BaseUser
16 from mygpo.utils import linearize
17 from mygpo.core.proxy import DocumentABCMeta, proxy_object
18 from mygpo.decorators import repeat_on_conflict
19 from mygpo.users.ratings import RatingMixin
20 from mygpo.users.sync import SyncedDevicesMixin
21 from mygpo.db.couchdb.podcast import podcasts_by_id, podcasts_to_dict
22 from mygpo.db.couchdb.user import user_history, device_history
26 RE_DEVICE_UID = re.compile(r'^[\w.-]+$')
28 # TODO: derive from ValidationException?
29 class InvalidEpisodeActionAttributes(ValueError):
30 """ raised when the attribues of an episode action fail validation """
33 class DeviceUIDException(Exception):
34 pass
37 class DeviceDoesNotExist(Exception):
38 pass
41 class DeviceDeletedException(DeviceDoesNotExist):
42 pass
45 class Suggestions(Document, RatingMixin):
46 user = StringProperty(required=True)
47 user_oldid = IntegerProperty()
48 podcasts = StringListProperty()
49 blacklist = StringListProperty()
52 def get_podcasts(self, count=None):
53 user = User.get(self.user)
54 subscriptions = user.get_subscribed_podcast_ids()
56 ids = filter(lambda x: not x in self.blacklist + subscriptions, self.podcasts)
57 if count:
58 ids = ids[:count]
59 return filter(lambda x: x and x.title, podcasts_by_id(ids))
62 def __repr__(self):
63 if not self._id:
64 return super(Suggestions, self).__repr__()
65 else:
66 return '%d Suggestions for %s (%s)' % \
67 (len(self.podcasts), self.user, self._id)
70 class EpisodeAction(DocumentSchema):
71 """
72 One specific action to an episode. Must
73 always be part of a EpisodeUserState
74 """
76 action = StringProperty(required=True)
78 # walltime of the event (assigned by the uploading client, defaults to now)
79 timestamp = DateTimeProperty(required=True, default=datetime.utcnow)
81 # upload time of the event
82 upload_timestamp = IntegerProperty(required=True)
84 device_oldid = IntegerProperty(required=False)
85 device = StringProperty()
86 started = IntegerProperty()
87 playmark = IntegerProperty()
88 total = IntegerProperty()
90 def __eq__(self, other):
91 if not isinstance(other, EpisodeAction):
92 return False
93 vals = ('action', 'timestamp', 'device', 'started', 'playmark',
94 'total')
95 return all([getattr(self, v, None) == getattr(other, v, None) for v in vals])
98 def to_history_entry(self):
99 entry = HistoryEntry()
100 entry.action = self.action
101 entry.timestamp = self.timestamp
102 entry.device_id = self.device
103 entry.started = self.started
104 entry.position = self.playmark
105 entry.total = self.total
106 return entry
110 def validate_time_values(self):
111 """ Validates allowed combinations of time-values """
113 PLAY_ACTION_KEYS = ('playmark', 'started', 'total')
115 # Key found, but must not be supplied (no play action!)
116 if self.action != 'play':
117 for key in PLAY_ACTION_KEYS:
118 if getattr(self, key, None) is not None:
119 raise InvalidEpisodeActionAttributes('%s only allowed in play actions' % key)
121 # Sanity check: If started or total are given, require playmark
122 if ((self.started is not None) or (self.total is not None)) and \
123 self.playmark is None:
124 raise InvalidEpisodeActionAttributes('started and total require position')
126 # Sanity check: total and playmark can only appear together
127 if ((self.total is not None) or (self.started is not None)) and \
128 ((self.total is None) or (self.started is None)):
129 raise InvalidEpisodeActionAttributes('total and started can only appear together')
132 def __repr__(self):
133 return '%s-Action on %s at %s (in %s)' % \
134 (self.action, self.device, self.timestamp, self._id)
137 def __hash__(self):
138 return hash(frozenset([self.action, self.timestamp, self.device,
139 self.started, self.playmark, self.total]))
142 class Chapter(Document):
143 """ A user-entered episode chapter """
145 device = StringProperty()
146 created = DateTimeProperty()
147 start = IntegerProperty(required=True)
148 end = IntegerProperty(required=True)
149 label = StringProperty()
150 advertisement = BooleanProperty()
153 def __repr__(self):
154 return '<%s %s (%d-%d)>' % (self.__class__.__name__, self.label,
155 self.start, self.end)
158 class EpisodeUserState(Document):
160 Contains everything a user has done with an Episode
163 episode = StringProperty(required=True)
164 actions = SchemaListProperty(EpisodeAction)
165 settings = DictProperty()
166 user_oldid = IntegerProperty()
167 user = StringProperty(required=True)
168 ref_url = StringProperty(required=True)
169 podcast_ref_url = StringProperty(required=True)
170 merged_ids = StringListProperty()
171 chapters = SchemaListProperty(Chapter)
172 podcast = StringProperty(required=True)
176 def add_actions(self, actions):
177 map(EpisodeAction.validate_time_values, actions)
178 self.actions = list(self.actions) + actions
179 self.actions = list(set(self.actions))
180 self.actions = sorted(self.actions, key=lambda x: x.timestamp)
183 def is_favorite(self):
184 return self.settings.get('is_favorite', False)
187 def set_favorite(self, set_to=True):
188 self.settings['is_favorite'] = set_to
191 def update_chapters(self, add=[], rem=[]):
192 """ Updates the Chapter list
194 * add contains the chapters to be added
196 * rem contains tuples of (start, end) times. Chapters that match
197 both endpoints will be removed
200 @repeat_on_conflict(['state'])
201 def update(state):
202 for chapter in add:
203 self.chapters = self.chapters + [chapter]
205 for start, end in rem:
206 keep = lambda c: c.start != start or c.end != end
207 self.chapters = filter(keep, self.chapters)
209 self.save()
211 update(state=self)
214 def get_history_entries(self):
215 return imap(EpisodeAction.to_history_entry, self.actions)
218 def __repr__(self):
219 return 'Episode-State %s (in %s)' % \
220 (self.episode, self._id)
222 def __eq__(self, other):
223 if not isinstance(other, EpisodeUserState):
224 return False
226 return (self.episode == other.episode and
227 self.user == other.user)
231 class SubscriptionAction(Document):
232 action = StringProperty()
233 timestamp = DateTimeProperty(default=datetime.utcnow)
234 device = StringProperty()
237 __metaclass__ = DocumentABCMeta
240 def __cmp__(self, other):
241 return cmp(self.timestamp, other.timestamp)
243 def __eq__(self, other):
244 return self.action == other.action and \
245 self.timestamp == other.timestamp and \
246 self.device == other.device
248 def __hash__(self):
249 return hash(self.action) + hash(self.timestamp) + hash(self.device)
251 def __repr__(self):
252 return '<SubscriptionAction %s on %s at %s>' % (
253 self.action, self.device, self.timestamp)
256 class PodcastUserState(Document):
258 Contains everything that a user has done
259 with a specific podcast and all its episodes
262 podcast = StringProperty(required=True)
263 user_oldid = IntegerProperty()
264 user = StringProperty(required=True)
265 settings = DictProperty()
266 actions = SchemaListProperty(SubscriptionAction)
267 tags = StringListProperty()
268 ref_url = StringProperty(required=True)
269 disabled_devices = StringListProperty()
270 merged_ids = StringListProperty()
273 def remove_device(self, device):
275 Removes all actions from the podcast state that refer to the
276 given device
278 self.actions = filter(lambda a: a.device != device.id, self.actions)
281 def subscribe(self, device):
282 action = SubscriptionAction()
283 action.action = 'subscribe'
284 action.device = device.id
285 self.add_actions([action])
288 def unsubscribe(self, device):
289 action = SubscriptionAction()
290 action.action = 'unsubscribe'
291 action.device = device.id
292 self.add_actions([action])
295 def add_actions(self, actions):
296 self.actions = list(set(self.actions + actions))
297 self.actions = sorted(self.actions)
300 def add_tags(self, tags):
301 self.tags = list(set(self.tags + tags))
304 def set_device_state(self, devices):
305 disabled_devices = [device.id for device in devices if device.deleted]
306 self.disabled_devices = disabled_devices
309 def get_change_between(self, device_id, since, until):
311 Returns the change of the subscription status for the given device
312 between the two timestamps.
314 The change is given as either 'subscribe' (the podcast has been
315 subscribed), 'unsubscribed' (the podcast has been unsubscribed) or
316 None (no change)
319 device_actions = filter(lambda x: x.device == device_id, self.actions)
320 before = filter(lambda x: x.timestamp <= since, device_actions)
321 after = filter(lambda x: x.timestamp <= until, device_actions)
323 # nothing happened, so there can be no change
324 if not after:
325 return None
327 then = before[-1] if before else None
328 now = after[-1]
330 if then is None:
331 if now.action != 'unsubscribe':
332 return now.action
333 elif then.action != now.action:
334 return now.action
335 return None
338 def get_subscribed_device_ids(self):
339 """ device Ids on which the user subscribed to the podcast """
340 devices = set()
342 for action in self.actions:
343 if action.action == "subscribe":
344 if not action.device in self.disabled_devices:
345 devices.add(action.device)
346 else:
347 if action.device in devices:
348 devices.remove(action.device)
350 return devices
354 def is_public(self):
355 return self.settings.get('public_subscription', True)
358 def __eq__(self, other):
359 if other is None:
360 return False
362 return self.podcast == other.podcast and \
363 self.user == other.user
365 def __repr__(self):
366 return 'Podcast %s for User %s (%s)' % \
367 (self.podcast, self.user, self._id)
370 class Device(Document):
371 id = StringProperty(default=lambda: uuid.uuid4().hex)
372 oldid = IntegerProperty(required=False)
373 uid = StringProperty(required=True)
374 name = StringProperty(required=True, default='New Device')
375 type = StringProperty(required=True, default='other')
376 settings = DictProperty()
377 deleted = BooleanProperty(default=False)
378 user_agent = StringProperty()
381 def get_subscription_changes(self, since, until):
383 Returns the subscription changes for the device as two lists.
384 The first lists contains the Ids of the podcasts that have been
385 subscribed to, the second list of those that have been unsubscribed
386 from.
389 from mygpo.db.couchdb.podcast_state import podcast_states_for_device
391 add, rem = [], []
392 podcast_states = podcast_states_for_device(self.id)
393 for p_state in podcast_states:
394 change = p_state.get_change_between(self.id, since, until)
395 if change == 'subscribe':
396 add.append( p_state.ref_url )
397 elif change == 'unsubscribe':
398 rem.append( p_state.ref_url )
400 return add, rem
403 def get_latest_changes(self):
405 from mygpo.db.couchdb.podcast_state import podcast_states_for_device
407 podcast_states = podcast_states_for_device(self.id)
408 for p_state in podcast_states:
409 actions = filter(lambda x: x.device == self.id, reversed(p_state.actions))
410 if actions:
411 yield (p_state.podcast, actions[0])
414 def get_subscribed_podcast_states(self):
415 r = PodcastUserState.view('subscriptions/by_device',
416 startkey = [self.id, None],
417 endkey = [self.id, {}],
418 include_docs = True
420 return list(r)
423 def get_subscribed_podcast_ids(self):
424 states = self.get_subscribed_podcast_states()
425 return [state.podcast for state in states]
428 def get_subscribed_podcasts(self):
429 """ Returns all subscribed podcasts for the device
431 The attribute "url" contains the URL that was used when subscribing to
432 the podcast """
434 states = self.get_subscribed_podcast_states()
435 podcast_ids = [state.podcast for state in states]
436 podcasts = podcasts_to_dict(podcast_ids)
438 for state in states:
439 podcast = proxy_object(podcasts[state.podcast], url=state.ref_url)
440 podcasts[state.podcast] = podcast
442 return podcasts.values()
445 def __hash__(self):
446 return hash(frozenset([self.id, self.uid, self.name, self.type, self.deleted]))
449 def __eq__(self, other):
450 return self.id == other.id
453 def __repr__(self):
454 return '<{cls} {id}>'.format(cls=self.__class__.__name__, id=self.id)
457 def __str__(self):
458 return self.name
460 def __unicode__(self):
461 return self.name
465 TOKEN_NAMES = ('subscriptions_token', 'favorite_feeds_token',
466 'publisher_update_token', 'userpage_token')
469 class TokenException(Exception):
470 pass
473 class User(BaseUser, SyncedDevicesMixin):
474 oldid = IntegerProperty()
475 settings = DictProperty()
476 devices = SchemaListProperty(Device)
477 published_objects = StringListProperty()
478 deleted = BooleanProperty(default=False)
479 suggestions_up_to_date = BooleanProperty(default=False)
480 twitter = StringProperty()
481 about = StringProperty()
483 # token for accessing subscriptions of this use
484 subscriptions_token = StringProperty(default=None)
486 # token for accessing the favorite-episodes feed of this user
487 favorite_feeds_token = StringProperty(default=None)
489 # token for automatically updating feeds published by this user
490 publisher_update_token = StringProperty(default=None)
492 # token for accessing the userpage of this user
493 userpage_token = StringProperty(default=None)
495 class Meta:
496 app_label = 'users'
499 def create_new_token(self, token_name, length=32):
500 """ creates a new random token """
502 if token_name not in TOKEN_NAMES:
503 raise TokenException('Invalid token name %s' % token_name)
505 token = "".join(random.sample(string.letters+string.digits, length))
506 setattr(self, token_name, token)
510 @repeat_on_conflict(['self'])
511 def get_token(self, token_name):
512 """ returns a token, and generate those that are still missing """
514 generated = False
516 if token_name not in TOKEN_NAMES:
517 raise TokenException('Invalid token name %s' % token_name)
519 for tn in TOKEN_NAMES:
520 if getattr(self, tn) is None:
521 self.create_new_token(tn)
522 generated = True
524 if generated:
525 self.save()
527 return getattr(self, token_name)
531 @property
532 def active_devices(self):
533 not_deleted = lambda d: not d.deleted
534 return filter(not_deleted, self.devices)
537 @property
538 def inactive_devices(self):
539 deleted = lambda d: d.deleted
540 return filter(deleted, self.devices)
543 def get_devices_by_id(self):
544 return dict( (device.id, device) for device in self.devices)
547 def get_device(self, id):
549 if not hasattr(self, '__device_by_id'):
550 self.__devices_by_id = dict( (d.id, d) for d in self.devices)
552 return self.__devices_by_id.get(id, None)
555 def get_device_by_uid(self, uid, only_active=True):
557 if not hasattr(self, '__devices_by_uio'):
558 self.__devices_by_uid = dict( (d.uid, d) for d in self.devices)
560 try:
561 device = self.__devices_by_uid[uid]
563 if only_active and device.deleted:
564 raise DeviceDeletedException(
565 'Device with UID %s is deleted' % uid)
567 return device
569 except KeyError as e:
570 raise DeviceDoesNotExist('There is no device with UID %s' % uid)
573 @repeat_on_conflict(['self'])
574 def update_device(self, device):
575 """ Sets the device and saves the user """
576 self.set_device(device)
577 self.save()
580 def set_device(self, device):
582 if not RE_DEVICE_UID.match(device.uid):
583 raise DeviceUIDException(u"'{uid} is not a valid device ID".format(
584 uid=device.uid))
586 devices = list(self.devices)
587 ids = [x.id for x in devices]
588 if not device.id in ids:
589 devices.append(device)
590 self.devices = devices
591 return
593 index = ids.index(device.id)
594 devices.pop(index)
595 devices.insert(index, device)
596 self.devices = devices
599 def remove_device(self, device):
600 devices = list(self.devices)
601 ids = [x.id for x in devices]
602 if not device.id in ids:
603 return
605 index = ids.index(device.id)
606 devices.pop(index)
607 self.devices = devices
609 if self.is_synced(device):
610 self.unsync_device(device)
613 def get_subscriptions_by_device(self, public=None):
614 from mygpo.db.couchdb.podcast_state import subscriptions_by_user
615 get_dev = itemgetter(2)
616 groups = collections.defaultdict(list)
617 subscriptions = subscriptions_by_user(self, public=public)
618 subscriptions = sorted(subscriptions, key=get_dev)
620 for public, podcast_id, device_id in subscriptions:
621 groups[device_id].append(podcast_id)
623 return groups
626 def get_subscribed_podcast_states(self, public=None):
628 Returns the Ids of all subscribed podcasts
631 r = PodcastUserState.view('subscriptions/by_user',
632 startkey = [self._id, public, None, None],
633 endkey = [self._id+'ZZZ', None, None, None],
634 reduce = False,
635 include_docs = True
638 return set(r)
641 def get_subscribed_podcast_ids(self, public=None):
642 states = self.get_subscribed_podcast_states(public=public)
643 return [state.podcast for state in states]
647 def get_subscribed_podcasts(self, public=None):
648 """ Returns all subscribed podcasts for the user
650 The attribute "url" contains the URL that was used when subscribing to
651 the podcast """
653 states = self.get_subscribed_podcast_states(public=public)
654 podcast_ids = [state.podcast for state in states]
655 podcasts = podcasts_to_dict(podcast_ids)
657 for state in states:
658 podcast = proxy_object(podcasts[state.podcast], url=state.ref_url)
659 podcasts[state.podcast] = podcast
661 return podcasts.values()
665 def get_subscription_history(self, device_id=None, reverse=False, public=None):
666 """ Returns chronologically ordered subscription history entries
668 Setting device_id restricts the actions to a certain device
671 from mygpo.db.couchdb.podcast_state import podcast_states_for_user, \
672 podcast_states_for_device
674 def action_iter(state):
675 for action in sorted(state.actions, reverse=reverse):
676 if device_id is not None and device_id != action.device:
677 continue
679 if public is not None and state.is_public() != public:
680 continue
682 entry = HistoryEntry()
683 entry.timestamp = action.timestamp
684 entry.action = action.action
685 entry.podcast_id = state.podcast
686 entry.device_id = action.device
687 yield entry
689 if device_id is None:
690 podcast_states = podcast_states_for_user(self)
691 else:
692 podcast_states = podcast_states_for_device(device_id)
694 # create an action_iter for each PodcastUserState
695 subscription_action_lists = [action_iter(x) for x in podcast_states]
697 action_cmp_key = lambda x: x.timestamp
699 # Linearize their subscription-actions
700 return linearize(action_cmp_key, subscription_action_lists, reverse)
703 def get_global_subscription_history(self, public=None):
704 """ Actions that added/removed podcasts from the subscription list
706 Returns an iterator of all subscription actions that either
707 * added subscribed a podcast that hasn't been subscribed directly
708 before the action (but could have been subscribed) earlier
709 * removed a subscription of the podcast is not longer subscribed
710 after the action
713 subscriptions = collections.defaultdict(int)
715 for entry in self.get_subscription_history(public=public):
716 if entry.action == 'subscribe':
717 subscriptions[entry.podcast_id] += 1
719 # a new subscription has been added
720 if subscriptions[entry.podcast_id] == 1:
721 yield entry
723 elif entry.action == 'unsubscribe':
724 subscriptions[entry.podcast_id] -= 1
726 # the last subscription has been removed
727 if subscriptions[entry.podcast_id] == 0:
728 yield entry
732 def get_newest_episodes(self, max_date, max_per_podcast=5):
733 """ Returns the newest episodes of all subscribed podcasts
735 Only max_per_podcast episodes per podcast are loaded. Episodes with
736 release dates above max_date are discarded.
738 This method returns a generator that produces the newest episodes.
740 The number of required DB queries is equal to the number of (distinct)
741 podcasts of all consumed episodes (max: number of subscribed podcasts),
742 plus a constant number of initial queries (when the first episode is
743 consumed). """
745 cmp_key = lambda episode: episode.released or datetime(2000, 01, 01)
747 podcasts = list(self.get_subscribed_podcasts())
748 podcasts = filter(lambda p: p.latest_episode_timestamp, podcasts)
749 podcasts = sorted(podcasts, key=lambda p: p.latest_episode_timestamp,
750 reverse=True)
752 podcast_dict = dict((p.get_id(), p) for p in podcasts)
754 # contains the un-yielded episodes, newest first
755 episodes = []
757 for podcast in podcasts:
759 yielded_episodes = 0
761 for episode in episodes:
762 # determine for which episodes there won't be a new episodes
763 # that is newer; those can be yielded
764 if episode.released > podcast.latest_episode_timestamp:
765 p = podcast_dict.get(episode.podcast, None)
766 yield proxy_object(episode, podcast=p)
767 yielded_episodes += 1
768 else:
769 break
771 # remove the episodes that have been yielded before
772 episodes = episodes[yielded_episodes:]
774 # fetch and merge episodes for the next podcast
775 from mygpo.db.couchdb.episode import episodes_for_podcast
776 new_episodes = episodes_for_podcast(podcast, since=1,
777 until=max_date, descending=True, limit=max_per_podcast)
778 episodes = sorted(episodes+new_episodes, key=cmp_key, reverse=True)
781 # yield the remaining episodes
782 for episode in episodes:
783 podcast = podcast_dict.get(episode.podcast, None)
784 yield proxy_object(episode, podcast=podcast)
789 def save(self, *args, **kwargs):
791 from mygpo.db.couchdb.podcast_state import podcast_states_for_user
793 super(User, self).save(*args, **kwargs)
795 podcast_states = podcast_states_for_user(self)
796 for state in podcast_states:
797 @repeat_on_conflict(['state'])
798 def _update_state(state):
799 old_devs = set(state.disabled_devices)
800 state.set_device_state(self.devices)
802 if old_devs != set(state.disabled_devices):
803 state.save()
805 _update_state(state)
810 def __eq__(self, other):
811 if not other:
812 return False
814 # ensure that other isn't AnonymousUser
815 return other.is_authenticated() and self._id == other._id
818 def __ne__(self, other):
819 return not(self == other)
822 def __repr__(self):
823 return 'User %s' % self._id
826 class History(object):
828 def __init__(self, user, device):
829 self.user = user
830 self.device = device
833 def __getitem__(self, key):
835 if isinstance(key, slice):
836 start = key.start or 0
837 length = key.stop - start
838 else:
839 start = key
840 length = 1
842 if self.device:
843 return device_history(self.user, self.device, start, length)
845 else:
846 return user_history(self.user, start, length)
850 class HistoryEntry(object):
851 """ A class that can represent subscription and episode actions """
854 @classmethod
855 def from_action_dict(cls, action):
857 entry = HistoryEntry()
859 if 'timestamp' in action:
860 ts = action.pop('timestamp')
861 entry.timestamp = dateutil.parser.parse(ts)
863 for key, value in action.items():
864 setattr(entry, key, value)
866 return entry
869 @property
870 def playmark(self):
871 return getattr(self, 'position', None)
874 @classmethod
875 def fetch_data(cls, user, entries,
876 podcasts=None, episodes=None):
877 """ Efficiently loads additional data for a number of entries """
879 if podcasts is None:
880 # load podcast data
881 podcast_ids = [getattr(x, 'podcast_id', None) for x in entries]
882 podcast_ids = filter(None, podcast_ids)
883 podcasts = podcasts_to_dict(podcast_ids)
885 if episodes is None:
886 from mygpo.db.couchdb.episode import episodes_to_dict
887 # load episode data
888 episode_ids = [getattr(x, 'episode_id', None) for x in entries]
889 episode_ids = filter(None, episode_ids)
890 episodes = episodes_to_dict(episode_ids)
892 # load device data
893 # does not need pre-populated data because no db-access is required
894 device_ids = [getattr(x, 'device_id', None) for x in entries]
895 device_ids = filter(None, device_ids)
896 devices = dict([ (id, user.get_device(id)) for id in device_ids])
899 for entry in entries:
900 podcast_id = getattr(entry, 'podcast_id', None)
901 entry.podcast = podcasts.get(podcast_id, None)
903 episode_id = getattr(entry, 'episode_id', None)
904 entry.episode = episodes.get(episode_id, None)
906 if hasattr(entry, 'user'):
907 entry.user = user
909 device = devices.get(getattr(entry, 'device_id', None), None)
910 entry.device = device
913 return entries