Merge branch 'upload-timestamps' of github.com:gpodder/mygpo into upload-timestamps
[mygpo.git] / mygpo / users / models.py
blob97175a91da0b2b73cae8f381a81c3807220c63be
1 import re
2 import uuid, collections
3 from datetime import datetime
4 import dateutil.parser
5 from itertools import imap
6 from operator import itemgetter
7 import random
8 import string
10 from couchdbkit.ext.django.schema import *
12 from django.core.cache import cache
14 from django_couchdb_utils.registration.models import User as BaseUser
16 from mygpo.utils import linearize, get_timestamp
17 from mygpo.core.proxy import DocumentABCMeta, proxy_object
18 from mygpo.decorators import repeat_on_conflict
19 from mygpo.users.ratings import RatingMixin
20 from mygpo.users.sync import SyncedDevicesMixin
21 from mygpo.db.couchdb.podcast import podcasts_by_id, podcasts_to_dict
22 from mygpo.db.couchdb.user import user_history, device_history
26 RE_DEVICE_UID = re.compile(r'^[\w.-]+$')
28 # TODO: derive from ValidationException?
29 class InvalidEpisodeActionAttributes(ValueError):
30 """ raised when the attribues of an episode action fail validation """
33 class DeviceUIDException(Exception):
34 pass
37 class DeviceDoesNotExist(Exception):
38 pass
41 class DeviceDeletedException(DeviceDoesNotExist):
42 pass
45 class Suggestions(Document, RatingMixin):
46 user = StringProperty(required=True)
47 user_oldid = IntegerProperty()
48 podcasts = StringListProperty()
49 blacklist = StringListProperty()
52 def get_podcasts(self, count=None):
53 user = User.get(self.user)
54 subscriptions = user.get_subscribed_podcast_ids()
56 ids = filter(lambda x: not x in self.blacklist + subscriptions, self.podcasts)
57 if count:
58 ids = ids[:count]
59 return filter(lambda x: x and x.title, podcasts_by_id(ids))
62 def __repr__(self):
63 if not self._id:
64 return super(Suggestions, self).__repr__()
65 else:
66 return '%d Suggestions for %s (%s)' % \
67 (len(self.podcasts), self.user, self._id)
70 def default_upload_timestamp():
71 ts = datetime.utcnow()
72 return get_timestamp(ts)
75 class EpisodeAction(DocumentSchema):
76 """
77 One specific action to an episode. Must
78 always be part of a EpisodeUserState
79 """
81 action = StringProperty(required=True)
83 # walltime of the event (assigned by the uploading client, defaults to now)
84 timestamp = DateTimeProperty(required=True, default=datetime.utcnow)
86 # upload time of the event
87 upload_timestamp = IntegerProperty(required=True,
88 default=default_upload_timestamp)
90 device_oldid = IntegerProperty(required=False)
91 device = StringProperty()
92 started = IntegerProperty()
93 playmark = IntegerProperty()
94 total = IntegerProperty()
96 def __eq__(self, other):
97 if not isinstance(other, EpisodeAction):
98 return False
99 vals = ('action', 'timestamp', 'device', 'started', 'playmark',
100 'total')
101 return all([getattr(self, v, None) == getattr(other, v, None) for v in vals])
104 def to_history_entry(self):
105 entry = HistoryEntry()
106 entry.action = self.action
107 entry.timestamp = self.timestamp
108 entry.device_id = self.device
109 entry.started = self.started
110 entry.position = self.playmark
111 entry.total = self.total
112 return entry
116 def validate_time_values(self):
117 """ Validates allowed combinations of time-values """
119 PLAY_ACTION_KEYS = ('playmark', 'started', 'total')
121 # Key found, but must not be supplied (no play action!)
122 if self.action != 'play':
123 for key in PLAY_ACTION_KEYS:
124 if getattr(self, key, None) is not None:
125 raise InvalidEpisodeActionAttributes('%s only allowed in play actions' % key)
127 # Sanity check: If started or total are given, require playmark
128 if ((self.started is not None) or (self.total is not None)) and \
129 self.playmark is None:
130 raise InvalidEpisodeActionAttributes('started and total require position')
132 # Sanity check: total and playmark can only appear together
133 if ((self.total is not None) or (self.started is not None)) and \
134 ((self.total is None) or (self.started is None)):
135 raise InvalidEpisodeActionAttributes('total and started can only appear together')
138 def __repr__(self):
139 return '%s-Action on %s at %s (in %s)' % \
140 (self.action, self.device, self.timestamp, self._id)
143 def __hash__(self):
144 return hash(frozenset([self.action, self.timestamp, self.device,
145 self.started, self.playmark, self.total]))
148 class Chapter(Document):
149 """ A user-entered episode chapter """
151 device = StringProperty()
152 created = DateTimeProperty()
153 start = IntegerProperty(required=True)
154 end = IntegerProperty(required=True)
155 label = StringProperty()
156 advertisement = BooleanProperty()
159 def __repr__(self):
160 return '<%s %s (%d-%d)>' % (self.__class__.__name__, self.label,
161 self.start, self.end)
164 class EpisodeUserState(Document):
166 Contains everything a user has done with an Episode
169 episode = StringProperty(required=True)
170 actions = SchemaListProperty(EpisodeAction)
171 settings = DictProperty()
172 user_oldid = IntegerProperty()
173 user = StringProperty(required=True)
174 ref_url = StringProperty(required=True)
175 podcast_ref_url = StringProperty(required=True)
176 merged_ids = StringListProperty()
177 chapters = SchemaListProperty(Chapter)
178 podcast = StringProperty(required=True)
182 def add_actions(self, actions):
183 map(EpisodeAction.validate_time_values, actions)
184 self.actions = list(self.actions) + actions
185 self.actions = list(set(self.actions))
186 self.actions = sorted(self.actions, key=lambda x: x.timestamp)
189 def is_favorite(self):
190 return self.settings.get('is_favorite', False)
193 def set_favorite(self, set_to=True):
194 self.settings['is_favorite'] = set_to
197 def update_chapters(self, add=[], rem=[]):
198 """ Updates the Chapter list
200 * add contains the chapters to be added
202 * rem contains tuples of (start, end) times. Chapters that match
203 both endpoints will be removed
206 @repeat_on_conflict(['state'])
207 def update(state):
208 for chapter in add:
209 self.chapters = self.chapters + [chapter]
211 for start, end in rem:
212 keep = lambda c: c.start != start or c.end != end
213 self.chapters = filter(keep, self.chapters)
215 self.save()
217 update(state=self)
220 def get_history_entries(self):
221 return imap(EpisodeAction.to_history_entry, self.actions)
224 def __repr__(self):
225 return 'Episode-State %s (in %s)' % \
226 (self.episode, self._id)
228 def __eq__(self, other):
229 if not isinstance(other, EpisodeUserState):
230 return False
232 return (self.episode == other.episode and
233 self.user == other.user)
237 class SubscriptionAction(Document):
238 action = StringProperty()
239 timestamp = DateTimeProperty(default=datetime.utcnow)
240 device = StringProperty()
243 __metaclass__ = DocumentABCMeta
246 def __cmp__(self, other):
247 return cmp(self.timestamp, other.timestamp)
249 def __eq__(self, other):
250 return self.action == other.action and \
251 self.timestamp == other.timestamp and \
252 self.device == other.device
254 def __hash__(self):
255 return hash(self.action) + hash(self.timestamp) + hash(self.device)
257 def __repr__(self):
258 return '<SubscriptionAction %s on %s at %s>' % (
259 self.action, self.device, self.timestamp)
262 class PodcastUserState(Document):
264 Contains everything that a user has done
265 with a specific podcast and all its episodes
268 podcast = StringProperty(required=True)
269 user_oldid = IntegerProperty()
270 user = StringProperty(required=True)
271 settings = DictProperty()
272 actions = SchemaListProperty(SubscriptionAction)
273 tags = StringListProperty()
274 ref_url = StringProperty(required=True)
275 disabled_devices = StringListProperty()
276 merged_ids = StringListProperty()
279 def remove_device(self, device):
281 Removes all actions from the podcast state that refer to the
282 given device
284 self.actions = filter(lambda a: a.device != device.id, self.actions)
287 def subscribe(self, device):
288 action = SubscriptionAction()
289 action.action = 'subscribe'
290 action.device = device.id
291 self.add_actions([action])
294 def unsubscribe(self, device):
295 action = SubscriptionAction()
296 action.action = 'unsubscribe'
297 action.device = device.id
298 self.add_actions([action])
301 def add_actions(self, actions):
302 self.actions = list(set(self.actions + actions))
303 self.actions = sorted(self.actions)
306 def add_tags(self, tags):
307 self.tags = list(set(self.tags + tags))
310 def set_device_state(self, devices):
311 disabled_devices = [device.id for device in devices if device.deleted]
312 self.disabled_devices = disabled_devices
315 def get_change_between(self, device_id, since, until):
317 Returns the change of the subscription status for the given device
318 between the two timestamps.
320 The change is given as either 'subscribe' (the podcast has been
321 subscribed), 'unsubscribed' (the podcast has been unsubscribed) or
322 None (no change)
325 device_actions = filter(lambda x: x.device == device_id, self.actions)
326 before = filter(lambda x: x.timestamp <= since, device_actions)
327 after = filter(lambda x: x.timestamp <= until, device_actions)
329 # nothing happened, so there can be no change
330 if not after:
331 return None
333 then = before[-1] if before else None
334 now = after[-1]
336 if then is None:
337 if now.action != 'unsubscribe':
338 return now.action
339 elif then.action != now.action:
340 return now.action
341 return None
344 def get_subscribed_device_ids(self):
345 """ device Ids on which the user subscribed to the podcast """
346 devices = set()
348 for action in self.actions:
349 if action.action == "subscribe":
350 if not action.device in self.disabled_devices:
351 devices.add(action.device)
352 else:
353 if action.device in devices:
354 devices.remove(action.device)
356 return devices
360 def is_public(self):
361 return self.settings.get('public_subscription', True)
364 def __eq__(self, other):
365 if other is None:
366 return False
368 return self.podcast == other.podcast and \
369 self.user == other.user
371 def __repr__(self):
372 return 'Podcast %s for User %s (%s)' % \
373 (self.podcast, self.user, self._id)
376 class Device(Document):
377 id = StringProperty(default=lambda: uuid.uuid4().hex)
378 oldid = IntegerProperty(required=False)
379 uid = StringProperty(required=True)
380 name = StringProperty(required=True, default='New Device')
381 type = StringProperty(required=True, default='other')
382 settings = DictProperty()
383 deleted = BooleanProperty(default=False)
384 user_agent = StringProperty()
387 def get_subscription_changes(self, since, until):
389 Returns the subscription changes for the device as two lists.
390 The first lists contains the Ids of the podcasts that have been
391 subscribed to, the second list of those that have been unsubscribed
392 from.
395 from mygpo.db.couchdb.podcast_state import podcast_states_for_device
397 add, rem = [], []
398 podcast_states = podcast_states_for_device(self.id)
399 for p_state in podcast_states:
400 change = p_state.get_change_between(self.id, since, until)
401 if change == 'subscribe':
402 add.append( p_state.ref_url )
403 elif change == 'unsubscribe':
404 rem.append( p_state.ref_url )
406 return add, rem
409 def get_latest_changes(self):
411 from mygpo.db.couchdb.podcast_state import podcast_states_for_device
413 podcast_states = podcast_states_for_device(self.id)
414 for p_state in podcast_states:
415 actions = filter(lambda x: x.device == self.id, reversed(p_state.actions))
416 if actions:
417 yield (p_state.podcast, actions[0])
420 def get_subscribed_podcast_states(self):
421 r = PodcastUserState.view('subscriptions/by_device',
422 startkey = [self.id, None],
423 endkey = [self.id, {}],
424 include_docs = True
426 return list(r)
429 def get_subscribed_podcast_ids(self):
430 states = self.get_subscribed_podcast_states()
431 return [state.podcast for state in states]
434 def get_subscribed_podcasts(self):
435 """ Returns all subscribed podcasts for the device
437 The attribute "url" contains the URL that was used when subscribing to
438 the podcast """
440 states = self.get_subscribed_podcast_states()
441 podcast_ids = [state.podcast for state in states]
442 podcasts = podcasts_to_dict(podcast_ids)
444 for state in states:
445 podcast = proxy_object(podcasts[state.podcast], url=state.ref_url)
446 podcasts[state.podcast] = podcast
448 return podcasts.values()
451 def __hash__(self):
452 return hash(frozenset([self.id, self.uid, self.name, self.type, self.deleted]))
455 def __eq__(self, other):
456 return self.id == other.id
459 def __repr__(self):
460 return '<{cls} {id}>'.format(cls=self.__class__.__name__, id=self.id)
463 def __str__(self):
464 return self.name
466 def __unicode__(self):
467 return self.name
471 TOKEN_NAMES = ('subscriptions_token', 'favorite_feeds_token',
472 'publisher_update_token', 'userpage_token')
475 class TokenException(Exception):
476 pass
479 class User(BaseUser, SyncedDevicesMixin):
480 oldid = IntegerProperty()
481 settings = DictProperty()
482 devices = SchemaListProperty(Device)
483 published_objects = StringListProperty()
484 deleted = BooleanProperty(default=False)
485 suggestions_up_to_date = BooleanProperty(default=False)
487 # token for accessing subscriptions of this use
488 subscriptions_token = StringProperty(default=None)
490 # token for accessing the favorite-episodes feed of this user
491 favorite_feeds_token = StringProperty(default=None)
493 # token for automatically updating feeds published by this user
494 publisher_update_token = StringProperty(default=None)
496 # token for accessing the userpage of this user
497 userpage_token = StringProperty(default=None)
499 class Meta:
500 app_label = 'users'
503 def create_new_token(self, token_name, length=32):
504 """ creates a new random token """
506 if token_name not in TOKEN_NAMES:
507 raise TokenException('Invalid token name %s' % token_name)
509 token = "".join(random.sample(string.letters+string.digits, length))
510 setattr(self, token_name, token)
514 def get_token(self, token_name):
515 """ returns a token, and generate those that are still missing """
517 generated = False
519 if token_name not in TOKEN_NAMES:
520 raise TokenException('Invalid token name %s' % token_name)
522 for tn in TOKEN_NAMES:
523 if getattr(self, tn) is None:
524 self.create_new_token(tn)
525 generated = True
527 if generated:
528 self.save()
530 return getattr(self, token_name)
534 @property
535 def active_devices(self):
536 not_deleted = lambda d: not d.deleted
537 return filter(not_deleted, self.devices)
540 @property
541 def inactive_devices(self):
542 deleted = lambda d: d.deleted
543 return filter(deleted, self.devices)
546 def get_devices_by_id(self):
547 return dict( (device.id, device) for device in self.devices)
550 def get_device(self, id):
552 if not hasattr(self, '__device_by_id'):
553 self.__devices_by_id = dict( (d.id, d) for d in self.devices)
555 return self.__devices_by_id.get(id, None)
558 def get_device_by_uid(self, uid, only_active=True):
560 if not hasattr(self, '__devices_by_uio'):
561 self.__devices_by_uid = dict( (d.uid, d) for d in self.devices)
563 try:
564 device = self.__devices_by_uid[uid]
566 if only_active and device.deleted:
567 raise DeviceDeletedException(
568 'Device with UID %s is deleted' % uid)
570 return device
572 except KeyError as e:
573 raise DeviceDoesNotExist('There is no device with UID %s' % uid)
576 def update_device(self, device):
577 """ Sets the device and saves the user """
579 @repeat_on_conflict(['user'])
580 def _update(user, device):
581 user.set_device(device)
582 user.save()
584 _update(user=self, device=device)
587 def set_device(self, device):
589 if not RE_DEVICE_UID.match(device.uid):
590 raise DeviceUIDException(u"'{uid} is not a valid device ID".format(
591 uid=device.uid))
593 devices = list(self.devices)
594 ids = [x.id for x in devices]
595 if not device.id in ids:
596 devices.append(device)
597 self.devices = devices
598 return
600 index = ids.index(device.id)
601 devices.pop(index)
602 devices.insert(index, device)
603 self.devices = devices
606 def remove_device(self, device):
607 devices = list(self.devices)
608 ids = [x.id for x in devices]
609 if not device.id in ids:
610 return
612 index = ids.index(device.id)
613 devices.pop(index)
614 self.devices = devices
616 if self.is_synced(device):
617 self.unsync_device(device)
620 def get_subscriptions_by_device(self, public=None):
621 from mygpo.db.couchdb.podcast_state import subscriptions_by_user
622 get_dev = itemgetter(2)
623 groups = collections.defaultdict(list)
624 subscriptions = subscriptions_by_user(self, public=public)
625 subscriptions = sorted(subscriptions, key=get_dev)
627 for public, podcast_id, device_id in subscriptions:
628 groups[device_id].append(podcast_id)
630 return groups
633 def get_subscribed_podcast_states(self, public=None):
635 Returns the Ids of all subscribed podcasts
638 r = PodcastUserState.view('subscriptions/by_user',
639 startkey = [self._id, public, None, None],
640 endkey = [self._id+'ZZZ', None, None, None],
641 reduce = False,
642 include_docs = True
645 return set(r)
648 def get_subscribed_podcast_ids(self, public=None):
649 states = self.get_subscribed_podcast_states(public=public)
650 return [state.podcast for state in states]
654 def get_subscribed_podcasts(self, public=None):
655 """ Returns all subscribed podcasts for the user
657 The attribute "url" contains the URL that was used when subscribing to
658 the podcast """
660 states = self.get_subscribed_podcast_states(public=public)
661 podcast_ids = [state.podcast for state in states]
662 podcasts = podcasts_to_dict(podcast_ids)
664 for state in states:
665 podcast = proxy_object(podcasts[state.podcast], url=state.ref_url)
666 podcasts[state.podcast] = podcast
668 return podcasts.values()
672 def get_subscription_history(self, device_id=None, reverse=False, public=None):
673 """ Returns chronologically ordered subscription history entries
675 Setting device_id restricts the actions to a certain device
678 from mygpo.db.couchdb.podcast_state import podcast_states_for_user, \
679 podcast_states_for_device
681 def action_iter(state):
682 for action in sorted(state.actions, reverse=reverse):
683 if device_id is not None and device_id != action.device:
684 continue
686 if public is not None and state.is_public() != public:
687 continue
689 entry = HistoryEntry()
690 entry.timestamp = action.timestamp
691 entry.action = action.action
692 entry.podcast_id = state.podcast
693 entry.device_id = action.device
694 yield entry
696 if device_id is None:
697 podcast_states = podcast_states_for_user(self)
698 else:
699 podcast_states = podcast_states_for_device(device_id)
701 # create an action_iter for each PodcastUserState
702 subscription_action_lists = [action_iter(x) for x in podcast_states]
704 action_cmp_key = lambda x: x.timestamp
706 # Linearize their subscription-actions
707 return linearize(action_cmp_key, subscription_action_lists, reverse)
710 def get_global_subscription_history(self, public=None):
711 """ Actions that added/removed podcasts from the subscription list
713 Returns an iterator of all subscription actions that either
714 * added subscribed a podcast that hasn't been subscribed directly
715 before the action (but could have been subscribed) earlier
716 * removed a subscription of the podcast is not longer subscribed
717 after the action
720 subscriptions = collections.defaultdict(int)
722 for entry in self.get_subscription_history(public=public):
723 if entry.action == 'subscribe':
724 subscriptions[entry.podcast_id] += 1
726 # a new subscription has been added
727 if subscriptions[entry.podcast_id] == 1:
728 yield entry
730 elif entry.action == 'unsubscribe':
731 subscriptions[entry.podcast_id] -= 1
733 # the last subscription has been removed
734 if subscriptions[entry.podcast_id] == 0:
735 yield entry
739 def get_newest_episodes(self, max_date, max_per_podcast=5):
740 """ Returns the newest episodes of all subscribed podcasts
742 Only max_per_podcast episodes per podcast are loaded. Episodes with
743 release dates above max_date are discarded.
745 This method returns a generator that produces the newest episodes.
747 The number of required DB queries is equal to the number of (distinct)
748 podcasts of all consumed episodes (max: number of subscribed podcasts),
749 plus a constant number of initial queries (when the first episode is
750 consumed). """
752 cmp_key = lambda episode: episode.released or datetime(2000, 01, 01)
754 podcasts = list(self.get_subscribed_podcasts())
755 podcasts = filter(lambda p: p.latest_episode_timestamp, podcasts)
756 podcasts = sorted(podcasts, key=lambda p: p.latest_episode_timestamp,
757 reverse=True)
759 podcast_dict = dict((p.get_id(), p) for p in podcasts)
761 # contains the un-yielded episodes, newest first
762 episodes = []
764 for podcast in podcasts:
766 yielded_episodes = 0
768 for episode in episodes:
769 # determine for which episodes there won't be a new episodes
770 # that is newer; those can be yielded
771 if episode.released > podcast.latest_episode_timestamp:
772 p = podcast_dict.get(episode.podcast, None)
773 yield proxy_object(episode, podcast=p)
774 yielded_episodes += 1
775 else:
776 break
778 # remove the episodes that have been yielded before
779 episodes = episodes[yielded_episodes:]
781 # fetch and merge episodes for the next podcast
782 from mygpo.db.couchdb.episode import episodes_for_podcast
783 new_episodes = episodes_for_podcast(podcast, since=1,
784 until=max_date, descending=True, limit=max_per_podcast)
785 episodes = sorted(episodes+new_episodes, key=cmp_key, reverse=True)
788 # yield the remaining episodes
789 for episode in episodes:
790 podcast = podcast_dict.get(episode.podcast, None)
791 yield proxy_object(episode, podcast=podcast)
796 def save(self, *args, **kwargs):
798 from mygpo.db.couchdb.podcast_state import podcast_states_for_user
800 super(User, self).save(*args, **kwargs)
802 podcast_states = podcast_states_for_user(self)
803 for state in podcast_states:
804 @repeat_on_conflict(['state'])
805 def _update_state(state):
806 old_devs = set(state.disabled_devices)
807 state.set_device_state(self.devices)
809 if old_devs != set(state.disabled_devices):
810 state.save()
812 _update_state(state=state)
817 def __eq__(self, other):
818 if not other:
819 return False
821 # ensure that other isn't AnonymousUser
822 return other.is_authenticated() and self._id == other._id
825 def __ne__(self, other):
826 return not(self == other)
829 def __repr__(self):
830 return 'User %s' % self._id
833 class History(object):
835 def __init__(self, user, device):
836 self.user = user
837 self.device = device
840 def __getitem__(self, key):
842 if isinstance(key, slice):
843 start = key.start or 0
844 length = key.stop - start
845 else:
846 start = key
847 length = 1
849 if self.device:
850 return device_history(self.user, self.device, start, length)
852 else:
853 return user_history(self.user, start, length)
857 class HistoryEntry(object):
858 """ A class that can represent subscription and episode actions """
861 @classmethod
862 def from_action_dict(cls, action):
864 entry = HistoryEntry()
866 if 'timestamp' in action:
867 ts = action.pop('timestamp')
868 entry.timestamp = dateutil.parser.parse(ts)
870 for key, value in action.items():
871 setattr(entry, key, value)
873 return entry
876 @property
877 def playmark(self):
878 return getattr(self, 'position', None)
881 @classmethod
882 def fetch_data(cls, user, entries,
883 podcasts=None, episodes=None):
884 """ Efficiently loads additional data for a number of entries """
886 if podcasts is None:
887 # load podcast data
888 podcast_ids = [getattr(x, 'podcast_id', None) for x in entries]
889 podcast_ids = filter(None, podcast_ids)
890 podcasts = podcasts_to_dict(podcast_ids)
892 if episodes is None:
893 from mygpo.db.couchdb.episode import episodes_to_dict
894 # load episode data
895 episode_ids = [getattr(x, 'episode_id', None) for x in entries]
896 episode_ids = filter(None, episode_ids)
897 episodes = episodes_to_dict(episode_ids)
899 # load device data
900 # does not need pre-populated data because no db-access is required
901 device_ids = [getattr(x, 'device_id', None) for x in entries]
902 device_ids = filter(None, device_ids)
903 devices = dict([ (id, user.get_device(id)) for id in device_ids])
906 for entry in entries:
907 podcast_id = getattr(entry, 'podcast_id', None)
908 entry.podcast = podcasts.get(podcast_id, None)
910 episode_id = getattr(entry, 'episode_id', None)
911 entry.episode = episodes.get(episode_id, None)
913 if hasattr(entry, 'user'):
914 entry.user = user
916 device = devices.get(getattr(entry, 'device_id', None), None)
917 entry.device = device
920 return entries