[Migration] remove CouchDB fulltext index
[mygpo.git] / mygpo / users / models.py
blobf9f73d7d359bafd3d8b729063388dd2f6fba2b63
1 import re
2 import uuid
3 import collections
4 from datetime import datetime
5 import dateutil.parser
6 from itertools import imap
7 from operator import itemgetter
8 import random
9 import string
11 from couchdbkit.ext.django.schema import *
13 from django.core.cache import cache
15 from django_couchdb_utils.registration.models import User as BaseUser
17 from mygpo.podcasts.models import Podcast
18 from mygpo.utils import linearize
19 from mygpo.core.proxy import DocumentABCMeta, proxy_object
20 from mygpo.decorators import repeat_on_conflict
21 from mygpo.users.ratings import RatingMixin
22 from mygpo.users.sync import SyncedDevicesMixin
23 from mygpo.users.subscriptions import subscription_changes, podcasts_for_states
24 from mygpo.users.settings import FAV_FLAG, PUBLIC_SUB_PODCAST, SettingsMixin
25 from mygpo.db.couchdb.user import user_history, device_history, \
26 create_missing_user_tokens
28 # make sure this code is executed at startup
29 from mygpo.users.signals import *
32 RE_DEVICE_UID = re.compile(r'^[\w.-]+$')
34 # TODO: derive from ValidationException?
35 class InvalidEpisodeActionAttributes(ValueError):
36 """ raised when the attribues of an episode action fail validation """
39 class SubscriptionException(Exception):
40 """ raised when a subscription can not be modified """
43 class DeviceUIDException(Exception):
44 pass
47 class DeviceDoesNotExist(Exception):
48 pass
51 class DeviceDeletedException(DeviceDoesNotExist):
52 pass
55 class Suggestions(Document, RatingMixin):
56 user = StringProperty(required=True)
57 user_oldid = IntegerProperty()
58 podcasts = StringListProperty()
59 blacklist = StringListProperty()
62 def get_podcasts(self, count=None):
63 user = User.get(self.user)
64 subscriptions = user.get_subscribed_podcast_ids()
66 ids = filter(lambda x: not x in self.blacklist + subscriptions, self.podcasts)
67 if count:
68 ids = ids[:count]
69 return filter(lambda x: x and x.title, Podcast.objects.filter(id__in=ids))
72 def __repr__(self):
73 if not self._id:
74 return super(Suggestions, self).__repr__()
75 else:
76 return '%d Suggestions for %s (%s)' % \
77 (len(self.podcasts), self.user, self._id)
80 class EpisodeAction(DocumentSchema):
81 """
82 One specific action to an episode. Must
83 always be part of a EpisodeUserState
84 """
86 action = StringProperty(required=True)
88 # walltime of the event (assigned by the uploading client, defaults to now)
89 timestamp = DateTimeProperty(required=True, default=datetime.utcnow)
91 # upload time of the event
92 upload_timestamp = IntegerProperty(required=True)
94 device_oldid = IntegerProperty(required=False)
95 device = StringProperty()
96 started = IntegerProperty()
97 playmark = IntegerProperty()
98 total = IntegerProperty()
100 def __eq__(self, other):
101 if not isinstance(other, EpisodeAction):
102 return False
103 vals = ('action', 'timestamp', 'device', 'started', 'playmark',
104 'total')
105 return all([getattr(self, v, None) == getattr(other, v, None) for v in vals])
108 def to_history_entry(self):
109 entry = HistoryEntry()
110 entry.action = self.action
111 entry.timestamp = self.timestamp
112 entry.device_id = self.device
113 entry.started = self.started
114 entry.position = self.playmark
115 entry.total = self.total
116 return entry
120 def validate_time_values(self):
121 """ Validates allowed combinations of time-values """
123 PLAY_ACTION_KEYS = ('playmark', 'started', 'total')
125 # Key found, but must not be supplied (no play action!)
126 if self.action != 'play':
127 for key in PLAY_ACTION_KEYS:
128 if getattr(self, key, None) is not None:
129 raise InvalidEpisodeActionAttributes('%s only allowed in play actions' % key)
131 # Sanity check: If started or total are given, require playmark
132 if ((self.started is not None) or (self.total is not None)) and \
133 self.playmark is None:
134 raise InvalidEpisodeActionAttributes('started and total require position')
136 # Sanity check: total and playmark can only appear together
137 if ((self.total is not None) or (self.started is not None)) and \
138 ((self.total is None) or (self.started is None)):
139 raise InvalidEpisodeActionAttributes('total and started can only appear together')
142 def __repr__(self):
143 return '%s-Action on %s at %s (in %s)' % \
144 (self.action, self.device, self.timestamp, self._id)
147 def __hash__(self):
148 return hash(frozenset([self.action, self.timestamp, self.device,
149 self.started, self.playmark, self.total]))
152 class Chapter(Document):
153 """ A user-entered episode chapter """
155 device = StringProperty()
156 created = DateTimeProperty()
157 start = IntegerProperty(required=True)
158 end = IntegerProperty(required=True)
159 label = StringProperty()
160 advertisement = BooleanProperty()
163 def __repr__(self):
164 return '<%s %s (%d-%d)>' % (self.__class__.__name__, self.label,
165 self.start, self.end)
168 class EpisodeUserState(Document, SettingsMixin):
170 Contains everything a user has done with an Episode
173 episode = StringProperty(required=True)
174 actions = SchemaListProperty(EpisodeAction)
175 user_oldid = IntegerProperty()
176 user = StringProperty(required=True)
177 ref_url = StringProperty(required=True)
178 podcast_ref_url = StringProperty(required=True)
179 merged_ids = StringListProperty()
180 chapters = SchemaListProperty(Chapter)
181 podcast = StringProperty(required=True)
185 def add_actions(self, actions):
186 map(EpisodeAction.validate_time_values, actions)
187 self.actions = list(self.actions) + actions
188 self.actions = list(set(self.actions))
189 self.actions = sorted(self.actions, key=lambda x: x.timestamp)
192 def is_favorite(self):
193 return self.get_wksetting(FAV_FLAG)
196 def set_favorite(self, set_to=True):
197 self.settings[FAV_FLAG.name] = set_to
200 def get_history_entries(self):
201 return imap(EpisodeAction.to_history_entry, self.actions)
204 def __repr__(self):
205 return 'Episode-State %s (in %s)' % \
206 (self.episode, self._id)
208 def __eq__(self, other):
209 if not isinstance(other, EpisodeUserState):
210 return False
212 return (self.episode == other.episode and
213 self.user == other.user)
217 class SubscriptionAction(Document):
218 action = StringProperty()
219 timestamp = DateTimeProperty(default=datetime.utcnow)
220 device = StringProperty()
223 __metaclass__ = DocumentABCMeta
226 def __cmp__(self, other):
227 return cmp(self.timestamp, other.timestamp)
229 def __eq__(self, other):
230 return self.action == other.action and \
231 self.timestamp == other.timestamp and \
232 self.device == other.device
234 def __hash__(self):
235 return hash(self.action) + hash(self.timestamp) + hash(self.device)
237 def __repr__(self):
238 return '<SubscriptionAction %s on %s at %s>' % (
239 self.action, self.device, self.timestamp)
242 class PodcastUserState(Document, SettingsMixin):
244 Contains everything that a user has done
245 with a specific podcast and all its episodes
248 podcast = StringProperty(required=True)
249 user_oldid = IntegerProperty()
250 user = StringProperty(required=True)
251 actions = SchemaListProperty(SubscriptionAction)
252 tags = StringListProperty()
253 ref_url = StringProperty(required=True)
254 disabled_devices = StringListProperty()
255 merged_ids = StringListProperty()
258 def remove_device(self, device):
260 Removes all actions from the podcast state that refer to the
261 given device
263 self.actions = filter(lambda a: a.device != device.id, self.actions)
266 def subscribe(self, device):
267 action = SubscriptionAction()
268 action.action = 'subscribe'
269 action.device = device.id
270 self.add_actions([action])
273 def unsubscribe(self, device):
274 action = SubscriptionAction()
275 action.action = 'unsubscribe'
276 action.device = device.id
277 self.add_actions([action])
280 def add_actions(self, actions):
281 self.actions = list(set(self.actions + actions))
282 self.actions = sorted(self.actions)
285 def add_tags(self, tags):
286 self.tags = list(set(self.tags + tags))
289 def set_device_state(self, devices):
290 disabled_devices = [device.id for device in devices if device.deleted]
291 self.disabled_devices = disabled_devices
294 def get_change_between(self, device_id, since, until):
296 Returns the change of the subscription status for the given device
297 between the two timestamps.
299 The change is given as either 'subscribe' (the podcast has been
300 subscribed), 'unsubscribed' (the podcast has been unsubscribed) or
301 None (no change)
304 device_actions = filter(lambda x: x.device == device_id, self.actions)
305 before = filter(lambda x: x.timestamp <= since, device_actions)
306 after = filter(lambda x: x.timestamp <= until, device_actions)
308 # nothing happened, so there can be no change
309 if not after:
310 return None
312 then = before[-1] if before else None
313 now = after[-1]
315 if then is None:
316 if now.action != 'unsubscribe':
317 return now.action
318 elif then.action != now.action:
319 return now.action
320 return None
323 def get_subscribed_device_ids(self):
324 """ device Ids on which the user subscribed to the podcast """
325 devices = set()
327 for action in self.actions:
328 if action.action == "subscribe":
329 if not action.device in self.disabled_devices:
330 devices.add(action.device)
331 else:
332 if action.device in devices:
333 devices.remove(action.device)
335 return devices
338 def is_subscribed_on(self, device):
339 """ checks if the podcast is subscribed on the given device """
341 for action in reversed(self.actions):
342 if not action.device == device.id:
343 continue
345 # we only need to check the latest action for the device
346 return (action.action == 'subscribe')
348 # we haven't found any matching action
349 return False
352 def is_public(self):
353 return self.get_wksetting(PUBLIC_SUB_PODCAST)
356 def __eq__(self, other):
357 if other is None:
358 return False
360 return self.podcast == other.podcast and \
361 self.user == other.user
363 def __repr__(self):
364 return 'Podcast %s for User %s (%s)' % \
365 (self.podcast, self.user, self._id)
368 class Device(Document, SettingsMixin):
369 id = StringProperty(default=lambda: uuid.uuid4().hex)
370 oldid = IntegerProperty(required=False)
371 uid = StringProperty(required=True)
372 name = StringProperty(required=True, default='New Device')
373 type = StringProperty(required=True, default='other')
374 deleted = BooleanProperty(default=False)
375 user_agent = StringProperty()
378 def get_subscription_changes(self, since, until):
380 Returns the subscription changes for the device as two lists.
381 The first lists contains the Ids of the podcasts that have been
382 subscribed to, the second list of those that have been unsubscribed
383 from.
386 from mygpo.db.couchdb.podcast_state import podcast_states_for_device
387 podcast_states = podcast_states_for_device(self.id)
388 return subscription_changes(self.id, podcast_states, since, until)
391 def get_latest_changes(self):
393 from mygpo.db.couchdb.podcast_state import podcast_states_for_device
395 podcast_states = podcast_states_for_device(self.id)
396 for p_state in podcast_states:
397 actions = filter(lambda x: x.device == self.id, reversed(p_state.actions))
398 if actions:
399 yield (p_state.podcast, actions[0])
402 def get_subscribed_podcast_ids(self):
403 from mygpo.db.couchdb.podcast_state import get_subscribed_podcast_states_by_device
404 states = get_subscribed_podcast_states_by_device(self)
405 return [state.podcast for state in states]
408 def get_subscribed_podcasts(self):
409 """ Returns all subscribed podcasts for the device
411 The attribute "url" contains the URL that was used when subscribing to
412 the podcast """
414 from mygpo.db.couchdb.podcast_state import get_subscribed_podcast_states_by_device
415 states = get_subscribed_podcast_states_by_device(self)
416 return podcasts_for_states(states)
419 def __hash__(self):
420 return hash(frozenset([self.id, self.uid, self.name, self.type, self.deleted]))
423 def __eq__(self, other):
424 return self.id == other.id
427 def __repr__(self):
428 return '<{cls} {id}>'.format(cls=self.__class__.__name__, id=self.id)
431 def __str__(self):
432 return self.name
434 def __unicode__(self):
435 return self.name
439 TOKEN_NAMES = ('subscriptions_token', 'favorite_feeds_token',
440 'publisher_update_token', 'userpage_token')
443 class TokenException(Exception):
444 pass
447 class User(BaseUser, SyncedDevicesMixin, SettingsMixin):
448 oldid = IntegerProperty()
449 devices = SchemaListProperty(Device)
450 published_objects = StringListProperty()
451 deleted = BooleanProperty(default=False)
452 suggestions_up_to_date = BooleanProperty(default=False)
453 twitter = StringProperty()
454 about = StringProperty()
455 google_email = StringProperty()
457 # token for accessing subscriptions of this use
458 subscriptions_token = StringProperty(default=None)
460 # token for accessing the favorite-episodes feed of this user
461 favorite_feeds_token = StringProperty(default=None)
463 # token for automatically updating feeds published by this user
464 publisher_update_token = StringProperty(default=None)
466 # token for accessing the userpage of this user
467 userpage_token = StringProperty(default=None)
469 class Meta:
470 app_label = 'users'
473 def create_new_token(self, token_name, length=32):
474 """ creates a new random token """
476 if token_name not in TOKEN_NAMES:
477 raise TokenException('Invalid token name %s' % token_name)
479 token = "".join(random.sample(string.letters+string.digits, length))
480 setattr(self, token_name, token)
484 @repeat_on_conflict(['self'])
485 def get_token(self, token_name):
486 """ returns a token, and generate those that are still missing """
488 generated = False
490 if token_name not in TOKEN_NAMES:
491 raise TokenException('Invalid token name %s' % token_name)
493 create_missing_user_tokens(self)
495 return getattr(self, token_name)
499 @property
500 def active_devices(self):
501 not_deleted = lambda d: not d.deleted
502 return filter(not_deleted, self.devices)
505 @property
506 def inactive_devices(self):
507 deleted = lambda d: d.deleted
508 return filter(deleted, self.devices)
511 def get_devices_by_id(self, device_ids=None):
512 """ Returns a dict of {devices_id: device} """
513 if device_ids is None:
514 # return all devices
515 devices = self.devices
516 else:
517 devices = self.get_devices(device_ids)
519 return {device.id: device for device in devices}
522 def get_device(self, id):
524 if not hasattr(self, '__device_by_id'):
525 self.__devices_by_id = self.get_devices_by_id()
527 return self.__devices_by_id.get(id, None)
530 def get_devices(self, ids):
531 return filter(None, (self.get_device(dev_id) for dev_id in ids))
534 def get_device_by_uid(self, uid, only_active=True):
536 if not hasattr(self, '__devices_by_uio'):
537 self.__devices_by_uid = dict( (d.uid, d) for d in self.devices)
539 try:
540 device = self.__devices_by_uid[uid]
542 if only_active and device.deleted:
543 raise DeviceDeletedException(
544 'Device with UID %s is deleted' % uid)
546 return device
548 except KeyError as e:
549 raise DeviceDoesNotExist('There is no device with UID %s' % uid)
552 def set_device(self, device):
554 if not RE_DEVICE_UID.match(device.uid):
555 raise DeviceUIDException(u"'{uid} is not a valid device ID".format(
556 uid=device.uid))
558 devices = list(self.devices)
559 ids = [x.id for x in devices]
560 if not device.id in ids:
561 devices.append(device)
562 self.devices = devices
563 return
565 index = ids.index(device.id)
566 devices.pop(index)
567 devices.insert(index, device)
568 self.devices = devices
571 def remove_device(self, device):
572 devices = list(self.devices)
573 ids = [x.id for x in devices]
574 if not device.id in ids:
575 return
577 index = ids.index(device.id)
578 devices.pop(index)
579 self.devices = devices
581 if self.is_synced(device):
582 self.unsync_device(device)
585 def get_subscriptions_by_device(self, public=None):
586 from mygpo.db.couchdb.podcast_state import subscriptions_by_user
587 get_dev = itemgetter(2)
588 groups = collections.defaultdict(list)
589 subscriptions = subscriptions_by_user(self, public=public)
590 subscriptions = sorted(subscriptions, key=get_dev)
592 for public, podcast_id, device_id in subscriptions:
593 groups[device_id].append(podcast_id)
595 return groups
597 def get_subscribed_podcast_ids(self, public=None):
598 from mygpo.db.couchdb.podcast_state import get_subscribed_podcast_states_by_user
599 states = get_subscribed_podcast_states_by_user(self, public)
600 return [state.podcast for state in states]
604 def get_subscribed_podcasts(self, public=None):
605 """ Returns all subscribed podcasts for the user
607 The attribute "url" contains the URL that was used when subscribing to
608 the podcast """
610 from mygpo.db.couchdb.podcast_state import get_subscribed_podcast_states_by_user
611 states = get_subscribed_podcast_states_by_user(self, public)
612 podcast_ids = [state.podcast for state in states]
613 podcasts = Podcast.objects.get(id__in=podcast_ids)
614 podcasts = {podcast.id: podcast for podcast in podcasts}
616 for state in states:
617 podcast = podcasts.get(state.podcast, None)
618 if podcast is None:
619 continue
621 podcast = proxy_object(podcast, url=state.ref_url)
622 podcasts[state.podcast] = podcast
624 return set(podcasts.values())
628 def get_subscription_history(self, device_id=None, reverse=False, public=None):
629 """ Returns chronologically ordered subscription history entries
631 Setting device_id restricts the actions to a certain device
634 from mygpo.db.couchdb.podcast_state import podcast_states_for_user, \
635 podcast_states_for_device
637 def action_iter(state):
638 for action in sorted(state.actions, reverse=reverse):
639 if device_id is not None and device_id != action.device:
640 continue
642 if public is not None and state.is_public() != public:
643 continue
645 entry = HistoryEntry()
646 entry.timestamp = action.timestamp
647 entry.action = action.action
648 entry.podcast_id = state.podcast
649 entry.device_id = action.device
650 yield entry
652 if device_id is None:
653 podcast_states = podcast_states_for_user(self)
654 else:
655 podcast_states = podcast_states_for_device(device_id)
657 # create an action_iter for each PodcastUserState
658 subscription_action_lists = [action_iter(x) for x in podcast_states]
660 action_cmp_key = lambda x: x.timestamp
662 # Linearize their subscription-actions
663 return linearize(action_cmp_key, subscription_action_lists, reverse)
666 def get_global_subscription_history(self, public=None):
667 """ Actions that added/removed podcasts from the subscription list
669 Returns an iterator of all subscription actions that either
670 * added subscribed a podcast that hasn't been subscribed directly
671 before the action (but could have been subscribed) earlier
672 * removed a subscription of the podcast is not longer subscribed
673 after the action
676 subscriptions = collections.defaultdict(int)
678 for entry in self.get_subscription_history(public=public):
679 if entry.action == 'subscribe':
680 subscriptions[entry.podcast_id] += 1
682 # a new subscription has been added
683 if subscriptions[entry.podcast_id] == 1:
684 yield entry
686 elif entry.action == 'unsubscribe':
687 subscriptions[entry.podcast_id] -= 1
689 # the last subscription has been removed
690 if subscriptions[entry.podcast_id] == 0:
691 yield entry
695 def get_newest_episodes(self, max_date, max_per_podcast=5):
696 """ Returns the newest episodes of all subscribed podcasts
698 Only max_per_podcast episodes per podcast are loaded. Episodes with
699 release dates above max_date are discarded.
701 This method returns a generator that produces the newest episodes.
703 The number of required DB queries is equal to the number of (distinct)
704 podcasts of all consumed episodes (max: number of subscribed podcasts),
705 plus a constant number of initial queries (when the first episode is
706 consumed). """
708 cmp_key = lambda episode: episode.released or datetime(2000, 01, 01)
710 podcasts = list(self.get_subscribed_podcasts())
711 podcasts = filter(lambda p: p.latest_episode_timestamp, podcasts)
712 podcasts = sorted(podcasts, key=lambda p: p.latest_episode_timestamp,
713 reverse=True)
715 podcast_dict = dict((p.get_id(), p) for p in podcasts)
717 # contains the un-yielded episodes, newest first
718 episodes = []
720 for podcast in podcasts:
722 yielded_episodes = 0
724 for episode in episodes:
725 # determine for which episodes there won't be a new episodes
726 # that is newer; those can be yielded
727 if episode.released > podcast.latest_episode_timestamp:
728 p = podcast_dict.get(episode.podcast, None)
729 yield proxy_object(episode, podcast=p)
730 yielded_episodes += 1
731 else:
732 break
734 # remove the episodes that have been yielded before
735 episodes = episodes[yielded_episodes:]
737 # fetch and merge episodes for the next podcast
738 # TODO: max_per_podcast
739 new_episodes = podcast.episode_set.filter(release__isnull=False, released__lt=max_date)[:max_per_podcast]
740 episodes = sorted(episodes+new_episodes, key=cmp_key, reverse=True)
743 # yield the remaining episodes
744 for episode in episodes:
745 podcast = podcast_dict.get(episode.podcast, None)
746 yield proxy_object(episode, podcast=podcast)
749 def __eq__(self, other):
750 if not other:
751 return False
753 # ensure that other isn't AnonymousUser
754 return other.is_authenticated() and self._id == other._id
757 def __ne__(self, other):
758 return not(self == other)
761 def __repr__(self):
762 return 'User %s' % self._id
765 class History(object):
767 def __init__(self, user, device):
768 self.user = user
769 self.device = device
772 def __getitem__(self, key):
774 if isinstance(key, slice):
775 start = key.start or 0
776 length = key.stop - start
777 else:
778 start = key
779 length = 1
781 if self.device:
782 return device_history(self.user, self.device, start, length)
784 else:
785 return user_history(self.user, start, length)
789 class HistoryEntry(object):
790 """ A class that can represent subscription and episode actions """
793 @classmethod
794 def from_action_dict(cls, action):
796 entry = HistoryEntry()
798 if 'timestamp' in action:
799 ts = action.pop('timestamp')
800 entry.timestamp = dateutil.parser.parse(ts)
802 for key, value in action.items():
803 setattr(entry, key, value)
805 return entry
808 @property
809 def playmark(self):
810 return getattr(self, 'position', None)
813 @classmethod
814 def fetch_data(cls, user, entries,
815 podcasts=None, episodes=None):
816 """ Efficiently loads additional data for a number of entries """
818 if podcasts is None:
819 # load podcast data
820 podcast_ids = [getattr(x, 'podcast_id', None) for x in entries]
821 podcast_ids = filter(None, podcast_ids)
822 podcasts = Podcast.objects.filter(id__in=podcast_ids)
823 podcasts = {podcast.id: podcast for podcast in podcasts}
825 if episodes is None:
826 # load episode data
827 episode_ids = [getattr(x, 'episode_id', None) for x in entries]
828 episode_ids = filter(None, episode_ids)
829 episodes = Episode.objects.filter(id__in=episode_ids)
830 episodes = {episode.id: episode for episode in episodes}
832 # load device data
833 # does not need pre-populated data because no db-access is required
834 device_ids = [getattr(x, 'device_id', None) for x in entries]
835 device_ids = filter(None, device_ids)
836 devices = dict([ (id, user.get_device(id)) for id in device_ids])
839 for entry in entries:
840 podcast_id = getattr(entry, 'podcast_id', None)
841 entry.podcast = podcasts.get(podcast_id, None)
843 episode_id = getattr(entry, 'episode_id', None)
844 entry.episode = episodes.get(episode_id, None)
846 if hasattr(entry, 'user'):
847 entry.user = user
849 device = devices.get(getattr(entry, 'device_id', None), None)
850 entry.device = device
853 return entries