2 import uuid
, collections
3 from datetime
import datetime
5 from itertools
import imap
6 from operator
import itemgetter
10 from couchdbkit
.ext
.django
.schema
import *
12 from django
.core
.cache
import cache
14 from django_couchdb_utils
.registration
.models
import User
as BaseUser
16 from mygpo
.core
.models
import Podcast
17 from mygpo
.utils
import linearize
18 from mygpo
.core
.proxy
import DocumentABCMeta
, proxy_object
19 from mygpo
.decorators
import repeat_on_conflict
20 from mygpo
.users
.ratings
import RatingMixin
21 from mygpo
.users
.sync
import SyncedDevicesMixin
22 from mygpo
.db
.couchdb
.podcast
import podcasts_by_id
, podcasts_to_dict
23 from mygpo
.db
.couchdb
.user
import user_history
, device_history
26 RE_DEVICE_UID
= re
.compile(r
'^[\w.-]+$')
29 class InvalidEpisodeActionAttributes(ValueError):
30 """ raised when the attribues of an episode action fail validation """
33 class DeviceUIDException(Exception):
37 class DeviceDoesNotExist(Exception):
41 class DeviceDeletedException(DeviceDoesNotExist
):
45 class Suggestions(Document
, RatingMixin
):
46 user
= StringProperty(required
=True)
47 user_oldid
= IntegerProperty()
48 podcasts
= StringListProperty()
49 blacklist
= StringListProperty()
52 def get_podcasts(self
, count
=None):
53 user
= User
.get(self
.user
)
54 subscriptions
= user
.get_subscribed_podcast_ids()
56 ids
= filter(lambda x
: not x
in self
.blacklist
+ subscriptions
, self
.podcasts
)
59 return filter(lambda x
: x
and x
.title
, podcasts_by_id(ids
))
64 return super(Suggestions
, self
).__repr
__()
66 return '%d Suggestions for %s (%s)' % \
67 (len(self
.podcasts
), self
.user
, self
._id
)
70 class EpisodeAction(DocumentSchema
):
72 One specific action to an episode. Must
73 always be part of a EpisodeUserState
76 action
= StringProperty(required
=True)
77 timestamp
= DateTimeProperty(required
=True, default
=datetime
.utcnow
)
78 device_oldid
= IntegerProperty(required
=False)
79 device
= StringProperty()
80 started
= IntegerProperty()
81 playmark
= IntegerProperty()
82 total
= IntegerProperty()
84 def __eq__(self
, other
):
85 if not isinstance(other
, EpisodeAction
):
87 vals
= ('action', 'timestamp', 'device', 'started', 'playmark',
89 return all([getattr(self
, v
, None) == getattr(other
, v
, None) for v
in vals
])
92 def to_history_entry(self
):
93 entry
= HistoryEntry()
94 entry
.action
= self
.action
95 entry
.timestamp
= self
.timestamp
96 entry
.device_id
= self
.device
97 entry
.started
= self
.started
98 entry
.position
= self
.playmark
99 entry
.total
= self
.total
104 def validate_time_values(self
):
105 """ Validates allowed combinations of time-values """
107 PLAY_ACTION_KEYS
= ('playmark', 'started', 'total')
109 # Key found, but must not be supplied (no play action!)
110 if self
.action
!= 'play':
111 for key
in PLAY_ACTION_KEYS
:
112 if getattr(self
, key
, None) is not None:
113 raise InvalidEpisodeActionAttributes('%s only allowed in play actions' % key
)
115 # Sanity check: If started or total are given, require playmark
116 if ((self
.started
is not None) or (self
.total
is not None)) and \
117 self
.playmark
is None:
118 raise InvalidEpisodeActionAttributes('started and total require position')
120 # Sanity check: total and playmark can only appear together
121 if ((self
.total
is not None) or (self
.started
is not None)) and \
122 ((self
.total
is None) or (self
.started
is None)):
123 raise InvalidEpisodeActionAttributes('total and started can only appear together')
127 return '%s-Action on %s at %s (in %s)' % \
128 (self
.action
, self
.device
, self
.timestamp
, self
._id
)
132 return hash(frozenset([self
.action
, self
.timestamp
, self
.device
,
133 self
.started
, self
.playmark
, self
.total
]))
136 class Chapter(Document
):
137 """ A user-entered episode chapter """
139 device
= StringProperty()
140 created
= DateTimeProperty()
141 start
= IntegerProperty(required
=True)
142 end
= IntegerProperty(required
=True)
143 label
= StringProperty()
144 advertisement
= BooleanProperty()
148 return '<%s %s (%d-%d)>' % (self
.__class
__.__name
__, self
.label
,
149 self
.start
, self
.end
)
152 class EpisodeUserState(Document
):
154 Contains everything a user has done with an Episode
157 episode
= StringProperty(required
=True)
158 actions
= SchemaListProperty(EpisodeAction
)
159 settings
= DictProperty()
160 user_oldid
= IntegerProperty()
161 user
= StringProperty(required
=True)
162 ref_url
= StringProperty(required
=True)
163 podcast_ref_url
= StringProperty(required
=True)
164 merged_ids
= StringListProperty()
165 chapters
= SchemaListProperty(Chapter
)
166 podcast
= StringProperty(required
=True)
170 def add_actions(self
, actions
):
171 map(EpisodeAction
.validate_time_values
, actions
)
172 self
.actions
= list(self
.actions
) + actions
173 self
.actions
= list(set(self
.actions
))
174 self
.actions
= sorted(self
.actions
, key
=lambda x
: x
.timestamp
)
177 def is_favorite(self
):
178 return self
.settings
.get('is_favorite', False)
181 def set_favorite(self
, set_to
=True):
182 self
.settings
['is_favorite'] = set_to
185 def update_chapters(self
, add
=[], rem
=[]):
186 """ Updates the Chapter list
188 * add contains the chapters to be added
190 * rem contains tuples of (start, end) times. Chapters that match
191 both endpoints will be removed
194 @repeat_on_conflict(['state'])
197 self
.chapters
= self
.chapters
+ [chapter
]
199 for start
, end
in rem
:
200 keep
= lambda c
: c
.start
!= start
or c
.end
!= end
201 self
.chapters
= filter(keep
, self
.chapters
)
208 def get_history_entries(self
):
209 return imap(EpisodeAction
.to_history_entry
, self
.actions
)
213 return 'Episode-State %s (in %s)' % \
214 (self
.episode
, self
._id
)
216 def __eq__(self
, other
):
217 if not isinstance(other
, EpisodeUserState
):
220 return (self
.episode
== other
.episode
and
221 self
.user
== other
.user
)
225 class SubscriptionAction(Document
):
226 action
= StringProperty()
227 timestamp
= DateTimeProperty(default
=datetime
.utcnow
)
228 device
= StringProperty()
231 __metaclass__
= DocumentABCMeta
234 def __cmp__(self
, other
):
235 return cmp(self
.timestamp
, other
.timestamp
)
237 def __eq__(self
, other
):
238 return self
.action
== other
.action
and \
239 self
.timestamp
== other
.timestamp
and \
240 self
.device
== other
.device
243 return hash(self
.action
) + hash(self
.timestamp
) + hash(self
.device
)
246 return '<SubscriptionAction %s on %s at %s>' % (
247 self
.action
, self
.device
, self
.timestamp
)
250 class PodcastUserState(Document
):
252 Contains everything that a user has done
253 with a specific podcast and all its episodes
256 podcast
= StringProperty(required
=True)
257 user_oldid
= IntegerProperty()
258 user
= StringProperty(required
=True)
259 settings
= DictProperty()
260 actions
= SchemaListProperty(SubscriptionAction
)
261 tags
= StringListProperty()
262 ref_url
= StringProperty(required
=True)
263 disabled_devices
= StringListProperty()
264 merged_ids
= StringListProperty()
267 def remove_device(self
, device
):
269 Removes all actions from the podcast state that refer to the
272 self
.actions
= filter(lambda a
: a
.device
!= device
.id, self
.actions
)
275 def subscribe(self
, device
):
276 action
= SubscriptionAction()
277 action
.action
= 'subscribe'
278 action
.device
= device
.id
279 self
.add_actions([action
])
282 def unsubscribe(self
, device
):
283 action
= SubscriptionAction()
284 action
.action
= 'unsubscribe'
285 action
.device
= device
.id
286 self
.add_actions([action
])
289 def add_actions(self
, actions
):
290 self
.actions
= list(set(self
.actions
+ actions
))
291 self
.actions
= sorted(self
.actions
)
294 def add_tags(self
, tags
):
295 self
.tags
= list(set(self
.tags
+ tags
))
298 def set_device_state(self
, devices
):
299 disabled_devices
= [device
.id for device
in devices
if device
.deleted
]
300 self
.disabled_devices
= disabled_devices
303 def get_change_between(self
, device_id
, since
, until
):
305 Returns the change of the subscription status for the given device
306 between the two timestamps.
308 The change is given as either 'subscribe' (the podcast has been
309 subscribed), 'unsubscribed' (the podcast has been unsubscribed) or
313 device_actions
= filter(lambda x
: x
.device
== device_id
, self
.actions
)
314 before
= filter(lambda x
: x
.timestamp
<= since
, device_actions
)
315 after
= filter(lambda x
: x
.timestamp
<= until
, device_actions
)
317 # nothing happened, so there can be no change
321 then
= before
[-1] if before
else None
325 if now
.action
!= 'unsubscribe':
327 elif then
.action
!= now
.action
:
332 def get_subscribed_device_ids(self
):
333 """ device Ids on which the user subscribed to the podcast """
336 for action
in self
.actions
:
337 if action
.action
== "subscribe":
338 if not action
.device
in self
.disabled_devices
:
339 devices
.add(action
.device
)
341 if action
.device
in devices
:
342 devices
.remove(action
.device
)
349 return self
.settings
.get('public_subscription', True)
352 def __eq__(self
, other
):
356 return self
.podcast
== other
.podcast
and \
357 self
.user
== other
.user
360 return 'Podcast %s for User %s (%s)' % \
361 (self
.podcast
, self
.user
, self
._id
)
364 class Device(Document
):
365 id = StringProperty(default
=lambda: uuid
.uuid4().hex)
366 oldid
= IntegerProperty(required
=False)
367 uid
= StringProperty(required
=True)
368 name
= StringProperty(required
=True, default
='New Device')
369 type = StringProperty(required
=True, default
='other')
370 settings
= DictProperty()
371 deleted
= BooleanProperty(default
=False)
372 user_agent
= StringProperty()
375 def get_subscription_changes(self
, since
, until
):
377 Returns the subscription changes for the device as two lists.
378 The first lists contains the Ids of the podcasts that have been
379 subscribed to, the second list of those that have been unsubscribed
383 from mygpo
.db
.couchdb
.podcast_state
import podcast_states_for_device
386 podcast_states
= podcast_states_for_device(self
.id)
387 for p_state
in podcast_states
:
388 change
= p_state
.get_change_between(self
.id, since
, until
)
389 if change
== 'subscribe':
390 add
.append( p_state
.ref_url
)
391 elif change
== 'unsubscribe':
392 rem
.append( p_state
.ref_url
)
397 def get_latest_changes(self
):
399 from mygpo
.db
.couchdb
.podcast_state
import podcast_states_for_device
401 podcast_states
= podcast_states_for_device(self
.id)
402 for p_state
in podcast_states
:
403 actions
= filter(lambda x
: x
.device
== self
.id, reversed(p_state
.actions
))
405 yield (p_state
.podcast
, actions
[0])
408 def get_subscribed_podcast_states(self
):
409 r
= PodcastUserState
.view('subscriptions/by_device',
410 startkey
= [self
.id, None],
411 endkey
= [self
.id, {}],
417 def get_subscribed_podcast_ids(self
):
418 states
= self
.get_subscribed_podcast_states()
419 return [state
.podcast
for state
in states
]
422 def get_subscribed_podcasts(self
):
423 """ Returns all subscribed podcasts for the device
425 The attribute "url" contains the URL that was used when subscribing to
428 states
= self
.get_subscribed_podcast_states()
429 podcast_ids
= [state
.podcast
for state
in states
]
430 podcasts
= podcasts_to_dict(podcast_ids
)
433 podcast
= proxy_object(podcasts
[state
.podcast
], url
=state
.ref_url
)
434 podcasts
[state
.podcast
] = podcast
436 return podcasts
.values()
440 return hash(frozenset([self
.id, self
.uid
, self
.name
, self
.type, self
.deleted
]))
443 def __eq__(self
, other
):
444 return self
.id == other
.id
448 return '<{cls} {id}>'.format(cls
=self
.__class
__.__name
__, id=self
.id)
454 def __unicode__(self
):
459 TOKEN_NAMES
= ('subscriptions_token', 'favorite_feeds_token',
460 'publisher_update_token', 'userpage_token')
463 class TokenException(Exception):
467 class User(BaseUser
, SyncedDevicesMixin
):
468 oldid
= IntegerProperty()
469 settings
= DictProperty()
470 devices
= SchemaListProperty(Device
)
471 published_objects
= StringListProperty()
472 deleted
= BooleanProperty(default
=False)
473 suggestions_up_to_date
= BooleanProperty(default
=False)
474 twitter
= StringProperty()
475 about
= StringProperty()
477 # token for accessing subscriptions of this use
478 subscriptions_token
= StringProperty(default
=None)
480 # token for accessing the favorite-episodes feed of this user
481 favorite_feeds_token
= StringProperty(default
=None)
483 # token for automatically updating feeds published by this user
484 publisher_update_token
= StringProperty(default
=None)
486 # token for accessing the userpage of this user
487 userpage_token
= StringProperty(default
=None)
493 def create_new_token(self
, token_name
, length
=32):
494 """ creates a new random token """
496 if token_name
not in TOKEN_NAMES
:
497 raise TokenException('Invalid token name %s' % token_name
)
499 token
= "".join(random
.sample(string
.letters
+string
.digits
, length
))
500 setattr(self
, token_name
, token
)
504 def get_token(self
, token_name
):
505 """ returns a token, and generate those that are still missing """
509 if token_name
not in TOKEN_NAMES
:
510 raise TokenException('Invalid token name %s' % token_name
)
512 for tn
in TOKEN_NAMES
:
513 if getattr(self
, tn
) is None:
514 self
.create_new_token(tn
)
520 return getattr(self
, token_name
)
525 def active_devices(self
):
526 not_deleted
= lambda d
: not d
.deleted
527 return filter(not_deleted
, self
.devices
)
531 def inactive_devices(self
):
532 deleted
= lambda d
: d
.deleted
533 return filter(deleted
, self
.devices
)
536 def get_devices_by_id(self
):
537 return dict( (device
.id, device
) for device
in self
.devices
)
540 def get_device(self
, id):
542 if not hasattr(self
, '__device_by_id'):
543 self
.__devices
_by
_id
= dict( (d
.id, d
) for d
in self
.devices
)
545 return self
.__devices
_by
_id
.get(id, None)
548 def get_device_by_uid(self
, uid
, only_active
=True):
550 if not hasattr(self
, '__devices_by_uio'):
551 self
.__devices
_by
_uid
= dict( (d
.uid
, d
) for d
in self
.devices
)
554 device
= self
.__devices
_by
_uid
[uid
]
556 if only_active
and device
.deleted
:
557 raise DeviceDeletedException(
558 'Device with UID %s is deleted' % uid
)
562 except KeyError as e
:
563 raise DeviceDoesNotExist('There is no device with UID %s' % uid
)
566 def update_device(self
, device
):
567 """ Sets the device and saves the user """
569 @repeat_on_conflict(['user'])
570 def _update(user
, device
):
571 user
.set_device(device
)
574 _update(user
=self
, device
=device
)
577 def set_device(self
, device
):
579 if not RE_DEVICE_UID
.match(device
.uid
):
580 raise DeviceUIDException(u
"'{uid} is not a valid device ID".format(
583 devices
= list(self
.devices
)
584 ids
= [x
.id for x
in devices
]
585 if not device
.id in ids
:
586 devices
.append(device
)
587 self
.devices
= devices
590 index
= ids
.index(device
.id)
592 devices
.insert(index
, device
)
593 self
.devices
= devices
596 def remove_device(self
, device
):
597 devices
= list(self
.devices
)
598 ids
= [x
.id for x
in devices
]
599 if not device
.id in ids
:
602 index
= ids
.index(device
.id)
604 self
.devices
= devices
606 if self
.is_synced(device
):
607 self
.unsync_device(device
)
610 def get_subscriptions_by_device(self
, public
=None):
611 from mygpo
.db
.couchdb
.podcast_state
import subscriptions_by_user
612 get_dev
= itemgetter(2)
613 groups
= collections
.defaultdict(list)
614 subscriptions
= subscriptions_by_user(self
, public
=public
)
615 subscriptions
= sorted(subscriptions
, key
=get_dev
)
617 for public
, podcast_id
, device_id
in subscriptions
:
618 groups
[device_id
].append(podcast_id
)
623 def get_subscribed_podcast_states(self
, public
=None):
625 Returns the Ids of all subscribed podcasts
628 r
= PodcastUserState
.view('subscriptions/by_user',
629 startkey
= [self
._id
, public
, None, None],
630 endkey
= [self
._id
+'ZZZ', None, None, None],
638 def get_subscribed_podcast_ids(self
, public
=None):
639 states
= self
.get_subscribed_podcast_states(public
=public
)
640 return [state
.podcast
for state
in states
]
644 def get_subscribed_podcasts(self
, public
=None):
645 """ Returns all subscribed podcasts for the user
647 The attribute "url" contains the URL that was used when subscribing to
650 states
= self
.get_subscribed_podcast_states(public
=public
)
651 podcast_ids
= [state
.podcast
for state
in states
]
652 podcasts
= podcasts_to_dict(podcast_ids
)
655 podcast
= proxy_object(podcasts
[state
.podcast
], url
=state
.ref_url
)
656 podcasts
[state
.podcast
] = podcast
658 return podcasts
.values()
662 def get_subscription_history(self
, device_id
=None, reverse
=False, public
=None):
663 """ Returns chronologically ordered subscription history entries
665 Setting device_id restricts the actions to a certain device
668 from mygpo
.db
.couchdb
.podcast_state
import podcast_states_for_user
, \
669 podcast_states_for_device
671 def action_iter(state
):
672 for action
in sorted(state
.actions
, reverse
=reverse
):
673 if device_id
is not None and device_id
!= action
.device
:
676 if public
is not None and state
.is_public() != public
:
679 entry
= HistoryEntry()
680 entry
.timestamp
= action
.timestamp
681 entry
.action
= action
.action
682 entry
.podcast_id
= state
.podcast
683 entry
.device_id
= action
.device
686 if device_id
is None:
687 podcast_states
= podcast_states_for_user(self
)
689 podcast_states
= podcast_states_for_device(device_id
)
691 # create an action_iter for each PodcastUserState
692 subscription_action_lists
= [action_iter(x
) for x
in podcast_states
]
694 action_cmp_key
= lambda x
: x
.timestamp
696 # Linearize their subscription-actions
697 return linearize(action_cmp_key
, subscription_action_lists
, reverse
)
700 def get_global_subscription_history(self
, public
=None):
701 """ Actions that added/removed podcasts from the subscription list
703 Returns an iterator of all subscription actions that either
704 * added subscribed a podcast that hasn't been subscribed directly
705 before the action (but could have been subscribed) earlier
706 * removed a subscription of the podcast is not longer subscribed
710 subscriptions
= collections
.defaultdict(int)
712 for entry
in self
.get_subscription_history(public
=public
):
713 if entry
.action
== 'subscribe':
714 subscriptions
[entry
.podcast_id
] += 1
716 # a new subscription has been added
717 if subscriptions
[entry
.podcast_id
] == 1:
720 elif entry
.action
== 'unsubscribe':
721 subscriptions
[entry
.podcast_id
] -= 1
723 # the last subscription has been removed
724 if subscriptions
[entry
.podcast_id
] == 0:
729 def get_newest_episodes(self
, max_date
, max_per_podcast
=5):
730 """ Returns the newest episodes of all subscribed podcasts
732 Only max_per_podcast episodes per podcast are loaded. Episodes with
733 release dates above max_date are discarded.
735 This method returns a generator that produces the newest episodes.
737 The number of required DB queries is equal to the number of (distinct)
738 podcasts of all consumed episodes (max: number of subscribed podcasts),
739 plus a constant number of initial queries (when the first episode is
742 cmp_key
= lambda episode
: episode
.released
or datetime(2000, 01, 01)
744 podcasts
= list(self
.get_subscribed_podcasts())
745 podcasts
= filter(lambda p
: p
.latest_episode_timestamp
, podcasts
)
746 podcasts
= sorted(podcasts
, key
=lambda p
: p
.latest_episode_timestamp
,
749 podcast_dict
= dict((p
.get_id(), p
) for p
in podcasts
)
751 # contains the un-yielded episodes, newest first
754 for podcast
in podcasts
:
758 for episode
in episodes
:
759 # determine for which episodes there won't be a new episodes
760 # that is newer; those can be yielded
761 if episode
.released
> podcast
.latest_episode_timestamp
:
762 p
= podcast_dict
.get(episode
.podcast
, None)
763 yield proxy_object(episode
, podcast
=p
)
764 yielded_episodes
+= 1
768 # remove the episodes that have been yielded before
769 episodes
= episodes
[yielded_episodes
:]
771 # fetch and merge episodes for the next podcast
772 from mygpo
.db
.couchdb
.episode
import episodes_for_podcast
773 new_episodes
= episodes_for_podcast(podcast
, since
=1,
774 until
=max_date
, descending
=True, limit
=max_per_podcast
)
775 episodes
= sorted(episodes
+new_episodes
, key
=cmp_key
, reverse
=True)
778 # yield the remaining episodes
779 for episode
in episodes
:
780 podcast
= podcast_dict
.get(episode
.podcast
, None)
781 yield proxy_object(episode
, podcast
=podcast
)
786 def save(self
, *args
, **kwargs
):
788 from mygpo
.db
.couchdb
.podcast_state
import podcast_states_for_user
790 super(User
, self
).save(*args
, **kwargs
)
792 podcast_states
= podcast_states_for_user(self
)
793 for state
in podcast_states
:
794 @repeat_on_conflict(['state'])
795 def _update_state(state
):
796 old_devs
= set(state
.disabled_devices
)
797 state
.set_device_state(self
.devices
)
799 if old_devs
!= set(state
.disabled_devices
):
802 _update_state(state
=state
)
807 def __eq__(self
, other
):
811 # ensure that other isn't AnonymousUser
812 return other
.is_authenticated() and self
._id
== other
._id
815 def __ne__(self
, other
):
816 return not(self
== other
)
820 return 'User %s' % self
._id
823 class History(object):
825 def __init__(self
, user
, device
):
830 def __getitem__(self
, key
):
832 if isinstance(key
, slice):
833 start
= key
.start
or 0
834 length
= key
.stop
- start
840 return device_history(self
.user
, self
.device
, start
, length
)
843 return user_history(self
.user
, start
, length
)
847 class HistoryEntry(object):
848 """ A class that can represent subscription and episode actions """
852 def from_action_dict(cls
, action
):
854 entry
= HistoryEntry()
856 if 'timestamp' in action
:
857 ts
= action
.pop('timestamp')
858 entry
.timestamp
= dateutil
.parser
.parse(ts
)
860 for key
, value
in action
.items():
861 setattr(entry
, key
, value
)
868 return getattr(self
, 'position', None)
872 def fetch_data(cls
, user
, entries
,
873 podcasts
=None, episodes
=None):
874 """ Efficiently loads additional data for a number of entries """
878 podcast_ids
= [getattr(x
, 'podcast_id', None) for x
in entries
]
879 podcast_ids
= filter(None, podcast_ids
)
880 podcasts
= podcasts_to_dict(podcast_ids
)
883 from mygpo
.db
.couchdb
.episode
import episodes_to_dict
885 episode_ids
= [getattr(x
, 'episode_id', None) for x
in entries
]
886 episode_ids
= filter(None, episode_ids
)
887 episodes
= episodes_to_dict(episode_ids
)
890 # does not need pre-populated data because no db-access is required
891 device_ids
= [getattr(x
, 'device_id', None) for x
in entries
]
892 device_ids
= filter(None, device_ids
)
893 devices
= dict([ (id, user
.get_device(id)) for id in device_ids
])
896 for entry
in entries
:
897 podcast_id
= getattr(entry
, 'podcast_id', None)
898 entry
.podcast
= podcasts
.get(podcast_id
, None)
900 episode_id
= getattr(entry
, 'episode_id', None)
901 entry
.episode
= episodes
.get(episode_id
, None)
903 if hasattr(entry
, 'user'):
906 device
= devices
.get(getattr(entry
, 'device_id', None), None)
907 entry
.device
= device