2 import uuid
, collections
3 from datetime
import datetime
5 from itertools
import imap
6 from operator
import itemgetter
10 from couchdbkit
.ext
.django
.schema
import *
12 from django
.core
.cache
import cache
14 from django_couchdb_utils
.registration
.models
import User
as BaseUser
16 from mygpo
.utils
import linearize
17 from mygpo
.core
.proxy
import DocumentABCMeta
, proxy_object
18 from mygpo
.decorators
import repeat_on_conflict
19 from mygpo
.users
.ratings
import RatingMixin
20 from mygpo
.users
.sync
import SyncedDevicesMixin
21 from mygpo
.db
.couchdb
.podcast
import podcasts_by_id
, podcasts_to_dict
22 from mygpo
.db
.couchdb
.user
import user_history
, device_history
26 RE_DEVICE_UID
= re
.compile(r
'^[\w.-]+$')
28 # TODO: derive from ValidationException?
29 class InvalidEpisodeActionAttributes(ValueError):
30 """ raised when the attribues of an episode action fail validation """
33 class DeviceUIDException(Exception):
37 class DeviceDoesNotExist(Exception):
41 class DeviceDeletedException(DeviceDoesNotExist
):
45 class Suggestions(Document
, RatingMixin
):
46 user
= StringProperty(required
=True)
47 user_oldid
= IntegerProperty()
48 podcasts
= StringListProperty()
49 blacklist
= StringListProperty()
52 def get_podcasts(self
, count
=None):
53 user
= User
.get(self
.user
)
54 subscriptions
= user
.get_subscribed_podcast_ids()
56 ids
= filter(lambda x
: not x
in self
.blacklist
+ subscriptions
, self
.podcasts
)
59 return filter(lambda x
: x
and x
.title
, podcasts_by_id(ids
))
64 return super(Suggestions
, self
).__repr
__()
66 return '%d Suggestions for %s (%s)' % \
67 (len(self
.podcasts
), self
.user
, self
._id
)
70 class EpisodeAction(DocumentSchema
):
72 One specific action to an episode. Must
73 always be part of a EpisodeUserState
76 action
= StringProperty(required
=True)
78 # walltime of the event (assigned by the uploading client, defaults to now)
79 timestamp
= DateTimeProperty(required
=True, default
=datetime
.utcnow
)
81 # upload time of the event
82 upload_timestamp
= IntegerProperty(required
=True)
84 device_oldid
= IntegerProperty(required
=False)
85 device
= StringProperty()
86 started
= IntegerProperty()
87 playmark
= IntegerProperty()
88 total
= IntegerProperty()
90 def __eq__(self
, other
):
91 if not isinstance(other
, EpisodeAction
):
93 vals
= ('action', 'timestamp', 'device', 'started', 'playmark',
95 return all([getattr(self
, v
, None) == getattr(other
, v
, None) for v
in vals
])
98 def to_history_entry(self
):
99 entry
= HistoryEntry()
100 entry
.action
= self
.action
101 entry
.timestamp
= self
.timestamp
102 entry
.device_id
= self
.device
103 entry
.started
= self
.started
104 entry
.position
= self
.playmark
105 entry
.total
= self
.total
110 def validate_time_values(self
):
111 """ Validates allowed combinations of time-values """
113 PLAY_ACTION_KEYS
= ('playmark', 'started', 'total')
115 # Key found, but must not be supplied (no play action!)
116 if self
.action
!= 'play':
117 for key
in PLAY_ACTION_KEYS
:
118 if getattr(self
, key
, None) is not None:
119 raise InvalidEpisodeActionAttributes('%s only allowed in play actions' % key
)
121 # Sanity check: If started or total are given, require playmark
122 if ((self
.started
is not None) or (self
.total
is not None)) and \
123 self
.playmark
is None:
124 raise InvalidEpisodeActionAttributes('started and total require position')
126 # Sanity check: total and playmark can only appear together
127 if ((self
.total
is not None) or (self
.started
is not None)) and \
128 ((self
.total
is None) or (self
.started
is None)):
129 raise InvalidEpisodeActionAttributes('total and started can only appear together')
133 return '%s-Action on %s at %s (in %s)' % \
134 (self
.action
, self
.device
, self
.timestamp
, self
._id
)
138 return hash(frozenset([self
.action
, self
.timestamp
, self
.device
,
139 self
.started
, self
.playmark
, self
.total
]))
142 class Chapter(Document
):
143 """ A user-entered episode chapter """
145 device
= StringProperty()
146 created
= DateTimeProperty()
147 start
= IntegerProperty(required
=True)
148 end
= IntegerProperty(required
=True)
149 label
= StringProperty()
150 advertisement
= BooleanProperty()
154 return '<%s %s (%d-%d)>' % (self
.__class
__.__name
__, self
.label
,
155 self
.start
, self
.end
)
158 class EpisodeUserState(Document
):
160 Contains everything a user has done with an Episode
163 episode
= StringProperty(required
=True)
164 actions
= SchemaListProperty(EpisodeAction
)
165 settings
= DictProperty()
166 user_oldid
= IntegerProperty()
167 user
= StringProperty(required
=True)
168 ref_url
= StringProperty(required
=True)
169 podcast_ref_url
= StringProperty(required
=True)
170 merged_ids
= StringListProperty()
171 chapters
= SchemaListProperty(Chapter
)
172 podcast
= StringProperty(required
=True)
176 def add_actions(self
, actions
):
177 map(EpisodeAction
.validate_time_values
, actions
)
178 self
.actions
= list(self
.actions
) + actions
179 self
.actions
= list(set(self
.actions
))
180 self
.actions
= sorted(self
.actions
, key
=lambda x
: x
.timestamp
)
183 def is_favorite(self
):
184 return self
.settings
.get('is_favorite', False)
187 def set_favorite(self
, set_to
=True):
188 self
.settings
['is_favorite'] = set_to
191 def update_chapters(self
, add
=[], rem
=[]):
192 """ Updates the Chapter list
194 * add contains the chapters to be added
196 * rem contains tuples of (start, end) times. Chapters that match
197 both endpoints will be removed
200 @repeat_on_conflict(['state'])
203 self
.chapters
= self
.chapters
+ [chapter
]
205 for start
, end
in rem
:
206 keep
= lambda c
: c
.start
!= start
or c
.end
!= end
207 self
.chapters
= filter(keep
, self
.chapters
)
214 def get_history_entries(self
):
215 return imap(EpisodeAction
.to_history_entry
, self
.actions
)
219 return 'Episode-State %s (in %s)' % \
220 (self
.episode
, self
._id
)
222 def __eq__(self
, other
):
223 if not isinstance(other
, EpisodeUserState
):
226 return (self
.episode
== other
.episode
and
227 self
.user
== other
.user
)
231 class SubscriptionAction(Document
):
232 action
= StringProperty()
233 timestamp
= DateTimeProperty(default
=datetime
.utcnow
)
234 device
= StringProperty()
237 __metaclass__
= DocumentABCMeta
240 def __cmp__(self
, other
):
241 return cmp(self
.timestamp
, other
.timestamp
)
243 def __eq__(self
, other
):
244 return self
.action
== other
.action
and \
245 self
.timestamp
== other
.timestamp
and \
246 self
.device
== other
.device
249 return hash(self
.action
) + hash(self
.timestamp
) + hash(self
.device
)
252 return '<SubscriptionAction %s on %s at %s>' % (
253 self
.action
, self
.device
, self
.timestamp
)
256 class PodcastUserState(Document
):
258 Contains everything that a user has done
259 with a specific podcast and all its episodes
262 podcast
= StringProperty(required
=True)
263 user_oldid
= IntegerProperty()
264 user
= StringProperty(required
=True)
265 settings
= DictProperty()
266 actions
= SchemaListProperty(SubscriptionAction
)
267 tags
= StringListProperty()
268 ref_url
= StringProperty(required
=True)
269 disabled_devices
= StringListProperty()
270 merged_ids
= StringListProperty()
273 def remove_device(self
, device
):
275 Removes all actions from the podcast state that refer to the
278 self
.actions
= filter(lambda a
: a
.device
!= device
.id, self
.actions
)
281 def subscribe(self
, device
):
282 action
= SubscriptionAction()
283 action
.action
= 'subscribe'
284 action
.device
= device
.id
285 self
.add_actions([action
])
288 def unsubscribe(self
, device
):
289 action
= SubscriptionAction()
290 action
.action
= 'unsubscribe'
291 action
.device
= device
.id
292 self
.add_actions([action
])
295 def add_actions(self
, actions
):
296 self
.actions
= list(set(self
.actions
+ actions
))
297 self
.actions
= sorted(self
.actions
)
300 def add_tags(self
, tags
):
301 self
.tags
= list(set(self
.tags
+ tags
))
304 def set_device_state(self
, devices
):
305 disabled_devices
= [device
.id for device
in devices
if device
.deleted
]
306 self
.disabled_devices
= disabled_devices
309 def get_change_between(self
, device_id
, since
, until
):
311 Returns the change of the subscription status for the given device
312 between the two timestamps.
314 The change is given as either 'subscribe' (the podcast has been
315 subscribed), 'unsubscribed' (the podcast has been unsubscribed) or
319 device_actions
= filter(lambda x
: x
.device
== device_id
, self
.actions
)
320 before
= filter(lambda x
: x
.timestamp
<= since
, device_actions
)
321 after
= filter(lambda x
: x
.timestamp
<= until
, device_actions
)
323 # nothing happened, so there can be no change
327 then
= before
[-1] if before
else None
331 if now
.action
!= 'unsubscribe':
333 elif then
.action
!= now
.action
:
338 def get_subscribed_device_ids(self
):
339 """ device Ids on which the user subscribed to the podcast """
342 for action
in self
.actions
:
343 if action
.action
== "subscribe":
344 if not action
.device
in self
.disabled_devices
:
345 devices
.add(action
.device
)
347 if action
.device
in devices
:
348 devices
.remove(action
.device
)
355 return self
.settings
.get('public_subscription', True)
358 def __eq__(self
, other
):
362 return self
.podcast
== other
.podcast
and \
363 self
.user
== other
.user
366 return 'Podcast %s for User %s (%s)' % \
367 (self
.podcast
, self
.user
, self
._id
)
370 class Device(Document
):
371 id = StringProperty(default
=lambda: uuid
.uuid4().hex)
372 oldid
= IntegerProperty(required
=False)
373 uid
= StringProperty(required
=True)
374 name
= StringProperty(required
=True, default
='New Device')
375 type = StringProperty(required
=True, default
='other')
376 settings
= DictProperty()
377 deleted
= BooleanProperty(default
=False)
378 user_agent
= StringProperty()
381 def get_subscription_changes(self
, since
, until
):
383 Returns the subscription changes for the device as two lists.
384 The first lists contains the Ids of the podcasts that have been
385 subscribed to, the second list of those that have been unsubscribed
389 from mygpo
.db
.couchdb
.podcast_state
import podcast_states_for_device
392 podcast_states
= podcast_states_for_device(self
.id)
393 for p_state
in podcast_states
:
394 change
= p_state
.get_change_between(self
.id, since
, until
)
395 if change
== 'subscribe':
396 add
.append( p_state
.ref_url
)
397 elif change
== 'unsubscribe':
398 rem
.append( p_state
.ref_url
)
403 def get_latest_changes(self
):
405 from mygpo
.db
.couchdb
.podcast_state
import podcast_states_for_device
407 podcast_states
= podcast_states_for_device(self
.id)
408 for p_state
in podcast_states
:
409 actions
= filter(lambda x
: x
.device
== self
.id, reversed(p_state
.actions
))
411 yield (p_state
.podcast
, actions
[0])
414 def get_subscribed_podcast_states(self
):
415 r
= PodcastUserState
.view('subscriptions/by_device',
416 startkey
= [self
.id, None],
417 endkey
= [self
.id, {}],
423 def get_subscribed_podcast_ids(self
):
424 states
= self
.get_subscribed_podcast_states()
425 return [state
.podcast
for state
in states
]
428 def get_subscribed_podcasts(self
):
429 """ Returns all subscribed podcasts for the device
431 The attribute "url" contains the URL that was used when subscribing to
434 states
= self
.get_subscribed_podcast_states()
435 podcast_ids
= [state
.podcast
for state
in states
]
436 podcasts
= podcasts_to_dict(podcast_ids
)
439 podcast
= proxy_object(podcasts
[state
.podcast
], url
=state
.ref_url
)
440 podcasts
[state
.podcast
] = podcast
442 return podcasts
.values()
446 return hash(frozenset([self
.id, self
.uid
, self
.name
, self
.type, self
.deleted
]))
449 def __eq__(self
, other
):
450 return self
.id == other
.id
454 return '<{cls} {id}>'.format(cls
=self
.__class
__.__name
__, id=self
.id)
460 def __unicode__(self
):
465 TOKEN_NAMES
= ('subscriptions_token', 'favorite_feeds_token',
466 'publisher_update_token', 'userpage_token')
469 class TokenException(Exception):
473 class User(BaseUser
, SyncedDevicesMixin
):
474 oldid
= IntegerProperty()
475 settings
= DictProperty()
476 devices
= SchemaListProperty(Device
)
477 published_objects
= StringListProperty()
478 deleted
= BooleanProperty(default
=False)
479 suggestions_up_to_date
= BooleanProperty(default
=False)
480 twitter
= StringProperty()
481 about
= StringProperty()
483 # token for accessing subscriptions of this use
484 subscriptions_token
= StringProperty(default
=None)
486 # token for accessing the favorite-episodes feed of this user
487 favorite_feeds_token
= StringProperty(default
=None)
489 # token for automatically updating feeds published by this user
490 publisher_update_token
= StringProperty(default
=None)
492 # token for accessing the userpage of this user
493 userpage_token
= StringProperty(default
=None)
499 def create_new_token(self
, token_name
, length
=32):
500 """ creates a new random token """
502 if token_name
not in TOKEN_NAMES
:
503 raise TokenException('Invalid token name %s' % token_name
)
505 token
= "".join(random
.sample(string
.letters
+string
.digits
, length
))
506 setattr(self
, token_name
, token
)
510 @repeat_on_conflict(['self'])
511 def get_token(self
, token_name
):
512 """ returns a token, and generate those that are still missing """
516 if token_name
not in TOKEN_NAMES
:
517 raise TokenException('Invalid token name %s' % token_name
)
519 for tn
in TOKEN_NAMES
:
520 if getattr(self
, tn
) is None:
521 self
.create_new_token(tn
)
527 return getattr(self
, token_name
)
532 def active_devices(self
):
533 not_deleted
= lambda d
: not d
.deleted
534 return filter(not_deleted
, self
.devices
)
538 def inactive_devices(self
):
539 deleted
= lambda d
: d
.deleted
540 return filter(deleted
, self
.devices
)
543 def get_devices_by_id(self
):
544 return dict( (device
.id, device
) for device
in self
.devices
)
547 def get_device(self
, id):
549 if not hasattr(self
, '__device_by_id'):
550 self
.__devices
_by
_id
= dict( (d
.id, d
) for d
in self
.devices
)
552 return self
.__devices
_by
_id
.get(id, None)
555 def get_device_by_uid(self
, uid
, only_active
=True):
557 if not hasattr(self
, '__devices_by_uio'):
558 self
.__devices
_by
_uid
= dict( (d
.uid
, d
) for d
in self
.devices
)
561 device
= self
.__devices
_by
_uid
[uid
]
563 if only_active
and device
.deleted
:
564 raise DeviceDeletedException(
565 'Device with UID %s is deleted' % uid
)
569 except KeyError as e
:
570 raise DeviceDoesNotExist('There is no device with UID %s' % uid
)
573 @repeat_on_conflict(['self'])
574 def update_device(self
, device
):
575 """ Sets the device and saves the user """
576 self
.set_device(device
)
580 def set_device(self
, device
):
582 if not RE_DEVICE_UID
.match(device
.uid
):
583 raise DeviceUIDException(u
"'{uid} is not a valid device ID".format(
586 devices
= list(self
.devices
)
587 ids
= [x
.id for x
in devices
]
588 if not device
.id in ids
:
589 devices
.append(device
)
590 self
.devices
= devices
593 index
= ids
.index(device
.id)
595 devices
.insert(index
, device
)
596 self
.devices
= devices
599 def remove_device(self
, device
):
600 devices
= list(self
.devices
)
601 ids
= [x
.id for x
in devices
]
602 if not device
.id in ids
:
605 index
= ids
.index(device
.id)
607 self
.devices
= devices
609 if self
.is_synced(device
):
610 self
.unsync_device(device
)
613 def get_subscriptions_by_device(self
, public
=None):
614 from mygpo
.db
.couchdb
.podcast_state
import subscriptions_by_user
615 get_dev
= itemgetter(2)
616 groups
= collections
.defaultdict(list)
617 subscriptions
= subscriptions_by_user(self
, public
=public
)
618 subscriptions
= sorted(subscriptions
, key
=get_dev
)
620 for public
, podcast_id
, device_id
in subscriptions
:
621 groups
[device_id
].append(podcast_id
)
626 def get_subscribed_podcast_states(self
, public
=None):
628 Returns the Ids of all subscribed podcasts
631 r
= PodcastUserState
.view('subscriptions/by_user',
632 startkey
= [self
._id
, public
, None, None],
633 endkey
= [self
._id
+'ZZZ', None, None, None],
641 def get_subscribed_podcast_ids(self
, public
=None):
642 states
= self
.get_subscribed_podcast_states(public
=public
)
643 return [state
.podcast
for state
in states
]
647 def get_subscribed_podcasts(self
, public
=None):
648 """ Returns all subscribed podcasts for the user
650 The attribute "url" contains the URL that was used when subscribing to
653 states
= self
.get_subscribed_podcast_states(public
=public
)
654 podcast_ids
= [state
.podcast
for state
in states
]
655 podcasts
= podcasts_to_dict(podcast_ids
)
658 podcast
= proxy_object(podcasts
[state
.podcast
], url
=state
.ref_url
)
659 podcasts
[state
.podcast
] = podcast
661 return podcasts
.values()
665 def get_subscription_history(self
, device_id
=None, reverse
=False, public
=None):
666 """ Returns chronologically ordered subscription history entries
668 Setting device_id restricts the actions to a certain device
671 from mygpo
.db
.couchdb
.podcast_state
import podcast_states_for_user
, \
672 podcast_states_for_device
674 def action_iter(state
):
675 for action
in sorted(state
.actions
, reverse
=reverse
):
676 if device_id
is not None and device_id
!= action
.device
:
679 if public
is not None and state
.is_public() != public
:
682 entry
= HistoryEntry()
683 entry
.timestamp
= action
.timestamp
684 entry
.action
= action
.action
685 entry
.podcast_id
= state
.podcast
686 entry
.device_id
= action
.device
689 if device_id
is None:
690 podcast_states
= podcast_states_for_user(self
)
692 podcast_states
= podcast_states_for_device(device_id
)
694 # create an action_iter for each PodcastUserState
695 subscription_action_lists
= [action_iter(x
) for x
in podcast_states
]
697 action_cmp_key
= lambda x
: x
.timestamp
699 # Linearize their subscription-actions
700 return linearize(action_cmp_key
, subscription_action_lists
, reverse
)
703 def get_global_subscription_history(self
, public
=None):
704 """ Actions that added/removed podcasts from the subscription list
706 Returns an iterator of all subscription actions that either
707 * added subscribed a podcast that hasn't been subscribed directly
708 before the action (but could have been subscribed) earlier
709 * removed a subscription of the podcast is not longer subscribed
713 subscriptions
= collections
.defaultdict(int)
715 for entry
in self
.get_subscription_history(public
=public
):
716 if entry
.action
== 'subscribe':
717 subscriptions
[entry
.podcast_id
] += 1
719 # a new subscription has been added
720 if subscriptions
[entry
.podcast_id
] == 1:
723 elif entry
.action
== 'unsubscribe':
724 subscriptions
[entry
.podcast_id
] -= 1
726 # the last subscription has been removed
727 if subscriptions
[entry
.podcast_id
] == 0:
732 def get_newest_episodes(self
, max_date
, max_per_podcast
=5):
733 """ Returns the newest episodes of all subscribed podcasts
735 Only max_per_podcast episodes per podcast are loaded. Episodes with
736 release dates above max_date are discarded.
738 This method returns a generator that produces the newest episodes.
740 The number of required DB queries is equal to the number of (distinct)
741 podcasts of all consumed episodes (max: number of subscribed podcasts),
742 plus a constant number of initial queries (when the first episode is
745 cmp_key
= lambda episode
: episode
.released
or datetime(2000, 01, 01)
747 podcasts
= list(self
.get_subscribed_podcasts())
748 podcasts
= filter(lambda p
: p
.latest_episode_timestamp
, podcasts
)
749 podcasts
= sorted(podcasts
, key
=lambda p
: p
.latest_episode_timestamp
,
752 podcast_dict
= dict((p
.get_id(), p
) for p
in podcasts
)
754 # contains the un-yielded episodes, newest first
757 for podcast
in podcasts
:
761 for episode
in episodes
:
762 # determine for which episodes there won't be a new episodes
763 # that is newer; those can be yielded
764 if episode
.released
> podcast
.latest_episode_timestamp
:
765 p
= podcast_dict
.get(episode
.podcast
, None)
766 yield proxy_object(episode
, podcast
=p
)
767 yielded_episodes
+= 1
771 # remove the episodes that have been yielded before
772 episodes
= episodes
[yielded_episodes
:]
774 # fetch and merge episodes for the next podcast
775 from mygpo
.db
.couchdb
.episode
import episodes_for_podcast
776 new_episodes
= episodes_for_podcast(podcast
, since
=1,
777 until
=max_date
, descending
=True, limit
=max_per_podcast
)
778 episodes
= sorted(episodes
+new_episodes
, key
=cmp_key
, reverse
=True)
781 # yield the remaining episodes
782 for episode
in episodes
:
783 podcast
= podcast_dict
.get(episode
.podcast
, None)
784 yield proxy_object(episode
, podcast
=podcast
)
789 def save(self
, *args
, **kwargs
):
791 from mygpo
.db
.couchdb
.podcast_state
import podcast_states_for_user
793 super(User
, self
).save(*args
, **kwargs
)
795 podcast_states
= podcast_states_for_user(self
)
796 for state
in podcast_states
:
797 @repeat_on_conflict(['state'])
798 def _update_state(state
):
799 old_devs
= set(state
.disabled_devices
)
800 state
.set_device_state(self
.devices
)
802 if old_devs
!= set(state
.disabled_devices
):
810 def __eq__(self
, other
):
814 # ensure that other isn't AnonymousUser
815 return other
.is_authenticated() and self
._id
== other
._id
818 def __ne__(self
, other
):
819 return not(self
== other
)
823 return 'User %s' % self
._id
826 class History(object):
828 def __init__(self
, user
, device
):
833 def __getitem__(self
, key
):
835 if isinstance(key
, slice):
836 start
= key
.start
or 0
837 length
= key
.stop
- start
843 return device_history(self
.user
, self
.device
, start
, length
)
846 return user_history(self
.user
, start
, length
)
850 class HistoryEntry(object):
851 """ A class that can represent subscription and episode actions """
855 def from_action_dict(cls
, action
):
857 entry
= HistoryEntry()
859 if 'timestamp' in action
:
860 ts
= action
.pop('timestamp')
861 entry
.timestamp
= dateutil
.parser
.parse(ts
)
863 for key
, value
in action
.items():
864 setattr(entry
, key
, value
)
871 return getattr(self
, 'position', None)
875 def fetch_data(cls
, user
, entries
,
876 podcasts
=None, episodes
=None):
877 """ Efficiently loads additional data for a number of entries """
881 podcast_ids
= [getattr(x
, 'podcast_id', None) for x
in entries
]
882 podcast_ids
= filter(None, podcast_ids
)
883 podcasts
= podcasts_to_dict(podcast_ids
)
886 from mygpo
.db
.couchdb
.episode
import episodes_to_dict
888 episode_ids
= [getattr(x
, 'episode_id', None) for x
in entries
]
889 episode_ids
= filter(None, episode_ids
)
890 episodes
= episodes_to_dict(episode_ids
)
893 # does not need pre-populated data because no db-access is required
894 device_ids
= [getattr(x
, 'device_id', None) for x
in entries
]
895 device_ids
= filter(None, device_ids
)
896 devices
= dict([ (id, user
.get_device(id)) for id in device_ids
])
899 for entry
in entries
:
900 podcast_id
= getattr(entry
, 'podcast_id', None)
901 entry
.podcast
= podcasts
.get(podcast_id
, None)
903 episode_id
= getattr(entry
, 'episode_id', None)
904 entry
.episode
= episodes
.get(episode_id
, None)
906 if hasattr(entry
, 'user'):
909 device
= devices
.get(getattr(entry
, 'device_id', None), None)
910 entry
.device
= device