4 from datetime
import datetime
6 from itertools
import imap
7 from operator
import itemgetter
11 from couchdbkit
.ext
.django
.schema
import *
13 from django
.core
.cache
import cache
15 from django_couchdb_utils
.registration
.models
import User
as BaseUser
17 from mygpo
.utils
import linearize
18 from mygpo
.core
.proxy
import DocumentABCMeta
, proxy_object
19 from mygpo
.decorators
import repeat_on_conflict
20 from mygpo
.users
.ratings
import RatingMixin
21 from mygpo
.users
.sync
import SyncedDevicesMixin
22 from mygpo
.users
.subscriptions
import subscription_changes
, podcasts_for_states
23 from mygpo
.users
.settings
import FAV_FLAG
, PUBLIC_SUB_PODCAST
, SettingsMixin
24 from mygpo
.db
.couchdb
.podcast
import podcasts_by_id
, podcasts_to_dict
25 from mygpo
.db
.couchdb
.user
import user_history
, device_history
27 # make sure this code is executed at startup
28 from mygpo
.users
.signals
import *
31 RE_DEVICE_UID
= re
.compile(r
'^[\w.-]+$')
33 # TODO: derive from ValidationException?
34 class InvalidEpisodeActionAttributes(ValueError):
35 """ raised when the attribues of an episode action fail validation """
38 class DeviceUIDException(Exception):
42 class DeviceDoesNotExist(Exception):
46 class DeviceDeletedException(DeviceDoesNotExist
):
50 class Suggestions(Document
, RatingMixin
):
51 user
= StringProperty(required
=True)
52 user_oldid
= IntegerProperty()
53 podcasts
= StringListProperty()
54 blacklist
= StringListProperty()
57 def get_podcasts(self
, count
=None):
58 user
= User
.get(self
.user
)
59 subscriptions
= user
.get_subscribed_podcast_ids()
61 ids
= filter(lambda x
: not x
in self
.blacklist
+ subscriptions
, self
.podcasts
)
64 return filter(lambda x
: x
and x
.title
, podcasts_by_id(ids
))
69 return super(Suggestions
, self
).__repr
__()
71 return '%d Suggestions for %s (%s)' % \
72 (len(self
.podcasts
), self
.user
, self
._id
)
75 class EpisodeAction(DocumentSchema
):
77 One specific action to an episode. Must
78 always be part of a EpisodeUserState
81 action
= StringProperty(required
=True)
83 # walltime of the event (assigned by the uploading client, defaults to now)
84 timestamp
= DateTimeProperty(required
=True, default
=datetime
.utcnow
)
86 # upload time of the event
87 upload_timestamp
= IntegerProperty(required
=True)
89 device_oldid
= IntegerProperty(required
=False)
90 device
= StringProperty()
91 started
= IntegerProperty()
92 playmark
= IntegerProperty()
93 total
= IntegerProperty()
95 def __eq__(self
, other
):
96 if not isinstance(other
, EpisodeAction
):
98 vals
= ('action', 'timestamp', 'device', 'started', 'playmark',
100 return all([getattr(self
, v
, None) == getattr(other
, v
, None) for v
in vals
])
103 def to_history_entry(self
):
104 entry
= HistoryEntry()
105 entry
.action
= self
.action
106 entry
.timestamp
= self
.timestamp
107 entry
.device_id
= self
.device
108 entry
.started
= self
.started
109 entry
.position
= self
.playmark
110 entry
.total
= self
.total
115 def validate_time_values(self
):
116 """ Validates allowed combinations of time-values """
118 PLAY_ACTION_KEYS
= ('playmark', 'started', 'total')
120 # Key found, but must not be supplied (no play action!)
121 if self
.action
!= 'play':
122 for key
in PLAY_ACTION_KEYS
:
123 if getattr(self
, key
, None) is not None:
124 raise InvalidEpisodeActionAttributes('%s only allowed in play actions' % key
)
126 # Sanity check: If started or total are given, require playmark
127 if ((self
.started
is not None) or (self
.total
is not None)) and \
128 self
.playmark
is None:
129 raise InvalidEpisodeActionAttributes('started and total require position')
131 # Sanity check: total and playmark can only appear together
132 if ((self
.total
is not None) or (self
.started
is not None)) and \
133 ((self
.total
is None) or (self
.started
is None)):
134 raise InvalidEpisodeActionAttributes('total and started can only appear together')
138 return '%s-Action on %s at %s (in %s)' % \
139 (self
.action
, self
.device
, self
.timestamp
, self
._id
)
143 return hash(frozenset([self
.action
, self
.timestamp
, self
.device
,
144 self
.started
, self
.playmark
, self
.total
]))
147 class Chapter(Document
):
148 """ A user-entered episode chapter """
150 device
= StringProperty()
151 created
= DateTimeProperty()
152 start
= IntegerProperty(required
=True)
153 end
= IntegerProperty(required
=True)
154 label
= StringProperty()
155 advertisement
= BooleanProperty()
159 return '<%s %s (%d-%d)>' % (self
.__class
__.__name
__, self
.label
,
160 self
.start
, self
.end
)
163 class EpisodeUserState(Document
, SettingsMixin
):
165 Contains everything a user has done with an Episode
168 episode
= StringProperty(required
=True)
169 actions
= SchemaListProperty(EpisodeAction
)
170 user_oldid
= IntegerProperty()
171 user
= StringProperty(required
=True)
172 ref_url
= StringProperty(required
=True)
173 podcast_ref_url
= StringProperty(required
=True)
174 merged_ids
= StringListProperty()
175 chapters
= SchemaListProperty(Chapter
)
176 podcast
= StringProperty(required
=True)
180 def add_actions(self
, actions
):
181 map(EpisodeAction
.validate_time_values
, actions
)
182 self
.actions
= list(self
.actions
) + actions
183 self
.actions
= list(set(self
.actions
))
184 self
.actions
= sorted(self
.actions
, key
=lambda x
: x
.timestamp
)
187 def is_favorite(self
):
188 return self
.get_wksetting(FAV_FLAG
)
191 def set_favorite(self
, set_to
=True):
192 self
.settings
[FAV_FLAG
.name
] = set_to
195 def update_chapters(self
, add
=[], rem
=[]):
196 """ Updates the Chapter list
198 * add contains the chapters to be added
200 * rem contains tuples of (start, end) times. Chapters that match
201 both endpoints will be removed
204 @repeat_on_conflict(['state'])
207 self
.chapters
= self
.chapters
+ [chapter
]
209 for start
, end
in rem
:
210 keep
= lambda c
: c
.start
!= start
or c
.end
!= end
211 self
.chapters
= filter(keep
, self
.chapters
)
218 def get_history_entries(self
):
219 return imap(EpisodeAction
.to_history_entry
, self
.actions
)
223 return 'Episode-State %s (in %s)' % \
224 (self
.episode
, self
._id
)
226 def __eq__(self
, other
):
227 if not isinstance(other
, EpisodeUserState
):
230 return (self
.episode
== other
.episode
and
231 self
.user
== other
.user
)
235 class SubscriptionAction(Document
):
236 action
= StringProperty()
237 timestamp
= DateTimeProperty(default
=datetime
.utcnow
)
238 device
= StringProperty()
241 __metaclass__
= DocumentABCMeta
244 def __cmp__(self
, other
):
245 return cmp(self
.timestamp
, other
.timestamp
)
247 def __eq__(self
, other
):
248 return self
.action
== other
.action
and \
249 self
.timestamp
== other
.timestamp
and \
250 self
.device
== other
.device
253 return hash(self
.action
) + hash(self
.timestamp
) + hash(self
.device
)
256 return '<SubscriptionAction %s on %s at %s>' % (
257 self
.action
, self
.device
, self
.timestamp
)
260 class PodcastUserState(Document
, SettingsMixin
):
262 Contains everything that a user has done
263 with a specific podcast and all its episodes
266 podcast
= StringProperty(required
=True)
267 user_oldid
= IntegerProperty()
268 user
= StringProperty(required
=True)
269 actions
= SchemaListProperty(SubscriptionAction
)
270 tags
= StringListProperty()
271 ref_url
= StringProperty(required
=True)
272 disabled_devices
= StringListProperty()
273 merged_ids
= StringListProperty()
276 def remove_device(self
, device
):
278 Removes all actions from the podcast state that refer to the
281 self
.actions
= filter(lambda a
: a
.device
!= device
.id, self
.actions
)
284 def subscribe(self
, device
):
285 action
= SubscriptionAction()
286 action
.action
= 'subscribe'
287 action
.device
= device
.id
288 self
.add_actions([action
])
291 def unsubscribe(self
, device
):
292 action
= SubscriptionAction()
293 action
.action
= 'unsubscribe'
294 action
.device
= device
.id
295 self
.add_actions([action
])
298 def add_actions(self
, actions
):
299 self
.actions
= list(set(self
.actions
+ actions
))
300 self
.actions
= sorted(self
.actions
)
303 def add_tags(self
, tags
):
304 self
.tags
= list(set(self
.tags
+ tags
))
307 def set_device_state(self
, devices
):
308 disabled_devices
= [device
.id for device
in devices
if device
.deleted
]
309 self
.disabled_devices
= disabled_devices
312 def get_change_between(self
, device_id
, since
, until
):
314 Returns the change of the subscription status for the given device
315 between the two timestamps.
317 The change is given as either 'subscribe' (the podcast has been
318 subscribed), 'unsubscribed' (the podcast has been unsubscribed) or
322 device_actions
= filter(lambda x
: x
.device
== device_id
, self
.actions
)
323 before
= filter(lambda x
: x
.timestamp
<= since
, device_actions
)
324 after
= filter(lambda x
: x
.timestamp
<= until
, device_actions
)
326 # nothing happened, so there can be no change
330 then
= before
[-1] if before
else None
334 if now
.action
!= 'unsubscribe':
336 elif then
.action
!= now
.action
:
341 def get_subscribed_device_ids(self
):
342 """ device Ids on which the user subscribed to the podcast """
345 for action
in self
.actions
:
346 if action
.action
== "subscribe":
347 if not action
.device
in self
.disabled_devices
:
348 devices
.add(action
.device
)
350 if action
.device
in devices
:
351 devices
.remove(action
.device
)
356 def is_subscribed_on(self
, device
):
357 """ checks if the podcast is subscribed on the given device """
359 for action
in reversed(self
.actions
):
360 if not action
.device
== device
.id:
363 # we only need to check the latest action for the device
364 return (action
.action
== 'subscribe')
366 # we haven't found any matching action
371 return self
.get_wksetting(PUBLIC_SUB_PODCAST
)
374 def __eq__(self
, other
):
378 return self
.podcast
== other
.podcast
and \
379 self
.user
== other
.user
382 return 'Podcast %s for User %s (%s)' % \
383 (self
.podcast
, self
.user
, self
._id
)
386 class Device(Document
, SettingsMixin
):
387 id = StringProperty(default
=lambda: uuid
.uuid4().hex)
388 oldid
= IntegerProperty(required
=False)
389 uid
= StringProperty(required
=True)
390 name
= StringProperty(required
=True, default
='New Device')
391 type = StringProperty(required
=True, default
='other')
392 deleted
= BooleanProperty(default
=False)
393 user_agent
= StringProperty()
396 def get_subscription_changes(self
, since
, until
):
398 Returns the subscription changes for the device as two lists.
399 The first lists contains the Ids of the podcasts that have been
400 subscribed to, the second list of those that have been unsubscribed
404 from mygpo
.db
.couchdb
.podcast_state
import podcast_states_for_device
405 podcast_states
= podcast_states_for_device(self
.id)
406 return subscription_changes(self
.id, podcast_states
, since
, until
)
409 def get_latest_changes(self
):
411 from mygpo
.db
.couchdb
.podcast_state
import podcast_states_for_device
413 podcast_states
= podcast_states_for_device(self
.id)
414 for p_state
in podcast_states
:
415 actions
= filter(lambda x
: x
.device
== self
.id, reversed(p_state
.actions
))
417 yield (p_state
.podcast
, actions
[0])
420 def get_subscribed_podcast_states(self
):
421 r
= PodcastUserState
.view('subscriptions/by_device',
422 startkey
= [self
.id, None],
423 endkey
= [self
.id, {}],
429 def get_subscribed_podcast_ids(self
):
430 states
= self
.get_subscribed_podcast_states()
431 return [state
.podcast
for state
in states
]
434 def get_subscribed_podcasts(self
):
435 """ Returns all subscribed podcasts for the device
437 The attribute "url" contains the URL that was used when subscribing to
440 states
= self
.get_subscribed_podcast_states()
441 return podcasts_for_states(states
)
445 return hash(frozenset([self
.id, self
.uid
, self
.name
, self
.type, self
.deleted
]))
448 def __eq__(self
, other
):
449 return self
.id == other
.id
453 return '<{cls} {id}>'.format(cls
=self
.__class
__.__name
__, id=self
.id)
459 def __unicode__(self
):
464 TOKEN_NAMES
= ('subscriptions_token', 'favorite_feeds_token',
465 'publisher_update_token', 'userpage_token')
468 class TokenException(Exception):
472 class User(BaseUser
, SyncedDevicesMixin
, SettingsMixin
):
473 oldid
= IntegerProperty()
474 devices
= SchemaListProperty(Device
)
475 published_objects
= StringListProperty()
476 deleted
= BooleanProperty(default
=False)
477 suggestions_up_to_date
= BooleanProperty(default
=False)
478 twitter
= StringProperty()
479 about
= StringProperty()
480 google_email
= StringProperty()
482 # token for accessing subscriptions of this use
483 subscriptions_token
= StringProperty(default
=None)
485 # token for accessing the favorite-episodes feed of this user
486 favorite_feeds_token
= StringProperty(default
=None)
488 # token for automatically updating feeds published by this user
489 publisher_update_token
= StringProperty(default
=None)
491 # token for accessing the userpage of this user
492 userpage_token
= StringProperty(default
=None)
498 def create_new_token(self
, token_name
, length
=32):
499 """ creates a new random token """
501 if token_name
not in TOKEN_NAMES
:
502 raise TokenException('Invalid token name %s' % token_name
)
504 token
= "".join(random
.sample(string
.letters
+string
.digits
, length
))
505 setattr(self
, token_name
, token
)
509 @repeat_on_conflict(['self'])
510 def get_token(self
, token_name
):
511 """ returns a token, and generate those that are still missing """
515 if token_name
not in TOKEN_NAMES
:
516 raise TokenException('Invalid token name %s' % token_name
)
518 for tn
in TOKEN_NAMES
:
519 if getattr(self
, tn
) is None:
520 self
.create_new_token(tn
)
526 return getattr(self
, token_name
)
531 def active_devices(self
):
532 not_deleted
= lambda d
: not d
.deleted
533 return filter(not_deleted
, self
.devices
)
537 def inactive_devices(self
):
538 deleted
= lambda d
: d
.deleted
539 return filter(deleted
, self
.devices
)
542 def get_devices_by_id(self
):
543 return dict( (device
.id, device
) for device
in self
.devices
)
546 def get_device(self
, id):
548 if not hasattr(self
, '__device_by_id'):
549 self
.__devices
_by
_id
= dict( (d
.id, d
) for d
in self
.devices
)
551 return self
.__devices
_by
_id
.get(id, None)
554 def get_device_by_uid(self
, uid
, only_active
=True):
556 if not hasattr(self
, '__devices_by_uio'):
557 self
.__devices
_by
_uid
= dict( (d
.uid
, d
) for d
in self
.devices
)
560 device
= self
.__devices
_by
_uid
[uid
]
562 if only_active
and device
.deleted
:
563 raise DeviceDeletedException(
564 'Device with UID %s is deleted' % uid
)
568 except KeyError as e
:
569 raise DeviceDoesNotExist('There is no device with UID %s' % uid
)
572 @repeat_on_conflict(['self'])
573 def update_device(self
, device
):
574 """ Sets the device and saves the user """
575 self
.set_device(device
)
579 def set_device(self
, device
):
581 if not RE_DEVICE_UID
.match(device
.uid
):
582 raise DeviceUIDException(u
"'{uid} is not a valid device ID".format(
585 devices
= list(self
.devices
)
586 ids
= [x
.id for x
in devices
]
587 if not device
.id in ids
:
588 devices
.append(device
)
589 self
.devices
= devices
592 index
= ids
.index(device
.id)
594 devices
.insert(index
, device
)
595 self
.devices
= devices
598 def remove_device(self
, device
):
599 devices
= list(self
.devices
)
600 ids
= [x
.id for x
in devices
]
601 if not device
.id in ids
:
604 index
= ids
.index(device
.id)
606 self
.devices
= devices
608 if self
.is_synced(device
):
609 self
.unsync_device(device
)
612 def get_subscriptions_by_device(self
, public
=None):
613 from mygpo
.db
.couchdb
.podcast_state
import subscriptions_by_user
614 get_dev
= itemgetter(2)
615 groups
= collections
.defaultdict(list)
616 subscriptions
= subscriptions_by_user(self
, public
=public
)
617 subscriptions
= sorted(subscriptions
, key
=get_dev
)
619 for public
, podcast_id
, device_id
in subscriptions
:
620 groups
[device_id
].append(podcast_id
)
625 def get_subscribed_podcast_states(self
, public
=None):
627 Returns the Ids of all subscribed podcasts
630 r
= PodcastUserState
.view('subscriptions/by_user',
631 startkey
= [self
._id
, public
, None, None],
632 endkey
= [self
._id
+'ZZZ', None, None, None],
640 def get_subscribed_podcast_ids(self
, public
=None):
641 states
= self
.get_subscribed_podcast_states(public
=public
)
642 return [state
.podcast
for state
in states
]
646 def get_subscribed_podcasts(self
, public
=None):
647 """ Returns all subscribed podcasts for the user
649 The attribute "url" contains the URL that was used when subscribing to
652 states
= self
.get_subscribed_podcast_states(public
=public
)
653 podcast_ids
= [state
.podcast
for state
in states
]
654 podcasts
= podcasts_to_dict(podcast_ids
)
657 podcast
= proxy_object(podcasts
[state
.podcast
], url
=state
.ref_url
)
658 podcasts
[state
.podcast
] = podcast
660 return podcasts
.values()
664 def get_subscription_history(self
, device_id
=None, reverse
=False, public
=None):
665 """ Returns chronologically ordered subscription history entries
667 Setting device_id restricts the actions to a certain device
670 from mygpo
.db
.couchdb
.podcast_state
import podcast_states_for_user
, \
671 podcast_states_for_device
673 def action_iter(state
):
674 for action
in sorted(state
.actions
, reverse
=reverse
):
675 if device_id
is not None and device_id
!= action
.device
:
678 if public
is not None and state
.is_public() != public
:
681 entry
= HistoryEntry()
682 entry
.timestamp
= action
.timestamp
683 entry
.action
= action
.action
684 entry
.podcast_id
= state
.podcast
685 entry
.device_id
= action
.device
688 if device_id
is None:
689 podcast_states
= podcast_states_for_user(self
)
691 podcast_states
= podcast_states_for_device(device_id
)
693 # create an action_iter for each PodcastUserState
694 subscription_action_lists
= [action_iter(x
) for x
in podcast_states
]
696 action_cmp_key
= lambda x
: x
.timestamp
698 # Linearize their subscription-actions
699 return linearize(action_cmp_key
, subscription_action_lists
, reverse
)
702 def get_global_subscription_history(self
, public
=None):
703 """ Actions that added/removed podcasts from the subscription list
705 Returns an iterator of all subscription actions that either
706 * added subscribed a podcast that hasn't been subscribed directly
707 before the action (but could have been subscribed) earlier
708 * removed a subscription of the podcast is not longer subscribed
712 subscriptions
= collections
.defaultdict(int)
714 for entry
in self
.get_subscription_history(public
=public
):
715 if entry
.action
== 'subscribe':
716 subscriptions
[entry
.podcast_id
] += 1
718 # a new subscription has been added
719 if subscriptions
[entry
.podcast_id
] == 1:
722 elif entry
.action
== 'unsubscribe':
723 subscriptions
[entry
.podcast_id
] -= 1
725 # the last subscription has been removed
726 if subscriptions
[entry
.podcast_id
] == 0:
731 def get_newest_episodes(self
, max_date
, max_per_podcast
=5):
732 """ Returns the newest episodes of all subscribed podcasts
734 Only max_per_podcast episodes per podcast are loaded. Episodes with
735 release dates above max_date are discarded.
737 This method returns a generator that produces the newest episodes.
739 The number of required DB queries is equal to the number of (distinct)
740 podcasts of all consumed episodes (max: number of subscribed podcasts),
741 plus a constant number of initial queries (when the first episode is
744 cmp_key
= lambda episode
: episode
.released
or datetime(2000, 01, 01)
746 podcasts
= list(self
.get_subscribed_podcasts())
747 podcasts
= filter(lambda p
: p
.latest_episode_timestamp
, podcasts
)
748 podcasts
= sorted(podcasts
, key
=lambda p
: p
.latest_episode_timestamp
,
751 podcast_dict
= dict((p
.get_id(), p
) for p
in podcasts
)
753 # contains the un-yielded episodes, newest first
756 for podcast
in podcasts
:
760 for episode
in episodes
:
761 # determine for which episodes there won't be a new episodes
762 # that is newer; those can be yielded
763 if episode
.released
> podcast
.latest_episode_timestamp
:
764 p
= podcast_dict
.get(episode
.podcast
, None)
765 yield proxy_object(episode
, podcast
=p
)
766 yielded_episodes
+= 1
770 # remove the episodes that have been yielded before
771 episodes
= episodes
[yielded_episodes
:]
773 # fetch and merge episodes for the next podcast
774 from mygpo
.db
.couchdb
.episode
import episodes_for_podcast
775 new_episodes
= episodes_for_podcast(podcast
, since
=1,
776 until
=max_date
, descending
=True, limit
=max_per_podcast
)
777 episodes
= sorted(episodes
+new_episodes
, key
=cmp_key
, reverse
=True)
780 # yield the remaining episodes
781 for episode
in episodes
:
782 podcast
= podcast_dict
.get(episode
.podcast
, None)
783 yield proxy_object(episode
, podcast
=podcast
)
786 def __eq__(self
, other
):
790 # ensure that other isn't AnonymousUser
791 return other
.is_authenticated() and self
._id
== other
._id
794 def __ne__(self
, other
):
795 return not(self
== other
)
799 return 'User %s' % self
._id
802 class History(object):
804 def __init__(self
, user
, device
):
809 def __getitem__(self
, key
):
811 if isinstance(key
, slice):
812 start
= key
.start
or 0
813 length
= key
.stop
- start
819 return device_history(self
.user
, self
.device
, start
, length
)
822 return user_history(self
.user
, start
, length
)
826 class HistoryEntry(object):
827 """ A class that can represent subscription and episode actions """
831 def from_action_dict(cls
, action
):
833 entry
= HistoryEntry()
835 if 'timestamp' in action
:
836 ts
= action
.pop('timestamp')
837 entry
.timestamp
= dateutil
.parser
.parse(ts
)
839 for key
, value
in action
.items():
840 setattr(entry
, key
, value
)
847 return getattr(self
, 'position', None)
851 def fetch_data(cls
, user
, entries
,
852 podcasts
=None, episodes
=None):
853 """ Efficiently loads additional data for a number of entries """
857 podcast_ids
= [getattr(x
, 'podcast_id', None) for x
in entries
]
858 podcast_ids
= filter(None, podcast_ids
)
859 podcasts
= podcasts_to_dict(podcast_ids
)
862 from mygpo
.db
.couchdb
.episode
import episodes_to_dict
864 episode_ids
= [getattr(x
, 'episode_id', None) for x
in entries
]
865 episode_ids
= filter(None, episode_ids
)
866 episodes
= episodes_to_dict(episode_ids
)
869 # does not need pre-populated data because no db-access is required
870 device_ids
= [getattr(x
, 'device_id', None) for x
in entries
]
871 device_ids
= filter(None, device_ids
)
872 devices
= dict([ (id, user
.get_device(id)) for id in device_ids
])
875 for entry
in entries
:
876 podcast_id
= getattr(entry
, 'podcast_id', None)
877 entry
.podcast
= podcasts
.get(podcast_id
, None)
879 episode_id
= getattr(entry
, 'episode_id', None)
880 entry
.episode
= episodes
.get(episode_id
, None)
882 if hasattr(entry
, 'user'):
885 device
= devices
.get(getattr(entry
, 'device_id', None), None)
886 entry
.device
= device