2 import uuid
, collections
3 from datetime
import datetime
5 from itertools
import imap
6 from operator
import itemgetter
10 from couchdbkit
.ext
.django
.schema
import *
12 from django
.core
.cache
import cache
14 from django_couchdb_utils
.registration
.models
import User
as BaseUser
16 from mygpo
.core
.models
import Podcast
17 from mygpo
.utils
import linearize
18 from mygpo
.core
.proxy
import DocumentABCMeta
, proxy_object
19 from mygpo
.decorators
import repeat_on_conflict
20 from mygpo
.users
.ratings
import RatingMixin
21 from mygpo
.users
.sync
import SyncedDevicesMixin
22 from mygpo
.users
.settings
import FAV_FLAG
, PUBLIC_SUB_PODCAST
, SettingsMixin
23 from mygpo
.db
.couchdb
.podcast
import podcasts_by_id
, podcasts_to_dict
24 from mygpo
.db
.couchdb
.user
import user_history
, device_history
27 RE_DEVICE_UID
= re
.compile(r
'^[\w.-]+$')
30 class InvalidEpisodeActionAttributes(ValueError):
31 """ raised when the attribues of an episode action fail validation """
34 class DeviceUIDException(Exception):
38 class DeviceDoesNotExist(Exception):
42 class DeviceDeletedException(DeviceDoesNotExist
):
46 class Suggestions(Document
, RatingMixin
):
47 user
= StringProperty(required
=True)
48 user_oldid
= IntegerProperty()
49 podcasts
= StringListProperty()
50 blacklist
= StringListProperty()
53 def get_podcasts(self
, count
=None):
54 user
= User
.get(self
.user
)
55 subscriptions
= user
.get_subscribed_podcast_ids()
57 ids
= filter(lambda x
: not x
in self
.blacklist
+ subscriptions
, self
.podcasts
)
60 return filter(lambda x
: x
and x
.title
, podcasts_by_id(ids
))
65 return super(Suggestions
, self
).__repr
__()
67 return '%d Suggestions for %s (%s)' % \
68 (len(self
.podcasts
), self
.user
, self
._id
)
71 class EpisodeAction(DocumentSchema
):
73 One specific action to an episode. Must
74 always be part of a EpisodeUserState
77 action
= StringProperty(required
=True)
78 timestamp
= DateTimeProperty(required
=True, default
=datetime
.utcnow
)
79 device_oldid
= IntegerProperty(required
=False)
80 device
= StringProperty()
81 started
= IntegerProperty()
82 playmark
= IntegerProperty()
83 total
= IntegerProperty()
85 def __eq__(self
, other
):
86 if not isinstance(other
, EpisodeAction
):
88 vals
= ('action', 'timestamp', 'device', 'started', 'playmark',
90 return all([getattr(self
, v
, None) == getattr(other
, v
, None) for v
in vals
])
93 def to_history_entry(self
):
94 entry
= HistoryEntry()
95 entry
.action
= self
.action
96 entry
.timestamp
= self
.timestamp
97 entry
.device_id
= self
.device
98 entry
.started
= self
.started
99 entry
.position
= self
.playmark
100 entry
.total
= self
.total
105 def validate_time_values(self
):
106 """ Validates allowed combinations of time-values """
108 PLAY_ACTION_KEYS
= ('playmark', 'started', 'total')
110 # Key found, but must not be supplied (no play action!)
111 if self
.action
!= 'play':
112 for key
in PLAY_ACTION_KEYS
:
113 if getattr(self
, key
, None) is not None:
114 raise InvalidEpisodeActionAttributes('%s only allowed in play actions' % key
)
116 # Sanity check: If started or total are given, require playmark
117 if ((self
.started
is not None) or (self
.total
is not None)) and \
118 self
.playmark
is None:
119 raise InvalidEpisodeActionAttributes('started and total require position')
121 # Sanity check: total and playmark can only appear together
122 if ((self
.total
is not None) or (self
.started
is not None)) and \
123 ((self
.total
is None) or (self
.started
is None)):
124 raise InvalidEpisodeActionAttributes('total and started can only appear together')
128 return '%s-Action on %s at %s (in %s)' % \
129 (self
.action
, self
.device
, self
.timestamp
, self
._id
)
133 return hash(frozenset([self
.action
, self
.timestamp
, self
.device
,
134 self
.started
, self
.playmark
, self
.total
]))
137 class Chapter(Document
):
138 """ A user-entered episode chapter """
140 device
= StringProperty()
141 created
= DateTimeProperty()
142 start
= IntegerProperty(required
=True)
143 end
= IntegerProperty(required
=True)
144 label
= StringProperty()
145 advertisement
= BooleanProperty()
149 return '<%s %s (%d-%d)>' % (self
.__class
__.__name
__, self
.label
,
150 self
.start
, self
.end
)
153 class EpisodeUserState(Document
, SettingsMixin
):
155 Contains everything a user has done with an Episode
158 episode
= StringProperty(required
=True)
159 actions
= SchemaListProperty(EpisodeAction
)
160 user_oldid
= IntegerProperty()
161 user
= StringProperty(required
=True)
162 ref_url
= StringProperty(required
=True)
163 podcast_ref_url
= StringProperty(required
=True)
164 merged_ids
= StringListProperty()
165 chapters
= SchemaListProperty(Chapter
)
166 podcast
= StringProperty(required
=True)
170 def add_actions(self
, actions
):
171 map(EpisodeAction
.validate_time_values
, actions
)
172 self
.actions
= list(self
.actions
) + actions
173 self
.actions
= list(set(self
.actions
))
174 self
.actions
= sorted(self
.actions
, key
=lambda x
: x
.timestamp
)
177 def is_favorite(self
):
178 return self
.get_wksetting(FAV_FLAG
)
181 def set_favorite(self
, set_to
=True):
182 self
.settings
[FAV_FLAG
.name
] = set_to
185 def update_chapters(self
, add
=[], rem
=[]):
186 """ Updates the Chapter list
188 * add contains the chapters to be added
190 * rem contains tuples of (start, end) times. Chapters that match
191 both endpoints will be removed
194 @repeat_on_conflict(['state'])
197 self
.chapters
= self
.chapters
+ [chapter
]
199 for start
, end
in rem
:
200 keep
= lambda c
: c
.start
!= start
or c
.end
!= end
201 self
.chapters
= filter(keep
, self
.chapters
)
208 def get_history_entries(self
):
209 return imap(EpisodeAction
.to_history_entry
, self
.actions
)
213 return 'Episode-State %s (in %s)' % \
214 (self
.episode
, self
._id
)
216 def __eq__(self
, other
):
217 if not isinstance(other
, EpisodeUserState
):
220 return (self
.episode
== other
.episode
and
221 self
.user
== other
.user
)
225 class SubscriptionAction(Document
):
226 action
= StringProperty()
227 timestamp
= DateTimeProperty(default
=datetime
.utcnow
)
228 device
= StringProperty()
231 __metaclass__
= DocumentABCMeta
234 def __cmp__(self
, other
):
235 return cmp(self
.timestamp
, other
.timestamp
)
237 def __eq__(self
, other
):
238 return self
.action
== other
.action
and \
239 self
.timestamp
== other
.timestamp
and \
240 self
.device
== other
.device
243 return hash(self
.action
) + hash(self
.timestamp
) + hash(self
.device
)
246 return '<SubscriptionAction %s on %s at %s>' % (
247 self
.action
, self
.device
, self
.timestamp
)
250 class PodcastUserState(Document
, SettingsMixin
):
252 Contains everything that a user has done
253 with a specific podcast and all its episodes
256 podcast
= StringProperty(required
=True)
257 user_oldid
= IntegerProperty()
258 user
= StringProperty(required
=True)
259 actions
= SchemaListProperty(SubscriptionAction
)
260 tags
= StringListProperty()
261 ref_url
= StringProperty(required
=True)
262 disabled_devices
= StringListProperty()
263 merged_ids
= StringListProperty()
266 def remove_device(self
, device
):
268 Removes all actions from the podcast state that refer to the
271 self
.actions
= filter(lambda a
: a
.device
!= device
.id, self
.actions
)
274 def subscribe(self
, device
):
275 action
= SubscriptionAction()
276 action
.action
= 'subscribe'
277 action
.device
= device
.id
278 self
.add_actions([action
])
281 def unsubscribe(self
, device
):
282 action
= SubscriptionAction()
283 action
.action
= 'unsubscribe'
284 action
.device
= device
.id
285 self
.add_actions([action
])
288 def add_actions(self
, actions
):
289 self
.actions
= list(set(self
.actions
+ actions
))
290 self
.actions
= sorted(self
.actions
)
293 def add_tags(self
, tags
):
294 self
.tags
= list(set(self
.tags
+ tags
))
297 def set_device_state(self
, devices
):
298 disabled_devices
= [device
.id for device
in devices
if device
.deleted
]
299 self
.disabled_devices
= disabled_devices
302 def get_change_between(self
, device_id
, since
, until
):
304 Returns the change of the subscription status for the given device
305 between the two timestamps.
307 The change is given as either 'subscribe' (the podcast has been
308 subscribed), 'unsubscribed' (the podcast has been unsubscribed) or
312 device_actions
= filter(lambda x
: x
.device
== device_id
, self
.actions
)
313 before
= filter(lambda x
: x
.timestamp
<= since
, device_actions
)
314 after
= filter(lambda x
: x
.timestamp
<= until
, device_actions
)
316 # nothing happened, so there can be no change
320 then
= before
[-1] if before
else None
324 if now
.action
!= 'unsubscribe':
326 elif then
.action
!= now
.action
:
331 def get_subscribed_device_ids(self
):
332 """ device Ids on which the user subscribed to the podcast """
335 for action
in self
.actions
:
336 if action
.action
== "subscribe":
337 if not action
.device
in self
.disabled_devices
:
338 devices
.add(action
.device
)
340 if action
.device
in devices
:
341 devices
.remove(action
.device
)
348 return self
.get_wksetting(PUBLIC_SUB_PODCAST
)
351 def __eq__(self
, other
):
355 return self
.podcast
== other
.podcast
and \
356 self
.user
== other
.user
359 return 'Podcast %s for User %s (%s)' % \
360 (self
.podcast
, self
.user
, self
._id
)
363 class Device(Document
, SettingsMixin
):
364 id = StringProperty(default
=lambda: uuid
.uuid4().hex)
365 oldid
= IntegerProperty(required
=False)
366 uid
= StringProperty(required
=True)
367 name
= StringProperty(required
=True, default
='New Device')
368 type = StringProperty(required
=True, default
='other')
369 deleted
= BooleanProperty(default
=False)
370 user_agent
= StringProperty()
373 def get_subscription_changes(self
, since
, until
):
375 Returns the subscription changes for the device as two lists.
376 The first lists contains the Ids of the podcasts that have been
377 subscribed to, the second list of those that have been unsubscribed
381 from mygpo
.db
.couchdb
.podcast_state
import podcast_states_for_device
384 podcast_states
= podcast_states_for_device(self
.id)
385 for p_state
in podcast_states
:
386 change
= p_state
.get_change_between(self
.id, since
, until
)
387 if change
== 'subscribe':
388 add
.append( p_state
.ref_url
)
389 elif change
== 'unsubscribe':
390 rem
.append( p_state
.ref_url
)
395 def get_latest_changes(self
):
397 from mygpo
.db
.couchdb
.podcast_state
import podcast_states_for_device
399 podcast_states
= podcast_states_for_device(self
.id)
400 for p_state
in podcast_states
:
401 actions
= filter(lambda x
: x
.device
== self
.id, reversed(p_state
.actions
))
403 yield (p_state
.podcast
, actions
[0])
406 def get_subscribed_podcast_states(self
):
407 r
= PodcastUserState
.view('subscriptions/by_device',
408 startkey
= [self
.id, None],
409 endkey
= [self
.id, {}],
415 def get_subscribed_podcast_ids(self
):
416 states
= self
.get_subscribed_podcast_states()
417 return [state
.podcast
for state
in states
]
420 def get_subscribed_podcasts(self
):
421 """ Returns all subscribed podcasts for the device
423 The attribute "url" contains the URL that was used when subscribing to
426 states
= self
.get_subscribed_podcast_states()
427 podcast_ids
= [state
.podcast
for state
in states
]
428 podcasts
= podcasts_to_dict(podcast_ids
)
431 podcast
= proxy_object(podcasts
[state
.podcast
], url
=state
.ref_url
)
432 podcasts
[state
.podcast
] = podcast
434 return podcasts
.values()
438 return hash(frozenset([self
.id, self
.uid
, self
.name
, self
.type, self
.deleted
]))
441 def __eq__(self
, other
):
442 return self
.id == other
.id
446 return '<{cls} {id}>'.format(cls
=self
.__class
__.__name
__, id=self
.id)
452 def __unicode__(self
):
457 TOKEN_NAMES
= ('subscriptions_token', 'favorite_feeds_token',
458 'publisher_update_token', 'userpage_token')
461 class TokenException(Exception):
465 class User(BaseUser
, SyncedDevicesMixin
, SettingsMixin
):
466 oldid
= IntegerProperty()
467 devices
= SchemaListProperty(Device
)
468 published_objects
= StringListProperty()
469 deleted
= BooleanProperty(default
=False)
470 suggestions_up_to_date
= BooleanProperty(default
=False)
471 twitter
= StringProperty()
472 about
= StringProperty()
474 # token for accessing subscriptions of this use
475 subscriptions_token
= StringProperty(default
=None)
477 # token for accessing the favorite-episodes feed of this user
478 favorite_feeds_token
= StringProperty(default
=None)
480 # token for automatically updating feeds published by this user
481 publisher_update_token
= StringProperty(default
=None)
483 # token for accessing the userpage of this user
484 userpage_token
= StringProperty(default
=None)
490 def create_new_token(self
, token_name
, length
=32):
491 """ creates a new random token """
493 if token_name
not in TOKEN_NAMES
:
494 raise TokenException('Invalid token name %s' % token_name
)
496 token
= "".join(random
.sample(string
.letters
+string
.digits
, length
))
497 setattr(self
, token_name
, token
)
501 def get_token(self
, token_name
):
502 """ returns a token, and generate those that are still missing """
506 if token_name
not in TOKEN_NAMES
:
507 raise TokenException('Invalid token name %s' % token_name
)
509 for tn
in TOKEN_NAMES
:
510 if getattr(self
, tn
) is None:
511 self
.create_new_token(tn
)
517 return getattr(self
, token_name
)
522 def active_devices(self
):
523 not_deleted
= lambda d
: not d
.deleted
524 return filter(not_deleted
, self
.devices
)
528 def inactive_devices(self
):
529 deleted
= lambda d
: d
.deleted
530 return filter(deleted
, self
.devices
)
533 def get_devices_by_id(self
):
534 return dict( (device
.id, device
) for device
in self
.devices
)
537 def get_device(self
, id):
539 if not hasattr(self
, '__device_by_id'):
540 self
.__devices
_by
_id
= dict( (d
.id, d
) for d
in self
.devices
)
542 return self
.__devices
_by
_id
.get(id, None)
545 def get_device_by_uid(self
, uid
, only_active
=True):
547 if not hasattr(self
, '__devices_by_uio'):
548 self
.__devices
_by
_uid
= dict( (d
.uid
, d
) for d
in self
.devices
)
551 device
= self
.__devices
_by
_uid
[uid
]
553 if only_active
and device
.deleted
:
554 raise DeviceDeletedException(
555 'Device with UID %s is deleted' % uid
)
559 except KeyError as e
:
560 raise DeviceDoesNotExist('There is no device with UID %s' % uid
)
563 def update_device(self
, device
):
564 """ Sets the device and saves the user """
566 @repeat_on_conflict(['user'])
567 def _update(user
, device
):
568 user
.set_device(device
)
571 _update(user
=self
, device
=device
)
574 def set_device(self
, device
):
576 if not RE_DEVICE_UID
.match(device
.uid
):
577 raise DeviceUIDException(u
"'{uid} is not a valid device ID".format(
580 devices
= list(self
.devices
)
581 ids
= [x
.id for x
in devices
]
582 if not device
.id in ids
:
583 devices
.append(device
)
584 self
.devices
= devices
587 index
= ids
.index(device
.id)
589 devices
.insert(index
, device
)
590 self
.devices
= devices
593 def remove_device(self
, device
):
594 devices
= list(self
.devices
)
595 ids
= [x
.id for x
in devices
]
596 if not device
.id in ids
:
599 index
= ids
.index(device
.id)
601 self
.devices
= devices
603 if self
.is_synced(device
):
604 self
.unsync_device(device
)
607 def get_subscriptions_by_device(self
, public
=None):
608 from mygpo
.db
.couchdb
.podcast_state
import subscriptions_by_user
609 get_dev
= itemgetter(2)
610 groups
= collections
.defaultdict(list)
611 subscriptions
= subscriptions_by_user(self
, public
=public
)
612 subscriptions
= sorted(subscriptions
, key
=get_dev
)
614 for public
, podcast_id
, device_id
in subscriptions
:
615 groups
[device_id
].append(podcast_id
)
620 def get_subscribed_podcast_states(self
, public
=None):
622 Returns the Ids of all subscribed podcasts
625 r
= PodcastUserState
.view('subscriptions/by_user',
626 startkey
= [self
._id
, public
, None, None],
627 endkey
= [self
._id
+'ZZZ', None, None, None],
635 def get_subscribed_podcast_ids(self
, public
=None):
636 states
= self
.get_subscribed_podcast_states(public
=public
)
637 return [state
.podcast
for state
in states
]
641 def get_subscribed_podcasts(self
, public
=None):
642 """ Returns all subscribed podcasts for the user
644 The attribute "url" contains the URL that was used when subscribing to
647 states
= self
.get_subscribed_podcast_states(public
=public
)
648 podcast_ids
= [state
.podcast
for state
in states
]
649 podcasts
= podcasts_to_dict(podcast_ids
)
652 podcast
= proxy_object(podcasts
[state
.podcast
], url
=state
.ref_url
)
653 podcasts
[state
.podcast
] = podcast
655 return podcasts
.values()
659 def get_subscription_history(self
, device_id
=None, reverse
=False, public
=None):
660 """ Returns chronologically ordered subscription history entries
662 Setting device_id restricts the actions to a certain device
665 from mygpo
.db
.couchdb
.podcast_state
import podcast_states_for_user
, \
666 podcast_states_for_device
668 def action_iter(state
):
669 for action
in sorted(state
.actions
, reverse
=reverse
):
670 if device_id
is not None and device_id
!= action
.device
:
673 if public
is not None and state
.is_public() != public
:
676 entry
= HistoryEntry()
677 entry
.timestamp
= action
.timestamp
678 entry
.action
= action
.action
679 entry
.podcast_id
= state
.podcast
680 entry
.device_id
= action
.device
683 if device_id
is None:
684 podcast_states
= podcast_states_for_user(self
)
686 podcast_states
= podcast_states_for_device(device_id
)
688 # create an action_iter for each PodcastUserState
689 subscription_action_lists
= [action_iter(x
) for x
in podcast_states
]
691 action_cmp_key
= lambda x
: x
.timestamp
693 # Linearize their subscription-actions
694 return linearize(action_cmp_key
, subscription_action_lists
, reverse
)
697 def get_global_subscription_history(self
, public
=None):
698 """ Actions that added/removed podcasts from the subscription list
700 Returns an iterator of all subscription actions that either
701 * added subscribed a podcast that hasn't been subscribed directly
702 before the action (but could have been subscribed) earlier
703 * removed a subscription of the podcast is not longer subscribed
707 subscriptions
= collections
.defaultdict(int)
709 for entry
in self
.get_subscription_history(public
=public
):
710 if entry
.action
== 'subscribe':
711 subscriptions
[entry
.podcast_id
] += 1
713 # a new subscription has been added
714 if subscriptions
[entry
.podcast_id
] == 1:
717 elif entry
.action
== 'unsubscribe':
718 subscriptions
[entry
.podcast_id
] -= 1
720 # the last subscription has been removed
721 if subscriptions
[entry
.podcast_id
] == 0:
726 def get_newest_episodes(self
, max_date
, max_per_podcast
=5):
727 """ Returns the newest episodes of all subscribed podcasts
729 Only max_per_podcast episodes per podcast are loaded. Episodes with
730 release dates above max_date are discarded.
732 This method returns a generator that produces the newest episodes.
734 The number of required DB queries is equal to the number of (distinct)
735 podcasts of all consumed episodes (max: number of subscribed podcasts),
736 plus a constant number of initial queries (when the first episode is
739 cmp_key
= lambda episode
: episode
.released
or datetime(2000, 01, 01)
741 podcasts
= list(self
.get_subscribed_podcasts())
742 podcasts
= filter(lambda p
: p
.latest_episode_timestamp
, podcasts
)
743 podcasts
= sorted(podcasts
, key
=lambda p
: p
.latest_episode_timestamp
,
746 podcast_dict
= dict((p
.get_id(), p
) for p
in podcasts
)
748 # contains the un-yielded episodes, newest first
751 for podcast
in podcasts
:
755 for episode
in episodes
:
756 # determine for which episodes there won't be a new episodes
757 # that is newer; those can be yielded
758 if episode
.released
> podcast
.latest_episode_timestamp
:
759 p
= podcast_dict
.get(episode
.podcast
, None)
760 yield proxy_object(episode
, podcast
=p
)
761 yielded_episodes
+= 1
765 # remove the episodes that have been yielded before
766 episodes
= episodes
[yielded_episodes
:]
768 # fetch and merge episodes for the next podcast
769 from mygpo
.db
.couchdb
.episode
import episodes_for_podcast
770 new_episodes
= episodes_for_podcast(podcast
, since
=1,
771 until
=max_date
, descending
=True, limit
=max_per_podcast
)
772 episodes
= sorted(episodes
+new_episodes
, key
=cmp_key
, reverse
=True)
775 # yield the remaining episodes
776 for episode
in episodes
:
777 podcast
= podcast_dict
.get(episode
.podcast
, None)
778 yield proxy_object(episode
, podcast
=podcast
)
783 def save(self
, *args
, **kwargs
):
785 from mygpo
.db
.couchdb
.podcast_state
import podcast_states_for_user
787 super(User
, self
).save(*args
, **kwargs
)
789 podcast_states
= podcast_states_for_user(self
)
790 for state
in podcast_states
:
791 @repeat_on_conflict(['state'])
792 def _update_state(state
):
793 old_devs
= set(state
.disabled_devices
)
794 state
.set_device_state(self
.devices
)
796 if old_devs
!= set(state
.disabled_devices
):
799 _update_state(state
=state
)
804 def __eq__(self
, other
):
808 # ensure that other isn't AnonymousUser
809 return other
.is_authenticated() and self
._id
== other
._id
812 def __ne__(self
, other
):
813 return not(self
== other
)
817 return 'User %s' % self
._id
820 class History(object):
822 def __init__(self
, user
, device
):
827 def __getitem__(self
, key
):
829 if isinstance(key
, slice):
830 start
= key
.start
or 0
831 length
= key
.stop
- start
837 return device_history(self
.user
, self
.device
, start
, length
)
840 return user_history(self
.user
, start
, length
)
844 class HistoryEntry(object):
845 """ A class that can represent subscription and episode actions """
849 def from_action_dict(cls
, action
):
851 entry
= HistoryEntry()
853 if 'timestamp' in action
:
854 ts
= action
.pop('timestamp')
855 entry
.timestamp
= dateutil
.parser
.parse(ts
)
857 for key
, value
in action
.items():
858 setattr(entry
, key
, value
)
865 return getattr(self
, 'position', None)
869 def fetch_data(cls
, user
, entries
,
870 podcasts
=None, episodes
=None):
871 """ Efficiently loads additional data for a number of entries """
875 podcast_ids
= [getattr(x
, 'podcast_id', None) for x
in entries
]
876 podcast_ids
= filter(None, podcast_ids
)
877 podcasts
= podcasts_to_dict(podcast_ids
)
880 from mygpo
.db
.couchdb
.episode
import episodes_to_dict
882 episode_ids
= [getattr(x
, 'episode_id', None) for x
in entries
]
883 episode_ids
= filter(None, episode_ids
)
884 episodes
= episodes_to_dict(episode_ids
)
887 # does not need pre-populated data because no db-access is required
888 device_ids
= [getattr(x
, 'device_id', None) for x
in entries
]
889 device_ids
= filter(None, device_ids
)
890 devices
= dict([ (id, user
.get_device(id)) for id in device_ids
])
893 for entry
in entries
:
894 podcast_id
= getattr(entry
, 'podcast_id', None)
895 entry
.podcast
= podcasts
.get(podcast_id
, None)
897 episode_id
= getattr(entry
, 'episode_id', None)
898 entry
.episode
= episodes
.get(episode_id
, None)
900 if hasattr(entry
, 'user'):
903 device
= devices
.get(getattr(entry
, 'device_id', None), None)
904 entry
.device
= device