4 from datetime
import datetime
6 from itertools
import imap
7 from operator
import itemgetter
11 from couchdbkit
.ext
.django
.schema
import *
13 from django
.core
.cache
import cache
15 from django_couchdb_utils
.registration
.models
import User
as BaseUser
17 from mygpo
.podcasts
.models
import Podcast
18 from mygpo
.utils
import linearize
19 from mygpo
.core
.proxy
import DocumentABCMeta
, proxy_object
20 from mygpo
.decorators
import repeat_on_conflict
21 from mygpo
.users
.ratings
import RatingMixin
22 from mygpo
.users
.sync
import SyncedDevicesMixin
23 from mygpo
.users
.subscriptions
import subscription_changes
, podcasts_for_states
24 from mygpo
.users
.settings
import FAV_FLAG
, PUBLIC_SUB_PODCAST
, SettingsMixin
25 from mygpo
.db
.couchdb
.user
import user_history
, device_history
, \
26 create_missing_user_tokens
28 # make sure this code is executed at startup
29 from mygpo
.users
.signals
import *
32 RE_DEVICE_UID
= re
.compile(r
'^[\w.-]+$')
34 # TODO: derive from ValidationException?
35 class InvalidEpisodeActionAttributes(ValueError):
36 """ raised when the attribues of an episode action fail validation """
39 class SubscriptionException(Exception):
40 """ raised when a subscription can not be modified """
43 class DeviceUIDException(Exception):
47 class DeviceDoesNotExist(Exception):
51 class DeviceDeletedException(DeviceDoesNotExist
):
55 class Suggestions(Document
, RatingMixin
):
56 user
= StringProperty(required
=True)
57 user_oldid
= IntegerProperty()
58 podcasts
= StringListProperty()
59 blacklist
= StringListProperty()
62 def get_podcasts(self
, count
=None):
63 user
= User
.get(self
.user
)
64 subscriptions
= user
.get_subscribed_podcast_ids()
66 ids
= filter(lambda x
: not x
in self
.blacklist
+ subscriptions
, self
.podcasts
)
69 return filter(lambda x
: x
and x
.title
, Podcast
.objects
.filter(id__in
=ids
))
74 return super(Suggestions
, self
).__repr
__()
76 return '%d Suggestions for %s (%s)' % \
77 (len(self
.podcasts
), self
.user
, self
._id
)
80 class EpisodeAction(DocumentSchema
):
82 One specific action to an episode. Must
83 always be part of a EpisodeUserState
86 action
= StringProperty(required
=True)
88 # walltime of the event (assigned by the uploading client, defaults to now)
89 timestamp
= DateTimeProperty(required
=True, default
=datetime
.utcnow
)
91 # upload time of the event
92 upload_timestamp
= IntegerProperty(required
=True)
94 device_oldid
= IntegerProperty(required
=False)
95 device
= StringProperty()
96 started
= IntegerProperty()
97 playmark
= IntegerProperty()
98 total
= IntegerProperty()
100 def __eq__(self
, other
):
101 if not isinstance(other
, EpisodeAction
):
103 vals
= ('action', 'timestamp', 'device', 'started', 'playmark',
105 return all([getattr(self
, v
, None) == getattr(other
, v
, None) for v
in vals
])
108 def to_history_entry(self
):
109 entry
= HistoryEntry()
110 entry
.action
= self
.action
111 entry
.timestamp
= self
.timestamp
112 entry
.device_id
= self
.device
113 entry
.started
= self
.started
114 entry
.position
= self
.playmark
115 entry
.total
= self
.total
120 def validate_time_values(self
):
121 """ Validates allowed combinations of time-values """
123 PLAY_ACTION_KEYS
= ('playmark', 'started', 'total')
125 # Key found, but must not be supplied (no play action!)
126 if self
.action
!= 'play':
127 for key
in PLAY_ACTION_KEYS
:
128 if getattr(self
, key
, None) is not None:
129 raise InvalidEpisodeActionAttributes('%s only allowed in play actions' % key
)
131 # Sanity check: If started or total are given, require playmark
132 if ((self
.started
is not None) or (self
.total
is not None)) and \
133 self
.playmark
is None:
134 raise InvalidEpisodeActionAttributes('started and total require position')
136 # Sanity check: total and playmark can only appear together
137 if ((self
.total
is not None) or (self
.started
is not None)) and \
138 ((self
.total
is None) or (self
.started
is None)):
139 raise InvalidEpisodeActionAttributes('total and started can only appear together')
143 return '%s-Action on %s at %s (in %s)' % \
144 (self
.action
, self
.device
, self
.timestamp
, self
._id
)
148 return hash(frozenset([self
.action
, self
.timestamp
, self
.device
,
149 self
.started
, self
.playmark
, self
.total
]))
152 class Chapter(Document
):
153 """ A user-entered episode chapter """
155 device
= StringProperty()
156 created
= DateTimeProperty()
157 start
= IntegerProperty(required
=True)
158 end
= IntegerProperty(required
=True)
159 label
= StringProperty()
160 advertisement
= BooleanProperty()
164 return '<%s %s (%d-%d)>' % (self
.__class
__.__name
__, self
.label
,
165 self
.start
, self
.end
)
168 class EpisodeUserState(Document
, SettingsMixin
):
170 Contains everything a user has done with an Episode
173 episode
= StringProperty(required
=True)
174 actions
= SchemaListProperty(EpisodeAction
)
175 user_oldid
= IntegerProperty()
176 user
= StringProperty(required
=True)
177 ref_url
= StringProperty(required
=True)
178 podcast_ref_url
= StringProperty(required
=True)
179 merged_ids
= StringListProperty()
180 chapters
= SchemaListProperty(Chapter
)
181 podcast
= StringProperty(required
=True)
185 def add_actions(self
, actions
):
186 map(EpisodeAction
.validate_time_values
, actions
)
187 self
.actions
= list(self
.actions
) + actions
188 self
.actions
= list(set(self
.actions
))
189 self
.actions
= sorted(self
.actions
, key
=lambda x
: x
.timestamp
)
192 def is_favorite(self
):
193 return self
.get_wksetting(FAV_FLAG
)
196 def set_favorite(self
, set_to
=True):
197 self
.settings
[FAV_FLAG
.name
] = set_to
200 def get_history_entries(self
):
201 return imap(EpisodeAction
.to_history_entry
, self
.actions
)
205 return 'Episode-State %s (in %s)' % \
206 (self
.episode
, self
._id
)
208 def __eq__(self
, other
):
209 if not isinstance(other
, EpisodeUserState
):
212 return (self
.episode
== other
.episode
and
213 self
.user
== other
.user
)
217 class SubscriptionAction(Document
):
218 action
= StringProperty()
219 timestamp
= DateTimeProperty(default
=datetime
.utcnow
)
220 device
= StringProperty()
223 __metaclass__
= DocumentABCMeta
226 def __cmp__(self
, other
):
227 return cmp(self
.timestamp
, other
.timestamp
)
229 def __eq__(self
, other
):
230 return self
.action
== other
.action
and \
231 self
.timestamp
== other
.timestamp
and \
232 self
.device
== other
.device
235 return hash(self
.action
) + hash(self
.timestamp
) + hash(self
.device
)
238 return '<SubscriptionAction %s on %s at %s>' % (
239 self
.action
, self
.device
, self
.timestamp
)
242 class PodcastUserState(Document
, SettingsMixin
):
244 Contains everything that a user has done
245 with a specific podcast and all its episodes
248 podcast
= StringProperty(required
=True)
249 user_oldid
= IntegerProperty()
250 user
= StringProperty(required
=True)
251 actions
= SchemaListProperty(SubscriptionAction
)
252 tags
= StringListProperty()
253 ref_url
= StringProperty(required
=True)
254 disabled_devices
= StringListProperty()
255 merged_ids
= StringListProperty()
258 def remove_device(self
, device
):
260 Removes all actions from the podcast state that refer to the
263 self
.actions
= filter(lambda a
: a
.device
!= device
.id, self
.actions
)
266 def subscribe(self
, device
):
267 action
= SubscriptionAction()
268 action
.action
= 'subscribe'
269 action
.device
= device
.id
270 self
.add_actions([action
])
273 def unsubscribe(self
, device
):
274 action
= SubscriptionAction()
275 action
.action
= 'unsubscribe'
276 action
.device
= device
.id
277 self
.add_actions([action
])
280 def add_actions(self
, actions
):
281 self
.actions
= list(set(self
.actions
+ actions
))
282 self
.actions
= sorted(self
.actions
)
285 def add_tags(self
, tags
):
286 self
.tags
= list(set(self
.tags
+ tags
))
289 def set_device_state(self
, devices
):
290 disabled_devices
= [device
.id for device
in devices
if device
.deleted
]
291 self
.disabled_devices
= disabled_devices
294 def get_change_between(self
, device_id
, since
, until
):
296 Returns the change of the subscription status for the given device
297 between the two timestamps.
299 The change is given as either 'subscribe' (the podcast has been
300 subscribed), 'unsubscribed' (the podcast has been unsubscribed) or
304 device_actions
= filter(lambda x
: x
.device
== device_id
, self
.actions
)
305 before
= filter(lambda x
: x
.timestamp
<= since
, device_actions
)
306 after
= filter(lambda x
: x
.timestamp
<= until
, device_actions
)
308 # nothing happened, so there can be no change
312 then
= before
[-1] if before
else None
316 if now
.action
!= 'unsubscribe':
318 elif then
.action
!= now
.action
:
323 def get_subscribed_device_ids(self
):
324 """ device Ids on which the user subscribed to the podcast """
327 for action
in self
.actions
:
328 if action
.action
== "subscribe":
329 if not action
.device
in self
.disabled_devices
:
330 devices
.add(action
.device
)
332 if action
.device
in devices
:
333 devices
.remove(action
.device
)
338 def is_subscribed_on(self
, device
):
339 """ checks if the podcast is subscribed on the given device """
341 for action
in reversed(self
.actions
):
342 if not action
.device
== device
.id:
345 # we only need to check the latest action for the device
346 return (action
.action
== 'subscribe')
348 # we haven't found any matching action
353 return self
.get_wksetting(PUBLIC_SUB_PODCAST
)
356 def __eq__(self
, other
):
360 return self
.podcast
== other
.podcast
and \
361 self
.user
== other
.user
364 return 'Podcast %s for User %s (%s)' % \
365 (self
.podcast
, self
.user
, self
._id
)
368 class Device(Document
, SettingsMixin
):
369 id = StringProperty(default
=lambda: uuid
.uuid4().hex)
370 oldid
= IntegerProperty(required
=False)
371 uid
= StringProperty(required
=True)
372 name
= StringProperty(required
=True, default
='New Device')
373 type = StringProperty(required
=True, default
='other')
374 deleted
= BooleanProperty(default
=False)
375 user_agent
= StringProperty()
378 def get_subscription_changes(self
, since
, until
):
380 Returns the subscription changes for the device as two lists.
381 The first lists contains the Ids of the podcasts that have been
382 subscribed to, the second list of those that have been unsubscribed
386 from mygpo
.db
.couchdb
.podcast_state
import podcast_states_for_device
387 podcast_states
= podcast_states_for_device(self
.id)
388 return subscription_changes(self
.id, podcast_states
, since
, until
)
391 def get_latest_changes(self
):
393 from mygpo
.db
.couchdb
.podcast_state
import podcast_states_for_device
395 podcast_states
= podcast_states_for_device(self
.id)
396 for p_state
in podcast_states
:
397 actions
= filter(lambda x
: x
.device
== self
.id, reversed(p_state
.actions
))
399 yield (p_state
.podcast
, actions
[0])
402 def get_subscribed_podcast_ids(self
):
403 from mygpo
.db
.couchdb
.podcast_state
import get_subscribed_podcast_states_by_device
404 states
= get_subscribed_podcast_states_by_device(self
)
405 return [state
.podcast
for state
in states
]
408 def get_subscribed_podcasts(self
):
409 """ Returns all subscribed podcasts for the device
411 The attribute "url" contains the URL that was used when subscribing to
414 from mygpo
.db
.couchdb
.podcast_state
import get_subscribed_podcast_states_by_device
415 states
= get_subscribed_podcast_states_by_device(self
)
416 return podcasts_for_states(states
)
420 return hash(frozenset([self
.id, self
.uid
, self
.name
, self
.type, self
.deleted
]))
423 def __eq__(self
, other
):
424 return self
.id == other
.id
428 return '<{cls} {id}>'.format(cls
=self
.__class
__.__name
__, id=self
.id)
434 def __unicode__(self
):
439 TOKEN_NAMES
= ('subscriptions_token', 'favorite_feeds_token',
440 'publisher_update_token', 'userpage_token')
443 class TokenException(Exception):
447 class User(BaseUser
, SyncedDevicesMixin
, SettingsMixin
):
448 oldid
= IntegerProperty()
449 devices
= SchemaListProperty(Device
)
450 published_objects
= StringListProperty()
451 deleted
= BooleanProperty(default
=False)
452 suggestions_up_to_date
= BooleanProperty(default
=False)
453 twitter
= StringProperty()
454 about
= StringProperty()
455 google_email
= StringProperty()
457 # token for accessing subscriptions of this use
458 subscriptions_token
= StringProperty(default
=None)
460 # token for accessing the favorite-episodes feed of this user
461 favorite_feeds_token
= StringProperty(default
=None)
463 # token for automatically updating feeds published by this user
464 publisher_update_token
= StringProperty(default
=None)
466 # token for accessing the userpage of this user
467 userpage_token
= StringProperty(default
=None)
473 def create_new_token(self
, token_name
, length
=32):
474 """ creates a new random token """
476 if token_name
not in TOKEN_NAMES
:
477 raise TokenException('Invalid token name %s' % token_name
)
479 token
= "".join(random
.sample(string
.letters
+string
.digits
, length
))
480 setattr(self
, token_name
, token
)
484 @repeat_on_conflict(['self'])
485 def get_token(self
, token_name
):
486 """ returns a token, and generate those that are still missing """
490 if token_name
not in TOKEN_NAMES
:
491 raise TokenException('Invalid token name %s' % token_name
)
493 create_missing_user_tokens(self
)
495 return getattr(self
, token_name
)
500 def active_devices(self
):
501 not_deleted
= lambda d
: not d
.deleted
502 return filter(not_deleted
, self
.devices
)
506 def inactive_devices(self
):
507 deleted
= lambda d
: d
.deleted
508 return filter(deleted
, self
.devices
)
511 def get_devices_by_id(self
, device_ids
=None):
512 """ Returns a dict of {devices_id: device} """
513 if device_ids
is None:
515 devices
= self
.devices
517 devices
= self
.get_devices(device_ids
)
519 return {device
.id: device
for device
in devices
}
522 def get_device(self
, id):
524 if not hasattr(self
, '__device_by_id'):
525 self
.__devices
_by
_id
= self
.get_devices_by_id()
527 return self
.__devices
_by
_id
.get(id, None)
530 def get_devices(self
, ids
):
531 return filter(None, (self
.get_device(dev_id
) for dev_id
in ids
))
534 def get_device_by_uid(self
, uid
, only_active
=True):
536 if not hasattr(self
, '__devices_by_uio'):
537 self
.__devices
_by
_uid
= dict( (d
.uid
, d
) for d
in self
.devices
)
540 device
= self
.__devices
_by
_uid
[uid
]
542 if only_active
and device
.deleted
:
543 raise DeviceDeletedException(
544 'Device with UID %s is deleted' % uid
)
548 except KeyError as e
:
549 raise DeviceDoesNotExist('There is no device with UID %s' % uid
)
552 def set_device(self
, device
):
554 if not RE_DEVICE_UID
.match(device
.uid
):
555 raise DeviceUIDException(u
"'{uid} is not a valid device ID".format(
558 devices
= list(self
.devices
)
559 ids
= [x
.id for x
in devices
]
560 if not device
.id in ids
:
561 devices
.append(device
)
562 self
.devices
= devices
565 index
= ids
.index(device
.id)
567 devices
.insert(index
, device
)
568 self
.devices
= devices
571 def remove_device(self
, device
):
572 devices
= list(self
.devices
)
573 ids
= [x
.id for x
in devices
]
574 if not device
.id in ids
:
577 index
= ids
.index(device
.id)
579 self
.devices
= devices
581 if self
.is_synced(device
):
582 self
.unsync_device(device
)
585 def get_subscriptions_by_device(self
, public
=None):
586 from mygpo
.db
.couchdb
.podcast_state
import subscriptions_by_user
587 get_dev
= itemgetter(2)
588 groups
= collections
.defaultdict(list)
589 subscriptions
= subscriptions_by_user(self
, public
=public
)
590 subscriptions
= sorted(subscriptions
, key
=get_dev
)
592 for public
, podcast_id
, device_id
in subscriptions
:
593 groups
[device_id
].append(podcast_id
)
597 def get_subscribed_podcast_ids(self
, public
=None):
598 from mygpo
.db
.couchdb
.podcast_state
import get_subscribed_podcast_states_by_user
599 states
= get_subscribed_podcast_states_by_user(self
, public
)
600 return [state
.podcast
for state
in states
]
604 def get_subscribed_podcasts(self
, public
=None):
605 """ Returns all subscribed podcasts for the user
607 The attribute "url" contains the URL that was used when subscribing to
610 from mygpo
.db
.couchdb
.podcast_state
import get_subscribed_podcast_states_by_user
611 states
= get_subscribed_podcast_states_by_user(self
, public
)
612 podcast_ids
= [state
.podcast
for state
in states
]
613 podcasts
= Podcast
.objects
.get(id__in
=podcast_ids
)
614 podcasts
= {podcast
.id: podcast
for podcast
in podcasts
}
617 podcast
= podcasts
.get(state
.podcast
, None)
621 podcast
= proxy_object(podcast
, url
=state
.ref_url
)
622 podcasts
[state
.podcast
] = podcast
624 return set(podcasts
.values())
628 def get_subscription_history(self
, device_id
=None, reverse
=False, public
=None):
629 """ Returns chronologically ordered subscription history entries
631 Setting device_id restricts the actions to a certain device
634 from mygpo
.db
.couchdb
.podcast_state
import podcast_states_for_user
, \
635 podcast_states_for_device
637 def action_iter(state
):
638 for action
in sorted(state
.actions
, reverse
=reverse
):
639 if device_id
is not None and device_id
!= action
.device
:
642 if public
is not None and state
.is_public() != public
:
645 entry
= HistoryEntry()
646 entry
.timestamp
= action
.timestamp
647 entry
.action
= action
.action
648 entry
.podcast_id
= state
.podcast
649 entry
.device_id
= action
.device
652 if device_id
is None:
653 podcast_states
= podcast_states_for_user(self
)
655 podcast_states
= podcast_states_for_device(device_id
)
657 # create an action_iter for each PodcastUserState
658 subscription_action_lists
= [action_iter(x
) for x
in podcast_states
]
660 action_cmp_key
= lambda x
: x
.timestamp
662 # Linearize their subscription-actions
663 return linearize(action_cmp_key
, subscription_action_lists
, reverse
)
666 def get_global_subscription_history(self
, public
=None):
667 """ Actions that added/removed podcasts from the subscription list
669 Returns an iterator of all subscription actions that either
670 * added subscribed a podcast that hasn't been subscribed directly
671 before the action (but could have been subscribed) earlier
672 * removed a subscription of the podcast is not longer subscribed
676 subscriptions
= collections
.defaultdict(int)
678 for entry
in self
.get_subscription_history(public
=public
):
679 if entry
.action
== 'subscribe':
680 subscriptions
[entry
.podcast_id
] += 1
682 # a new subscription has been added
683 if subscriptions
[entry
.podcast_id
] == 1:
686 elif entry
.action
== 'unsubscribe':
687 subscriptions
[entry
.podcast_id
] -= 1
689 # the last subscription has been removed
690 if subscriptions
[entry
.podcast_id
] == 0:
695 def get_newest_episodes(self
, max_date
, max_per_podcast
=5):
696 """ Returns the newest episodes of all subscribed podcasts
698 Only max_per_podcast episodes per podcast are loaded. Episodes with
699 release dates above max_date are discarded.
701 This method returns a generator that produces the newest episodes.
703 The number of required DB queries is equal to the number of (distinct)
704 podcasts of all consumed episodes (max: number of subscribed podcasts),
705 plus a constant number of initial queries (when the first episode is
708 cmp_key
= lambda episode
: episode
.released
or datetime(2000, 01, 01)
710 podcasts
= list(self
.get_subscribed_podcasts())
711 podcasts
= filter(lambda p
: p
.latest_episode_timestamp
, podcasts
)
712 podcasts
= sorted(podcasts
, key
=lambda p
: p
.latest_episode_timestamp
,
715 podcast_dict
= dict((p
.get_id(), p
) for p
in podcasts
)
717 # contains the un-yielded episodes, newest first
720 for podcast
in podcasts
:
724 for episode
in episodes
:
725 # determine for which episodes there won't be a new episodes
726 # that is newer; those can be yielded
727 if episode
.released
> podcast
.latest_episode_timestamp
:
728 p
= podcast_dict
.get(episode
.podcast
, None)
729 yield proxy_object(episode
, podcast
=p
)
730 yielded_episodes
+= 1
734 # remove the episodes that have been yielded before
735 episodes
= episodes
[yielded_episodes
:]
737 # fetch and merge episodes for the next podcast
738 # TODO: max_per_podcast
739 new_episodes
= podcast
.episode_set
.filter(release__isnull
=False, released__lt
=max_date
)[:max_per_podcast
]
740 episodes
= sorted(episodes
+new_episodes
, key
=cmp_key
, reverse
=True)
743 # yield the remaining episodes
744 for episode
in episodes
:
745 podcast
= podcast_dict
.get(episode
.podcast
, None)
746 yield proxy_object(episode
, podcast
=podcast
)
749 def __eq__(self
, other
):
753 # ensure that other isn't AnonymousUser
754 return other
.is_authenticated() and self
._id
== other
._id
757 def __ne__(self
, other
):
758 return not(self
== other
)
762 return 'User %s' % self
._id
765 class History(object):
767 def __init__(self
, user
, device
):
772 def __getitem__(self
, key
):
774 if isinstance(key
, slice):
775 start
= key
.start
or 0
776 length
= key
.stop
- start
782 return device_history(self
.user
, self
.device
, start
, length
)
785 return user_history(self
.user
, start
, length
)
789 class HistoryEntry(object):
790 """ A class that can represent subscription and episode actions """
794 def from_action_dict(cls
, action
):
796 entry
= HistoryEntry()
798 if 'timestamp' in action
:
799 ts
= action
.pop('timestamp')
800 entry
.timestamp
= dateutil
.parser
.parse(ts
)
802 for key
, value
in action
.items():
803 setattr(entry
, key
, value
)
810 return getattr(self
, 'position', None)
814 def fetch_data(cls
, user
, entries
,
815 podcasts
=None, episodes
=None):
816 """ Efficiently loads additional data for a number of entries """
820 podcast_ids
= [getattr(x
, 'podcast_id', None) for x
in entries
]
821 podcast_ids
= filter(None, podcast_ids
)
822 podcasts
= Podcast
.objects
.filter(id__in
=podcast_ids
)
823 podcasts
= {podcast
.id: podcast
for podcast
in podcasts
}
827 episode_ids
= [getattr(x
, 'episode_id', None) for x
in entries
]
828 episode_ids
= filter(None, episode_ids
)
829 episodes
= Episode
.objects
.filter(id__in
=episode_ids
)
830 episodes
= {episode
.id: episode
for episode
in episodes
}
833 # does not need pre-populated data because no db-access is required
834 device_ids
= [getattr(x
, 'device_id', None) for x
in entries
]
835 device_ids
= filter(None, device_ids
)
836 devices
= dict([ (id, user
.get_device(id)) for id in device_ids
])
839 for entry
in entries
:
840 podcast_id
= getattr(entry
, 'podcast_id', None)
841 entry
.podcast
= podcasts
.get(podcast_id
, None)
843 episode_id
= getattr(entry
, 'episode_id', None)
844 entry
.episode
= episodes
.get(episode_id
, None)
846 if hasattr(entry
, 'user'):
849 device
= devices
.get(getattr(entry
, 'device_id', None), None)
850 entry
.device
= device