4 from datetime
import datetime
6 from itertools
import imap
7 from operator
import itemgetter
11 from couchdbkit
.ext
.django
.schema
import *
13 from django
.core
.cache
import cache
15 from django_couchdb_utils
.registration
.models
import User
as BaseUser
17 from mygpo
.utils
import linearize
18 from mygpo
.core
.proxy
import DocumentABCMeta
, proxy_object
19 from mygpo
.decorators
import repeat_on_conflict
20 from mygpo
.users
.ratings
import RatingMixin
21 from mygpo
.users
.sync
import SyncedDevicesMixin
22 from mygpo
.users
.subscriptions
import subscription_changes
, podcasts_for_states
23 from mygpo
.users
.settings
import FAV_FLAG
, PUBLIC_SUB_PODCAST
, SettingsMixin
24 from mygpo
.db
.couchdb
.podcast
import podcasts_by_id
, podcasts_to_dict
25 from mygpo
.db
.couchdb
.user
import user_history
, device_history
, \
26 create_missing_user_tokens
28 # make sure this code is executed at startup
29 from mygpo
.users
.signals
import *
32 RE_DEVICE_UID
= re
.compile(r
'^[\w.-]+$')
34 # TODO: derive from ValidationException?
35 class InvalidEpisodeActionAttributes(ValueError):
36 """ raised when the attribues of an episode action fail validation """
39 class DeviceUIDException(Exception):
43 class DeviceDoesNotExist(Exception):
47 class DeviceDeletedException(DeviceDoesNotExist
):
51 class Suggestions(Document
, RatingMixin
):
52 user
= StringProperty(required
=True)
53 user_oldid
= IntegerProperty()
54 podcasts
= StringListProperty()
55 blacklist
= StringListProperty()
58 def get_podcasts(self
, count
=None):
59 user
= User
.get(self
.user
)
60 subscriptions
= user
.get_subscribed_podcast_ids()
62 ids
= filter(lambda x
: not x
in self
.blacklist
+ subscriptions
, self
.podcasts
)
65 return filter(lambda x
: x
and x
.title
, podcasts_by_id(ids
))
70 return super(Suggestions
, self
).__repr
__()
72 return '%d Suggestions for %s (%s)' % \
73 (len(self
.podcasts
), self
.user
, self
._id
)
76 class EpisodeAction(DocumentSchema
):
78 One specific action to an episode. Must
79 always be part of a EpisodeUserState
82 action
= StringProperty(required
=True)
84 # walltime of the event (assigned by the uploading client, defaults to now)
85 timestamp
= DateTimeProperty(required
=True, default
=datetime
.utcnow
)
87 # upload time of the event
88 upload_timestamp
= IntegerProperty(required
=True)
90 device_oldid
= IntegerProperty(required
=False)
91 device
= StringProperty()
92 started
= IntegerProperty()
93 playmark
= IntegerProperty()
94 total
= IntegerProperty()
96 def __eq__(self
, other
):
97 if not isinstance(other
, EpisodeAction
):
99 vals
= ('action', 'timestamp', 'device', 'started', 'playmark',
101 return all([getattr(self
, v
, None) == getattr(other
, v
, None) for v
in vals
])
104 def to_history_entry(self
):
105 entry
= HistoryEntry()
106 entry
.action
= self
.action
107 entry
.timestamp
= self
.timestamp
108 entry
.device_id
= self
.device
109 entry
.started
= self
.started
110 entry
.position
= self
.playmark
111 entry
.total
= self
.total
116 def validate_time_values(self
):
117 """ Validates allowed combinations of time-values """
119 PLAY_ACTION_KEYS
= ('playmark', 'started', 'total')
121 # Key found, but must not be supplied (no play action!)
122 if self
.action
!= 'play':
123 for key
in PLAY_ACTION_KEYS
:
124 if getattr(self
, key
, None) is not None:
125 raise InvalidEpisodeActionAttributes('%s only allowed in play actions' % key
)
127 # Sanity check: If started or total are given, require playmark
128 if ((self
.started
is not None) or (self
.total
is not None)) and \
129 self
.playmark
is None:
130 raise InvalidEpisodeActionAttributes('started and total require position')
132 # Sanity check: total and playmark can only appear together
133 if ((self
.total
is not None) or (self
.started
is not None)) and \
134 ((self
.total
is None) or (self
.started
is None)):
135 raise InvalidEpisodeActionAttributes('total and started can only appear together')
139 return '%s-Action on %s at %s (in %s)' % \
140 (self
.action
, self
.device
, self
.timestamp
, self
._id
)
144 return hash(frozenset([self
.action
, self
.timestamp
, self
.device
,
145 self
.started
, self
.playmark
, self
.total
]))
148 class Chapter(Document
):
149 """ A user-entered episode chapter """
151 device
= StringProperty()
152 created
= DateTimeProperty()
153 start
= IntegerProperty(required
=True)
154 end
= IntegerProperty(required
=True)
155 label
= StringProperty()
156 advertisement
= BooleanProperty()
160 return '<%s %s (%d-%d)>' % (self
.__class
__.__name
__, self
.label
,
161 self
.start
, self
.end
)
164 class EpisodeUserState(Document
, SettingsMixin
):
166 Contains everything a user has done with an Episode
169 episode
= StringProperty(required
=True)
170 actions
= SchemaListProperty(EpisodeAction
)
171 user_oldid
= IntegerProperty()
172 user
= StringProperty(required
=True)
173 ref_url
= StringProperty(required
=True)
174 podcast_ref_url
= StringProperty(required
=True)
175 merged_ids
= StringListProperty()
176 chapters
= SchemaListProperty(Chapter
)
177 podcast
= StringProperty(required
=True)
181 def add_actions(self
, actions
):
182 map(EpisodeAction
.validate_time_values
, actions
)
183 self
.actions
= list(self
.actions
) + actions
184 self
.actions
= list(set(self
.actions
))
185 self
.actions
= sorted(self
.actions
, key
=lambda x
: x
.timestamp
)
188 def is_favorite(self
):
189 return self
.get_wksetting(FAV_FLAG
)
192 def set_favorite(self
, set_to
=True):
193 self
.settings
[FAV_FLAG
.name
] = set_to
196 def get_history_entries(self
):
197 return imap(EpisodeAction
.to_history_entry
, self
.actions
)
201 return 'Episode-State %s (in %s)' % \
202 (self
.episode
, self
._id
)
204 def __eq__(self
, other
):
205 if not isinstance(other
, EpisodeUserState
):
208 return (self
.episode
== other
.episode
and
209 self
.user
== other
.user
)
213 class SubscriptionAction(Document
):
214 action
= StringProperty()
215 timestamp
= DateTimeProperty(default
=datetime
.utcnow
)
216 device
= StringProperty()
219 __metaclass__
= DocumentABCMeta
222 def __cmp__(self
, other
):
223 return cmp(self
.timestamp
, other
.timestamp
)
225 def __eq__(self
, other
):
226 return self
.action
== other
.action
and \
227 self
.timestamp
== other
.timestamp
and \
228 self
.device
== other
.device
231 return hash(self
.action
) + hash(self
.timestamp
) + hash(self
.device
)
234 return '<SubscriptionAction %s on %s at %s>' % (
235 self
.action
, self
.device
, self
.timestamp
)
238 class PodcastUserState(Document
, SettingsMixin
):
240 Contains everything that a user has done
241 with a specific podcast and all its episodes
244 podcast
= StringProperty(required
=True)
245 user_oldid
= IntegerProperty()
246 user
= StringProperty(required
=True)
247 actions
= SchemaListProperty(SubscriptionAction
)
248 tags
= StringListProperty()
249 ref_url
= StringProperty(required
=True)
250 disabled_devices
= StringListProperty()
251 merged_ids
= StringListProperty()
254 def remove_device(self
, device
):
256 Removes all actions from the podcast state that refer to the
259 self
.actions
= filter(lambda a
: a
.device
!= device
.id, self
.actions
)
262 def subscribe(self
, device
):
263 action
= SubscriptionAction()
264 action
.action
= 'subscribe'
265 action
.device
= device
.id
266 self
.add_actions([action
])
269 def unsubscribe(self
, device
):
270 action
= SubscriptionAction()
271 action
.action
= 'unsubscribe'
272 action
.device
= device
.id
273 self
.add_actions([action
])
276 def add_actions(self
, actions
):
277 self
.actions
= list(set(self
.actions
+ actions
))
278 self
.actions
= sorted(self
.actions
)
281 def add_tags(self
, tags
):
282 self
.tags
= list(set(self
.tags
+ tags
))
285 def set_device_state(self
, devices
):
286 disabled_devices
= [device
.id for device
in devices
if device
.deleted
]
287 self
.disabled_devices
= disabled_devices
290 def get_change_between(self
, device_id
, since
, until
):
292 Returns the change of the subscription status for the given device
293 between the two timestamps.
295 The change is given as either 'subscribe' (the podcast has been
296 subscribed), 'unsubscribed' (the podcast has been unsubscribed) or
300 device_actions
= filter(lambda x
: x
.device
== device_id
, self
.actions
)
301 before
= filter(lambda x
: x
.timestamp
<= since
, device_actions
)
302 after
= filter(lambda x
: x
.timestamp
<= until
, device_actions
)
304 # nothing happened, so there can be no change
308 then
= before
[-1] if before
else None
312 if now
.action
!= 'unsubscribe':
314 elif then
.action
!= now
.action
:
319 def get_subscribed_device_ids(self
):
320 """ device Ids on which the user subscribed to the podcast """
323 for action
in self
.actions
:
324 if action
.action
== "subscribe":
325 if not action
.device
in self
.disabled_devices
:
326 devices
.add(action
.device
)
328 if action
.device
in devices
:
329 devices
.remove(action
.device
)
334 def is_subscribed_on(self
, device
):
335 """ checks if the podcast is subscribed on the given device """
337 for action
in reversed(self
.actions
):
338 if not action
.device
== device
.id:
341 # we only need to check the latest action for the device
342 return (action
.action
== 'subscribe')
344 # we haven't found any matching action
349 return self
.get_wksetting(PUBLIC_SUB_PODCAST
)
352 def __eq__(self
, other
):
356 return self
.podcast
== other
.podcast
and \
357 self
.user
== other
.user
360 return 'Podcast %s for User %s (%s)' % \
361 (self
.podcast
, self
.user
, self
._id
)
364 class Device(Document
, SettingsMixin
):
365 id = StringProperty(default
=lambda: uuid
.uuid4().hex)
366 oldid
= IntegerProperty(required
=False)
367 uid
= StringProperty(required
=True)
368 name
= StringProperty(required
=True, default
='New Device')
369 type = StringProperty(required
=True, default
='other')
370 deleted
= BooleanProperty(default
=False)
371 user_agent
= StringProperty()
374 def get_subscription_changes(self
, since
, until
):
376 Returns the subscription changes for the device as two lists.
377 The first lists contains the Ids of the podcasts that have been
378 subscribed to, the second list of those that have been unsubscribed
382 from mygpo
.db
.couchdb
.podcast_state
import podcast_states_for_device
383 podcast_states
= podcast_states_for_device(self
.id)
384 return subscription_changes(self
.id, podcast_states
, since
, until
)
387 def get_latest_changes(self
):
389 from mygpo
.db
.couchdb
.podcast_state
import podcast_states_for_device
391 podcast_states
= podcast_states_for_device(self
.id)
392 for p_state
in podcast_states
:
393 actions
= filter(lambda x
: x
.device
== self
.id, reversed(p_state
.actions
))
395 yield (p_state
.podcast
, actions
[0])
398 def get_subscribed_podcast_ids(self
):
399 from mygpo
.db
.couchdb
.podcast_state
import get_subscribed_podcast_states_by_device
400 states
= get_subscribed_podcast_states_by_device(self
)
401 return [state
.podcast
for state
in states
]
404 def get_subscribed_podcasts(self
):
405 """ Returns all subscribed podcasts for the device
407 The attribute "url" contains the URL that was used when subscribing to
410 from mygpo
.db
.couchdb
.podcast_state
import get_subscribed_podcast_states_by_device
411 states
= get_subscribed_podcast_states_by_device(self
)
412 return podcasts_for_states(states
)
416 return hash(frozenset([self
.id, self
.uid
, self
.name
, self
.type, self
.deleted
]))
419 def __eq__(self
, other
):
420 return self
.id == other
.id
424 return '<{cls} {id}>'.format(cls
=self
.__class
__.__name
__, id=self
.id)
430 def __unicode__(self
):
435 TOKEN_NAMES
= ('subscriptions_token', 'favorite_feeds_token',
436 'publisher_update_token', 'userpage_token')
439 class TokenException(Exception):
443 class User(BaseUser
, SyncedDevicesMixin
, SettingsMixin
):
444 oldid
= IntegerProperty()
445 devices
= SchemaListProperty(Device
)
446 published_objects
= StringListProperty()
447 deleted
= BooleanProperty(default
=False)
448 suggestions_up_to_date
= BooleanProperty(default
=False)
449 twitter
= StringProperty()
450 about
= StringProperty()
451 google_email
= StringProperty()
453 # token for accessing subscriptions of this use
454 subscriptions_token
= StringProperty(default
=None)
456 # token for accessing the favorite-episodes feed of this user
457 favorite_feeds_token
= StringProperty(default
=None)
459 # token for automatically updating feeds published by this user
460 publisher_update_token
= StringProperty(default
=None)
462 # token for accessing the userpage of this user
463 userpage_token
= StringProperty(default
=None)
469 def create_new_token(self
, token_name
, length
=32):
470 """ creates a new random token """
472 if token_name
not in TOKEN_NAMES
:
473 raise TokenException('Invalid token name %s' % token_name
)
475 token
= "".join(random
.sample(string
.letters
+string
.digits
, length
))
476 setattr(self
, token_name
, token
)
480 @repeat_on_conflict(['self'])
481 def get_token(self
, token_name
):
482 """ returns a token, and generate those that are still missing """
486 if token_name
not in TOKEN_NAMES
:
487 raise TokenException('Invalid token name %s' % token_name
)
489 create_missing_user_tokens(self
)
491 return getattr(self
, token_name
)
496 def active_devices(self
):
497 not_deleted
= lambda d
: not d
.deleted
498 return filter(not_deleted
, self
.devices
)
502 def inactive_devices(self
):
503 deleted
= lambda d
: d
.deleted
504 return filter(deleted
, self
.devices
)
507 def get_devices_by_id(self
, device_ids
=None):
508 """ Returns a dict of {devices_id: device} """
509 if device_ids
is None:
511 devices
= self
.devices
513 devices
= self
.get_devices(device_ids
)
515 return {device
.id: device
for device
in devices
}
518 def get_device(self
, id):
520 if not hasattr(self
, '__device_by_id'):
521 self
.__devices
_by
_id
= self
.get_devices_by_id()
523 return self
.__devices
_by
_id
.get(id, None)
526 def get_devices(self
, ids
):
527 return filter(None, (self
.get_device(dev_id
) for dev_id
in ids
))
530 def get_device_by_uid(self
, uid
, only_active
=True):
532 if not hasattr(self
, '__devices_by_uio'):
533 self
.__devices
_by
_uid
= dict( (d
.uid
, d
) for d
in self
.devices
)
536 device
= self
.__devices
_by
_uid
[uid
]
538 if only_active
and device
.deleted
:
539 raise DeviceDeletedException(
540 'Device with UID %s is deleted' % uid
)
544 except KeyError as e
:
545 raise DeviceDoesNotExist('There is no device with UID %s' % uid
)
548 def set_device(self
, device
):
550 if not RE_DEVICE_UID
.match(device
.uid
):
551 raise DeviceUIDException(u
"'{uid} is not a valid device ID".format(
554 devices
= list(self
.devices
)
555 ids
= [x
.id for x
in devices
]
556 if not device
.id in ids
:
557 devices
.append(device
)
558 self
.devices
= devices
561 index
= ids
.index(device
.id)
563 devices
.insert(index
, device
)
564 self
.devices
= devices
567 def remove_device(self
, device
):
568 devices
= list(self
.devices
)
569 ids
= [x
.id for x
in devices
]
570 if not device
.id in ids
:
573 index
= ids
.index(device
.id)
575 self
.devices
= devices
577 if self
.is_synced(device
):
578 self
.unsync_device(device
)
581 def get_subscriptions_by_device(self
, public
=None):
582 from mygpo
.db
.couchdb
.podcast_state
import subscriptions_by_user
583 get_dev
= itemgetter(2)
584 groups
= collections
.defaultdict(list)
585 subscriptions
= subscriptions_by_user(self
, public
=public
)
586 subscriptions
= sorted(subscriptions
, key
=get_dev
)
588 for public
, podcast_id
, device_id
in subscriptions
:
589 groups
[device_id
].append(podcast_id
)
593 def get_subscribed_podcast_ids(self
, public
=None):
594 from mygpo
.db
.couchdb
.podcast_state
import get_subscribed_podcast_states_by_user
595 states
= get_subscribed_podcast_states_by_user(self
, public
)
596 return [state
.podcast
for state
in states
]
600 def get_subscribed_podcasts(self
, public
=None):
601 """ Returns all subscribed podcasts for the user
603 The attribute "url" contains the URL that was used when subscribing to
606 from mygpo
.db
.couchdb
.podcast_state
import get_subscribed_podcast_states_by_user
607 states
= get_subscribed_podcast_states_by_user(self
, public
)
608 podcast_ids
= [state
.podcast
for state
in states
]
609 podcasts
= podcasts_to_dict(podcast_ids
)
612 podcast
= podcasts
.get(state
.podcast
, None)
616 podcast
= proxy_object(podcast
, url
=state
.ref_url
)
617 podcasts
[state
.podcast
] = podcast
619 return set(podcasts
.values())
623 def get_subscription_history(self
, device_id
=None, reverse
=False, public
=None):
624 """ Returns chronologically ordered subscription history entries
626 Setting device_id restricts the actions to a certain device
629 from mygpo
.db
.couchdb
.podcast_state
import podcast_states_for_user
, \
630 podcast_states_for_device
632 def action_iter(state
):
633 for action
in sorted(state
.actions
, reverse
=reverse
):
634 if device_id
is not None and device_id
!= action
.device
:
637 if public
is not None and state
.is_public() != public
:
640 entry
= HistoryEntry()
641 entry
.timestamp
= action
.timestamp
642 entry
.action
= action
.action
643 entry
.podcast_id
= state
.podcast
644 entry
.device_id
= action
.device
647 if device_id
is None:
648 podcast_states
= podcast_states_for_user(self
)
650 podcast_states
= podcast_states_for_device(device_id
)
652 # create an action_iter for each PodcastUserState
653 subscription_action_lists
= [action_iter(x
) for x
in podcast_states
]
655 action_cmp_key
= lambda x
: x
.timestamp
657 # Linearize their subscription-actions
658 return linearize(action_cmp_key
, subscription_action_lists
, reverse
)
661 def get_global_subscription_history(self
, public
=None):
662 """ Actions that added/removed podcasts from the subscription list
664 Returns an iterator of all subscription actions that either
665 * added subscribed a podcast that hasn't been subscribed directly
666 before the action (but could have been subscribed) earlier
667 * removed a subscription of the podcast is not longer subscribed
671 subscriptions
= collections
.defaultdict(int)
673 for entry
in self
.get_subscription_history(public
=public
):
674 if entry
.action
== 'subscribe':
675 subscriptions
[entry
.podcast_id
] += 1
677 # a new subscription has been added
678 if subscriptions
[entry
.podcast_id
] == 1:
681 elif entry
.action
== 'unsubscribe':
682 subscriptions
[entry
.podcast_id
] -= 1
684 # the last subscription has been removed
685 if subscriptions
[entry
.podcast_id
] == 0:
690 def get_newest_episodes(self
, max_date
, max_per_podcast
=5):
691 """ Returns the newest episodes of all subscribed podcasts
693 Only max_per_podcast episodes per podcast are loaded. Episodes with
694 release dates above max_date are discarded.
696 This method returns a generator that produces the newest episodes.
698 The number of required DB queries is equal to the number of (distinct)
699 podcasts of all consumed episodes (max: number of subscribed podcasts),
700 plus a constant number of initial queries (when the first episode is
703 cmp_key
= lambda episode
: episode
.released
or datetime(2000, 01, 01)
705 podcasts
= list(self
.get_subscribed_podcasts())
706 podcasts
= filter(lambda p
: p
.latest_episode_timestamp
, podcasts
)
707 podcasts
= sorted(podcasts
, key
=lambda p
: p
.latest_episode_timestamp
,
710 podcast_dict
= dict((p
.get_id(), p
) for p
in podcasts
)
712 # contains the un-yielded episodes, newest first
715 for podcast
in podcasts
:
719 for episode
in episodes
:
720 # determine for which episodes there won't be a new episodes
721 # that is newer; those can be yielded
722 if episode
.released
> podcast
.latest_episode_timestamp
:
723 p
= podcast_dict
.get(episode
.podcast
, None)
724 yield proxy_object(episode
, podcast
=p
)
725 yielded_episodes
+= 1
729 # remove the episodes that have been yielded before
730 episodes
= episodes
[yielded_episodes
:]
732 # fetch and merge episodes for the next podcast
733 from mygpo
.db
.couchdb
.episode
import episodes_for_podcast
734 new_episodes
= episodes_for_podcast(podcast
, since
=1,
735 until
=max_date
, descending
=True, limit
=max_per_podcast
)
736 episodes
= sorted(episodes
+new_episodes
, key
=cmp_key
, reverse
=True)
739 # yield the remaining episodes
740 for episode
in episodes
:
741 podcast
= podcast_dict
.get(episode
.podcast
, None)
742 yield proxy_object(episode
, podcast
=podcast
)
745 def __eq__(self
, other
):
749 # ensure that other isn't AnonymousUser
750 return other
.is_authenticated() and self
._id
== other
._id
753 def __ne__(self
, other
):
754 return not(self
== other
)
758 return 'User %s' % self
._id
761 class History(object):
763 def __init__(self
, user
, device
):
768 def __getitem__(self
, key
):
770 if isinstance(key
, slice):
771 start
= key
.start
or 0
772 length
= key
.stop
- start
778 return device_history(self
.user
, self
.device
, start
, length
)
781 return user_history(self
.user
, start
, length
)
785 class HistoryEntry(object):
786 """ A class that can represent subscription and episode actions """
790 def from_action_dict(cls
, action
):
792 entry
= HistoryEntry()
794 if 'timestamp' in action
:
795 ts
= action
.pop('timestamp')
796 entry
.timestamp
= dateutil
.parser
.parse(ts
)
798 for key
, value
in action
.items():
799 setattr(entry
, key
, value
)
806 return getattr(self
, 'position', None)
810 def fetch_data(cls
, user
, entries
,
811 podcasts
=None, episodes
=None):
812 """ Efficiently loads additional data for a number of entries """
816 podcast_ids
= [getattr(x
, 'podcast_id', None) for x
in entries
]
817 podcast_ids
= filter(None, podcast_ids
)
818 podcasts
= podcasts_to_dict(podcast_ids
)
821 from mygpo
.db
.couchdb
.episode
import episodes_to_dict
823 episode_ids
= [getattr(x
, 'episode_id', None) for x
in entries
]
824 episode_ids
= filter(None, episode_ids
)
825 episodes
= episodes_to_dict(episode_ids
)
828 # does not need pre-populated data because no db-access is required
829 device_ids
= [getattr(x
, 'device_id', None) for x
in entries
]
830 device_ids
= filter(None, device_ids
)
831 devices
= dict([ (id, user
.get_device(id)) for id in device_ids
])
834 for entry
in entries
:
835 podcast_id
= getattr(entry
, 'podcast_id', None)
836 entry
.podcast
= podcasts
.get(podcast_id
, None)
838 episode_id
= getattr(entry
, 'episode_id', None)
839 entry
.episode
= episodes
.get(episode_id
, None)
841 if hasattr(entry
, 'user'):
844 device
= devices
.get(getattr(entry
, 'device_id', None), None)
845 entry
.device
= device