4 from datetime
import datetime
6 from itertools
import imap
7 from operator
import itemgetter
11 from couchdbkit
.ext
.django
.schema
import *
13 from django
.core
.cache
import cache
15 from django_couchdb_utils
.registration
.models
import User
as BaseUser
17 from mygpo
.utils
import linearize
18 from mygpo
.core
.proxy
import DocumentABCMeta
, proxy_object
19 from mygpo
.decorators
import repeat_on_conflict
20 from mygpo
.users
.ratings
import RatingMixin
21 from mygpo
.users
.sync
import SyncedDevicesMixin
22 from mygpo
.users
.settings
import FAV_FLAG
, PUBLIC_SUB_PODCAST
, SettingsMixin
23 from mygpo
.db
.couchdb
.podcast
import podcasts_by_id
, podcasts_to_dict
24 from mygpo
.db
.couchdb
.user
import user_history
, device_history
26 # make sure this code is executed at startup
27 from mygpo
.users
.signals
import *
30 RE_DEVICE_UID
= re
.compile(r
'^[\w.-]+$')
32 # TODO: derive from ValidationException?
33 class InvalidEpisodeActionAttributes(ValueError):
34 """ raised when the attribues of an episode action fail validation """
37 class DeviceUIDException(Exception):
41 class DeviceDoesNotExist(Exception):
45 class DeviceDeletedException(DeviceDoesNotExist
):
49 class Suggestions(Document
, RatingMixin
):
50 user
= StringProperty(required
=True)
51 user_oldid
= IntegerProperty()
52 podcasts
= StringListProperty()
53 blacklist
= StringListProperty()
56 def get_podcasts(self
, count
=None):
57 user
= User
.get(self
.user
)
58 subscriptions
= user
.get_subscribed_podcast_ids()
60 ids
= filter(lambda x
: not x
in self
.blacklist
+ subscriptions
, self
.podcasts
)
63 return filter(lambda x
: x
and x
.title
, podcasts_by_id(ids
))
68 return super(Suggestions
, self
).__repr
__()
70 return '%d Suggestions for %s (%s)' % \
71 (len(self
.podcasts
), self
.user
, self
._id
)
74 class EpisodeAction(DocumentSchema
):
76 One specific action to an episode. Must
77 always be part of a EpisodeUserState
80 action
= StringProperty(required
=True)
82 # walltime of the event (assigned by the uploading client, defaults to now)
83 timestamp
= DateTimeProperty(required
=True, default
=datetime
.utcnow
)
85 # upload time of the event
86 upload_timestamp
= IntegerProperty(required
=True)
88 device_oldid
= IntegerProperty(required
=False)
89 device
= StringProperty()
90 started
= IntegerProperty()
91 playmark
= IntegerProperty()
92 total
= IntegerProperty()
94 def __eq__(self
, other
):
95 if not isinstance(other
, EpisodeAction
):
97 vals
= ('action', 'timestamp', 'device', 'started', 'playmark',
99 return all([getattr(self
, v
, None) == getattr(other
, v
, None) for v
in vals
])
102 def to_history_entry(self
):
103 entry
= HistoryEntry()
104 entry
.action
= self
.action
105 entry
.timestamp
= self
.timestamp
106 entry
.device_id
= self
.device
107 entry
.started
= self
.started
108 entry
.position
= self
.playmark
109 entry
.total
= self
.total
114 def validate_time_values(self
):
115 """ Validates allowed combinations of time-values """
117 PLAY_ACTION_KEYS
= ('playmark', 'started', 'total')
119 # Key found, but must not be supplied (no play action!)
120 if self
.action
!= 'play':
121 for key
in PLAY_ACTION_KEYS
:
122 if getattr(self
, key
, None) is not None:
123 raise InvalidEpisodeActionAttributes('%s only allowed in play actions' % key
)
125 # Sanity check: If started or total are given, require playmark
126 if ((self
.started
is not None) or (self
.total
is not None)) and \
127 self
.playmark
is None:
128 raise InvalidEpisodeActionAttributes('started and total require position')
130 # Sanity check: total and playmark can only appear together
131 if ((self
.total
is not None) or (self
.started
is not None)) and \
132 ((self
.total
is None) or (self
.started
is None)):
133 raise InvalidEpisodeActionAttributes('total and started can only appear together')
137 return '%s-Action on %s at %s (in %s)' % \
138 (self
.action
, self
.device
, self
.timestamp
, self
._id
)
142 return hash(frozenset([self
.action
, self
.timestamp
, self
.device
,
143 self
.started
, self
.playmark
, self
.total
]))
146 class Chapter(Document
):
147 """ A user-entered episode chapter """
149 device
= StringProperty()
150 created
= DateTimeProperty()
151 start
= IntegerProperty(required
=True)
152 end
= IntegerProperty(required
=True)
153 label
= StringProperty()
154 advertisement
= BooleanProperty()
158 return '<%s %s (%d-%d)>' % (self
.__class
__.__name
__, self
.label
,
159 self
.start
, self
.end
)
162 class EpisodeUserState(Document
, SettingsMixin
):
164 Contains everything a user has done with an Episode
167 episode
= StringProperty(required
=True)
168 actions
= SchemaListProperty(EpisodeAction
)
169 user_oldid
= IntegerProperty()
170 user
= StringProperty(required
=True)
171 ref_url
= StringProperty(required
=True)
172 podcast_ref_url
= StringProperty(required
=True)
173 merged_ids
= StringListProperty()
174 chapters
= SchemaListProperty(Chapter
)
175 podcast
= StringProperty(required
=True)
179 def add_actions(self
, actions
):
180 map(EpisodeAction
.validate_time_values
, actions
)
181 self
.actions
= list(self
.actions
) + actions
182 self
.actions
= list(set(self
.actions
))
183 self
.actions
= sorted(self
.actions
, key
=lambda x
: x
.timestamp
)
186 def is_favorite(self
):
187 return self
.get_wksetting(FAV_FLAG
)
190 def set_favorite(self
, set_to
=True):
191 self
.settings
[FAV_FLAG
.name
] = set_to
194 def update_chapters(self
, add
=[], rem
=[]):
195 """ Updates the Chapter list
197 * add contains the chapters to be added
199 * rem contains tuples of (start, end) times. Chapters that match
200 both endpoints will be removed
203 @repeat_on_conflict(['state'])
206 self
.chapters
= self
.chapters
+ [chapter
]
208 for start
, end
in rem
:
209 keep
= lambda c
: c
.start
!= start
or c
.end
!= end
210 self
.chapters
= filter(keep
, self
.chapters
)
217 def get_history_entries(self
):
218 return imap(EpisodeAction
.to_history_entry
, self
.actions
)
222 return 'Episode-State %s (in %s)' % \
223 (self
.episode
, self
._id
)
225 def __eq__(self
, other
):
226 if not isinstance(other
, EpisodeUserState
):
229 return (self
.episode
== other
.episode
and
230 self
.user
== other
.user
)
234 class SubscriptionAction(Document
):
235 action
= StringProperty()
236 timestamp
= DateTimeProperty(default
=datetime
.utcnow
)
237 device
= StringProperty()
240 __metaclass__
= DocumentABCMeta
243 def __cmp__(self
, other
):
244 return cmp(self
.timestamp
, other
.timestamp
)
246 def __eq__(self
, other
):
247 return self
.action
== other
.action
and \
248 self
.timestamp
== other
.timestamp
and \
249 self
.device
== other
.device
252 return hash(self
.action
) + hash(self
.timestamp
) + hash(self
.device
)
255 return '<SubscriptionAction %s on %s at %s>' % (
256 self
.action
, self
.device
, self
.timestamp
)
259 class PodcastUserState(Document
, SettingsMixin
):
261 Contains everything that a user has done
262 with a specific podcast and all its episodes
265 podcast
= StringProperty(required
=True)
266 user_oldid
= IntegerProperty()
267 user
= StringProperty(required
=True)
268 actions
= SchemaListProperty(SubscriptionAction
)
269 tags
= StringListProperty()
270 ref_url
= StringProperty(required
=True)
271 disabled_devices
= StringListProperty()
272 merged_ids
= StringListProperty()
275 def remove_device(self
, device
):
277 Removes all actions from the podcast state that refer to the
280 self
.actions
= filter(lambda a
: a
.device
!= device
.id, self
.actions
)
283 def subscribe(self
, device
):
284 action
= SubscriptionAction()
285 action
.action
= 'subscribe'
286 action
.device
= device
.id
287 self
.add_actions([action
])
290 def unsubscribe(self
, device
):
291 action
= SubscriptionAction()
292 action
.action
= 'unsubscribe'
293 action
.device
= device
.id
294 self
.add_actions([action
])
297 def add_actions(self
, actions
):
298 self
.actions
= list(set(self
.actions
+ actions
))
299 self
.actions
= sorted(self
.actions
)
302 def add_tags(self
, tags
):
303 self
.tags
= list(set(self
.tags
+ tags
))
306 def set_device_state(self
, devices
):
307 disabled_devices
= [device
.id for device
in devices
if device
.deleted
]
308 self
.disabled_devices
= disabled_devices
311 def get_change_between(self
, device_id
, since
, until
):
313 Returns the change of the subscription status for the given device
314 between the two timestamps.
316 The change is given as either 'subscribe' (the podcast has been
317 subscribed), 'unsubscribed' (the podcast has been unsubscribed) or
321 device_actions
= filter(lambda x
: x
.device
== device_id
, self
.actions
)
322 before
= filter(lambda x
: x
.timestamp
<= since
, device_actions
)
323 after
= filter(lambda x
: x
.timestamp
<= until
, device_actions
)
325 # nothing happened, so there can be no change
329 then
= before
[-1] if before
else None
333 if now
.action
!= 'unsubscribe':
335 elif then
.action
!= now
.action
:
340 def get_subscribed_device_ids(self
):
341 """ device Ids on which the user subscribed to the podcast """
344 for action
in self
.actions
:
345 if action
.action
== "subscribe":
346 if not action
.device
in self
.disabled_devices
:
347 devices
.add(action
.device
)
349 if action
.device
in devices
:
350 devices
.remove(action
.device
)
357 return self
.get_wksetting(PUBLIC_SUB_PODCAST
)
360 def __eq__(self
, other
):
364 return self
.podcast
== other
.podcast
and \
365 self
.user
== other
.user
368 return 'Podcast %s for User %s (%s)' % \
369 (self
.podcast
, self
.user
, self
._id
)
372 class Device(Document
, SettingsMixin
):
373 id = StringProperty(default
=lambda: uuid
.uuid4().hex)
374 oldid
= IntegerProperty(required
=False)
375 uid
= StringProperty(required
=True)
376 name
= StringProperty(required
=True, default
='New Device')
377 type = StringProperty(required
=True, default
='other')
378 deleted
= BooleanProperty(default
=False)
379 user_agent
= StringProperty()
382 def get_subscription_changes(self
, since
, until
):
384 Returns the subscription changes for the device as two lists.
385 The first lists contains the Ids of the podcasts that have been
386 subscribed to, the second list of those that have been unsubscribed
390 from mygpo
.db
.couchdb
.podcast_state
import podcast_states_for_device
393 podcast_states
= podcast_states_for_device(self
.id)
394 for p_state
in podcast_states
:
395 change
= p_state
.get_change_between(self
.id, since
, until
)
396 if change
== 'subscribe':
397 add
.append( p_state
.ref_url
)
398 elif change
== 'unsubscribe':
399 rem
.append( p_state
.ref_url
)
404 def get_latest_changes(self
):
406 from mygpo
.db
.couchdb
.podcast_state
import podcast_states_for_device
408 podcast_states
= podcast_states_for_device(self
.id)
409 for p_state
in podcast_states
:
410 actions
= filter(lambda x
: x
.device
== self
.id, reversed(p_state
.actions
))
412 yield (p_state
.podcast
, actions
[0])
415 def get_subscribed_podcast_states(self
):
416 r
= PodcastUserState
.view('subscriptions/by_device',
417 startkey
= [self
.id, None],
418 endkey
= [self
.id, {}],
424 def get_subscribed_podcast_ids(self
):
425 states
= self
.get_subscribed_podcast_states()
426 return [state
.podcast
for state
in states
]
429 def get_subscribed_podcasts(self
):
430 """ Returns all subscribed podcasts for the device
432 The attribute "url" contains the URL that was used when subscribing to
435 states
= self
.get_subscribed_podcast_states()
436 podcast_ids
= [state
.podcast
for state
in states
]
437 podcasts
= podcasts_to_dict(podcast_ids
)
440 podcast
= proxy_object(podcasts
[state
.podcast
], url
=state
.ref_url
)
441 podcasts
[state
.podcast
] = podcast
443 return podcasts
.values()
447 return hash(frozenset([self
.id, self
.uid
, self
.name
, self
.type, self
.deleted
]))
450 def __eq__(self
, other
):
451 return self
.id == other
.id
455 return '<{cls} {id}>'.format(cls
=self
.__class
__.__name
__, id=self
.id)
461 def __unicode__(self
):
466 TOKEN_NAMES
= ('subscriptions_token', 'favorite_feeds_token',
467 'publisher_update_token', 'userpage_token')
470 class TokenException(Exception):
474 class User(BaseUser
, SyncedDevicesMixin
, SettingsMixin
):
475 oldid
= IntegerProperty()
476 devices
= SchemaListProperty(Device
)
477 published_objects
= StringListProperty()
478 deleted
= BooleanProperty(default
=False)
479 suggestions_up_to_date
= BooleanProperty(default
=False)
480 twitter
= StringProperty()
481 about
= StringProperty()
482 google_email
= StringProperty()
484 # token for accessing subscriptions of this use
485 subscriptions_token
= StringProperty(default
=None)
487 # token for accessing the favorite-episodes feed of this user
488 favorite_feeds_token
= StringProperty(default
=None)
490 # token for automatically updating feeds published by this user
491 publisher_update_token
= StringProperty(default
=None)
493 # token for accessing the userpage of this user
494 userpage_token
= StringProperty(default
=None)
500 def create_new_token(self
, token_name
, length
=32):
501 """ creates a new random token """
503 if token_name
not in TOKEN_NAMES
:
504 raise TokenException('Invalid token name %s' % token_name
)
506 token
= "".join(random
.sample(string
.letters
+string
.digits
, length
))
507 setattr(self
, token_name
, token
)
511 @repeat_on_conflict(['self'])
512 def get_token(self
, token_name
):
513 """ returns a token, and generate those that are still missing """
517 if token_name
not in TOKEN_NAMES
:
518 raise TokenException('Invalid token name %s' % token_name
)
520 for tn
in TOKEN_NAMES
:
521 if getattr(self
, tn
) is None:
522 self
.create_new_token(tn
)
528 return getattr(self
, token_name
)
533 def active_devices(self
):
534 not_deleted
= lambda d
: not d
.deleted
535 return filter(not_deleted
, self
.devices
)
539 def inactive_devices(self
):
540 deleted
= lambda d
: d
.deleted
541 return filter(deleted
, self
.devices
)
544 def get_devices_by_id(self
):
545 return dict( (device
.id, device
) for device
in self
.devices
)
548 def get_device(self
, id):
550 if not hasattr(self
, '__device_by_id'):
551 self
.__devices
_by
_id
= dict( (d
.id, d
) for d
in self
.devices
)
553 return self
.__devices
_by
_id
.get(id, None)
556 def get_device_by_uid(self
, uid
, only_active
=True):
558 if not hasattr(self
, '__devices_by_uio'):
559 self
.__devices
_by
_uid
= dict( (d
.uid
, d
) for d
in self
.devices
)
562 device
= self
.__devices
_by
_uid
[uid
]
564 if only_active
and device
.deleted
:
565 raise DeviceDeletedException(
566 'Device with UID %s is deleted' % uid
)
570 except KeyError as e
:
571 raise DeviceDoesNotExist('There is no device with UID %s' % uid
)
574 @repeat_on_conflict(['self'])
575 def update_device(self
, device
):
576 """ Sets the device and saves the user """
577 self
.set_device(device
)
581 def set_device(self
, device
):
583 if not RE_DEVICE_UID
.match(device
.uid
):
584 raise DeviceUIDException(u
"'{uid} is not a valid device ID".format(
587 devices
= list(self
.devices
)
588 ids
= [x
.id for x
in devices
]
589 if not device
.id in ids
:
590 devices
.append(device
)
591 self
.devices
= devices
594 index
= ids
.index(device
.id)
596 devices
.insert(index
, device
)
597 self
.devices
= devices
600 def remove_device(self
, device
):
601 devices
= list(self
.devices
)
602 ids
= [x
.id for x
in devices
]
603 if not device
.id in ids
:
606 index
= ids
.index(device
.id)
608 self
.devices
= devices
610 if self
.is_synced(device
):
611 self
.unsync_device(device
)
614 def get_subscriptions_by_device(self
, public
=None):
615 from mygpo
.db
.couchdb
.podcast_state
import subscriptions_by_user
616 get_dev
= itemgetter(2)
617 groups
= collections
.defaultdict(list)
618 subscriptions
= subscriptions_by_user(self
, public
=public
)
619 subscriptions
= sorted(subscriptions
, key
=get_dev
)
621 for public
, podcast_id
, device_id
in subscriptions
:
622 groups
[device_id
].append(podcast_id
)
627 def get_subscribed_podcast_states(self
, public
=None):
629 Returns the Ids of all subscribed podcasts
632 r
= PodcastUserState
.view('subscriptions/by_user',
633 startkey
= [self
._id
, public
, None, None],
634 endkey
= [self
._id
+'ZZZ', None, None, None],
642 def get_subscribed_podcast_ids(self
, public
=None):
643 states
= self
.get_subscribed_podcast_states(public
=public
)
644 return [state
.podcast
for state
in states
]
648 def get_subscribed_podcasts(self
, public
=None):
649 """ Returns all subscribed podcasts for the user
651 The attribute "url" contains the URL that was used when subscribing to
654 states
= self
.get_subscribed_podcast_states(public
=public
)
655 podcast_ids
= [state
.podcast
for state
in states
]
656 podcasts
= podcasts_to_dict(podcast_ids
)
659 podcast
= proxy_object(podcasts
[state
.podcast
], url
=state
.ref_url
)
660 podcasts
[state
.podcast
] = podcast
662 return podcasts
.values()
666 def get_subscription_history(self
, device_id
=None, reverse
=False, public
=None):
667 """ Returns chronologically ordered subscription history entries
669 Setting device_id restricts the actions to a certain device
672 from mygpo
.db
.couchdb
.podcast_state
import podcast_states_for_user
, \
673 podcast_states_for_device
675 def action_iter(state
):
676 for action
in sorted(state
.actions
, reverse
=reverse
):
677 if device_id
is not None and device_id
!= action
.device
:
680 if public
is not None and state
.is_public() != public
:
683 entry
= HistoryEntry()
684 entry
.timestamp
= action
.timestamp
685 entry
.action
= action
.action
686 entry
.podcast_id
= state
.podcast
687 entry
.device_id
= action
.device
690 if device_id
is None:
691 podcast_states
= podcast_states_for_user(self
)
693 podcast_states
= podcast_states_for_device(device_id
)
695 # create an action_iter for each PodcastUserState
696 subscription_action_lists
= [action_iter(x
) for x
in podcast_states
]
698 action_cmp_key
= lambda x
: x
.timestamp
700 # Linearize their subscription-actions
701 return linearize(action_cmp_key
, subscription_action_lists
, reverse
)
704 def get_global_subscription_history(self
, public
=None):
705 """ Actions that added/removed podcasts from the subscription list
707 Returns an iterator of all subscription actions that either
708 * added subscribed a podcast that hasn't been subscribed directly
709 before the action (but could have been subscribed) earlier
710 * removed a subscription of the podcast is not longer subscribed
714 subscriptions
= collections
.defaultdict(int)
716 for entry
in self
.get_subscription_history(public
=public
):
717 if entry
.action
== 'subscribe':
718 subscriptions
[entry
.podcast_id
] += 1
720 # a new subscription has been added
721 if subscriptions
[entry
.podcast_id
] == 1:
724 elif entry
.action
== 'unsubscribe':
725 subscriptions
[entry
.podcast_id
] -= 1
727 # the last subscription has been removed
728 if subscriptions
[entry
.podcast_id
] == 0:
733 def get_newest_episodes(self
, max_date
, max_per_podcast
=5):
734 """ Returns the newest episodes of all subscribed podcasts
736 Only max_per_podcast episodes per podcast are loaded. Episodes with
737 release dates above max_date are discarded.
739 This method returns a generator that produces the newest episodes.
741 The number of required DB queries is equal to the number of (distinct)
742 podcasts of all consumed episodes (max: number of subscribed podcasts),
743 plus a constant number of initial queries (when the first episode is
746 cmp_key
= lambda episode
: episode
.released
or datetime(2000, 01, 01)
748 podcasts
= list(self
.get_subscribed_podcasts())
749 podcasts
= filter(lambda p
: p
.latest_episode_timestamp
, podcasts
)
750 podcasts
= sorted(podcasts
, key
=lambda p
: p
.latest_episode_timestamp
,
753 podcast_dict
= dict((p
.get_id(), p
) for p
in podcasts
)
755 # contains the un-yielded episodes, newest first
758 for podcast
in podcasts
:
762 for episode
in episodes
:
763 # determine for which episodes there won't be a new episodes
764 # that is newer; those can be yielded
765 if episode
.released
> podcast
.latest_episode_timestamp
:
766 p
= podcast_dict
.get(episode
.podcast
, None)
767 yield proxy_object(episode
, podcast
=p
)
768 yielded_episodes
+= 1
772 # remove the episodes that have been yielded before
773 episodes
= episodes
[yielded_episodes
:]
775 # fetch and merge episodes for the next podcast
776 from mygpo
.db
.couchdb
.episode
import episodes_for_podcast
777 new_episodes
= episodes_for_podcast(podcast
, since
=1,
778 until
=max_date
, descending
=True, limit
=max_per_podcast
)
779 episodes
= sorted(episodes
+new_episodes
, key
=cmp_key
, reverse
=True)
782 # yield the remaining episodes
783 for episode
in episodes
:
784 podcast
= podcast_dict
.get(episode
.podcast
, None)
785 yield proxy_object(episode
, podcast
=podcast
)
788 def __eq__(self
, other
):
792 # ensure that other isn't AnonymousUser
793 return other
.is_authenticated() and self
._id
== other
._id
796 def __ne__(self
, other
):
797 return not(self
== other
)
801 return 'User %s' % self
._id
804 class History(object):
806 def __init__(self
, user
, device
):
811 def __getitem__(self
, key
):
813 if isinstance(key
, slice):
814 start
= key
.start
or 0
815 length
= key
.stop
- start
821 return device_history(self
.user
, self
.device
, start
, length
)
824 return user_history(self
.user
, start
, length
)
828 class HistoryEntry(object):
829 """ A class that can represent subscription and episode actions """
833 def from_action_dict(cls
, action
):
835 entry
= HistoryEntry()
837 if 'timestamp' in action
:
838 ts
= action
.pop('timestamp')
839 entry
.timestamp
= dateutil
.parser
.parse(ts
)
841 for key
, value
in action
.items():
842 setattr(entry
, key
, value
)
849 return getattr(self
, 'position', None)
853 def fetch_data(cls
, user
, entries
,
854 podcasts
=None, episodes
=None):
855 """ Efficiently loads additional data for a number of entries """
859 podcast_ids
= [getattr(x
, 'podcast_id', None) for x
in entries
]
860 podcast_ids
= filter(None, podcast_ids
)
861 podcasts
= podcasts_to_dict(podcast_ids
)
864 from mygpo
.db
.couchdb
.episode
import episodes_to_dict
866 episode_ids
= [getattr(x
, 'episode_id', None) for x
in entries
]
867 episode_ids
= filter(None, episode_ids
)
868 episodes
= episodes_to_dict(episode_ids
)
871 # does not need pre-populated data because no db-access is required
872 device_ids
= [getattr(x
, 'device_id', None) for x
in entries
]
873 device_ids
= filter(None, device_ids
)
874 devices
= dict([ (id, user
.get_device(id)) for id in device_ids
])
877 for entry
in entries
:
878 podcast_id
= getattr(entry
, 'podcast_id', None)
879 entry
.podcast
= podcasts
.get(podcast_id
, None)
881 episode_id
= getattr(entry
, 'episode_id', None)
882 entry
.episode
= episodes
.get(episode_id
, None)
884 if hasattr(entry
, 'user'):
887 device
= devices
.get(getattr(entry
, 'device_id', None), None)
888 entry
.device
= device