Merge branch 'flattr'
[mygpo.git] / mygpo / api / advanced / __init__.py
blob74c449d1e9c43876526f1174976f80983947ccb7
2 # This file is part of my.gpodder.org.
4 # my.gpodder.org is free software: you can redistribute it and/or modify it
5 # under the terms of the GNU Affero General Public License as published by
6 # the Free Software Foundation, either version 3 of the License, or (at your
7 # option) any later version.
9 # my.gpodder.org is distributed in the hope that it will be useful, but
10 # WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
11 # or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public
12 # License for more details.
14 # You should have received a copy of the GNU Affero General Public License
15 # along with my.gpodder.org. If not, see <http://www.gnu.org/licenses/>.
18 from functools import partial
19 from itertools import imap, chain
20 from collections import defaultdict, namedtuple
21 from datetime import datetime
23 import dateutil.parser
25 try:
26 import gevent
27 except ImportError:
28 gevent = None
30 from django.http import HttpResponse, HttpResponseBadRequest, Http404, HttpResponseNotFound
31 from django.contrib.sites.models import RequestSite
32 from django.db import IntegrityError
33 from django.views.decorators.csrf import csrf_exempt
34 from django.views.decorators.cache import never_cache
36 from mygpo.api.constants import EPISODE_ACTION_TYPES, DEVICE_TYPES
37 from mygpo.api.httpresponse import JsonResponse
38 from mygpo.api.sanitizing import sanitize_url, sanitize_urls
39 from mygpo.api.advanced.directory import episode_data, podcast_data
40 from mygpo.api.backend import get_device, BulkSubscribe
41 from mygpo.couch import BulkException, bulk_save_retry
42 from mygpo.log import log
43 from mygpo.utils import parse_time, format_time, parse_bool, get_timestamp
44 from mygpo.decorators import allowed_methods, repeat_on_conflict
45 from mygpo.core import models
46 from mygpo.core.models import SanitizingRule, Podcast
47 from mygpo.users.models import PodcastUserState, EpisodeAction, \
48 EpisodeUserState, DeviceDoesNotExist, DeviceUIDException, \
49 InvalidEpisodeActionAttributes
50 from mygpo.json import json, JSONDecodeError
51 from mygpo.api.basic_auth import require_valid_user, check_username
52 from mygpo.db.couchdb.episode import episode_by_id, \
53 favorite_episodes_for_user, episodes_for_podcast
54 from mygpo.db.couchdb.podcast import podcast_for_url
55 from mygpo.db.couchdb.podcast_state import subscribed_podcast_ids_by_device
56 from mygpo.db.couchdb.episode_state import get_podcasts_episode_states, \
57 episode_state_for_ref_urls, get_episode_actions
60 # keys that are allowed in episode actions
61 EPISODE_ACTION_KEYS = ('position', 'episode', 'action', 'device', 'timestamp',
62 'started', 'total', 'podcast')
65 @csrf_exempt
66 @require_valid_user
67 @check_username
68 @never_cache
69 @allowed_methods(['GET', 'POST'])
70 def subscriptions(request, username, device_uid):
72 now = datetime.now()
73 now_ = get_timestamp(now)
75 if request.method == 'GET':
77 try:
78 device = request.user.get_device_by_uid(device_uid)
79 except DeviceDoesNotExist as e:
80 return HttpResponseNotFound(str(e))
82 since_ = request.GET.get('since', None)
83 if since_ == None:
84 return HttpResponseBadRequest('parameter since missing')
85 try:
86 since = datetime.fromtimestamp(float(since_))
87 except ValueError:
88 return HttpResponseBadRequest('since-value is not a valid timestamp')
90 changes = get_subscription_changes(request.user, device, since, now)
92 return JsonResponse(changes)
94 elif request.method == 'POST':
95 d = get_device(request.user, device_uid,
96 request.META.get('HTTP_USER_AGENT', ''))
98 if not request.raw_post_data:
99 return HttpResponseBadRequest('POST data must not be empty')
101 actions = json.loads(request.raw_post_data)
102 add = actions['add'] if 'add' in actions else []
103 rem = actions['remove'] if 'remove' in actions else []
105 add = filter(None, add)
106 rem = filter(None, rem)
108 try:
109 update_urls = update_subscriptions(request.user, d, add, rem)
110 except IntegrityError, e:
111 return HttpResponseBadRequest(e)
113 return JsonResponse({
114 'timestamp': now_,
115 'update_urls': update_urls,
119 def update_subscriptions(user, device, add, remove):
121 for a in add:
122 if a in remove:
123 raise IntegrityError('can not add and remove %s at the same time' % a)
125 add_s = list(sanitize_urls(add, 'podcast'))
126 rem_s = list(sanitize_urls(remove, 'podcast'))
128 assert len(add) == len(add_s) and len(remove) == len(rem_s)
130 updated_urls = filter(lambda (a, b): a != b, zip(add + remove, add_s + rem_s))
132 add_s = filter(None, add_s)
133 rem_s = filter(None, rem_s)
135 # If two different URLs (in add and remove) have
136 # been sanitized to the same, we ignore the removal
137 rem_s = filter(lambda x: x not in add_s, rem_s)
139 subscriber = BulkSubscribe(user, device)
141 for a in add_s:
142 subscriber.add_action(a, 'subscribe')
144 for r in rem_s:
145 subscriber.add_action(r, 'unsubscribe')
147 try:
148 subscriber.execute()
149 except BulkException as be:
150 for err in be.errors:
151 log('Advanced API: %(username)s: Updating subscription for '
152 '%(podcast_url)s on %(device_uid)s failed: '
153 '%(rerror)s (%(reason)s)'.format(username=user.username,
154 podcast_url=err.doc, device_uid=device.uid,
155 error=err.error, reason=err.reason)
158 return updated_urls
161 def get_subscription_changes(user, device, since, until):
162 add_urls, rem_urls = device.get_subscription_changes(since, until)
163 until_ = get_timestamp(until)
164 return {'add': add_urls, 'remove': rem_urls, 'timestamp': until_}
167 @csrf_exempt
168 @require_valid_user
169 @check_username
170 @never_cache
171 @allowed_methods(['GET', 'POST'])
172 def episodes(request, username, version=1):
174 version = int(version)
175 now = datetime.now()
176 now_ = get_timestamp(now)
177 ua_string = request.META.get('HTTP_USER_AGENT', '')
179 if request.method == 'POST':
180 try:
181 actions = json.loads(request.raw_post_data)
182 except (JSONDecodeError, UnicodeDecodeError) as e:
183 msg = 'Advanced API: could not decode episode update POST data for user %s: %s' % (username, e)
184 log(msg)
185 return HttpResponseBadRequest(msg)
187 try:
188 update_urls = update_episodes(request.user, actions, now, ua_string)
189 except DeviceUIDException as e:
190 import traceback
191 log('could not update episodes for user %s: %s %s: %s' % (username, e, traceback.format_exc(), actions))
192 return HttpResponseBadRequest(str(e))
193 except InvalidEpisodeActionAttributes as e:
194 import traceback
195 log('could not update episodes for user %s: %s %s: %s' % (username, e, traceback.format_exc(), actions))
196 return HttpResponseBadRequest(str(e))
198 return JsonResponse({'timestamp': now_, 'update_urls': update_urls})
200 elif request.method == 'GET':
201 podcast_url= request.GET.get('podcast', None)
202 device_uid = request.GET.get('device', None)
203 since_ = request.GET.get('since', None)
204 aggregated = parse_bool(request.GET.get('aggregated', False))
206 try:
207 since = int(since_) if since_ else None
208 except ValueError:
209 return HttpResponseBadRequest('since-value is not a valid timestamp')
211 if podcast_url:
212 podcast = podcast_for_url(podcast_url)
213 if not podcast:
214 raise Http404
215 else:
216 podcast = None
218 if device_uid:
220 try:
221 device = request.user.get_device_by_uid(device_uid)
222 except DeviceDoesNotExist as e:
223 return HttpResponseNotFound(str(e))
225 else:
226 device = None
228 changes = get_episode_changes(request.user, podcast, device, since,
229 now_, aggregated, version)
231 return JsonResponse(changes)
235 def convert_position(action):
236 """ convert position parameter for API 1 compatibility """
237 pos = getattr(action, 'position', None)
238 if pos is not None:
239 action.position = format_time(pos)
240 return action
244 def get_episode_changes(user, podcast, device, since, until, aggregated, version):
246 devices = dict( (dev.id, dev.uid) for dev in user.devices )
248 args = {}
249 if podcast is not None:
250 args['podcast_id'] = podcast.get_id()
252 if device is not None:
253 args['device_id'] = device.id
255 actions = get_episode_actions(user._id, since, until, **args)
257 if version == 1:
258 actions = imap(convert_position, actions)
260 clean_data = partial(clean_episode_action_data,
261 user=user, devices=devices)
263 actions = map(clean_data, actions)
264 actions = filter(None, actions)
266 if aggregated:
267 actions = dict( (a['episode'], a) for a in actions ).values()
269 return {'actions': actions, 'timestamp': until}
274 def clean_episode_action_data(action, user, devices):
276 if None in (action.get('podcast', None), action.get('episode', None)):
277 return None
279 if 'device_id' in action:
280 device_id = action['device_id']
281 device_uid = devices.get(device_id)
282 if device_uid:
283 action['device'] = device_uid
285 del action['device_id']
287 # remove superfluous keys
288 for x in action.keys():
289 if x not in EPISODE_ACTION_KEYS:
290 del action[x]
292 # set missing keys to None
293 for x in EPISODE_ACTION_KEYS:
294 if x not in action:
295 action[x] = None
297 if action['action'] != 'play':
298 if 'position' in action:
299 del action['position']
301 if 'total' in action:
302 del action['total']
304 if 'started' in action:
305 del action['started']
307 if 'playmark' in action:
308 del action['playmark']
310 else:
311 action['position'] = action.get('position', False) or 0
313 return action
319 def update_episodes(user, actions, now, ua_string):
320 update_urls = []
322 grouped_actions = defaultdict(list)
324 # group all actions by their episode
325 for action in actions:
327 podcast_url = action['podcast']
328 podcast_url = sanitize_append(podcast_url, 'podcast', update_urls)
329 if podcast_url == '': continue
331 episode_url = action['episode']
332 episode_url = sanitize_append(episode_url, 'episode', update_urls)
333 if episode_url == '': continue
335 act = parse_episode_action(action, user, update_urls, now, ua_string)
336 grouped_actions[ (podcast_url, episode_url) ].append(act)
339 auto_flattr_episodes = []
341 # Prepare the updates for each episode state
342 obj_funs = []
344 for (p_url, e_url), action_list in grouped_actions.iteritems():
345 episode_state = episode_state_for_ref_urls(user, p_url, e_url)
347 if any(a.action == 'play' for a in actions):
348 auto_flattr_episodes.append(episode_state.episode)
350 fun = partial(update_episode_actions, action_list=action_list)
351 obj_funs.append( (episode_state, fun) )
353 bulk_save_retry(obj_funs)
355 for episode_id in auto_flattr_episodes:
356 auto_flattr_episode.delay(user, episode_id)
358 return update_urls
361 def update_episode_actions(episode_state, action_list):
362 """ Adds actions to the episode state and saves if necessary """
364 len1 = len(episode_state.actions)
365 episode_state.add_actions(action_list)
367 if len(episode_state.actions) == len1:
368 return None
370 return episode_state
374 def parse_episode_action(action, user, update_urls, now, ua_string):
375 action_str = action.get('action', None)
376 if not valid_episodeaction(action_str):
377 raise Exception('invalid action %s' % action_str)
379 new_action = EpisodeAction()
381 new_action.action = action['action']
383 if action.get('device', False):
384 device = get_device(user, action['device'], ua_string)
385 new_action.device = device.id
387 if action.get('timestamp', False):
388 new_action.timestamp = dateutil.parser.parse(action['timestamp'])
389 else:
390 new_action.timestamp = now
391 new_action.timestamp = new_action.timestamp.replace(microsecond=0)
393 new_action.upload_timestamp = get_timestamp(now)
395 new_action.started = action.get('started', None)
396 new_action.playmark = action.get('position', None)
397 new_action.total = action.get('total', None)
399 return new_action
402 @csrf_exempt
403 @require_valid_user
404 @check_username
405 @never_cache
406 # Workaround for mygpoclient 1.0: It uses "PUT" requests
407 # instead of "POST" requests for uploading device settings
408 @allowed_methods(['POST', 'PUT'])
409 def device(request, username, device_uid):
410 d = get_device(request.user, device_uid,
411 request.META.get('HTTP_USER_AGENT', ''))
413 data = json.loads(request.raw_post_data)
415 if 'caption' in data:
416 if not data['caption']:
417 return HttpResponseBadRequest('caption must not be empty')
418 d.name = data['caption']
420 if 'type' in data:
421 if not valid_devicetype(data['type']):
422 return HttpResponseBadRequest('invalid device type %s' % data['type'])
423 d.type = data['type']
426 request.user.update_device(d)
428 return HttpResponse()
431 def valid_devicetype(type):
432 for t in DEVICE_TYPES:
433 if t[0] == type:
434 return True
435 return False
437 def valid_episodeaction(type):
438 for t in EPISODE_ACTION_TYPES:
439 if t[0] == type:
440 return True
441 return False
444 @csrf_exempt
445 @require_valid_user
446 @check_username
447 @never_cache
448 @allowed_methods(['GET'])
449 def devices(request, username):
450 devices = filter(lambda d: not d.deleted, request.user.devices)
451 devices = map(device_data, devices)
452 return JsonResponse(devices)
455 def device_data(device):
456 return dict(
457 id = device.uid,
458 caption = device.name,
459 type = device.type,
460 subscriptions= len(subscribed_podcast_ids_by_device(device)),
465 def get_podcast_data(podcasts, domain, url):
466 """ Gets podcast data for a URL from a dict of podcasts """
467 podcast = podcasts.get(url)
468 return podcast_data(podcast, domain)
471 def get_episode_data(podcasts, domain, clean_action_data, include_actions, episode_status):
472 """ Get episode data for an episode status object """
473 podcast_id = episode_status.episode.podcast
474 podcast = podcasts.get(podcast_id, None)
475 t = episode_data(episode_status.episode, domain, podcast)
476 t['status'] = episode_status.status
478 # include latest action (bug 1419)
479 if include_actions and episode_status.action:
480 t['action'] = clean_action_data(episode_status.action)
482 return t
485 @csrf_exempt
486 @require_valid_user
487 @check_username
488 @never_cache
489 def updates(request, username, device_uid):
490 now = datetime.now()
491 now_ = get_timestamp(now)
493 try:
494 device = request.user.get_device_by_uid(device_uid)
495 except DeviceDoesNotExist as e:
496 return HttpResponseNotFound(str(e))
498 since_ = request.GET.get('since', None)
499 if since_ == None:
500 return HttpResponseBadRequest('parameter since missing')
501 try:
502 since = datetime.fromtimestamp(float(since_))
503 except ValueError:
504 return HttpResponseBadRequest('since-value is not a valid timestamp')
506 include_actions = parse_bool(request.GET.get('include_actions', False))
508 ret = get_subscription_changes(request.user, device, since, now)
509 domain = RequestSite(request).domain
511 subscriptions = list(device.get_subscribed_podcasts())
513 podcasts = dict( (p.url, p) for p in subscriptions )
514 prepare_podcast_data = partial(get_podcast_data, podcasts, domain)
516 ret['add'] = map(prepare_podcast_data, ret['add'])
518 devices = dict( (dev.id, dev.uid) for dev in request.user.devices )
519 clean_action_data = partial(clean_episode_action_data,
520 user=request.user, devices=devices)
522 # index subscribed podcasts by their Id for fast access
523 podcasts = dict( (p.get_id(), p) for p in subscriptions )
524 prepare_episode_data = partial(get_episode_data, podcasts, domain,
525 clean_action_data, include_actions)
527 episode_updates = get_episode_updates(request.user, subscriptions, since)
528 ret['updates'] = map(prepare_episode_data, episode_updates)
530 return JsonResponse(ret)
533 def get_episode_updates(user, subscribed_podcasts, since):
534 """ Returns the episode updates since the timestamp """
536 EpisodeStatus = namedtuple('EpisodeStatus', 'episode status action')
538 episode_status = {}
540 # get episodes
541 if gevent:
542 episode_jobs = [gevent.spawn(episodes_for_podcast, p, since) for p in
543 subscribed_podcasts]
544 gevent.joinall(episode_jobs)
545 episodes = chain.from_iterable(job.get() for job in episode_jobs)
547 else:
548 episodes = chain.from_iterable(episodes_for_podcast(p, since) for p
549 in subscribed_podcasts)
552 for episode in episodes:
553 episode_status[episode._id] = EpisodeStatus(episode, 'new', None)
556 # get episode states
557 if gevent:
558 e_action_jobs = [gevent.spawn(get_podcasts_episode_states, p, user._id)
559 for p in subscribed_podcasts]
560 gevent.joinall(e_action_jobs)
561 e_actions = chain.from_iterable(job.get() for job in e_action_jobs)
563 else:
564 e_actions = [get_podcasts_episode_states(p, user._id) for p
565 in subscribed_podcasts]
568 for action in e_actions:
569 e_id = action['episode_id']
571 if e_id in episode_status:
572 episode = episode_status[e_id].episode
573 else:
574 episode = episode_by_id(e_id)
576 episode_status[e_id] = EpisodeStatus(episode, action['action'], action)
578 return episode_status.itervalues()
581 @require_valid_user
582 @check_username
583 @never_cache
584 def favorites(request, username):
585 favorites = favorite_episodes_for_user(request.user)
586 domain = RequestSite(request).domain
587 e_data = lambda e: episode_data(e, domain)
588 ret = map(e_data, favorites)
589 return JsonResponse(ret)
592 def sanitize_append(url, obj_type, sanitized_list):
593 urls = sanitize_url(url, obj_type)
594 if url != urls:
595 sanitized_list.append( (url, urls) )
596 return urls