mygpo/api/advanced/__init__.py

   1 #
   2 # This file is part of my.gpodder.org.
   3 #
   4 # my.gpodder.org is free software: you can redistribute it and/or modify it
   5 # under the terms of the GNU Affero General Public License as published by
   6 # the Free Software Foundation, either version 3 of the License, or (at your
   7 # option) any later version.
   8 #
   9 # my.gpodder.org is distributed in the hope that it will be useful, but
  10 # WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
  11 # or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public
  12 # License for more details.
  13 #
  14 # You should have received a copy of the GNU Affero General Public License
  15 # along with my.gpodder.org. If not, see <http://www.gnu.org/licenses/>.
  16 #
  17
  18 from functools import partial
  19 from itertools import imap, chain
  20 from collections import defaultdict, namedtuple
  21 from datetime import datetime
  22
  23 import dateutil.parser
  24 import gevent
  25
  26 from django.http import HttpResponse, HttpResponseBadRequest, Http404, HttpResponseNotFound
  27 from django.contrib.sites.models import RequestSite
  28 from django.db import IntegrityError
  29 from django.views.decorators.csrf import csrf_exempt
  30 from django.views.decorators.cache import never_cache
  31
  32 from mygpo.api.constants import EPISODE_ACTION_TYPES, DEVICE_TYPES
  33 from mygpo.api.httpresponse import JsonResponse
  34 from mygpo.api.sanitizing import sanitize_url, sanitize_urls
  35 from mygpo.api.advanced.directory import episode_data, podcast_data
  36 from mygpo.api.backend import get_device, BulkSubscribe
  37 from mygpo.couch import BulkException, bulk_save_retry
  38 from mygpo.log import log
  39 from mygpo.utils import parse_time, format_time, parse_bool, get_timestamp
  40 from mygpo.decorators import allowed_methods, repeat_on_conflict
  41 from mygpo.core import models
  42 from mygpo.core.models import SanitizingRule, Podcast
  43 from mygpo.users.models import PodcastUserState, EpisodeAction, \
  44      EpisodeUserState, DeviceDoesNotExist, DeviceUIDException, \
  45      InvalidEpisodeActionAttributes
  46 from mygpo.json import json, JSONDecodeError
  47 from mygpo.api.basic_auth import require_valid_user, check_username
  48 from mygpo.db.couchdb.episode import episode_by_id, \
  49          favorite_episodes_for_user, episodes_for_podcast
  50 from mygpo.db.couchdb.podcast import podcast_for_url
  51 from mygpo.db.couchdb.podcast_state import subscribed_podcast_ids_by_device
  52 from mygpo.db.couchdb.episode_state import get_podcasts_episode_states, \
  53          episode_state_for_ref_urls, get_episode_actions
  54
  55
  56 # keys that are allowed in episode actions
  57 EPISODE_ACTION_KEYS = ('position', 'episode', 'action', 'device', 'timestamp',
  58                        'started', 'total', 'podcast')
  59
  60
  61 @csrf_exempt
  62 @require_valid_user
  63 @check_username
  64 @never_cache
  65 @allowed_methods(['GET', 'POST'])
  66 def subscriptions(request, username, device_uid):
  67
  68     now = datetime.now()
  69     now_ = get_timestamp(now)
  70
  71     if request.method == 'GET':
  72
  73         try:
  74             device = request.user.get_device_by_uid(device_uid)
  75         except DeviceDoesNotExist as e:
  76             return HttpResponseNotFound(str(e))
  77
  78         since_ = request.GET.get('since', None)
  79         if since_ == None:
  80             return HttpResponseBadRequest('parameter since missing')
  81         try:
  82             since = datetime.fromtimestamp(float(since_))
  83         except ValueError:
  84             return HttpResponseBadRequest('since-value is not a valid timestamp')
  85
  86         changes = get_subscription_changes(request.user, device, since, now)
  87
  88         return JsonResponse(changes)
  89
  90     elif request.method == 'POST':
  91         d = get_device(request.user, device_uid,
  92                 request.META.get('HTTP_USER_AGENT', ''))
  93
  94         if not request.raw_post_data:
  95             return HttpResponseBadRequest('POST data must not be empty')
  96
  97         actions = json.loads(request.raw_post_data)
  98         add = actions['add'] if 'add' in actions else []
  99         rem = actions['remove'] if 'remove' in actions else []
 100
 101         add = filter(None, add)
 102         rem = filter(None, rem)
 103
 104         try:
 105             update_urls = update_subscriptions(request.user, d, add, rem)
 106         except IntegrityError, e:
 107             return HttpResponseBadRequest(e)
 108
 109         return JsonResponse({
 110             'timestamp': now_,
 111             'update_urls': update_urls,
 112             })
 113
 114
 115 def update_subscriptions(user, device, add, remove):
 116
 117     for a in add:
 118         if a in remove:
 119             raise IntegrityError('can not add and remove %s at the same time' % a)
 120
 121     add_s = list(sanitize_urls(add, 'podcast'))
 122     rem_s = list(sanitize_urls(remove, 'podcast'))
 123
 124     assert len(add) == len(add_s) and len(remove) == len(rem_s)
 125
 126     updated_urls = filter(lambda (a, b): a != b, zip(add + remove, add_s + rem_s))
 127
 128     add_s = filter(None, add_s)
 129     rem_s = filter(None, rem_s)
 130
 131     # If two different URLs (in add and remove) have
 132     # been sanitized to the same, we ignore the removal
 133     rem_s = filter(lambda x: x not in add_s, rem_s)
 134
 135     subscriber = BulkSubscribe(user, device)
 136
 137     for a in add_s:
 138         subscriber.add_action(a, 'subscribe')
 139
 140     for r in rem_s:
 141         subscriber.add_action(r, 'unsubscribe')
 142
 143     try:
 144         subscriber.execute()
 145     except BulkException as be:
 146         for err in be.errors:
 147             log('Advanced API: %(username)s: Updating subscription for '
 148                     '%(podcast_url)s on %(device_uid)s failed: '
 149                     '%(rerror)s (%(reason)s)'.format(username=user.username,
 150                         podcast_url=err.doc, device_uid=device.uid,
 151                         error=err.error, reason=err.reason)
 152                 )
 153
 154     return updated_urls
 155
 156
 157 def get_subscription_changes(user, device, since, until):
 158     add_urls, rem_urls = device.get_subscription_changes(since, until)
 159     until_ = get_timestamp(until)
 160     return {'add': add_urls, 'remove': rem_urls, 'timestamp': until_}
 161
 162
 163 @csrf_exempt
 164 @require_valid_user
 165 @check_username
 166 @never_cache
 167 @allowed_methods(['GET', 'POST'])
 168 def episodes(request, username, version=1):
 169
 170     version = int(version)
 171     now = datetime.now()
 172     now_ = get_timestamp(now)
 173     ua_string = request.META.get('HTTP_USER_AGENT', '')
 174
 175     if request.method == 'POST':
 176         try:
 177             actions = json.loads(request.raw_post_data)
 178         except (JSONDecodeError, UnicodeDecodeError) as e:
 179             log('Advanced API: could not decode episode update POST data for user %s: %s' % (username, e))
 180             return HttpResponseBadRequest()
 181
 182         try:
 183             update_urls = update_episodes(request.user, actions, now, ua_string)
 184         except DeviceUIDException as e:
 185             import traceback
 186             log('could not update episodes for user %s: %s %s: %s' % (username, e, traceback.format_exc(), actions))
 187             return HttpResponseBadRequest(str(e))
 188         except InvalidEpisodeActionAttributes as e:
 189             import traceback
 190             log('could not update episodes for user %s: %s %s: %s' % (username, e, traceback.format_exc(), actions))
 191             return HttpResponseBadRequest(str(e))
 192
 193         return JsonResponse({'timestamp': now_, 'update_urls': update_urls})
 194
 195     elif request.method == 'GET':
 196         podcast_url= request.GET.get('podcast', None)
 197         device_uid = request.GET.get('device', None)
 198         since_     = request.GET.get('since', None)
 199         aggregated = parse_bool(request.GET.get('aggregated', False))
 200
 201         try:
 202             since = datetime.fromtimestamp(float(since_)) if since_ else None
 203         except ValueError:
 204             return HttpResponseBadRequest('since-value is not a valid timestamp')
 205
 206         if podcast_url:
 207             podcast = podcast_for_url(podcast_url)
 208             if not podcast:
 209                 raise Http404
 210         else:
 211             podcast = None
 212
 213         if device_uid:
 214
 215             try:
 216                 device = request.user.get_device_by_uid(device_uid)
 217             except DeviceDoesNotExist as e:
 218                 return HttpResponseNotFound(str(e))
 219
 220         else:
 221             device = None
 222
 223         changes = get_episode_changes(request.user, podcast, device, since,
 224                 now, aggregated, version)
 225
 226         return JsonResponse(changes)
 227
 228
 229
 230 def convert_position(action):
 231     """ convert position parameter for API 1 compatibility """
 232     pos = getattr(action, 'position', None)
 233     if pos is not None:
 234         action.position = format_time(pos)
 235     return action
 236
 237
 238
 239 def get_episode_changes(user, podcast, device, since, until, aggregated, version):
 240
 241     devices = dict( (dev.id, dev.uid) for dev in user.devices )
 242
 243     args = {}
 244     if podcast is not None:
 245         args['podcast_id'] = podcast.get_id()
 246
 247     if device is not None:
 248         args['device_id'] = device.id
 249
 250     actions = get_episode_actions(user._id, since, until, **args)
 251
 252     if version == 1:
 253         actions = imap(convert_position, actions)
 254
 255     clean_data = partial(clean_episode_action_data,
 256             user=user, devices=devices)
 257
 258     actions = map(clean_data, actions)
 259     actions = filter(None, actions)
 260
 261     if aggregated:
 262         actions = dict( (a['episode'], a) for a in actions ).values()
 263
 264     until_ = get_timestamp(until)
 265
 266     return {'actions': actions, 'timestamp': until_}
 267
 268
 269
 270
 271 def clean_episode_action_data(action, user, devices):
 272
 273     if None in (action.get('podcast', None), action.get('episode', None)):
 274         return None
 275
 276     if 'device_id' in action:
 277         device_id = action['device_id']
 278         device_uid = devices.get(device_id)
 279         if device_uid:
 280             action['device'] = device_uid
 281
 282         del action['device_id']
 283
 284     # remove superfluous keys
 285     for x in action.keys():
 286         if x not in EPISODE_ACTION_KEYS:
 287             del action[x]
 288
 289     # set missing keys to None
 290     for x in EPISODE_ACTION_KEYS:
 291         if x not in action:
 292             action[x] = None
 293
 294     if action['action'] != 'play':
 295         if 'position' in action:
 296             del action['position']
 297
 298         if 'total' in action:
 299             del action['total']
 300
 301         if 'started' in action:
 302             del action['started']
 303
 304         if 'playmark' in action:
 305             del action['playmark']
 306
 307     else:
 308         action['position'] = action.get('position', False) or 0
 309
 310     return action
 311
 312
 313
 314
 315
 316 def update_episodes(user, actions, now, ua_string):
 317     update_urls = []
 318
 319     grouped_actions = defaultdict(list)
 320
 321     # group all actions by their episode
 322     for action in actions:
 323
 324         podcast_url = action['podcast']
 325         podcast_url = sanitize_append(podcast_url, 'podcast', update_urls)
 326         if podcast_url == '': continue
 327
 328         episode_url = action['episode']
 329         episode_url = sanitize_append(episode_url, 'episode', update_urls)
 330         if episode_url == '': continue
 331
 332         act = parse_episode_action(action, user, update_urls, now, ua_string)
 333         grouped_actions[ (podcast_url, episode_url) ].append(act)
 334
 335
 336     auto_flattr_episodes = []
 337
 338     # Prepare the updates for each episode state
 339     obj_funs = []
 340
 341     for (p_url, e_url), action_list in grouped_actions.iteritems():
 342         episode_state = episode_state_for_ref_urls(user, p_url, e_url)
 343
 344         if any(a.action == 'play' for a in actions):
 345             auto_flattr_episodes.append(episode_state.episode)
 346
 347         fun = partial(update_episode_actions, action_list=action_list)
 348         obj_funs.append( (episode_state, fun) )
 349
 350     bulk_save_retry(obj_funs)
 351
 352     for episode_id in auto_flattr_episodes:
 353         auto_flattr_episode.delay(user, episode_id)
 354
 355     return update_urls
 356
 357
 358 def update_episode_actions(episode_state, action_list):
 359     """ Adds actions to the episode state and saves if necessary """
 360
 361     len1 = len(episode_state.actions)
 362     episode_state.add_actions(action_list)
 363
 364     if len(episode_state.actions) == len1:
 365         return None
 366
 367     return episode_state
 368
 369
 370
 371 def parse_episode_action(action, user, update_urls, now, ua_string):
 372     action_str  = action.get('action', None)
 373     if not valid_episodeaction(action_str):
 374         raise Exception('invalid action %s' % action_str)
 375
 376     new_action = EpisodeAction()
 377
 378     new_action.action = action['action']
 379
 380     if action.get('device', False):
 381         device = get_device(user, action['device'], ua_string)
 382         new_action.device = device.id
 383
 384     if action.get('timestamp', False):
 385         new_action.timestamp = dateutil.parser.parse(action['timestamp'])
 386     else:
 387         new_action.timestamp = now
 388     new_action.timestamp = new_action.timestamp.replace(microsecond=0)
 389
 390     new_action.started = action.get('started', None)
 391     new_action.playmark = action.get('position', None)
 392     new_action.total = action.get('total', None)
 393
 394     return new_action
 395
 396
 397 @csrf_exempt
 398 @require_valid_user
 399 @check_username
 400 @never_cache
 401 # Workaround for mygpoclient 1.0: It uses "PUT" requests
 402 # instead of "POST" requests for uploading device settings
 403 @allowed_methods(['POST', 'PUT'])
 404 def device(request, username, device_uid):
 405     d = get_device(request.user, device_uid,
 406             request.META.get('HTTP_USER_AGENT', ''))
 407
 408     data = json.loads(request.raw_post_data)
 409
 410     if 'caption' in data:
 411         if not data['caption']:
 412             return HttpResponseBadRequest('caption must not be empty')
 413         d.name = data['caption']
 414
 415     if 'type' in data:
 416         if not valid_devicetype(data['type']):
 417            return HttpResponseBadRequest('invalid device type %s' % data['type'])
 418         d.type = data['type']
 419
 420
 421     request.user.update_device(d)
 422
 423     return HttpResponse()
 424
 425
 426 def valid_devicetype(type):
 427     for t in DEVICE_TYPES:
 428         if t[0] == type:
 429             return True
 430     return False
 431
 432 def valid_episodeaction(type):
 433     for t in EPISODE_ACTION_TYPES:
 434         if t[0] == type:
 435             return True
 436     return False
 437
 438
 439 @csrf_exempt
 440 @require_valid_user
 441 @check_username
 442 @never_cache
 443 @allowed_methods(['GET'])
 444 def devices(request, username):
 445     devices = filter(lambda d: not d.deleted, request.user.devices)
 446     devices = map(device_data, devices)
 447     return JsonResponse(devices)
 448
 449
 450 def device_data(device):
 451     return dict(
 452         id           = device.uid,
 453         caption      = device.name,
 454         type         = device.type,
 455         subscriptions= len(subscribed_podcast_ids_by_device(device)),
 456     )
 457
 458
 459
 460 def get_podcast_data(podcasts, domain, url):
 461     """ Gets podcast data for a URL from a dict of podcasts """
 462     podcast = podcasts.get(url)
 463     return podcast_data(podcast, domain)
 464
 465
 466 def get_episode_data(podcasts, domain, clean_action_data, include_actions, episode_status):
 467     """ Get episode data for an episode status object """
 468     podcast_id = episode_status.episode.podcast
 469     podcast = podcasts.get(podcast_id, None)
 470     t = episode_data(episode_status.episode, domain, podcast)
 471     t['status'] = episode_status.status
 472
 473     # include latest action (bug 1419)
 474     if include_actions and episode_status.action:
 475         t['action'] = clean_action_data(episode_status.action)
 476
 477     return t
 478
 479
 480 @csrf_exempt
 481 @require_valid_user
 482 @check_username
 483 @never_cache
 484 def updates(request, username, device_uid):
 485     now = datetime.now()
 486     now_ = get_timestamp(now)
 487
 488     try:
 489         device = request.user.get_device_by_uid(device_uid)
 490     except DeviceDoesNotExist as e:
 491         return HttpResponseNotFound(str(e))
 492
 493     since_ = request.GET.get('since', None)
 494     if since_ == None:
 495         return HttpResponseBadRequest('parameter since missing')
 496     try:
 497         since = datetime.fromtimestamp(float(since_))
 498     except ValueError:
 499         return HttpResponseBadRequest('since-value is not a valid timestamp')
 500
 501     include_actions = parse_bool(request.GET.get('include_actions', False))
 502
 503     ret = get_subscription_changes(request.user, device, since, now)
 504     domain = RequestSite(request).domain
 505
 506     subscriptions = list(device.get_subscribed_podcasts())
 507
 508     podcasts = dict( (p.url, p) for p in subscriptions )
 509     prepare_podcast_data = partial(get_podcast_data, podcasts, domain)
 510
 511     ret['add'] = map(prepare_podcast_data, ret['add'])
 512
 513     devices = dict( (dev.id, dev.uid) for dev in request.user.devices )
 514     clean_action_data = partial(clean_episode_action_data,
 515             user=request.user, devices=devices)
 516
 517     # index subscribed podcasts by their Id for fast access
 518     podcasts = dict( (p.get_id(), p) for p in subscriptions )
 519     prepare_episode_data = partial(get_episode_data, podcasts, domain,
 520             clean_action_data, include_actions)
 521
 522     episode_updates = get_episode_updates(request.user, subscriptions, since)
 523     ret['updates'] = map(prepare_episode_data, episode_updates)
 524
 525     return JsonResponse(ret)
 526
 527
 528 def get_episode_updates(user, subscribed_podcasts, since):
 529     """ Returns the episode updates since the timestamp """
 530
 531     EpisodeStatus = namedtuple('EpisodeStatus', 'episode status action')
 532
 533     episode_status = {}
 534
 535     # get episodes
 536     episode_jobs = [gevent.spawn(episodes_for_podcast, p, since) for p in
 537         subscribed_podcasts]
 538     gevent.joinall(episode_jobs)
 539     episodes = chain.from_iterable(job.get() for job in episode_jobs)
 540
 541     for episode in episodes:
 542         episode_status[episode._id] = EpisodeStatus(episode, 'new', None)
 543
 544     # get episode states
 545     e_action_jobs = [gevent.spawn(get_podcasts_episode_states, p, user._id)
 546             for p in subscribed_podcasts]
 547     gevent.joinall(e_action_jobs)
 548     e_actions = chain.from_iterable(job.get() for job in e_action_jobs)
 549
 550     for action in e_actions:
 551         e_id = action['episode_id']
 552
 553         if e_id in episode_status:
 554             episode = episode_status[e_id].episode
 555         else:
 556             episode = episode_by_id(e_id)
 557
 558         episode_status[e_id] = EpisodeStatus(episode, action['action'], action)
 559
 560     return episode_status.itervalues()
 561
 562
 563 @require_valid_user
 564 @check_username
 565 @never_cache
 566 def favorites(request, username):
 567     favorites = favorite_episodes_for_user(request.user)
 568     domain = RequestSite(request).domain
 569     e_data = lambda e: episode_data(e, domain)
 570     ret = map(e_data, favorites)
 571     return JsonResponse(ret)
 572
 573
 574 def sanitize_append(url, obj_type, sanitized_list):
 575     urls = sanitize_url(url, obj_type)
 576     if url != urls:
 577         sanitized_list.append( (url, urls) )
 578     return urls