mygpo/api/advanced/__init__.py

   1 #
   2 # This file is part of my.gpodder.org.
   3 #
   4 # my.gpodder.org is free software: you can redistribute it and/or modify it
   5 # under the terms of the GNU Affero General Public License as published by
   6 # the Free Software Foundation, either version 3 of the License, or (at your
   7 # option) any later version.
   8 #
   9 # my.gpodder.org is distributed in the hope that it will be useful, but
  10 # WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
  11 # or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public
  12 # License for more details.
  13 #
  14 # You should have received a copy of the GNU Affero General Public License
  15 # along with my.gpodder.org. If not, see <http://www.gnu.org/licenses/>.
  16 #
  17
  18 from functools import partial
  19 from itertools import imap, chain
  20 from collections import defaultdict, namedtuple
  21 from datetime import datetime
  22
  23 import dateutil.parser
  24 import gevent
  25
  26 from django.http import HttpResponse, HttpResponseBadRequest, Http404, HttpResponseNotFound
  27 from django.contrib.sites.models import RequestSite
  28 from django.db import IntegrityError
  29 from django.views.decorators.csrf import csrf_exempt
  30 from django.views.decorators.cache import never_cache
  31
  32 from mygpo.api.constants import EPISODE_ACTION_TYPES, DEVICE_TYPES
  33 from mygpo.api.httpresponse import JsonResponse
  34 from mygpo.api.sanitizing import sanitize_url, sanitize_urls
  35 from mygpo.api.advanced.directory import episode_data, podcast_data
  36 from mygpo.api.backend import get_device, get_favorites
  37 from mygpo.log import log
  38 from mygpo.utils import parse_time, format_time, parse_bool, get_to_dict, get_timestamp
  39 from mygpo.decorators import allowed_methods, repeat_on_conflict
  40 from mygpo.core import models
  41 from mygpo.core.models import SanitizingRule, Podcast
  42 from mygpo.users.models import PodcastUserState, EpisodeAction, EpisodeUserState, DeviceDoesNotExist
  43 from mygpo.json import json, JSONDecodeError
  44 from mygpo.api.basic_auth import require_valid_user, check_username
  45 from mygpo.couchdb import bulk_save_retry
  46
  47
  48 # keys that are allowed in episode actions
  49 EPISODE_ACTION_KEYS = ('position', 'episode', 'action', 'device', 'timestamp',
  50                        'started', 'total', 'podcast')
  51
  52
  53 @csrf_exempt
  54 @require_valid_user
  55 @check_username
  56 @never_cache
  57 @allowed_methods(['GET', 'POST'])
  58 def subscriptions(request, username, device_uid):
  59
  60     now = datetime.now()
  61     now_ = get_timestamp(now)
  62
  63     if request.method == 'GET':
  64
  65         try:
  66             device = request.user.get_device_by_uid(device_uid)
  67         except DeviceDoesNotExist as e:
  68             return HttpResponseNotFound(str(e))
  69
  70         since_ = request.GET.get('since', None)
  71         if since_ == None:
  72             return HttpResponseBadRequest('parameter since missing')
  73         try:
  74             since = datetime.fromtimestamp(float(since_))
  75         except ValueError:
  76             return HttpResponseBadRequest('since-value is not a valid timestamp')
  77
  78         changes = get_subscription_changes(request.user, device, since, now)
  79
  80         return JsonResponse(changes)
  81
  82     elif request.method == 'POST':
  83         d = get_device(request.user, device_uid,
  84                 request.META.get('HTTP_USER_AGENT', ''))
  85
  86         if not request.raw_post_data:
  87             return HttpResponseBadRequest('POST data must not be empty')
  88
  89         actions = json.loads(request.raw_post_data)
  90         add = actions['add'] if 'add' in actions else []
  91         rem = actions['remove'] if 'remove' in actions else []
  92
  93         add = filter(None, add)
  94         rem = filter(None, rem)
  95
  96         try:
  97             update_urls = update_subscriptions(request.user, d, add, rem)
  98         except IntegrityError, e:
  99             return HttpResponseBadRequest(e)
 100
 101         return JsonResponse({
 102             'timestamp': now_,
 103             'update_urls': update_urls,
 104             })
 105
 106
 107 def update_subscriptions(user, device, add, remove):
 108
 109     for a in add:
 110         if a in remove:
 111             raise IntegrityError('can not add and remove %s at the same time' % a)
 112
 113     add_s = list(sanitize_urls(add, 'podcast'))
 114     rem_s = list(sanitize_urls(remove, 'podcast'))
 115
 116     assert len(add) == len(add_s) and len(remove) == len(rem_s)
 117
 118     updated_urls = filter(lambda (a, b): a != b, zip(add + remove, add_s + rem_s))
 119
 120     add_s = filter(None, add_s)
 121     rem_s = filter(None, rem_s)
 122
 123     # If two different URLs (in add and remove) have
 124     # been sanitized to the same, we ignore the removal
 125     rem_s = filter(lambda x: x not in add_s, rem_s)
 126
 127     for a in add_s:
 128         p = Podcast.for_url(a, create=True)
 129         try:
 130             p.subscribe(user, device)
 131         except Exception as e:
 132             log('Advanced API: %(username)s: could not subscribe to podcast %(podcast_url)s on device %(device_id)s: %(exception)s' %
 133                 {'username': user.username, 'podcast_url': p.url, 'device_id': device.id, 'exception': e})
 134
 135     for r in rem_s:
 136         p = Podcast.for_url(r, create=True)
 137         try:
 138             p.unsubscribe(user, device)
 139         except Exception as e:
 140             log('Advanced API: %(username)s: could not unsubscribe from podcast %(podcast_url)s on device %(device_id)s: %(exception)s' %
 141                 {'username': user.username, 'podcast_url': p.url, 'device_id': device.id, 'exception': e})
 142
 143     return updated_urls
 144
 145
 146 def get_subscription_changes(user, device, since, until):
 147     add, rem = device.get_subscription_changes(since, until)
 148
 149     podcast_ids = add + rem
 150     podcasts = get_to_dict(Podcast, podcast_ids, get_id=models.Podcast.get_id)
 151
 152     add_podcasts = filter(None, (podcasts.get(i, None) for i in add))
 153     rem_podcasts = filter(None, (podcasts.get(i, None) for i in rem))
 154     add_urls = [ podcast.url for podcast in add_podcasts]
 155     rem_urls = [ podcast.url for podcast in rem_podcasts]
 156
 157     until_ = get_timestamp(until)
 158     return {'add': add_urls, 'remove': rem_urls, 'timestamp': until_}
 159
 160
 161 @csrf_exempt
 162 @require_valid_user
 163 @check_username
 164 @never_cache
 165 @allowed_methods(['GET', 'POST'])
 166 def episodes(request, username, version=1):
 167
 168     version = int(version)
 169     now = datetime.now()
 170     now_ = get_timestamp(now)
 171     ua_string = request.META.get('HTTP_USER_AGENT', '')
 172
 173     if request.method == 'POST':
 174         try:
 175             actions = json.loads(request.raw_post_data)
 176         except (JSONDecodeError, UnicodeDecodeError) as e:
 177             log('Advanced API: could not decode episode update POST data for user %s: %s' % (username, e))
 178             return HttpResponseBadRequest()
 179
 180         try:
 181             update_urls = update_episodes(request.user, actions, now, ua_string)
 182         except Exception, e:
 183             import traceback
 184             log('could not update episodes for user %s: %s %s: %s' % (username, e, traceback.format_exc(), actions))
 185             return HttpResponseBadRequest(e)
 186
 187         return JsonResponse({'timestamp': now_, 'update_urls': update_urls})
 188
 189     elif request.method == 'GET':
 190         podcast_url= request.GET.get('podcast', None)
 191         device_uid = request.GET.get('device', None)
 192         since_     = request.GET.get('since', None)
 193         aggregated = parse_bool(request.GET.get('aggregated', False))
 194
 195         try:
 196             since = datetime.fromtimestamp(float(since_)) if since_ else None
 197         except ValueError:
 198             return HttpResponseBadRequest('since-value is not a valid timestamp')
 199
 200         if podcast_url:
 201             podcast = Podcast.for_url(podcast_url)
 202             if not podcast:
 203                 raise Http404
 204         else:
 205             podcast = None
 206
 207         if device_uid:
 208
 209             try:
 210                 device = request.user.get_device_by_uid(device_uid)
 211             except DeviceDoesNotExist as e:
 212                 return HttpResponseNotFound(str(e))
 213
 214         else:
 215             device = None
 216
 217         changes = get_episode_changes(request.user, podcast, device, since,
 218                 now, aggregated, version)
 219
 220         return JsonResponse(changes)
 221
 222
 223
 224 def convert_position(action):
 225     """ convert position parameter for API 1 compatibility """
 226     pos = getattr(action, 'position', None)
 227     if pos is not None:
 228         action.position = format_time(pos)
 229     return action
 230
 231
 232
 233 def get_episode_changes(user, podcast, device, since, until, aggregated, version):
 234
 235     devices = dict( (dev.id, dev.uid) for dev in user.devices )
 236
 237     args = {}
 238     if podcast is not None:
 239         args['podcast_id'] = podcast.get_id()
 240
 241     if device is not None:
 242         args['device_id'] = device.id
 243
 244     actions = EpisodeAction.filter(user._id, since, until, **args)
 245
 246     if version == 1:
 247         actions = imap(convert_position, actions)
 248
 249     clean_data = partial(clean_episode_action_data,
 250             user=user, devices=devices)
 251
 252     actions = map(clean_data, actions)
 253     actions = filter(None, actions)
 254
 255     if aggregated:
 256         actions = dict( (a['episode'], a) for a in actions ).values()
 257
 258     until_ = get_timestamp(until)
 259
 260     return {'actions': actions, 'timestamp': until_}
 261
 262
 263
 264
 265 def clean_episode_action_data(action, user, devices):
 266
 267     if None in (action.get('podcast', None), action.get('episode', None)):
 268         return None
 269
 270     if 'device_id' in action:
 271         device_id = action['device_id']
 272         device_uid = devices.get(device_id)
 273         if device_uid:
 274             action['device'] = device_uid
 275
 276         del action['device_id']
 277
 278     # remove superfluous keys
 279     for x in action.keys():
 280         if x not in EPISODE_ACTION_KEYS:
 281             del action[x]
 282
 283     # set missing keys to None
 284     for x in EPISODE_ACTION_KEYS:
 285         if x not in action:
 286             action[x] = None
 287
 288     if action['action'] != 'play':
 289         if 'position' in action:
 290             del action['position']
 291
 292         if 'total' in action:
 293             del action['total']
 294
 295         if 'started' in action:
 296             del action['started']
 297
 298         if 'playmark' in action:
 299             del action['playmark']
 300
 301     else:
 302         action['position'] = action.get('position', False) or 0
 303
 304     return action
 305
 306
 307
 308
 309
 310 def update_episodes(user, actions, now, ua_string):
 311     update_urls = []
 312
 313     grouped_actions = defaultdict(list)
 314
 315     # group all actions by their episode
 316     for action in actions:
 317
 318         podcast_url = action['podcast']
 319         podcast_url = sanitize_append(podcast_url, 'podcast', update_urls)
 320         if podcast_url == '': continue
 321
 322         episode_url = action['episode']
 323         episode_url = sanitize_append(episode_url, 'episode', update_urls)
 324         if episode_url == '': continue
 325
 326         act = parse_episode_action(action, user, update_urls, now, ua_string)
 327         grouped_actions[ (podcast_url, episode_url) ].append(act)
 328
 329     # Prepare the updates for each episode state
 330     obj_funs = []
 331
 332     for (p_url, e_url), action_list in grouped_actions.iteritems():
 333         episode_state = EpisodeUserState.for_ref_urls(user, p_url, e_url)
 334
 335         fun = partial(update_episode_actions, action_list=action_list)
 336         obj_funs.append( (episode_state, fun) )
 337
 338     db = EpisodeUserState.get_db()
 339     bulk_save_retry(db, obj_funs)
 340
 341     return update_urls
 342
 343
 344 def update_episode_actions(episode_state, action_list):
 345     """ Adds actions to the episode state and saves if necessary """
 346
 347     len1 = len(episode_state.actions)
 348     episode_state.add_actions(action_list)
 349
 350     if len(episode_state.actions) == len1:
 351         return None
 352
 353     return episode_state
 354
 355
 356
 357 def parse_episode_action(action, user, update_urls, now, ua_string):
 358     action_str  = action.get('action', None)
 359     if not valid_episodeaction(action_str):
 360         raise Exception('invalid action %s' % action_str)
 361
 362     new_action = EpisodeAction()
 363
 364     new_action.action = action['action']
 365
 366     if action.get('device', False):
 367         device = get_device(user, action['device'], ua_string)
 368         new_action.device = device.id
 369
 370     if action.get('timestamp', False):
 371         new_action.timestamp = dateutil.parser.parse(action['timestamp'])
 372     else:
 373         new_action.timestamp = now
 374     new_action.timestamp = new_action.timestamp.replace(microsecond=0)
 375
 376     new_action.started = action.get('started', None)
 377     new_action.playmark = action.get('position', None)
 378     new_action.total = action.get('total', None)
 379
 380     return new_action
 381
 382
 383 @csrf_exempt
 384 @require_valid_user
 385 @check_username
 386 @never_cache
 387 # Workaround for mygpoclient 1.0: It uses "PUT" requests
 388 # instead of "POST" requests for uploading device settings
 389 @allowed_methods(['POST', 'PUT'])
 390 def device(request, username, device_uid):
 391     d = get_device(request.user, device_uid,
 392             request.META.get('HTTP_USER_AGENT', ''))
 393
 394     data = json.loads(request.raw_post_data)
 395
 396     if 'caption' in data:
 397         if not data['caption']:
 398             return HttpResponseBadRequest('caption must not be empty')
 399         d.name = data['caption']
 400
 401     if 'type' in data:
 402         if not valid_devicetype(data['type']):
 403            return HttpResponseBadRequest('invalid device type %s' % data['type'])
 404         d.type = data['type']
 405
 406
 407     request.user.update_device(d)
 408
 409     return HttpResponse()
 410
 411
 412 def valid_devicetype(type):
 413     for t in DEVICE_TYPES:
 414         if t[0] == type:
 415             return True
 416     return False
 417
 418 def valid_episodeaction(type):
 419     for t in EPISODE_ACTION_TYPES:
 420         if t[0] == type:
 421             return True
 422     return False
 423
 424
 425 @csrf_exempt
 426 @require_valid_user
 427 @check_username
 428 @never_cache
 429 @allowed_methods(['GET'])
 430 def devices(request, username):
 431     devices = filter(lambda d: not d.deleted, request.user.devices)
 432     devices = map(device_data, devices)
 433     return JsonResponse(devices)
 434
 435
 436 def device_data(device):
 437     return dict(
 438         id           = device.uid,
 439         caption      = device.name,
 440         type         = device.type,
 441         subscriptions= len(device.get_subscribed_podcast_ids())
 442     )
 443
 444
 445
 446 def get_podcast_data(podcasts, domain, url):
 447     """ Gets podcast data for a URL from a dict of podcasts """
 448     podcast = podcasts.get(url)
 449     return podcast_data(podcast, domain)
 450
 451
 452 def get_episode_data(podcasts, domain, clean_action_data, include_actions, episode_status):
 453     """ Get episode data for an episode status object """
 454     podcast_id = episode_status.episode.podcast
 455     podcast = podcasts.get(podcast_id, None)
 456     t = episode_data(episode_status.episode, domain, podcast)
 457     t['status'] = episode_status.status
 458
 459     # include latest action (bug 1419)
 460     if include_actions and episode_status.action:
 461         t['action'] = clean_action_data(episode_status.action)
 462
 463     return t
 464
 465
 466 @csrf_exempt
 467 @require_valid_user
 468 @check_username
 469 @never_cache
 470 def updates(request, username, device_uid):
 471     now = datetime.now()
 472     now_ = get_timestamp(now)
 473
 474     try:
 475         device = request.user.get_device_by_uid(device_uid)
 476     except DeviceDoesNotExist as e:
 477         return HttpResponseNotFound(str(e))
 478
 479     since_ = request.GET.get('since', None)
 480     if since_ == None:
 481         return HttpResponseBadRequest('parameter since missing')
 482     try:
 483         since = datetime.fromtimestamp(float(since_))
 484     except ValueError:
 485         return HttpResponseBadRequest('since-value is not a valid timestamp')
 486
 487     include_actions = parse_bool(request.GET.get('include_actions', False))
 488
 489     ret = get_subscription_changes(request.user, device, since, now)
 490     domain = RequestSite(request).domain
 491
 492     subscriptions = list(device.get_subscribed_podcasts())
 493
 494     podcasts = dict( (p.url, p) for p in subscriptions )
 495     prepare_podcast_data = partial(get_podcast_data, podcasts, domain)
 496
 497     ret['add'] = map(prepare_podcast_data, ret['add'])
 498
 499     devices = dict( (dev.id, dev.uid) for dev in request.user.devices )
 500     clean_action_data = partial(clean_episode_action_data,
 501             user=request.user, devices=devices)
 502
 503     # index subscribed podcasts by their Id for fast access
 504     podcasts = dict( (p.get_id(), p) for p in subscriptions )
 505     prepare_episode_data = partial(get_episode_data, podcasts, domain,
 506             clean_action_data, include_actions)
 507
 508     episode_updates = get_episode_updates(request.user, subscriptions, since)
 509     ret['updates'] = map(prepare_episode_data, episode_updates)
 510
 511     return JsonResponse(ret)
 512
 513
 514 def get_episode_updates(user, subscribed_podcasts, since):
 515     """ Returns the episode updates since the timestamp """
 516
 517     EpisodeStatus = namedtuple('EpisodeStatus', 'episode status action')
 518
 519     episode_status = {}
 520
 521     # get episodes
 522     episode_jobs = [gevent.spawn(p.get_episodes, since) for p in
 523         subscribed_podcasts]
 524     gevent.joinall(episode_jobs)
 525     episodes = chain.from_iterable(job.get() for job in episode_jobs)
 526
 527     for episode in episodes:
 528         episode_status[episode._id] = EpisodeStatus(episode, 'new', None)
 529
 530     # get episode states
 531     e_action_jobs = [gevent.spawn(p.get_episode_states, user._id) for p in
 532         subscribed_podcasts]
 533     gevent.joinall(e_action_jobs)
 534     e_actions = chain.from_iterable(job.get() for job in e_action_jobs)
 535
 536     for action in e_actions:
 537         e_id = action['episode_id']
 538
 539         if e_id in episode_status:
 540             episode = episode_status[e_id].episode
 541         else:
 542             episode = models.Episode.get(e_id)
 543
 544         episode_status[e_id] = EpisodeStatus(episode, action['action'], action)
 545
 546     return episode_status.itervalues()
 547
 548
 549 @require_valid_user
 550 @check_username
 551 @never_cache
 552 def favorites(request, username):
 553     favorites = get_favorites(request.user)
 554     domain = RequestSite(request).domain
 555     e_data = lambda e: episode_data(e, domain)
 556     ret = map(e_data, favorites)
 557     return JsonResponse(ret)
 558
 559
 560 def sanitize_append(url, obj_type, sanitized_list):
 561     urls = sanitize_url(url, obj_type)
 562     if url != urls:
 563         sanitized_list.append( (url, urls) )
 564     return urls