mygpo/api/advanced/__init__.py

   1 #
   2 # This file is part of my.gpodder.org.
   3 #
   4 # my.gpodder.org is free software: you can redistribute it and/or modify it
   5 # under the terms of the GNU Affero General Public License as published by
   6 # the Free Software Foundation, either version 3 of the License, or (at your
   7 # option) any later version.
   8 #
   9 # my.gpodder.org is distributed in the hope that it will be useful, but
  10 # WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
  11 # or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public
  12 # License for more details.
  13 #
  14 # You should have received a copy of the GNU Affero General Public License
  15 # along with my.gpodder.org. If not, see <http://www.gnu.org/licenses/>.
  16 #
  17
  18 from functools import partial
  19 from itertools import imap, chain
  20 from collections import defaultdict, namedtuple
  21 from datetime import datetime
  22 from importlib import import_module
  23
  24 import dateutil.parser
  25
  26 try:
  27     import gevent
  28 except ImportError:
  29     gevent = None
  30
  31 from django.http import HttpResponse, HttpResponseBadRequest, Http404, HttpResponseNotFound
  32 from django.contrib.sites.models import RequestSite
  33 from django.views.decorators.csrf import csrf_exempt
  34 from django.views.decorators.cache import never_cache
  35 from django.utils.decorators import method_decorator
  36 from django.views.generic.base import View
  37 from django.conf import settings as dsettings
  38
  39 from mygpo.api.constants import EPISODE_ACTION_TYPES, DEVICE_TYPES
  40 from mygpo.api.httpresponse import JsonResponse
  41 from mygpo.api.advanced.directory import episode_data, podcast_data
  42 from mygpo.api.backend import get_device, BulkSubscribe
  43 from mygpo.utils import parse_time, format_time, parse_bool, get_timestamp, \
  44     parse_request_body, normalize_feed_url
  45 from mygpo.decorators import allowed_methods, repeat_on_conflict
  46 from mygpo.core import models
  47 from mygpo.core.tasks import auto_flattr_episode
  48 from mygpo.users.models import PodcastUserState, EpisodeAction, \
  49      EpisodeUserState, DeviceDoesNotExist, DeviceUIDException, \
  50      InvalidEpisodeActionAttributes
  51 from mygpo.users.settings import FLATTR_AUTO
  52 from mygpo.core.json import JSONDecodeError
  53 from mygpo.api.basic_auth import require_valid_user, check_username
  54 from mygpo.db.couchdb import BulkException, bulk_save_retry
  55 from mygpo.db.couchdb.episode import episode_by_id, \
  56          favorite_episodes_for_user, episodes_for_podcast
  57 from mygpo.db.couchdb.podcast import podcast_for_url
  58 from mygpo.db.couchdb.podcast_state import subscribed_podcast_ids_by_device
  59 from mygpo.db.couchdb.episode_state import get_podcasts_episode_states, \
  60          episode_state_for_ref_urls, get_episode_actions
  61
  62
  63 import logging
  64 logger = logging.getLogger(__name__)
  65
  66
  67 # keys that are allowed in episode actions
  68 EPISODE_ACTION_KEYS = ('position', 'episode', 'action', 'device', 'timestamp',
  69                        'started', 'total', 'podcast')
  70
  71
  72 @csrf_exempt
  73 @require_valid_user
  74 @check_username
  75 @never_cache
  76 @allowed_methods(['GET', 'POST'])
  77 def subscriptions(request, username, device_uid):
  78
  79     now = datetime.now()
  80     now_ = get_timestamp(now)
  81
  82     if request.method == 'GET':
  83
  84         try:
  85             device = request.user.get_device_by_uid(device_uid)
  86         except DeviceDoesNotExist as e:
  87             return HttpResponseNotFound(str(e))
  88
  89         since_ = request.GET.get('since', None)
  90         if since_ is None:
  91             return HttpResponseBadRequest('parameter since missing')
  92         try:
  93             since = datetime.fromtimestamp(float(since_))
  94         except ValueError:
  95             return HttpResponseBadRequest('since-value is not a valid timestamp')
  96
  97         changes = get_subscription_changes(request.user, device, since, now)
  98
  99         return JsonResponse(changes)
 100
 101     elif request.method == 'POST':
 102         d = get_device(request.user, device_uid,
 103                 request.META.get('HTTP_USER_AGENT', ''))
 104
 105         if not request.body:
 106             return HttpResponseBadRequest('POST data must not be empty')
 107
 108         try:
 109             actions = parse_request_body(request)
 110         except (JSONDecodeError, UnicodeDecodeError, ValueError) as e:
 111             msg = (u'Could not decode subscription update POST data for ' +
 112                    'user %s: %s') % (username,
 113                    request.body.decode('ascii', errors='replace'))
 114             logger.exception(msg)
 115             return HttpResponseBadRequest(msg)
 116
 117         add = actions['add'] if 'add' in actions else []
 118         rem = actions['remove'] if 'remove' in actions else []
 119
 120         add = filter(None, add)
 121         rem = filter(None, rem)
 122
 123         try:
 124             update_urls = update_subscriptions(request.user, d, add, rem)
 125         except ValueError, e:
 126             return HttpResponseBadRequest(e)
 127
 128         return JsonResponse({
 129             'timestamp': now_,
 130             'update_urls': update_urls,
 131             })
 132
 133
 134 def update_subscriptions(user, device, add, remove):
 135
 136     for a in add:
 137         if a in remove:
 138             raise ValueError('can not add and remove %s at the same time' % a)
 139
 140     add_s = map(normalize_feed_url, add)
 141     rem_s = map(normalize_feed_url, remove)
 142
 143     assert len(add) == len(add_s) and len(remove) == len(rem_s)
 144
 145     updated_urls = filter(lambda (a, b): a != b, zip(add + remove, add_s + rem_s))
 146
 147     add_s = filter(None, add_s)
 148     rem_s = filter(None, rem_s)
 149
 150     # If two different URLs (in add and remove) have
 151     # been sanitized to the same, we ignore the removal
 152     rem_s = filter(lambda x: x not in add_s, rem_s)
 153
 154     subscriber = BulkSubscribe(user, device)
 155
 156     for a in add_s:
 157         subscriber.add_action(a, 'subscribe')
 158
 159     for r in rem_s:
 160         subscriber.add_action(r, 'unsubscribe')
 161
 162     try:
 163         subscriber.execute()
 164     except BulkException as be:
 165         for err in be.errors:
 166             loger.error('Advanced API: %(username)s: Updating subscription for '
 167                     '%(podcast_url)s on %(device_uid)s failed: '
 168                     '%(rerror)s (%(reason)s)'.format(username=user.username,
 169                         podcast_url=err.doc, device_uid=device.uid,
 170                         error=err.error, reason=err.reason)
 171                 )
 172
 173     return updated_urls
 174
 175
 176 def get_subscription_changes(user, device, since, until):
 177     add_urls, rem_urls = device.get_subscription_changes(since, until)
 178     until_ = get_timestamp(until)
 179     return {'add': add_urls, 'remove': rem_urls, 'timestamp': until_}
 180
 181
 182 @csrf_exempt
 183 @require_valid_user
 184 @check_username
 185 @never_cache
 186 @allowed_methods(['GET', 'POST'])
 187 def episodes(request, username, version=1):
 188
 189     version = int(version)
 190     now = datetime.now()
 191     now_ = get_timestamp(now)
 192     ua_string = request.META.get('HTTP_USER_AGENT', '')
 193
 194     if request.method == 'POST':
 195         try:
 196             actions = parse_request_body(request)
 197         except (JSONDecodeError, UnicodeDecodeError, ValueError) as e:
 198             msg = ('Could not decode episode update POST data for ' +
 199                    'user %s: %s') % (username,
 200                    request.body.decode('ascii', errors='replace'))
 201             logger.exception(msg)
 202             return HttpResponseBadRequest(msg)
 203
 204         logger.info('start: user %s: %d actions from %s' % (request.user._id, len(actions), ua_string))
 205
 206         # handle in background
 207         if len(actions) > dsettings.API_ACTIONS_MAX_NONBG:
 208             bg_handler = dsettings.API_ACTIONS_BG_HANDLER
 209             if bg_handler is not None:
 210
 211                 modname, funname = bg_handler.rsplit('.', 1)
 212                 mod = import_module(modname)
 213                 fun = getattr(mod, funname)
 214
 215                 fun(request.user, actions, now, ua_string)
 216
 217                 # TODO: return 202 Accepted
 218                 return JsonResponse({'timestamp': now_, 'update_urls': []})
 219
 220
 221         try:
 222             update_urls = update_episodes(request.user, actions, now, ua_string)
 223         except DeviceUIDException as e:
 224             logger.warn('invalid device UID while uploading episode actions for user %s', username)
 225             return HttpResponseBadRequest(str(e))
 226
 227         except InvalidEpisodeActionAttributes as e:
 228             logger.exception('invalid episode action attributes while uploading episode actions for user %s: %s' % (username,))
 229             return HttpResponseBadRequest(str(e))
 230
 231         logger.info('done:  user %s: %d actions from %s' % (request.user._id, len(actions), ua_string))
 232         return JsonResponse({'timestamp': now_, 'update_urls': update_urls})
 233
 234     elif request.method == 'GET':
 235         podcast_url= request.GET.get('podcast', None)
 236         device_uid = request.GET.get('device', None)
 237         since_     = request.GET.get('since', None)
 238         aggregated = parse_bool(request.GET.get('aggregated', False))
 239
 240         try:
 241             since = int(since_) if since_ else None
 242         except ValueError:
 243             return HttpResponseBadRequest('since-value is not a valid timestamp')
 244
 245         if podcast_url:
 246             podcast = podcast_for_url(podcast_url)
 247             if not podcast:
 248                 raise Http404
 249         else:
 250             podcast = None
 251
 252         if device_uid:
 253
 254             try:
 255                 device = request.user.get_device_by_uid(device_uid)
 256             except DeviceDoesNotExist as e:
 257                 return HttpResponseNotFound(str(e))
 258
 259         else:
 260             device = None
 261
 262         changes = get_episode_changes(request.user, podcast, device, since,
 263                 now_, aggregated, version)
 264
 265         return JsonResponse(changes)
 266
 267
 268
 269 def convert_position(action):
 270     """ convert position parameter for API 1 compatibility """
 271     pos = getattr(action, 'position', None)
 272     if pos is not None:
 273         action.position = format_time(pos)
 274     return action
 275
 276
 277
 278 def get_episode_changes(user, podcast, device, since, until, aggregated, version):
 279
 280     devices = dict( (dev.id, dev.uid) for dev in user.devices )
 281
 282     args = {}
 283     if podcast is not None:
 284         args['podcast_id'] = podcast.get_id()
 285
 286     if device is not None:
 287         args['device_id'] = device.id
 288
 289     actions = get_episode_actions(user._id, since, until, **args)
 290
 291     if version == 1:
 292         actions = imap(convert_position, actions)
 293
 294     clean_data = partial(clean_episode_action_data,
 295             user=user, devices=devices)
 296
 297     actions = map(clean_data, actions)
 298     actions = filter(None, actions)
 299
 300     if aggregated:
 301         actions = dict( (a['episode'], a) for a in actions ).values()
 302
 303     return {'actions': actions, 'timestamp': until}
 304
 305
 306
 307
 308 def clean_episode_action_data(action, user, devices):
 309
 310     if None in (action.get('podcast', None), action.get('episode', None)):
 311         return None
 312
 313     if 'device_id' in action:
 314         device_id = action['device_id']
 315         device_uid = devices.get(device_id)
 316         if device_uid:
 317             action['device'] = device_uid
 318
 319         del action['device_id']
 320
 321     # remove superfluous keys
 322     for x in action.keys():
 323         if x not in EPISODE_ACTION_KEYS:
 324             del action[x]
 325
 326     # set missing keys to None
 327     for x in EPISODE_ACTION_KEYS:
 328         if x not in action:
 329             action[x] = None
 330
 331     if action['action'] != 'play':
 332         if 'position' in action:
 333             del action['position']
 334
 335         if 'total' in action:
 336             del action['total']
 337
 338         if 'started' in action:
 339             del action['started']
 340
 341         if 'playmark' in action:
 342             del action['playmark']
 343
 344     else:
 345         action['position'] = action.get('position', False) or 0
 346
 347     return action
 348
 349
 350
 351
 352
 353 def update_episodes(user, actions, now, ua_string):
 354     update_urls = []
 355
 356     grouped_actions = defaultdict(list)
 357
 358     # group all actions by their episode
 359     for action in actions:
 360
 361         podcast_url = action['podcast']
 362         podcast_url = sanitize_append(podcast_url, update_urls)
 363         if podcast_url == '':
 364             continue
 365
 366         episode_url = action['episode']
 367         episode_url = sanitize_append(episode_url, update_urls)
 368         if episode_url == '':
 369             continue
 370
 371         act = parse_episode_action(action, user, update_urls, now, ua_string)
 372         grouped_actions[ (podcast_url, episode_url) ].append(act)
 373
 374
 375     auto_flattr_episodes = []
 376
 377     # Prepare the updates for each episode state
 378     obj_funs = []
 379
 380     for (p_url, e_url), action_list in grouped_actions.iteritems():
 381         episode_state = episode_state_for_ref_urls(user, p_url, e_url)
 382
 383         if any(a['action'] == 'play' for a in actions):
 384             auto_flattr_episodes.append(episode_state.episode)
 385
 386         fun = partial(update_episode_actions, action_list=action_list)
 387         obj_funs.append( (episode_state, fun) )
 388
 389     bulk_save_retry(obj_funs)
 390
 391     if user.get_wksetting(FLATTR_AUTO):
 392         for episode_id in auto_flattr_episodes:
 393             auto_flattr_episode.delay(user, episode_id)
 394
 395     return update_urls
 396
 397
 398 def update_episode_actions(episode_state, action_list):
 399     """ Adds actions to the episode state and saves if necessary """
 400
 401     len1 = len(episode_state.actions)
 402     episode_state.add_actions(action_list)
 403
 404     if len(episode_state.actions) == len1:
 405         return None
 406
 407     return episode_state
 408
 409
 410
 411 def parse_episode_action(action, user, update_urls, now, ua_string):
 412     action_str  = action.get('action', None)
 413     if not valid_episodeaction(action_str):
 414         raise Exception('invalid action %s' % action_str)
 415
 416     new_action = EpisodeAction()
 417
 418     new_action.action = action['action']
 419
 420     if action.get('device', False):
 421         device = get_device(user, action['device'], ua_string)
 422         new_action.device = device.id
 423
 424     if action.get('timestamp', False):
 425         new_action.timestamp = dateutil.parser.parse(action['timestamp'])
 426     else:
 427         new_action.timestamp = now
 428     new_action.timestamp = new_action.timestamp.replace(microsecond=0)
 429
 430     new_action.upload_timestamp = get_timestamp(now)
 431
 432     new_action.started = action.get('started', None)
 433     new_action.playmark = action.get('position', None)
 434     new_action.total = action.get('total', None)
 435
 436     return new_action
 437
 438
 439 @csrf_exempt
 440 @require_valid_user
 441 @check_username
 442 @never_cache
 443 # Workaround for mygpoclient 1.0: It uses "PUT" requests
 444 # instead of "POST" requests for uploading device settings
 445 @allowed_methods(['POST', 'PUT'])
 446 def device(request, username, device_uid):
 447     d = get_device(request.user, device_uid,
 448             request.META.get('HTTP_USER_AGENT', ''))
 449
 450     try:
 451         data = parse_request_body(request)
 452     except (JSONDecodeError, UnicodeDecodeError, ValueError) as e:
 453         msg = ('Could not decode device update POST data for ' +
 454                'user %s: %s') % (username,
 455                request.body.decode('ascii', errors='replace'))
 456         logger.exception(msg)
 457         return HttpResponseBadRequest(msg)
 458
 459     if 'caption' in data:
 460         if not data['caption']:
 461             return HttpResponseBadRequest('caption must not be empty')
 462         d.name = data['caption']
 463
 464     if 'type' in data:
 465         if not valid_devicetype(data['type']):
 466             return HttpResponseBadRequest('invalid device type %s' % data['type'])
 467         d.type = data['type']
 468
 469
 470     request.user.update_device(d)
 471
 472     return HttpResponse()
 473
 474
 475 def valid_devicetype(type):
 476     for t in DEVICE_TYPES:
 477         if t[0] == type:
 478             return True
 479     return False
 480
 481 def valid_episodeaction(type):
 482     for t in EPISODE_ACTION_TYPES:
 483         if t[0] == type:
 484             return True
 485     return False
 486
 487
 488 @csrf_exempt
 489 @require_valid_user
 490 @check_username
 491 @never_cache
 492 @allowed_methods(['GET'])
 493 def devices(request, username):
 494     devices = filter(lambda d: not d.deleted, request.user.devices)
 495     devices = map(device_data, devices)
 496     return JsonResponse(devices)
 497
 498
 499 def device_data(device):
 500     return dict(
 501         id           = device.uid,
 502         caption      = device.name,
 503         type         = device.type,
 504         subscriptions= len(subscribed_podcast_ids_by_device(device)),
 505     )
 506
 507
 508
 509 def get_podcast_data(podcasts, domain, url):
 510     """ Gets podcast data for a URL from a dict of podcasts """
 511     podcast = podcasts.get(url)
 512     return podcast_data(podcast, domain)
 513
 514
 515 def get_episode_data(podcasts, domain, clean_action_data, include_actions, episode_status):
 516     """ Get episode data for an episode status object """
 517     podcast_id = episode_status.episode.podcast
 518     podcast = podcasts.get(podcast_id, None)
 519     t = episode_data(episode_status.episode, domain, podcast)
 520     t['status'] = episode_status.status
 521
 522     # include latest action (bug 1419)
 523     if include_actions and episode_status.action:
 524         t['action'] = clean_action_data(episode_status.action)
 525
 526     return t
 527
 528
 529
 530 class DeviceUpdates(View):
 531
 532     @method_decorator(csrf_exempt)
 533     @method_decorator(require_valid_user)
 534     @method_decorator(check_username)
 535     @method_decorator(never_cache)
 536     def get(self, request, username, device_uid):
 537         now = datetime.now()
 538         now_ = get_timestamp(now)
 539
 540         try:
 541             device = request.user.get_device_by_uid(device_uid)
 542         except DeviceDoesNotExist as e:
 543             return HttpResponseNotFound(str(e))
 544
 545         since_ = request.GET.get('since', None)
 546         if since_ is None:
 547             return HttpResponseBadRequest('parameter since missing')
 548         try:
 549             since = datetime.fromtimestamp(float(since_))
 550         except ValueError:
 551             return HttpResponseBadRequest("'since' is not a valid timestamp")
 552
 553         include_actions = parse_bool(request.GET.get('include_actions', False))
 554
 555         ret = get_subscription_changes(request.user, device, since, now)
 556         domain = RequestSite(request).domain
 557
 558         subscriptions = list(device.get_subscribed_podcasts())
 559
 560         podcasts = dict( (p.url, p) for p in subscriptions )
 561         prepare_podcast_data = partial(get_podcast_data, podcasts, domain)
 562
 563         ret['add'] = map(prepare_podcast_data, ret['add'])
 564
 565         devices = dict( (dev.id, dev.uid) for dev in request.user.devices )
 566         clean_action_data = partial(clean_episode_action_data,
 567                 user=request.user, devices=devices)
 568
 569         # index subscribed podcasts by their Id for fast access
 570         podcasts = dict( (p.get_id(), p) for p in subscriptions )
 571         prepare_episode_data = partial(get_episode_data, podcasts, domain,
 572                 clean_action_data, include_actions)
 573
 574         episode_updates = self.get_episode_updates(request.user,
 575                 subscriptions, since)
 576         ret['updates'] = map(prepare_episode_data, episode_updates)
 577
 578         return JsonResponse(ret)
 579
 580
 581     def get_episode_updates(self, user, subscribed_podcasts, since,
 582             max_per_podcast=5):
 583         """ Returns the episode updates since the timestamp """
 584
 585         EpisodeStatus = namedtuple('EpisodeStatus', 'episode status action')
 586
 587         episode_status = {}
 588
 589         # get episodes
 590         if gevent:
 591             episode_jobs = [gevent.spawn(episodes_for_podcast, p, since,
 592                     limit=max_per_podcast) for p in subscribed_podcasts]
 593             gevent.joinall(episode_jobs)
 594             episodes = chain.from_iterable(job.get() for job in episode_jobs)
 595
 596         else:
 597             episodes = chain.from_iterable(episodes_for_podcast(p, since,
 598                     limit=max_per_podcast) for p in subscribed_podcasts)
 599
 600
 601         for episode in episodes:
 602             episode_status[episode._id] = EpisodeStatus(episode, 'new', None)
 603
 604
 605         # get episode states
 606         if gevent:
 607             e_action_jobs = [gevent.spawn(get_podcasts_episode_states, p,
 608                     user._id) for p in subscribed_podcasts]
 609             gevent.joinall(e_action_jobs)
 610             e_actions = chain.from_iterable(job.get() for job in e_action_jobs)
 611
 612         else:
 613             e_actions = chain.from_iterable(get_podcasts_episode_states(p,
 614                     user._id) for p in subscribed_podcasts)
 615
 616
 617         for action in e_actions:
 618             e_id = action['episode_id']
 619
 620             if e_id in episode_status:
 621                 episode = episode_status[e_id].episode
 622             else:
 623                 episode = episode_by_id(e_id)
 624
 625             episode_status[e_id] = EpisodeStatus(episode, action['action'],
 626                     action)
 627
 628         return episode_status.itervalues()
 629
 630
 631 @require_valid_user
 632 @check_username
 633 @never_cache
 634 def favorites(request, username):
 635     favorites = favorite_episodes_for_user(request.user)
 636     domain = RequestSite(request).domain
 637     e_data = lambda e: episode_data(e, domain)
 638     ret = map(e_data, favorites)
 639     return JsonResponse(ret)
 640
 641
 642 def sanitize_append(url, sanitized_list):
 643     urls = normalize_feed_url(url)
 644     if url != urls:
 645         sanitized_list.append( (url, urls or '') )
 646     return urls