mygpo/api/advanced/__init__.py

   1 #
   2 # This file is part of my.gpodder.org.
   3 #
   4 # my.gpodder.org is free software: you can redistribute it and/or modify it
   5 # under the terms of the GNU Affero General Public License as published by
   6 # the Free Software Foundation, either version 3 of the License, or (at your
   7 # option) any later version.
   8 #
   9 # my.gpodder.org is distributed in the hope that it will be useful, but
  10 # WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
  11 # or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public
  12 # License for more details.
  13 #
  14 # You should have received a copy of the GNU Affero General Public License
  15 # along with my.gpodder.org. If not, see <http://www.gnu.org/licenses/>.
  16 #
  17
  18 from functools import partial
  19 from itertools import imap
  20 from collections import defaultdict, namedtuple
  21 from datetime import datetime
  22 from importlib import import_module
  23
  24 import dateutil.parser
  25
  26 from django.http import HttpResponse, HttpResponseBadRequest, Http404, HttpResponseNotFound
  27 from django.contrib.sites.models import RequestSite
  28 from django.views.decorators.csrf import csrf_exempt
  29 from django.views.decorators.cache import never_cache
  30 from django.conf import settings as dsettings
  31
  32 from mygpo.api.constants import EPISODE_ACTION_TYPES, DEVICE_TYPES
  33 from mygpo.api.httpresponse import JsonResponse
  34 from mygpo.api.advanced.directory import episode_data
  35 from mygpo.api.backend import get_device, BulkSubscribe
  36 from mygpo.utils import format_time, parse_bool, get_timestamp, \
  37     parse_request_body, normalize_feed_url
  38 from mygpo.decorators import allowed_methods, cors_origin
  39 from mygpo.core.tasks import auto_flattr_episode
  40 from mygpo.users.models import EpisodeAction, \
  41      DeviceDoesNotExist, DeviceUIDException, \
  42      InvalidEpisodeActionAttributes
  43 from mygpo.users.settings import FLATTR_AUTO
  44 from mygpo.core.json import JSONDecodeError
  45 from mygpo.api.basic_auth import require_valid_user, check_username
  46 from mygpo.db.couchdb import BulkException, bulk_save_retry, \
  47     get_userdata_database
  48 from mygpo.db.couchdb.episode import favorite_episodes_for_user
  49 from mygpo.db.couchdb.podcast import podcast_for_url
  50 from mygpo.db.couchdb.podcast_state import subscribed_podcast_ids_by_device
  51 from mygpo.db.couchdb.episode_state import episode_state_for_ref_urls, \
  52     get_episode_actions
  53 from mygpo.db.couchdb.user import set_device
  54
  55
  56 import logging
  57 logger = logging.getLogger(__name__)
  58
  59
  60 # keys that are allowed in episode actions
  61 EPISODE_ACTION_KEYS = ('position', 'episode', 'action', 'device', 'timestamp',
  62                        'started', 'total', 'podcast')
  63
  64
  65 @csrf_exempt
  66 @require_valid_user
  67 @check_username
  68 @never_cache
  69 @allowed_methods(['GET', 'POST'])
  70 @cors_origin()
  71 def subscriptions(request, username, device_uid):
  72
  73     now = datetime.now()
  74     now_ = get_timestamp(now)
  75
  76     if request.method == 'GET':
  77
  78         try:
  79             device = request.user.get_device_by_uid(device_uid)
  80         except DeviceDoesNotExist as e:
  81             return HttpResponseNotFound(str(e))
  82
  83         since_ = request.GET.get('since', None)
  84         if since_ is None:
  85             return HttpResponseBadRequest('parameter since missing')
  86         try:
  87             since = datetime.fromtimestamp(float(since_))
  88         except ValueError:
  89             return HttpResponseBadRequest('since-value is not a valid timestamp')
  90
  91         changes = get_subscription_changes(request.user, device, since, now)
  92
  93         return JsonResponse(changes)
  94
  95     elif request.method == 'POST':
  96         d = get_device(request.user, device_uid,
  97                 request.META.get('HTTP_USER_AGENT', ''))
  98
  99         if not request.body:
 100             return HttpResponseBadRequest('POST data must not be empty')
 101
 102         try:
 103             actions = parse_request_body(request)
 104         except (JSONDecodeError, UnicodeDecodeError, ValueError) as e:
 105             msg = (u'Could not decode subscription update POST data for ' +
 106                    'user %s: %s') % (username,
 107                    request.body.decode('ascii', errors='replace'))
 108             logger.warn(msg, exc_info=True)
 109             return HttpResponseBadRequest(msg)
 110
 111         add = actions['add'] if 'add' in actions else []
 112         rem = actions['remove'] if 'remove' in actions else []
 113
 114         add = filter(None, add)
 115         rem = filter(None, rem)
 116
 117         try:
 118             update_urls = update_subscriptions(request.user, d, add, rem)
 119         except ValueError, e:
 120             return HttpResponseBadRequest(e)
 121
 122         return JsonResponse({
 123             'timestamp': now_,
 124             'update_urls': update_urls,
 125             })
 126
 127
 128 def update_subscriptions(user, device, add, remove):
 129
 130     for a in add:
 131         if a in remove:
 132             raise ValueError('can not add and remove %s at the same time' % a)
 133
 134     add_s = map(normalize_feed_url, add)
 135     rem_s = map(normalize_feed_url, remove)
 136
 137     assert len(add) == len(add_s) and len(remove) == len(rem_s)
 138
 139     updated_urls = filter(lambda (a, b): a != b, zip(add + remove, add_s + rem_s))
 140
 141     add_s = filter(None, add_s)
 142     rem_s = filter(None, rem_s)
 143
 144     # If two different URLs (in add and remove) have
 145     # been sanitized to the same, we ignore the removal
 146     rem_s = filter(lambda x: x not in add_s, rem_s)
 147
 148     subscriber = BulkSubscribe(user, device)
 149
 150     for a in add_s:
 151         subscriber.add_action(a, 'subscribe')
 152
 153     for r in rem_s:
 154         subscriber.add_action(r, 'unsubscribe')
 155
 156     try:
 157         subscriber.execute()
 158     except BulkException as be:
 159         for err in be.errors:
 160             loger.error('Advanced API: %(username)s: Updating subscription for '
 161                     '%(podcast_url)s on %(device_uid)s failed: '
 162                     '%(rerror)s (%(reason)s)'.format(username=user.username,
 163                         podcast_url=err.doc, device_uid=device.uid,
 164                         error=err.error, reason=err.reason)
 165                 )
 166
 167     return updated_urls
 168
 169
 170 def get_subscription_changes(user, device, since, until):
 171     add_urls, rem_urls = device.get_subscription_changes(since, until)
 172     until_ = get_timestamp(until)
 173     return {'add': add_urls, 'remove': rem_urls, 'timestamp': until_}
 174
 175
 176 @csrf_exempt
 177 @require_valid_user
 178 @check_username
 179 @never_cache
 180 @allowed_methods(['GET', 'POST'])
 181 @cors_origin()
 182 def episodes(request, username, version=1):
 183
 184     version = int(version)
 185     now = datetime.now()
 186     now_ = get_timestamp(now)
 187     ua_string = request.META.get('HTTP_USER_AGENT', '')
 188
 189     if request.method == 'POST':
 190         try:
 191             actions = parse_request_body(request)
 192         except (JSONDecodeError, UnicodeDecodeError, ValueError) as e:
 193             msg = ('Could not decode episode update POST data for ' +
 194                    'user %s: %s') % (username,
 195                    request.body.decode('ascii', errors='replace'))
 196             logger.warn(msg, exc_info=True)
 197             return HttpResponseBadRequest(msg)
 198
 199         logger.info('start: user %s: %d actions from %s' % (request.user._id, len(actions), ua_string))
 200
 201         # handle in background
 202         if len(actions) > dsettings.API_ACTIONS_MAX_NONBG:
 203             bg_handler = dsettings.API_ACTIONS_BG_HANDLER
 204             if bg_handler is not None:
 205
 206                 modname, funname = bg_handler.rsplit('.', 1)
 207                 mod = import_module(modname)
 208                 fun = getattr(mod, funname)
 209
 210                 fun(request.user, actions, now, ua_string)
 211
 212                 # TODO: return 202 Accepted
 213                 return JsonResponse({'timestamp': now_, 'update_urls': []})
 214
 215
 216         try:
 217             update_urls = update_episodes(request.user, actions, now, ua_string)
 218         except DeviceUIDException as e:
 219             logger.warn('invalid device UID while uploading episode actions for user %s', username)
 220             return HttpResponseBadRequest(str(e))
 221
 222         except InvalidEpisodeActionAttributes as e:
 223             msg = 'invalid episode action attributes while uploading episode actions for user %s' % (username,)
 224             logger.warn(msg, exc_info=True)
 225             return HttpResponseBadRequest(str(e))
 226
 227         logger.info('done:  user %s: %d actions from %s' % (request.user._id, len(actions), ua_string))
 228         return JsonResponse({'timestamp': now_, 'update_urls': update_urls})
 229
 230     elif request.method == 'GET':
 231         podcast_url= request.GET.get('podcast', None)
 232         device_uid = request.GET.get('device', None)
 233         since_     = request.GET.get('since', None)
 234         aggregated = parse_bool(request.GET.get('aggregated', False))
 235
 236         try:
 237             since = int(since_) if since_ else None
 238         except ValueError:
 239             return HttpResponseBadRequest('since-value is not a valid timestamp')
 240
 241         if podcast_url:
 242             podcast = podcast_for_url(podcast_url)
 243             if not podcast:
 244                 raise Http404
 245         else:
 246             podcast = None
 247
 248         if device_uid:
 249
 250             try:
 251                 device = request.user.get_device_by_uid(device_uid)
 252             except DeviceDoesNotExist as e:
 253                 return HttpResponseNotFound(str(e))
 254
 255         else:
 256             device = None
 257
 258         changes = get_episode_changes(request.user, podcast, device, since,
 259                 now_, aggregated, version)
 260
 261         return JsonResponse(changes)
 262
 263
 264
 265 def convert_position(action):
 266     """ convert position parameter for API 1 compatibility """
 267     pos = getattr(action, 'position', None)
 268     if pos is not None:
 269         action.position = format_time(pos)
 270     return action
 271
 272
 273
 274 def get_episode_changes(user, podcast, device, since, until, aggregated, version):
 275
 276     devices = dict( (dev.id, dev.uid) for dev in user.devices )
 277
 278     args = {}
 279     if podcast is not None:
 280         args['podcast_id'] = podcast.get_id()
 281
 282     if device is not None:
 283         args['device_id'] = device.id
 284
 285     actions = get_episode_actions(user._id, since, until, **args)
 286
 287     if version == 1:
 288         actions = imap(convert_position, actions)
 289
 290     clean_data = partial(clean_episode_action_data,
 291             user=user, devices=devices)
 292
 293     actions = map(clean_data, actions)
 294     actions = filter(None, actions)
 295
 296     if aggregated:
 297         actions = dict( (a['episode'], a) for a in actions ).values()
 298
 299     return {'actions': actions, 'timestamp': until}
 300
 301
 302
 303
 304 def clean_episode_action_data(action, user, devices):
 305
 306     if None in (action.get('podcast', None), action.get('episode', None)):
 307         return None
 308
 309     if 'device_id' in action:
 310         device_id = action['device_id']
 311         device_uid = devices.get(device_id)
 312         if device_uid:
 313             action['device'] = device_uid
 314
 315         del action['device_id']
 316
 317     # remove superfluous keys
 318     for x in action.keys():
 319         if x not in EPISODE_ACTION_KEYS:
 320             del action[x]
 321
 322     # set missing keys to None
 323     for x in EPISODE_ACTION_KEYS:
 324         if x not in action:
 325             action[x] = None
 326
 327     if action['action'] != 'play':
 328         if 'position' in action:
 329             del action['position']
 330
 331         if 'total' in action:
 332             del action['total']
 333
 334         if 'started' in action:
 335             del action['started']
 336
 337         if 'playmark' in action:
 338             del action['playmark']
 339
 340     else:
 341         action['position'] = action.get('position', False) or 0
 342
 343     return action
 344
 345
 346
 347
 348
 349 def update_episodes(user, actions, now, ua_string):
 350     update_urls = []
 351
 352     grouped_actions = defaultdict(list)
 353
 354     # group all actions by their episode
 355     for action in actions:
 356
 357         podcast_url = action['podcast']
 358         podcast_url = sanitize_append(podcast_url, update_urls)
 359         if podcast_url == '':
 360             continue
 361
 362         episode_url = action['episode']
 363         episode_url = sanitize_append(episode_url, update_urls)
 364         if episode_url == '':
 365             continue
 366
 367         act = parse_episode_action(action, user, update_urls, now, ua_string)
 368         grouped_actions[ (podcast_url, episode_url) ].append(act)
 369
 370
 371     auto_flattr_episodes = []
 372
 373     # Prepare the updates for each episode state
 374     obj_funs = []
 375
 376     for (p_url, e_url), action_list in grouped_actions.iteritems():
 377         episode_state = episode_state_for_ref_urls(user, p_url, e_url)
 378
 379         if any(a['action'] == 'play' for a in actions):
 380             auto_flattr_episodes.append(episode_state.episode)
 381
 382         fun = partial(update_episode_actions, action_list=action_list)
 383         obj_funs.append( (episode_state, fun) )
 384
 385     udb = get_userdata_database()
 386     bulk_save_retry(obj_funs, udb)
 387
 388     if user.get_wksetting(FLATTR_AUTO):
 389         for episode_id in auto_flattr_episodes:
 390             auto_flattr_episode.delay(user, episode_id)
 391
 392     return update_urls
 393
 394
 395 def update_episode_actions(episode_state, action_list):
 396     """ Adds actions to the episode state and saves if necessary """
 397
 398     len1 = len(episode_state.actions)
 399     episode_state.add_actions(action_list)
 400
 401     if len(episode_state.actions) == len1:
 402         return None
 403
 404     return episode_state
 405
 406
 407
 408 def parse_episode_action(action, user, update_urls, now, ua_string):
 409     action_str  = action.get('action', None)
 410     if not valid_episodeaction(action_str):
 411         raise Exception('invalid action %s' % action_str)
 412
 413     new_action = EpisodeAction()
 414
 415     new_action.action = action['action']
 416
 417     if action.get('device', False):
 418         device = get_device(user, action['device'], ua_string)
 419         new_action.device = device.id
 420
 421     if action.get('timestamp', False):
 422         new_action.timestamp = dateutil.parser.parse(action['timestamp'])
 423     else:
 424         new_action.timestamp = now
 425     new_action.timestamp = new_action.timestamp.replace(microsecond=0)
 426
 427     new_action.upload_timestamp = get_timestamp(now)
 428
 429     new_action.started = action.get('started', None)
 430     new_action.playmark = action.get('position', None)
 431     new_action.total = action.get('total', None)
 432
 433     return new_action
 434
 435
 436 @csrf_exempt
 437 @require_valid_user
 438 @check_username
 439 @never_cache
 440 # Workaround for mygpoclient 1.0: It uses "PUT" requests
 441 # instead of "POST" requests for uploading device settings
 442 @allowed_methods(['POST', 'PUT'])
 443 @cors_origin()
 444 def device(request, username, device_uid):
 445     d = get_device(request.user, device_uid,
 446             request.META.get('HTTP_USER_AGENT', ''))
 447
 448     try:
 449         data = parse_request_body(request)
 450     except (JSONDecodeError, UnicodeDecodeError, ValueError) as e:
 451         msg = ('Could not decode device update POST data for ' +
 452                'user %s: %s') % (username,
 453                request.body.decode('ascii', errors='replace'))
 454         logger.warn(msg, exc_info=True)
 455         return HttpResponseBadRequest(msg)
 456
 457     if 'caption' in data:
 458         if not data['caption']:
 459             return HttpResponseBadRequest('caption must not be empty')
 460         d.name = data['caption']
 461
 462     if 'type' in data:
 463         if not valid_devicetype(data['type']):
 464             return HttpResponseBadRequest('invalid device type %s' % data['type'])
 465         d.type = data['type']
 466
 467
 468     set_device(request.user, d)
 469
 470     return HttpResponse()
 471
 472
 473 def valid_devicetype(type):
 474     for t in DEVICE_TYPES:
 475         if t[0] == type:
 476             return True
 477     return False
 478
 479 def valid_episodeaction(type):
 480     for t in EPISODE_ACTION_TYPES:
 481         if t[0] == type:
 482             return True
 483     return False
 484
 485
 486 @csrf_exempt
 487 @require_valid_user
 488 @check_username
 489 @never_cache
 490 @allowed_methods(['GET'])
 491 @cors_origin()
 492 def devices(request, username):
 493     devices = filter(lambda d: not d.deleted, request.user.devices)
 494     devices = map(device_data, devices)
 495     return JsonResponse(devices)
 496
 497
 498 def device_data(device):
 499     return dict(
 500         id           = device.uid,
 501         caption      = device.name,
 502         type         = device.type,
 503         subscriptions= len(subscribed_podcast_ids_by_device(device)),
 504     )
 505
 506
 507 @require_valid_user
 508 @check_username
 509 @never_cache
 510 @cors_origin()
 511 def favorites(request, username):
 512     favorites = favorite_episodes_for_user(request.user)
 513     domain = RequestSite(request).domain
 514     e_data = lambda e: episode_data(e, domain)
 515     ret = map(e_data, favorites)
 516     return JsonResponse(ret)
 517
 518
 519 def sanitize_append(url, sanitized_list):
 520     urls = normalize_feed_url(url)
 521     if url != urls:
 522         sanitized_list.append( (url, urls or '') )
 523     return urls