mygpo/api/advanced/__init__.py

   1 #
   2 # This file is part of my.gpodder.org.
   3 #
   4 # my.gpodder.org is free software: you can redistribute it and/or modify it
   5 # under the terms of the GNU Affero General Public License as published by
   6 # the Free Software Foundation, either version 3 of the License, or (at your
   7 # option) any later version.
   8 #
   9 # my.gpodder.org is distributed in the hope that it will be useful, but
  10 # WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
  11 # or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public
  12 # License for more details.
  13 #
  14 # You should have received a copy of the GNU Affero General Public License
  15 # along with my.gpodder.org. If not, see <http://www.gnu.org/licenses/>.
  16 #
  17
  18 from functools import partial
  19 from itertools import imap, chain
  20 from collections import defaultdict, namedtuple
  21 from datetime import datetime
  22
  23 import dateutil.parser
  24
  25 try:
  26     import gevent
  27 except ImportError:
  28     gevent = None
  29
  30 from django.http import HttpResponse, HttpResponseBadRequest, Http404, HttpResponseNotFound
  31 from django.contrib.sites.models import RequestSite
  32 from django.db import IntegrityError
  33 from django.views.decorators.csrf import csrf_exempt
  34 from django.views.decorators.cache import never_cache
  35
  36 from mygpo.api.constants import EPISODE_ACTION_TYPES, DEVICE_TYPES
  37 from mygpo.api.httpresponse import JsonResponse
  38 from mygpo.api.sanitizing import sanitize_url, sanitize_urls
  39 from mygpo.api.advanced.directory import episode_data, podcast_data
  40 from mygpo.api.backend import get_device, BulkSubscribe
  41 from mygpo.couch import BulkException, bulk_save_retry
  42 from mygpo.log import log
  43 from mygpo.utils import parse_time, format_time, parse_bool, get_timestamp
  44 from mygpo.decorators import allowed_methods, repeat_on_conflict
  45 from mygpo.core import models
  46 from mygpo.core.models import SanitizingRule, Podcast
  47 from mygpo.users.models import PodcastUserState, EpisodeAction, \
  48      EpisodeUserState, DeviceDoesNotExist, DeviceUIDException, \
  49      InvalidEpisodeActionAttributes
  50 from mygpo.json import json, JSONDecodeError
  51 from mygpo.api.basic_auth import require_valid_user, check_username
  52 from mygpo.db.couchdb.episode import episode_by_id, \
  53          favorite_episodes_for_user, episodes_for_podcast
  54 from mygpo.db.couchdb.podcast import podcast_for_url
  55 from mygpo.db.couchdb.podcast_state import subscribed_podcast_ids_by_device
  56 from mygpo.db.couchdb.episode_state import get_podcasts_episode_states, \
  57          episode_state_for_ref_urls, get_episode_actions
  58
  59
  60 # keys that are allowed in episode actions
  61 EPISODE_ACTION_KEYS = ('position', 'episode', 'action', 'device', 'timestamp',
  62                        'started', 'total', 'podcast')
  63
  64
  65 @csrf_exempt
  66 @require_valid_user
  67 @check_username
  68 @never_cache
  69 @allowed_methods(['GET', 'POST'])
  70 def subscriptions(request, username, device_uid):
  71
  72     now = datetime.now()
  73     now_ = get_timestamp(now)
  74
  75     if request.method == 'GET':
  76
  77         try:
  78             device = request.user.get_device_by_uid(device_uid)
  79         except DeviceDoesNotExist as e:
  80             return HttpResponseNotFound(str(e))
  81
  82         since_ = request.GET.get('since', None)
  83         if since_ == None:
  84             return HttpResponseBadRequest('parameter since missing')
  85         try:
  86             since = datetime.fromtimestamp(float(since_))
  87         except ValueError:
  88             return HttpResponseBadRequest('since-value is not a valid timestamp')
  89
  90         changes = get_subscription_changes(request.user, device, since, now)
  91
  92         return JsonResponse(changes)
  93
  94     elif request.method == 'POST':
  95         d = get_device(request.user, device_uid,
  96                 request.META.get('HTTP_USER_AGENT', ''))
  97
  98         if not request.raw_post_data:
  99             return HttpResponseBadRequest('POST data must not be empty')
 100
 101         actions = json.loads(request.raw_post_data)
 102         add = actions['add'] if 'add' in actions else []
 103         rem = actions['remove'] if 'remove' in actions else []
 104
 105         add = filter(None, add)
 106         rem = filter(None, rem)
 107
 108         try:
 109             update_urls = update_subscriptions(request.user, d, add, rem)
 110         except IntegrityError, e:
 111             return HttpResponseBadRequest(e)
 112
 113         return JsonResponse({
 114             'timestamp': now_,
 115             'update_urls': update_urls,
 116             })
 117
 118
 119 def update_subscriptions(user, device, add, remove):
 120
 121     for a in add:
 122         if a in remove:
 123             raise IntegrityError('can not add and remove %s at the same time' % a)
 124
 125     add_s = list(sanitize_urls(add, 'podcast'))
 126     rem_s = list(sanitize_urls(remove, 'podcast'))
 127
 128     assert len(add) == len(add_s) and len(remove) == len(rem_s)
 129
 130     updated_urls = filter(lambda (a, b): a != b, zip(add + remove, add_s + rem_s))
 131
 132     add_s = filter(None, add_s)
 133     rem_s = filter(None, rem_s)
 134
 135     # If two different URLs (in add and remove) have
 136     # been sanitized to the same, we ignore the removal
 137     rem_s = filter(lambda x: x not in add_s, rem_s)
 138
 139     subscriber = BulkSubscribe(user, device)
 140
 141     for a in add_s:
 142         subscriber.add_action(a, 'subscribe')
 143
 144     for r in rem_s:
 145         subscriber.add_action(r, 'unsubscribe')
 146
 147     try:
 148         subscriber.execute()
 149     except BulkException as be:
 150         for err in be.errors:
 151             log('Advanced API: %(username)s: Updating subscription for '
 152                     '%(podcast_url)s on %(device_uid)s failed: '
 153                     '%(rerror)s (%(reason)s)'.format(username=user.username,
 154                         podcast_url=err.doc, device_uid=device.uid,
 155                         error=err.error, reason=err.reason)
 156                 )
 157
 158     return updated_urls
 159
 160
 161 def get_subscription_changes(user, device, since, until):
 162     add_urls, rem_urls = device.get_subscription_changes(since, until)
 163     until_ = get_timestamp(until)
 164     return {'add': add_urls, 'remove': rem_urls, 'timestamp': until_}
 165
 166
 167 @csrf_exempt
 168 @require_valid_user
 169 @check_username
 170 @never_cache
 171 @allowed_methods(['GET', 'POST'])
 172 def episodes(request, username, version=1):
 173
 174     version = int(version)
 175     now = datetime.now()
 176     now_ = get_timestamp(now)
 177     ua_string = request.META.get('HTTP_USER_AGENT', '')
 178
 179     if request.method == 'POST':
 180         try:
 181             actions = json.loads(request.raw_post_data)
 182         except (JSONDecodeError, UnicodeDecodeError) as e:
 183             log('Advanced API: could not decode episode update POST data for user %s: %s' % (username, e))
 184             return HttpResponseBadRequest()
 185
 186         try:
 187             update_urls = update_episodes(request.user, actions, now, ua_string)
 188         except DeviceUIDException as e:
 189             import traceback
 190             log('could not update episodes for user %s: %s %s: %s' % (username, e, traceback.format_exc(), actions))
 191             return HttpResponseBadRequest(str(e))
 192         except InvalidEpisodeActionAttributes as e:
 193             import traceback
 194             log('could not update episodes for user %s: %s %s: %s' % (username, e, traceback.format_exc(), actions))
 195             return HttpResponseBadRequest(str(e))
 196
 197         return JsonResponse({'timestamp': now_, 'update_urls': update_urls})
 198
 199     elif request.method == 'GET':
 200         podcast_url= request.GET.get('podcast', None)
 201         device_uid = request.GET.get('device', None)
 202         since_     = request.GET.get('since', None)
 203         aggregated = parse_bool(request.GET.get('aggregated', False))
 204
 205         try:
 206             since = datetime.fromtimestamp(float(since_)) if since_ else None
 207         except ValueError:
 208             return HttpResponseBadRequest('since-value is not a valid timestamp')
 209
 210         if podcast_url:
 211             podcast = podcast_for_url(podcast_url)
 212             if not podcast:
 213                 raise Http404
 214         else:
 215             podcast = None
 216
 217         if device_uid:
 218
 219             try:
 220                 device = request.user.get_device_by_uid(device_uid)
 221             except DeviceDoesNotExist as e:
 222                 return HttpResponseNotFound(str(e))
 223
 224         else:
 225             device = None
 226
 227         changes = get_episode_changes(request.user, podcast, device, since,
 228                 now, aggregated, version)
 229
 230         return JsonResponse(changes)
 231
 232
 233
 234 def convert_position(action):
 235     """ convert position parameter for API 1 compatibility """
 236     pos = getattr(action, 'position', None)
 237     if pos is not None:
 238         action.position = format_time(pos)
 239     return action
 240
 241
 242
 243 def get_episode_changes(user, podcast, device, since, until, aggregated, version):
 244
 245     devices = dict( (dev.id, dev.uid) for dev in user.devices )
 246
 247     args = {}
 248     if podcast is not None:
 249         args['podcast_id'] = podcast.get_id()
 250
 251     if device is not None:
 252         args['device_id'] = device.id
 253
 254     actions = get_episode_actions(user._id, since, until, **args)
 255
 256     if version == 1:
 257         actions = imap(convert_position, actions)
 258
 259     clean_data = partial(clean_episode_action_data,
 260             user=user, devices=devices)
 261
 262     actions = map(clean_data, actions)
 263     actions = filter(None, actions)
 264
 265     if aggregated:
 266         actions = dict( (a['episode'], a) for a in actions ).values()
 267
 268     until_ = get_timestamp(until)
 269
 270     return {'actions': actions, 'timestamp': until_}
 271
 272
 273
 274
 275 def clean_episode_action_data(action, user, devices):
 276
 277     if None in (action.get('podcast', None), action.get('episode', None)):
 278         return None
 279
 280     if 'device_id' in action:
 281         device_id = action['device_id']
 282         device_uid = devices.get(device_id)
 283         if device_uid:
 284             action['device'] = device_uid
 285
 286         del action['device_id']
 287
 288     # remove superfluous keys
 289     for x in action.keys():
 290         if x not in EPISODE_ACTION_KEYS:
 291             del action[x]
 292
 293     # set missing keys to None
 294     for x in EPISODE_ACTION_KEYS:
 295         if x not in action:
 296             action[x] = None
 297
 298     if action['action'] != 'play':
 299         if 'position' in action:
 300             del action['position']
 301
 302         if 'total' in action:
 303             del action['total']
 304
 305         if 'started' in action:
 306             del action['started']
 307
 308         if 'playmark' in action:
 309             del action['playmark']
 310
 311     else:
 312         action['position'] = action.get('position', False) or 0
 313
 314     return action
 315
 316
 317
 318
 319
 320 def update_episodes(user, actions, now, ua_string):
 321     update_urls = []
 322
 323     grouped_actions = defaultdict(list)
 324
 325     # group all actions by their episode
 326     for action in actions:
 327
 328         podcast_url = action['podcast']
 329         podcast_url = sanitize_append(podcast_url, 'podcast', update_urls)
 330         if podcast_url == '': continue
 331
 332         episode_url = action['episode']
 333         episode_url = sanitize_append(episode_url, 'episode', update_urls)
 334         if episode_url == '': continue
 335
 336         act = parse_episode_action(action, user, update_urls, now, ua_string)
 337         grouped_actions[ (podcast_url, episode_url) ].append(act)
 338
 339
 340     auto_flattr_episodes = []
 341
 342     # Prepare the updates for each episode state
 343     obj_funs = []
 344
 345     for (p_url, e_url), action_list in grouped_actions.iteritems():
 346         episode_state = episode_state_for_ref_urls(user, p_url, e_url)
 347
 348         if any(a.action == 'play' for a in actions):
 349             auto_flattr_episodes.append(episode_state.episode)
 350
 351         fun = partial(update_episode_actions, action_list=action_list)
 352         obj_funs.append( (episode_state, fun) )
 353
 354     bulk_save_retry(obj_funs)
 355
 356     for episode_id in auto_flattr_episodes:
 357         auto_flattr_episode.delay(user, episode_id)
 358
 359     return update_urls
 360
 361
 362 def update_episode_actions(episode_state, action_list):
 363     """ Adds actions to the episode state and saves if necessary """
 364
 365     len1 = len(episode_state.actions)
 366     episode_state.add_actions(action_list)
 367
 368     if len(episode_state.actions) == len1:
 369         return None
 370
 371     return episode_state
 372
 373
 374
 375 def parse_episode_action(action, user, update_urls, now, ua_string):
 376     action_str  = action.get('action', None)
 377     if not valid_episodeaction(action_str):
 378         raise Exception('invalid action %s' % action_str)
 379
 380     new_action = EpisodeAction()
 381
 382     new_action.action = action['action']
 383
 384     if action.get('device', False):
 385         device = get_device(user, action['device'], ua_string)
 386         new_action.device = device.id
 387
 388     if action.get('timestamp', False):
 389         new_action.timestamp = dateutil.parser.parse(action['timestamp'])
 390     else:
 391         new_action.timestamp = now
 392     new_action.timestamp = new_action.timestamp.replace(microsecond=0)
 393
 394     new_action.started = action.get('started', None)
 395     new_action.playmark = action.get('position', None)
 396     new_action.total = action.get('total', None)
 397
 398     return new_action
 399
 400
 401 @csrf_exempt
 402 @require_valid_user
 403 @check_username
 404 @never_cache
 405 # Workaround for mygpoclient 1.0: It uses "PUT" requests
 406 # instead of "POST" requests for uploading device settings
 407 @allowed_methods(['POST', 'PUT'])
 408 def device(request, username, device_uid):
 409     d = get_device(request.user, device_uid,
 410             request.META.get('HTTP_USER_AGENT', ''))
 411
 412     data = json.loads(request.raw_post_data)
 413
 414     if 'caption' in data:
 415         if not data['caption']:
 416             return HttpResponseBadRequest('caption must not be empty')
 417         d.name = data['caption']
 418
 419     if 'type' in data:
 420         if not valid_devicetype(data['type']):
 421            return HttpResponseBadRequest('invalid device type %s' % data['type'])
 422         d.type = data['type']
 423
 424
 425     request.user.update_device(d)
 426
 427     return HttpResponse()
 428
 429
 430 def valid_devicetype(type):
 431     for t in DEVICE_TYPES:
 432         if t[0] == type:
 433             return True
 434     return False
 435
 436 def valid_episodeaction(type):
 437     for t in EPISODE_ACTION_TYPES:
 438         if t[0] == type:
 439             return True
 440     return False
 441
 442
 443 @csrf_exempt
 444 @require_valid_user
 445 @check_username
 446 @never_cache
 447 @allowed_methods(['GET'])
 448 def devices(request, username):
 449     devices = filter(lambda d: not d.deleted, request.user.devices)
 450     devices = map(device_data, devices)
 451     return JsonResponse(devices)
 452
 453
 454 def device_data(device):
 455     return dict(
 456         id           = device.uid,
 457         caption      = device.name,
 458         type         = device.type,
 459         subscriptions= len(subscribed_podcast_ids_by_device(device)),
 460     )
 461
 462
 463
 464 def get_podcast_data(podcasts, domain, url):
 465     """ Gets podcast data for a URL from a dict of podcasts """
 466     podcast = podcasts.get(url)
 467     return podcast_data(podcast, domain)
 468
 469
 470 def get_episode_data(podcasts, domain, clean_action_data, include_actions, episode_status):
 471     """ Get episode data for an episode status object """
 472     podcast_id = episode_status.episode.podcast
 473     podcast = podcasts.get(podcast_id, None)
 474     t = episode_data(episode_status.episode, domain, podcast)
 475     t['status'] = episode_status.status
 476
 477     # include latest action (bug 1419)
 478     if include_actions and episode_status.action:
 479         t['action'] = clean_action_data(episode_status.action)
 480
 481     return t
 482
 483
 484 @csrf_exempt
 485 @require_valid_user
 486 @check_username
 487 @never_cache
 488 def updates(request, username, device_uid):
 489     now = datetime.now()
 490     now_ = get_timestamp(now)
 491
 492     try:
 493         device = request.user.get_device_by_uid(device_uid)
 494     except DeviceDoesNotExist as e:
 495         return HttpResponseNotFound(str(e))
 496
 497     since_ = request.GET.get('since', None)
 498     if since_ == None:
 499         return HttpResponseBadRequest('parameter since missing')
 500     try:
 501         since = datetime.fromtimestamp(float(since_))
 502     except ValueError:
 503         return HttpResponseBadRequest('since-value is not a valid timestamp')
 504
 505     include_actions = parse_bool(request.GET.get('include_actions', False))
 506
 507     ret = get_subscription_changes(request.user, device, since, now)
 508     domain = RequestSite(request).domain
 509
 510     subscriptions = list(device.get_subscribed_podcasts())
 511
 512     podcasts = dict( (p.url, p) for p in subscriptions )
 513     prepare_podcast_data = partial(get_podcast_data, podcasts, domain)
 514
 515     ret['add'] = map(prepare_podcast_data, ret['add'])
 516
 517     devices = dict( (dev.id, dev.uid) for dev in request.user.devices )
 518     clean_action_data = partial(clean_episode_action_data,
 519             user=request.user, devices=devices)
 520
 521     # index subscribed podcasts by their Id for fast access
 522     podcasts = dict( (p.get_id(), p) for p in subscriptions )
 523     prepare_episode_data = partial(get_episode_data, podcasts, domain,
 524             clean_action_data, include_actions)
 525
 526     episode_updates = get_episode_updates(request.user, subscriptions, since)
 527     ret['updates'] = map(prepare_episode_data, episode_updates)
 528
 529     return JsonResponse(ret)
 530
 531
 532 def get_episode_updates(user, subscribed_podcasts, since):
 533     """ Returns the episode updates since the timestamp """
 534
 535     EpisodeStatus = namedtuple('EpisodeStatus', 'episode status action')
 536
 537     episode_status = {}
 538
 539     # get episodes
 540     if gevent:
 541         episode_jobs = [gevent.spawn(episodes_for_podcast, p, since) for p in
 542             subscribed_podcasts]
 543         gevent.joinall(episode_jobs)
 544         episodes = chain.from_iterable(job.get() for job in episode_jobs)
 545
 546     else:
 547         episodes = chain.from_iterable(episodes_for_podcast(p, since) for p
 548                 in subscribed_podcasts)
 549
 550
 551     for episode in episodes:
 552         episode_status[episode._id] = EpisodeStatus(episode, 'new', None)
 553
 554
 555     # get episode states
 556     if gevent:
 557         e_action_jobs = [gevent.spawn(get_podcasts_episode_states, p, user._id)
 558                 for p in subscribed_podcasts]
 559         gevent.joinall(e_action_jobs)
 560         e_actions = chain.from_iterable(job.get() for job in e_action_jobs)
 561
 562     else:
 563         e_actions = [get_podcasts_episode_states(p, user._id) for p
 564             in subscribed_podcasts]
 565
 566
 567     for action in e_actions:
 568         e_id = action['episode_id']
 569
 570         if e_id in episode_status:
 571             episode = episode_status[e_id].episode
 572         else:
 573             episode = episode_by_id(e_id)
 574
 575         episode_status[e_id] = EpisodeStatus(episode, action['action'], action)
 576
 577     return episode_status.itervalues()
 578
 579
 580 @require_valid_user
 581 @check_username
 582 @never_cache
 583 def favorites(request, username):
 584     favorites = favorite_episodes_for_user(request.user)
 585     domain = RequestSite(request).domain
 586     e_data = lambda e: episode_data(e, domain)
 587     ret = map(e_data, favorites)
 588     return JsonResponse(ret)
 589
 590
 591 def sanitize_append(url, obj_type, sanitized_list):
 592     urls = sanitize_url(url, obj_type)
 593     if url != urls:
 594         sanitized_list.append( (url, urls) )
 595     return urls