mygpo/api/advanced/__init__.py

   1 #
   2 # This file is part of my.gpodder.org.
   3 #
   4 # my.gpodder.org is free software: you can redistribute it and/or modify it
   5 # under the terms of the GNU Affero General Public License as published by
   6 # the Free Software Foundation, either version 3 of the License, or (at your
   7 # option) any later version.
   8 #
   9 # my.gpodder.org is distributed in the hope that it will be useful, but
  10 # WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
  11 # or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public
  12 # License for more details.
  13 #
  14 # You should have received a copy of the GNU Affero General Public License
  15 # along with my.gpodder.org. If not, see <http://www.gnu.org/licenses/>.
  16 #
  17
  18 from functools import partial
  19 from itertools import imap, chain
  20 from collections import defaultdict, namedtuple
  21 from datetime import datetime
  22
  23 import dateutil.parser
  24
  25 try:
  26     import gevent
  27 except ImportError:
  28     gevent = None
  29
  30 from django.http import HttpResponse, HttpResponseBadRequest, Http404, HttpResponseNotFound
  31 from django.contrib.sites.models import RequestSite
  32 from django.views.decorators.csrf import csrf_exempt
  33 from django.views.decorators.cache import never_cache
  34 from django.utils.decorators import method_decorator
  35 from django.views.generic.base import View
  36
  37 from mygpo.api.constants import EPISODE_ACTION_TYPES, DEVICE_TYPES
  38 from mygpo.api.httpresponse import JsonResponse
  39 from mygpo.api.sanitizing import sanitize_url, sanitize_urls
  40 from mygpo.api.advanced.directory import episode_data, podcast_data
  41 from mygpo.api.backend import get_device, BulkSubscribe
  42 from mygpo.log import log
  43 from mygpo.utils import parse_time, format_time, parse_bool, get_timestamp
  44 from mygpo.decorators import allowed_methods, repeat_on_conflict
  45 from mygpo.core import models
  46 from mygpo.core.models import SanitizingRule, Podcast
  47 from mygpo.core.tasks import auto_flattr_episode
  48 from mygpo.users.models import PodcastUserState, EpisodeAction, \
  49      EpisodeUserState, DeviceDoesNotExist, DeviceUIDException, \
  50      InvalidEpisodeActionAttributes
  51 from mygpo.users.settings import FLATTR_AUTO
  52 from mygpo.core.json import json, JSONDecodeError
  53 from mygpo.api.basic_auth import require_valid_user, check_username
  54 from mygpo.db.couchdb import BulkException, bulk_save_retry
  55 from mygpo.db.couchdb.episode import episode_by_id, \
  56          favorite_episodes_for_user, episodes_for_podcast
  57 from mygpo.db.couchdb.podcast import podcast_for_url
  58 from mygpo.db.couchdb.podcast_state import subscribed_podcast_ids_by_device
  59 from mygpo.db.couchdb.episode_state import get_podcasts_episode_states, \
  60          episode_state_for_ref_urls, get_episode_actions
  61
  62
  63 # keys that are allowed in episode actions
  64 EPISODE_ACTION_KEYS = ('position', 'episode', 'action', 'device', 'timestamp',
  65                        'started', 'total', 'podcast')
  66
  67
  68 @csrf_exempt
  69 @require_valid_user
  70 @check_username
  71 @never_cache
  72 @allowed_methods(['GET', 'POST'])
  73 def subscriptions(request, username, device_uid):
  74
  75     now = datetime.now()
  76     now_ = get_timestamp(now)
  77
  78     if request.method == 'GET':
  79
  80         try:
  81             device = request.user.get_device_by_uid(device_uid)
  82         except DeviceDoesNotExist as e:
  83             return HttpResponseNotFound(str(e))
  84
  85         since_ = request.GET.get('since', None)
  86         if since_ is None:
  87             return HttpResponseBadRequest('parameter since missing')
  88         try:
  89             since = datetime.fromtimestamp(float(since_))
  90         except ValueError:
  91             return HttpResponseBadRequest('since-value is not a valid timestamp')
  92
  93         changes = get_subscription_changes(request.user, device, since, now)
  94
  95         return JsonResponse(changes)
  96
  97     elif request.method == 'POST':
  98         d = get_device(request.user, device_uid,
  99                 request.META.get('HTTP_USER_AGENT', ''))
 100
 101         if not request.body:
 102             return HttpResponseBadRequest('POST data must not be empty')
 103
 104         actions = json.loads(request.body)
 105         add = actions['add'] if 'add' in actions else []
 106         rem = actions['remove'] if 'remove' in actions else []
 107
 108         add = filter(None, add)
 109         rem = filter(None, rem)
 110
 111         try:
 112             update_urls = update_subscriptions(request.user, d, add, rem)
 113         except ValueError, e:
 114             return HttpResponseBadRequest(e)
 115
 116         return JsonResponse({
 117             'timestamp': now_,
 118             'update_urls': update_urls,
 119             })
 120
 121
 122 def update_subscriptions(user, device, add, remove):
 123
 124     for a in add:
 125         if a in remove:
 126             raise ValueError('can not add and remove %s at the same time' % a)
 127
 128     add_s = list(sanitize_urls(add, 'podcast'))
 129     rem_s = list(sanitize_urls(remove, 'podcast'))
 130
 131     assert len(add) == len(add_s) and len(remove) == len(rem_s)
 132
 133     updated_urls = filter(lambda (a, b): a != b, zip(add + remove, add_s + rem_s))
 134
 135     add_s = filter(None, add_s)
 136     rem_s = filter(None, rem_s)
 137
 138     # If two different URLs (in add and remove) have
 139     # been sanitized to the same, we ignore the removal
 140     rem_s = filter(lambda x: x not in add_s, rem_s)
 141
 142     subscriber = BulkSubscribe(user, device)
 143
 144     for a in add_s:
 145         subscriber.add_action(a, 'subscribe')
 146
 147     for r in rem_s:
 148         subscriber.add_action(r, 'unsubscribe')
 149
 150     try:
 151         subscriber.execute()
 152     except BulkException as be:
 153         for err in be.errors:
 154             log('Advanced API: %(username)s: Updating subscription for '
 155                     '%(podcast_url)s on %(device_uid)s failed: '
 156                     '%(rerror)s (%(reason)s)'.format(username=user.username,
 157                         podcast_url=err.doc, device_uid=device.uid,
 158                         error=err.error, reason=err.reason)
 159                 )
 160
 161     return updated_urls
 162
 163
 164 def get_subscription_changes(user, device, since, until):
 165     add_urls, rem_urls = device.get_subscription_changes(since, until)
 166     until_ = get_timestamp(until)
 167     return {'add': add_urls, 'remove': rem_urls, 'timestamp': until_}
 168
 169
 170 @csrf_exempt
 171 @require_valid_user
 172 @check_username
 173 @never_cache
 174 @allowed_methods(['GET', 'POST'])
 175 def episodes(request, username, version=1):
 176
 177     version = int(version)
 178     now = datetime.now()
 179     now_ = get_timestamp(now)
 180     ua_string = request.META.get('HTTP_USER_AGENT', '')
 181
 182     if request.method == 'POST':
 183         try:
 184             actions = json.loads(request.body)
 185         except (JSONDecodeError, UnicodeDecodeError) as e:
 186             msg = 'Advanced API: could not decode episode update POST data for user %s: %s' % (username, e)
 187             log(msg)
 188             return HttpResponseBadRequest(msg)
 189
 190         try:
 191             update_urls = update_episodes(request.user, actions, now, ua_string)
 192         except DeviceUIDException as e:
 193             import traceback
 194             s = u'could not update episodes for user %s: %s %s: %s' % (username, e, traceback.format_exc(), actions)
 195             log(s.decode('utf-8', errors='ignore'))
 196             return HttpResponseBadRequest(str(e))
 197         except InvalidEpisodeActionAttributes as e:
 198             import traceback
 199             log(u'could not update episodes for user %s: %s %s: %s' % (username, e, traceback.format_exc(), actions))
 200             return HttpResponseBadRequest(str(e))
 201
 202         log('done:  user %s: %d actions from %s' % (request.user._id, len(actions), ua_string))
 203         return JsonResponse({'timestamp': now_, 'update_urls': update_urls})
 204
 205     elif request.method == 'GET':
 206         podcast_url= request.GET.get('podcast', None)
 207         device_uid = request.GET.get('device', None)
 208         since_     = request.GET.get('since', None)
 209         aggregated = parse_bool(request.GET.get('aggregated', False))
 210
 211         try:
 212             since = int(since_) if since_ else None
 213         except ValueError:
 214             return HttpResponseBadRequest('since-value is not a valid timestamp')
 215
 216         if podcast_url:
 217             podcast = podcast_for_url(podcast_url)
 218             if not podcast:
 219                 raise Http404
 220         else:
 221             podcast = None
 222
 223         if device_uid:
 224
 225             try:
 226                 device = request.user.get_device_by_uid(device_uid)
 227             except DeviceDoesNotExist as e:
 228                 return HttpResponseNotFound(str(e))
 229
 230         else:
 231             device = None
 232
 233         changes = get_episode_changes(request.user, podcast, device, since,
 234                 now_, aggregated, version)
 235
 236         return JsonResponse(changes)
 237
 238
 239
 240 def convert_position(action):
 241     """ convert position parameter for API 1 compatibility """
 242     pos = getattr(action, 'position', None)
 243     if pos is not None:
 244         action.position = format_time(pos)
 245     return action
 246
 247
 248
 249 def get_episode_changes(user, podcast, device, since, until, aggregated, version):
 250
 251     devices = dict( (dev.id, dev.uid) for dev in user.devices )
 252
 253     args = {}
 254     if podcast is not None:
 255         args['podcast_id'] = podcast.get_id()
 256
 257     if device is not None:
 258         args['device_id'] = device.id
 259
 260     actions = get_episode_actions(user._id, since, until, **args)
 261
 262     if version == 1:
 263         actions = imap(convert_position, actions)
 264
 265     clean_data = partial(clean_episode_action_data,
 266             user=user, devices=devices)
 267
 268     actions = map(clean_data, actions)
 269     actions = filter(None, actions)
 270
 271     if aggregated:
 272         actions = dict( (a['episode'], a) for a in actions ).values()
 273
 274     return {'actions': actions, 'timestamp': until}
 275
 276
 277
 278
 279 def clean_episode_action_data(action, user, devices):
 280
 281     if None in (action.get('podcast', None), action.get('episode', None)):
 282         return None
 283
 284     if 'device_id' in action:
 285         device_id = action['device_id']
 286         device_uid = devices.get(device_id)
 287         if device_uid:
 288             action['device'] = device_uid
 289
 290         del action['device_id']
 291
 292     # remove superfluous keys
 293     for x in action.keys():
 294         if x not in EPISODE_ACTION_KEYS:
 295             del action[x]
 296
 297     # set missing keys to None
 298     for x in EPISODE_ACTION_KEYS:
 299         if x not in action:
 300             action[x] = None
 301
 302     if action['action'] != 'play':
 303         if 'position' in action:
 304             del action['position']
 305
 306         if 'total' in action:
 307             del action['total']
 308
 309         if 'started' in action:
 310             del action['started']
 311
 312         if 'playmark' in action:
 313             del action['playmark']
 314
 315     else:
 316         action['position'] = action.get('position', False) or 0
 317
 318     return action
 319
 320
 321
 322
 323
 324 def update_episodes(user, actions, now, ua_string):
 325     update_urls = []
 326
 327     grouped_actions = defaultdict(list)
 328
 329     # group all actions by their episode
 330     for action in actions:
 331
 332         podcast_url = action['podcast']
 333         podcast_url = sanitize_append(podcast_url, 'podcast', update_urls)
 334         if podcast_url == '':
 335             continue
 336
 337         episode_url = action['episode']
 338         episode_url = sanitize_append(episode_url, 'episode', update_urls)
 339         if episode_url == '':
 340             continue
 341
 342         act = parse_episode_action(action, user, update_urls, now, ua_string)
 343         grouped_actions[ (podcast_url, episode_url) ].append(act)
 344
 345
 346     auto_flattr_episodes = []
 347
 348     # Prepare the updates for each episode state
 349     obj_funs = []
 350
 351     for (p_url, e_url), action_list in grouped_actions.iteritems():
 352         episode_state = episode_state_for_ref_urls(user, p_url, e_url)
 353
 354         if any(a['action'] == 'play' for a in actions):
 355             auto_flattr_episodes.append(episode_state.episode)
 356
 357         fun = partial(update_episode_actions, action_list=action_list)
 358         obj_funs.append( (episode_state, fun) )
 359
 360     bulk_save_retry(obj_funs)
 361
 362     if user.get_wksetting(FLATTR_AUTO):
 363         for episode_id in auto_flattr_episodes:
 364             auto_flattr_episode.delay(user, episode_id)
 365
 366     return update_urls
 367
 368
 369 def update_episode_actions(episode_state, action_list):
 370     """ Adds actions to the episode state and saves if necessary """
 371
 372     len1 = len(episode_state.actions)
 373     episode_state.add_actions(action_list)
 374
 375     if len(episode_state.actions) == len1:
 376         return None
 377
 378     return episode_state
 379
 380
 381
 382 def parse_episode_action(action, user, update_urls, now, ua_string):
 383     action_str  = action.get('action', None)
 384     if not valid_episodeaction(action_str):
 385         raise Exception('invalid action %s' % action_str)
 386
 387     new_action = EpisodeAction()
 388
 389     new_action.action = action['action']
 390
 391     if action.get('device', False):
 392         device = get_device(user, action['device'], ua_string)
 393         new_action.device = device.id
 394
 395     if action.get('timestamp', False):
 396         new_action.timestamp = dateutil.parser.parse(action['timestamp'])
 397     else:
 398         new_action.timestamp = now
 399     new_action.timestamp = new_action.timestamp.replace(microsecond=0)
 400
 401     new_action.upload_timestamp = get_timestamp(now)
 402
 403     new_action.started = action.get('started', None)
 404     new_action.playmark = action.get('position', None)
 405     new_action.total = action.get('total', None)
 406
 407     return new_action
 408
 409
 410 @csrf_exempt
 411 @require_valid_user
 412 @check_username
 413 @never_cache
 414 # Workaround for mygpoclient 1.0: It uses "PUT" requests
 415 # instead of "POST" requests for uploading device settings
 416 @allowed_methods(['POST', 'PUT'])
 417 def device(request, username, device_uid):
 418     d = get_device(request.user, device_uid,
 419             request.META.get('HTTP_USER_AGENT', ''))
 420
 421     data = json.loads(request.body)
 422
 423     if 'caption' in data:
 424         if not data['caption']:
 425             return HttpResponseBadRequest('caption must not be empty')
 426         d.name = data['caption']
 427
 428     if 'type' in data:
 429         if not valid_devicetype(data['type']):
 430             return HttpResponseBadRequest('invalid device type %s' % data['type'])
 431         d.type = data['type']
 432
 433
 434     request.user.update_device(d)
 435
 436     return HttpResponse()
 437
 438
 439 def valid_devicetype(type):
 440     for t in DEVICE_TYPES:
 441         if t[0] == type:
 442             return True
 443     return False
 444
 445 def valid_episodeaction(type):
 446     for t in EPISODE_ACTION_TYPES:
 447         if t[0] == type:
 448             return True
 449     return False
 450
 451
 452 @csrf_exempt
 453 @require_valid_user
 454 @check_username
 455 @never_cache
 456 @allowed_methods(['GET'])
 457 def devices(request, username):
 458     devices = filter(lambda d: not d.deleted, request.user.devices)
 459     devices = map(device_data, devices)
 460     return JsonResponse(devices)
 461
 462
 463 def device_data(device):
 464     return dict(
 465         id           = device.uid,
 466         caption      = device.name,
 467         type         = device.type,
 468         subscriptions= len(subscribed_podcast_ids_by_device(device)),
 469     )
 470
 471
 472
 473 def get_podcast_data(podcasts, domain, url):
 474     """ Gets podcast data for a URL from a dict of podcasts """
 475     podcast = podcasts.get(url)
 476     return podcast_data(podcast, domain)
 477
 478
 479 def get_episode_data(podcasts, domain, clean_action_data, include_actions, episode_status):
 480     """ Get episode data for an episode status object """
 481     podcast_id = episode_status.episode.podcast
 482     podcast = podcasts.get(podcast_id, None)
 483     t = episode_data(episode_status.episode, domain, podcast)
 484     t['status'] = episode_status.status
 485
 486     # include latest action (bug 1419)
 487     if include_actions and episode_status.action:
 488         t['action'] = clean_action_data(episode_status.action)
 489
 490     return t
 491
 492
 493
 494 class DeviceUpdates(View):
 495
 496     @method_decorator(csrf_exempt)
 497     @method_decorator(require_valid_user)
 498     @method_decorator(check_username)
 499     @method_decorator(never_cache)
 500     def get(self, request, username, device_uid):
 501         now = datetime.now()
 502         now_ = get_timestamp(now)
 503
 504         try:
 505             device = request.user.get_device_by_uid(device_uid)
 506         except DeviceDoesNotExist as e:
 507             return HttpResponseNotFound(str(e))
 508
 509         since_ = request.GET.get('since', None)
 510         if since_ is None:
 511             return HttpResponseBadRequest('parameter since missing')
 512         try:
 513             since = datetime.fromtimestamp(float(since_))
 514         except ValueError:
 515             return HttpResponseBadRequest("'since' is not a valid timestamp")
 516
 517         include_actions = parse_bool(request.GET.get('include_actions', False))
 518
 519         ret = get_subscription_changes(request.user, device, since, now)
 520         domain = RequestSite(request).domain
 521
 522         subscriptions = list(device.get_subscribed_podcasts())
 523
 524         podcasts = dict( (p.url, p) for p in subscriptions )
 525         prepare_podcast_data = partial(get_podcast_data, podcasts, domain)
 526
 527         ret['add'] = map(prepare_podcast_data, ret['add'])
 528
 529         devices = dict( (dev.id, dev.uid) for dev in request.user.devices )
 530         clean_action_data = partial(clean_episode_action_data,
 531                 user=request.user, devices=devices)
 532
 533         # index subscribed podcasts by their Id for fast access
 534         podcasts = dict( (p.get_id(), p) for p in subscriptions )
 535         prepare_episode_data = partial(get_episode_data, podcasts, domain,
 536                 clean_action_data, include_actions)
 537
 538         episode_updates = self.get_episode_updates(request.user,
 539                 subscriptions, since)
 540         ret['updates'] = map(prepare_episode_data, episode_updates)
 541
 542         return JsonResponse(ret)
 543
 544
 545     def get_episode_updates(self, user, subscribed_podcasts, since,
 546             max_per_podcast=5):
 547         """ Returns the episode updates since the timestamp """
 548
 549         EpisodeStatus = namedtuple('EpisodeStatus', 'episode status action')
 550
 551         episode_status = {}
 552
 553         # get episodes
 554         if gevent:
 555             episode_jobs = [gevent.spawn(episodes_for_podcast, p, since,
 556                     limit=max_per_podcast) for p in subscribed_podcasts]
 557             gevent.joinall(episode_jobs)
 558             episodes = chain.from_iterable(job.get() for job in episode_jobs)
 559
 560         else:
 561             episodes = chain.from_iterable(episodes_for_podcast(p, since,
 562                     limit=max_per_podcast) for p in subscribed_podcasts)
 563
 564
 565         for episode in episodes:
 566             episode_status[episode._id] = EpisodeStatus(episode, 'new', None)
 567
 568
 569         # get episode states
 570         if gevent:
 571             e_action_jobs = [gevent.spawn(get_podcasts_episode_states, p,
 572                     user._id) for p in subscribed_podcasts]
 573             gevent.joinall(e_action_jobs)
 574             e_actions = chain.from_iterable(job.get() for job in e_action_jobs)
 575
 576         else:
 577             e_actions = chain.from_iterable(get_podcasts_episode_states(p,
 578                     user._id) for p in subscribed_podcasts)
 579
 580
 581         for action in e_actions:
 582             e_id = action['episode_id']
 583
 584             if e_id in episode_status:
 585                 episode = episode_status[e_id].episode
 586             else:
 587                 episode = episode_by_id(e_id)
 588
 589             episode_status[e_id] = EpisodeStatus(episode, action['action'],
 590                     action)
 591
 592         return episode_status.itervalues()
 593
 594
 595 @require_valid_user
 596 @check_username
 597 @never_cache
 598 def favorites(request, username):
 599     favorites = favorite_episodes_for_user(request.user)
 600     domain = RequestSite(request).domain
 601     e_data = lambda e: episode_data(e, domain)
 602     ret = map(e_data, favorites)
 603     return JsonResponse(ret)
 604
 605
 606 def sanitize_append(url, obj_type, sanitized_list):
 607     urls = sanitize_url(url, obj_type)
 608     if url != urls:
 609         sanitized_list.append( (url, urls) )
 610     return urls