mygpo/api/advanced/__init__.py

   1 #
   2 # This file is part of my.gpodder.org.
   3 #
   4 # my.gpodder.org is free software: you can redistribute it and/or modify it
   5 # under the terms of the GNU Affero General Public License as published by
   6 # the Free Software Foundation, either version 3 of the License, or (at your
   7 # option) any later version.
   8 #
   9 # my.gpodder.org is distributed in the hope that it will be useful, but
  10 # WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
  11 # or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public
  12 # License for more details.
  13 #
  14 # You should have received a copy of the GNU Affero General Public License
  15 # along with my.gpodder.org. If not, see <http://www.gnu.org/licenses/>.
  16 #
  17
  18 from itertools import imap, chain
  19 from collections import defaultdict, namedtuple
  20 from mygpo.api.basic_auth import require_valid_user, check_username
  21 from django.http import HttpResponse, HttpResponseBadRequest
  22 from mygpo.api.models import Device, Podcast, Episode, EPISODE_ACTION_TYPES, DEVICE_TYPES
  23 from mygpo.api.httpresponse import JsonResponse
  24 from mygpo.api.sanitizing import sanitize_url, sanitize_urls
  25 from mygpo.api.advanced.directory import episode_data, podcast_data
  26 from mygpo.api.backend import get_device, get_favorites
  27 from django.shortcuts import get_object_or_404
  28 from django.contrib.sites.models import RequestSite
  29 from datetime import datetime
  30 import dateutil.parser
  31 from mygpo.log import log
  32 from mygpo.utils import parse_time, format_time, parse_bool, get_to_dict, get_timestamp
  33 from mygpo.decorators import allowed_methods, repeat_on_conflict
  34 from mygpo.core import models
  35 from mygpo.core.models import SanitizingRule
  36 from django.db import IntegrityError
  37 from django.views.decorators.csrf import csrf_exempt
  38 from mygpo.users.models import PodcastUserState, EpisodeAction, EpisodeUserState
  39 from mygpo import migrate
  40
  41 try:
  42     import simplejson as json
  43 except ImportError:
  44     import json
  45
  46
  47 # keys that are allowed in episode actions
  48 EPISODE_ACTION_KEYS = ('position', 'episode', 'action', 'device', 'timestamp',
  49                        'started', 'total', 'podcast')
  50
  51
  52 @csrf_exempt
  53 @require_valid_user
  54 @check_username
  55 @allowed_methods(['GET', 'POST'])
  56 def subscriptions(request, username, device_uid):
  57
  58     now = datetime.now()
  59     now_ = get_timestamp(now)
  60
  61     if request.method == 'GET':
  62         d = get_object_or_404(Device, user=request.user, uid=device_uid, deleted=False)
  63
  64         since_ = request.GET.get('since', None)
  65         if since_ == None:
  66             return HttpResponseBadRequest('parameter since missing')
  67         try:
  68             since = datetime.fromtimestamp(float(since_))
  69         except ValueError:
  70             return HttpResponseBadRequest('since-value is not a valid timestamp')
  71
  72         dev = migrate.get_or_migrate_device(d)
  73         changes = get_subscription_changes(request.user, dev, since, now)
  74
  75         return JsonResponse(changes)
  76
  77     elif request.method == 'POST':
  78         d = get_device(request.user, device_uid)
  79
  80         actions = json.loads(request.raw_post_data)
  81         add = actions['add'] if 'add' in actions else []
  82         rem = actions['remove'] if 'remove' in actions else []
  83
  84         add = filter(None, add)
  85         rem = filter(None, rem)
  86
  87         try:
  88             update_urls = update_subscriptions(request.user, d, add, rem)
  89         except IntegrityError, e:
  90             return HttpResponseBadRequest(e)
  91
  92         return JsonResponse({
  93             'timestamp': now_,
  94             'update_urls': update_urls,
  95             })
  96
  97
  98 def update_subscriptions(user, device, add, remove):
  99
 100     for a in add:
 101         if a in remove:
 102             raise IntegrityError('can not add and remove %s at the same time' % a)
 103
 104     add_s = list(sanitize_urls(add, 'podcast'))
 105     rem_s = list(sanitize_urls(remove, 'podcast'))
 106
 107     assert len(add) == len(add_s) and len(remove) == len(rem_s)
 108
 109     updated_urls = filter(lambda (a, b): a != b, zip(add + remove, add_s + rem_s))
 110
 111     add_s = filter(None, add_s)
 112     rem_s = filter(None, rem_s)
 113
 114     # If two different URLs (in add and remove) have
 115     # been sanitized to the same, we ignore the removal
 116     rem_s = filter(lambda x: x not in add_s, rem_s)
 117
 118     for a in add_s:
 119         p, p_created = Podcast.objects.get_or_create(url=a)
 120         p = migrate.get_or_migrate_podcast(p)
 121         try:
 122             p.subscribe(device)
 123         except Exception as e:
 124             log('Advanced API: %(username)s: could not subscribe to podcast %(podcast_url)s on device %(device_id)s: %(exception)s' %
 125                 {'username': user.username, 'podcast_url': p.url, 'device_id': device.id, 'exception': e})
 126
 127     for r in rem_s:
 128         p, p_created = Podcast.objects.get_or_create(url=r)
 129         p = migrate.get_or_migrate_podcast(p)
 130         try:
 131             p.unsubscribe(device)
 132         except Exception as e:
 133             log('Advanced API: %(username)s: could not unsubscribe from podcast %(podcast_url)s on device %(device_id)s: %(exception)s' %
 134                 {'username': user.username, 'podcast_url': p.url, 'device_id': device.id, 'exception': e})
 135
 136     return updated_urls
 137
 138
 139 def get_subscription_changes(user, device, since, until):
 140     add, rem = device.get_subscription_changes(since, until)
 141
 142     podcast_ids = add + rem
 143     podcasts = get_to_dict(models.Podcast, podcast_ids, get_id=models.Podcast.get_id)
 144
 145     add_podcasts = filter(None, (podcasts.get(i, None) for i in add))
 146     rem_podcasts = filter(None, (podcasts.get(i, None) for i in rem))
 147     add_urls = [ podcast.url for podcast in add_podcasts]
 148     rem_urls = [ podcast.url for podcast in rem_podcasts]
 149
 150     until_ = get_timestamp(until)
 151     return {'add': add_urls, 'remove': rem_urls, 'timestamp': until_}
 152
 153
 154 @csrf_exempt
 155 @require_valid_user
 156 @check_username
 157 @allowed_methods(['GET', 'POST'])
 158 def episodes(request, username, version=1):
 159
 160     version = int(version)
 161     now = datetime.now()
 162     now_ = get_timestamp(now)
 163
 164     if request.method == 'POST':
 165         try:
 166             actions = json.loads(request.raw_post_data)
 167         except KeyError, e:
 168             log('could not parse episode update info for user %s: %s' % (username, e))
 169             return HttpResponseBadRequest()
 170
 171         try:
 172             update_urls = update_episodes(request.user, actions, now)
 173         except Exception, e:
 174             import traceback
 175             log('could not update episodes for user %s: %s %s: %s' % (username, e, traceback.format_exc(), actions))
 176             return HttpResponseBadRequest(e)
 177
 178         return JsonResponse({'timestamp': now_, 'update_urls': update_urls})
 179
 180     elif request.method == 'GET':
 181         podcast_url= request.GET.get('podcast', None)
 182         device_uid = request.GET.get('device', None)
 183         since_     = request.GET.get('since', None)
 184         aggregated = parse_bool(request.GET.get('aggregated', False))
 185
 186         try:
 187             since = datetime.fromtimestamp(float(since_)) if since_ else None
 188         except ValueError:
 189             return HttpResponseBadRequest('since-value is not a valid timestamp')
 190
 191         podcast = get_object_or_404(Podcast, url=podcast_url) if podcast_url else None
 192         podcast = migrate.get_or_migrate_podcast(podcast) if podcast else None
 193
 194         device  = get_object_or_404(Device, user=request.user,uid=device_uid, deleted=False) if device_uid else None
 195
 196         return JsonResponse(get_episode_changes(request.user, podcast, device, since, now, aggregated, version))
 197
 198
 199 def get_episode_changes(user, podcast, device, since, until, aggregated, version):
 200
 201     new_user = migrate.get_or_migrate_user(user)
 202     devices = dict( (dev.oldid, dev.uid) for dev in new_user.devices )
 203
 204     args = {}
 205     if podcast is not None: args['podcast_id'] = podcast.get_id()
 206     if device is not None:  args['device_oldid'] = device.id
 207
 208     actions = EpisodeAction.filter(user.id, since, until, *args)
 209
 210     if version == 1:
 211         # convert position parameter for API 1 compatibility
 212         def convert_position(action):
 213             pos = action.get('position', None)
 214             if pos is not None:
 215                 action['position'] = format_time(pos)
 216             return action
 217
 218         actions = imap(convert_position, actions)
 219
 220
 221     def clean_data(action):
 222         action['podcast'] = action.get('podcast_url', None)
 223         action['episode'] = action.get('episode_url', None)
 224
 225         if None in (action['podcast'], action['episode']):
 226             return None
 227
 228         if 'device_oldid' in action:
 229             device_oldid = action['device_oldid']
 230             if not device_oldid in devices:
 231                 try:
 232                     dev = Device.objects.get(id=device_oldid)
 233                 except Device.DoesNotExist:
 234                     return None
 235
 236                 dev = migrate.get_or_migrate_device(dev, user=new_user)
 237                 action['device'] = dev.uid
 238             else:
 239                 action['device'] = devices[action['device_oldid']]
 240             del action['device_oldid']
 241
 242         # remove superfluous keys
 243         for x in action.keys():
 244             if x not in EPISODE_ACTION_KEYS:
 245                 del action[x]
 246
 247         # set missing keys to None
 248         for x in EPISODE_ACTION_KEYS:
 249             if x not in action:
 250                 action[x] = None
 251
 252         return action
 253
 254     actions = map(clean_data, actions)
 255     actions = filter(None, actions)
 256
 257     if aggregated:
 258         actions = dict( (a['episode'], a) for a in actions ).values()
 259
 260     until_ = get_timestamp(until)
 261
 262     return {'actions': actions, 'timestamp': until_}
 263
 264
 265 def update_episodes(user, actions, now):
 266     update_urls = []
 267
 268     grouped_actions = defaultdict(list)
 269
 270     # group all actions by their episode
 271     for action in actions:
 272
 273         podcast_url = action['podcast']
 274         podcast_url = sanitize_append(podcast_url, 'podcast', update_urls)
 275         if podcast_url == '': continue
 276
 277         episode_url = action['episode']
 278         episode_url = sanitize_append(episode_url, 'episode', update_urls)
 279         if episode_url == '': continue
 280
 281         new_user = migrate.get_or_migrate_user(user)
 282         act = parse_episode_action(action, new_user, update_urls, now)
 283         grouped_actions[ (podcast_url, episode_url) ].append(act)
 284
 285     # load the episode state only once for every episode
 286     for (p_url, e_url), action_list in grouped_actions.iteritems():
 287         episode_state = EpisodeUserState.for_ref_urls(user, p_url, e_url)
 288
 289         if isinstance(episode_state, dict):
 290             from mygpo.log import log
 291             log('episode_state (%s, %s, %s): %s' % (user,
 292                         p_url, e_url, episode_state))
 293
 294
 295         @repeat_on_conflict(['episode_state'])
 296         def _update(episode_state):
 297             changed = False
 298
 299             len1 = len(episode_state.actions)
 300             episode_state.add_actions(action_list)
 301             len2 = len(episode_state.actions)
 302
 303             if len1 < len2:
 304                 changed = True
 305
 306             if episode_state.ref_url != e_url:
 307                 episode_state.ref_url = e_url
 308                 changed = True
 309
 310             if episode_state.podcast_ref_url != p_url:
 311                 episode_state.podcast_ref_url = p_url
 312                 changed = True
 313
 314             if changed:
 315                 episode_state.save()
 316
 317
 318         _update(episode_state=episode_state)
 319
 320     return update_urls
 321
 322
 323 def parse_episode_action(action, user, update_urls, now):
 324     action_str  = action.get('action', None)
 325     if not valid_episodeaction(action_str):
 326         raise Exception('invalid action %s' % action_str)
 327
 328     new_action = EpisodeAction()
 329
 330     new_action.action = action['action']
 331
 332     if action.get('device', False):
 333         device = user.get_device_by_uid(action['device'])
 334         if device is None:
 335             from django.contrib.auth.models import User
 336             user_ = User.objects.get(id=user.oldid)
 337             dev, created = Device.objects.get_or_create(user=user_, uid=action['device'])
 338             device = migrate.get_or_migrate_device(dev, user)
 339         new_action.device_oldid = device.oldid
 340         new_action.device = device.id
 341
 342     if action.get('timestamp', False):
 343         new_action.timestamp = dateutil.parser.parse(action['timestamp'])
 344     else:
 345         new_action.timestamp = now
 346     new_action.timestamp = new_action.timestamp.replace(microsecond=0)
 347
 348     new_action.started = action.get('started', None)
 349     new_action.playmark = action.get('position', None)
 350     new_action.total = action.get('total', None)
 351
 352     return new_action
 353
 354
 355 @csrf_exempt
 356 @require_valid_user
 357 @check_username
 358 # Workaround for mygpoclient 1.0: It uses "PUT" requests
 359 # instead of "POST" requests for uploading device settings
 360 @allowed_methods(['POST', 'PUT'])
 361 def device(request, username, device_uid):
 362     d = get_device(request.user, device_uid)
 363
 364     data = json.loads(request.raw_post_data)
 365
 366     if 'caption' in data:
 367         d.name = data['caption']
 368
 369     if 'type' in data:
 370         if not valid_devicetype(data['type']):
 371            return HttpResponseBadRequest('invalid device type %s' % data['type'])
 372         d.type = data['type']
 373
 374     d.save()
 375
 376     return HttpResponse()
 377
 378
 379 def valid_devicetype(type):
 380     for t in DEVICE_TYPES:
 381         if t[0] == type:
 382             return True
 383     return False
 384
 385 def valid_episodeaction(type):
 386     for t in EPISODE_ACTION_TYPES:
 387         if t[0] == type:
 388             return True
 389     return False
 390
 391
 392 @csrf_exempt
 393 @require_valid_user
 394 @check_username
 395 @allowed_methods(['GET'])
 396 def devices(request, username):
 397     devices = Device.objects.filter(user=request.user, deleted=False)
 398     devices = map(migrate.get_or_migrate_device, devices)
 399     devices = map(device_data, devices)
 400
 401     return JsonResponse(devices)
 402
 403
 404 def device_data(device):
 405     return dict(
 406         id           = device.uid,
 407         caption      = device.name,
 408         type         = device.type,
 409         subscriptions= len(device.get_subscribed_podcast_ids())
 410     )
 411
 412
 413 @csrf_exempt
 414 @require_valid_user
 415 @check_username
 416 def updates(request, username, device_uid):
 417     now = datetime.now()
 418     now_ = get_timestamp(now)
 419
 420     device = get_object_or_404(Device, user=request.user, uid=device_uid)
 421
 422     since_ = request.GET.get('since', None)
 423     if since_ == None:
 424         return HttpResponseBadRequest('parameter since missing')
 425     try:
 426         since = datetime.fromtimestamp(float(since_))
 427     except ValueError:
 428         return HttpResponseBadRequest('since-value is not a valid timestamp')
 429
 430     dev = migrate.get_or_migrate_device(device)
 431     ret = get_subscription_changes(request.user, dev, since, now)
 432     domain = RequestSite(request).domain
 433
 434     subscriptions = dev.get_subscribed_podcasts()
 435
 436     podcasts = dict( (p.url, p) for p in subscriptions )
 437
 438     def prepare_podcast_data(url):
 439         podcast = podcasts.get(url)
 440         try:
 441             return podcast_data(podcast, domain)
 442         except ValueError:
 443             from mygpo.log import log
 444             log('updates: podcast is None for url %s and dict %s' %
 445                     (url, podcasts.keys()))
 446             for k,v in podcasts.items():
 447                 log('%s - %s' % (k, v))
 448
 449             raise
 450
 451     ret['add'] = map(prepare_podcast_data, ret['add'])
 452
 453
 454     # index subscribed podcasts by their Id for fast access
 455     podcasts = dict( (p.get_id(), p) for p in subscriptions )
 456
 457     def prepare_episode_data(episode_status):
 458         """ converts the data to primitives that converted to JSON """
 459         podcast_id = episode_status.episode.podcast
 460         podcast = podcasts.get(podcast_id, None)
 461         t = episode_data(episode_status.episode, domain, podcast)
 462         t['status'] = episode_status.status
 463         return t
 464
 465     episode_updates = get_episode_updates(request.user, subscriptions, since)
 466     ret['updates'] = map(prepare_episode_data, episode_updates)
 467
 468     return JsonResponse(ret)
 469
 470
 471 def get_episode_updates(user, subscribed_podcasts, since):
 472     """ Returns the episode updates since the timestamp """
 473
 474     EpisodeStatus = namedtuple('EpisodeStatus', 'episode status')
 475
 476     subscriptions_oldpodcasts = [p.get_old_obj() for p in subscribed_podcasts]
 477
 478     episode_status = {}
 479     #TODO: changes this to a get_multi when episodes have been migrated
 480     for e in Episode.objects.filter(podcast__in=subscriptions_oldpodcasts, timestamp__gte=since).order_by('timestamp'):
 481         episode = migrate.get_or_migrate_episode(e)
 482         episode_status[episode._id] = EpisodeStatus(episode, 'new')
 483
 484     e_actions = (p.get_episode_states(user.id) for p in subscribed_podcasts)
 485     e_actions = chain.from_iterable(e_actions)
 486
 487     for action in e_actions:
 488         e_id = action['episode_id']
 489
 490         if e_id in episode_status:
 491             episode = episode_status[e_id].episode
 492         else:
 493             episode = models.Episode.get(e_id)
 494
 495         episode_status[e_id] = EpisodeStatus(episode, action['action'])
 496
 497     return episode_status.itervalues()
 498
 499
 500 @require_valid_user
 501 @check_username
 502 def favorites(request, username):
 503     favorites = get_favorites(request.user)
 504     domain = RequestSite(request).domain
 505     e_data = lambda e: episode_data(e, domain)
 506     ret = map(e_data, favorites)
 507     return JsonResponse(ret)
 508
 509
 510 def sanitize_append(url, obj_type, sanitized_list):
 511     urls = sanitize_url(url, obj_type)
 512     if url != urls:
 513         sanitized_list.append( (url, urls) )
 514     return urls