2 # This file is part of my.gpodder.org.
4 # my.gpodder.org is free software: you can redistribute it and/or modify it
5 # under the terms of the GNU Affero General Public License as published by
6 # the Free Software Foundation, either version 3 of the License, or (at your
7 # option) any later version.
9 # my.gpodder.org is distributed in the hope that it will be useful, but
10 # WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
11 # or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public
12 # License for more details.
14 # You should have received a copy of the GNU Affero General Public License
15 # along with my.gpodder.org. If not, see <http://www.gnu.org/licenses/>.
18 from functools
import partial
19 from itertools
import imap
, chain
20 from collections
import defaultdict
, namedtuple
21 from datetime
import datetime
22 from importlib
import import_module
24 import dateutil
.parser
31 from django
.http
import HttpResponse
, HttpResponseBadRequest
, Http404
, HttpResponseNotFound
32 from django
.contrib
.sites
.models
import RequestSite
33 from django
.views
.decorators
.csrf
import csrf_exempt
34 from django
.views
.decorators
.cache
import never_cache
35 from django
.utils
.decorators
import method_decorator
36 from django
.views
.generic
.base
import View
37 from django
.conf
import settings
as dsettings
39 from mygpo
.api
.constants
import EPISODE_ACTION_TYPES
, DEVICE_TYPES
40 from mygpo
.api
.httpresponse
import JsonResponse
41 from mygpo
.api
.sanitizing
import sanitize_url
, sanitize_urls
42 from mygpo
.api
.advanced
.directory
import episode_data
, podcast_data
43 from mygpo
.api
.backend
import get_device
, BulkSubscribe
44 from mygpo
.utils
import parse_time
, format_time
, parse_bool
, get_timestamp
45 from mygpo
.decorators
import allowed_methods
, repeat_on_conflict
46 from mygpo
.core
import models
47 from mygpo
.core
.models
import SanitizingRule
, Podcast
48 from mygpo
.core
.tasks
import auto_flattr_episode
49 from mygpo
.users
.models
import PodcastUserState
, EpisodeAction
, \
50 EpisodeUserState
, DeviceDoesNotExist
, DeviceUIDException
, \
51 InvalidEpisodeActionAttributes
52 from mygpo
.users
.settings
import FLATTR_AUTO
53 from mygpo
.core
.json
import json
, JSONDecodeError
54 from mygpo
.api
.basic_auth
import require_valid_user
, check_username
55 from mygpo
.db
.couchdb
import BulkException
, bulk_save_retry
56 from mygpo
.db
.couchdb
.episode
import episode_by_id
, \
57 favorite_episodes_for_user
, episodes_for_podcast
58 from mygpo
.db
.couchdb
.podcast
import podcast_for_url
59 from mygpo
.db
.couchdb
.podcast_state
import subscribed_podcast_ids_by_device
60 from mygpo
.db
.couchdb
.episode_state
import get_podcasts_episode_states
, \
61 episode_state_for_ref_urls
, get_episode_actions
65 logger
= logging
.getLogger(__name__
)
68 # keys that are allowed in episode actions
69 EPISODE_ACTION_KEYS
= ('position', 'episode', 'action', 'device', 'timestamp',
70 'started', 'total', 'podcast')
77 @allowed_methods(['GET', 'POST'])
78 def subscriptions(request
, username
, device_uid
):
81 now_
= get_timestamp(now
)
83 if request
.method
== 'GET':
86 device
= request
.user
.get_device_by_uid(device_uid
)
87 except DeviceDoesNotExist
as e
:
88 return HttpResponseNotFound(str(e
))
90 since_
= request
.GET
.get('since', None)
92 return HttpResponseBadRequest('parameter since missing')
94 since
= datetime
.fromtimestamp(float(since_
))
96 return HttpResponseBadRequest('since-value is not a valid timestamp')
98 changes
= get_subscription_changes(request
.user
, device
, since
, now
)
100 return JsonResponse(changes
)
102 elif request
.method
== 'POST':
103 d
= get_device(request
.user
, device_uid
,
104 request
.META
.get('HTTP_USER_AGENT', ''))
107 return HttpResponseBadRequest('POST data must not be empty')
110 actions
= json
.loads(request
.body
)
111 except (JSONDecodeError
, UnicodeDecodeError, ValueError) as e
:
112 msg
= ('Could not decode subscription update POST data for ' +
113 'user %s: %s') % (username
, request
.body
)
114 logger
.exception(msg
)
115 return HttpResponseBadRequest(msg
)
117 add
= actions
['add'] if 'add' in actions
else []
118 rem
= actions
['remove'] if 'remove' in actions
else []
120 add
= filter(None, add
)
121 rem
= filter(None, rem
)
124 update_urls
= update_subscriptions(request
.user
, d
, add
, rem
)
125 except ValueError, e
:
126 return HttpResponseBadRequest(e
)
128 return JsonResponse({
130 'update_urls': update_urls
,
134 def update_subscriptions(user
, device
, add
, remove
):
138 raise ValueError('can not add and remove %s at the same time' % a
)
140 add_s
= list(sanitize_urls(add
, 'podcast'))
141 rem_s
= list(sanitize_urls(remove
, 'podcast'))
143 assert len(add
) == len(add_s
) and len(remove
) == len(rem_s
)
145 updated_urls
= filter(lambda (a
, b
): a
!= b
, zip(add
+ remove
, add_s
+ rem_s
))
147 add_s
= filter(None, add_s
)
148 rem_s
= filter(None, rem_s
)
150 # If two different URLs (in add and remove) have
151 # been sanitized to the same, we ignore the removal
152 rem_s
= filter(lambda x
: x
not in add_s
, rem_s
)
154 subscriber
= BulkSubscribe(user
, device
)
157 subscriber
.add_action(a
, 'subscribe')
160 subscriber
.add_action(r
, 'unsubscribe')
164 except BulkException
as be
:
165 for err
in be
.errors
:
166 loger
.error('Advanced API: %(username)s: Updating subscription for '
167 '%(podcast_url)s on %(device_uid)s failed: '
168 '%(rerror)s (%(reason)s)'.format(username
=user
.username
,
169 podcast_url
=err
.doc
, device_uid
=device
.uid
,
170 error
=err
.error
, reason
=err
.reason
)
176 def get_subscription_changes(user
, device
, since
, until
):
177 add_urls
, rem_urls
= device
.get_subscription_changes(since
, until
)
178 until_
= get_timestamp(until
)
179 return {'add': add_urls
, 'remove': rem_urls
, 'timestamp': until_
}
186 @allowed_methods(['GET', 'POST'])
187 def episodes(request
, username
, version
=1):
189 version
= int(version
)
191 now_
= get_timestamp(now
)
192 ua_string
= request
.META
.get('HTTP_USER_AGENT', '')
194 if request
.method
== 'POST':
196 actions
= json
.loads(request
.body
)
197 except (JSONDecodeError
, UnicodeDecodeError, ValueError) as e
:
198 msg
= ('Could not decode episode update POST data for ' +
199 'user %s: %s') % (username
, request
.body
)
200 logger
.exception(msg
)
201 return HttpResponseBadRequest(msg
)
203 logger
.info('start: user %s: %d actions from %s' % (request
.user
._id
, len(actions
), ua_string
))
205 # handle in background
206 if len(actions
) > dsettings
.API_ACTIONS_MAX_NONBG
:
207 bg_handler
= dsettings
.API_ACTIONS_BG_HANDLER
208 if bg_handler
is not None:
210 modname
, funname
= bg_handler
.rsplit('.', 1)
211 mod
= import_module(modname
)
212 fun
= getattr(mod
, funname
)
214 fun(request
.user
, actions
, now
, ua_string
)
216 # TODO: return 202 Accepted
217 return JsonResponse({'timestamp': now_
, 'update_urls': []})
221 update_urls
= update_episodes(request
.user
, actions
, now
, ua_string
)
222 except DeviceUIDException
as e
:
223 logger
.exception('invalid device UID while uploading episode actions for user %s' % (username
,))
224 return HttpResponseBadRequest(str(e
))
226 except InvalidEpisodeActionAttributes
as e
:
227 logger
.exception('invalid episode action attributes while uploading episode actions for user %s: %s' % (username
,))
228 return HttpResponseBadRequest(str(e
))
230 logger
.info('done: user %s: %d actions from %s' % (request
.user
._id
, len(actions
), ua_string
))
231 return JsonResponse({'timestamp': now_
, 'update_urls': update_urls
})
233 elif request
.method
== 'GET':
234 podcast_url
= request
.GET
.get('podcast', None)
235 device_uid
= request
.GET
.get('device', None)
236 since_
= request
.GET
.get('since', None)
237 aggregated
= parse_bool(request
.GET
.get('aggregated', False))
240 since
= int(since_
) if since_
else None
242 return HttpResponseBadRequest('since-value is not a valid timestamp')
245 podcast
= podcast_for_url(podcast_url
)
254 device
= request
.user
.get_device_by_uid(device_uid
)
255 except DeviceDoesNotExist
as e
:
256 return HttpResponseNotFound(str(e
))
261 changes
= get_episode_changes(request
.user
, podcast
, device
, since
,
262 now_
, aggregated
, version
)
264 return JsonResponse(changes
)
268 def convert_position(action
):
269 """ convert position parameter for API 1 compatibility """
270 pos
= getattr(action
, 'position', None)
272 action
.position
= format_time(pos
)
277 def get_episode_changes(user
, podcast
, device
, since
, until
, aggregated
, version
):
279 devices
= dict( (dev
.id, dev
.uid
) for dev
in user
.devices
)
282 if podcast
is not None:
283 args
['podcast_id'] = podcast
.get_id()
285 if device
is not None:
286 args
['device_id'] = device
.id
288 actions
= get_episode_actions(user
._id
, since
, until
, **args
)
291 actions
= imap(convert_position
, actions
)
293 clean_data
= partial(clean_episode_action_data
,
294 user
=user
, devices
=devices
)
296 actions
= map(clean_data
, actions
)
297 actions
= filter(None, actions
)
300 actions
= dict( (a
['episode'], a
) for a
in actions
).values()
302 return {'actions': actions
, 'timestamp': until
}
307 def clean_episode_action_data(action
, user
, devices
):
309 if None in (action
.get('podcast', None), action
.get('episode', None)):
312 if 'device_id' in action
:
313 device_id
= action
['device_id']
314 device_uid
= devices
.get(device_id
)
316 action
['device'] = device_uid
318 del action
['device_id']
320 # remove superfluous keys
321 for x
in action
.keys():
322 if x
not in EPISODE_ACTION_KEYS
:
325 # set missing keys to None
326 for x
in EPISODE_ACTION_KEYS
:
330 if action
['action'] != 'play':
331 if 'position' in action
:
332 del action
['position']
334 if 'total' in action
:
337 if 'started' in action
:
338 del action
['started']
340 if 'playmark' in action
:
341 del action
['playmark']
344 action
['position'] = action
.get('position', False) or 0
352 def update_episodes(user
, actions
, now
, ua_string
):
355 grouped_actions
= defaultdict(list)
357 # group all actions by their episode
358 for action
in actions
:
360 podcast_url
= action
['podcast']
361 podcast_url
= sanitize_append(podcast_url
, 'podcast', update_urls
)
362 if podcast_url
== '':
365 episode_url
= action
['episode']
366 episode_url
= sanitize_append(episode_url
, 'episode', update_urls
)
367 if episode_url
== '':
370 act
= parse_episode_action(action
, user
, update_urls
, now
, ua_string
)
371 grouped_actions
[ (podcast_url
, episode_url
) ].append(act
)
374 auto_flattr_episodes
= []
376 # Prepare the updates for each episode state
379 for (p_url
, e_url
), action_list
in grouped_actions
.iteritems():
380 episode_state
= episode_state_for_ref_urls(user
, p_url
, e_url
)
382 if any(a
['action'] == 'play' for a
in actions
):
383 auto_flattr_episodes
.append(episode_state
.episode
)
385 fun
= partial(update_episode_actions
, action_list
=action_list
)
386 obj_funs
.append( (episode_state
, fun
) )
388 bulk_save_retry(obj_funs
)
390 if user
.get_wksetting(FLATTR_AUTO
):
391 for episode_id
in auto_flattr_episodes
:
392 auto_flattr_episode
.delay(user
, episode_id
)
397 def update_episode_actions(episode_state
, action_list
):
398 """ Adds actions to the episode state and saves if necessary """
400 len1
= len(episode_state
.actions
)
401 episode_state
.add_actions(action_list
)
403 if len(episode_state
.actions
) == len1
:
410 def parse_episode_action(action
, user
, update_urls
, now
, ua_string
):
411 action_str
= action
.get('action', None)
412 if not valid_episodeaction(action_str
):
413 raise Exception('invalid action %s' % action_str
)
415 new_action
= EpisodeAction()
417 new_action
.action
= action
['action']
419 if action
.get('device', False):
420 device
= get_device(user
, action
['device'], ua_string
)
421 new_action
.device
= device
.id
423 if action
.get('timestamp', False):
424 new_action
.timestamp
= dateutil
.parser
.parse(action
['timestamp'])
426 new_action
.timestamp
= now
427 new_action
.timestamp
= new_action
.timestamp
.replace(microsecond
=0)
429 new_action
.upload_timestamp
= get_timestamp(now
)
431 new_action
.started
= action
.get('started', None)
432 new_action
.playmark
= action
.get('position', None)
433 new_action
.total
= action
.get('total', None)
442 # Workaround for mygpoclient 1.0: It uses "PUT" requests
443 # instead of "POST" requests for uploading device settings
444 @allowed_methods(['POST', 'PUT'])
445 def device(request
, username
, device_uid
):
446 d
= get_device(request
.user
, device_uid
,
447 request
.META
.get('HTTP_USER_AGENT', ''))
450 data
= json
.loads(request
.body
)
451 except (JSONDecodeError
, UnicodeDecodeError, ValueError) as e
:
452 msg
= ('Could not decode device update POST data for ' +
453 'user %s: %s') % (username
, request
.body
)
454 logger
.exception(msg
)
455 return HttpResponseBadRequest(msg
)
457 if 'caption' in data
:
458 if not data
['caption']:
459 return HttpResponseBadRequest('caption must not be empty')
460 d
.name
= data
['caption']
463 if not valid_devicetype(data
['type']):
464 return HttpResponseBadRequest('invalid device type %s' % data
['type'])
465 d
.type = data
['type']
468 request
.user
.update_device(d
)
470 return HttpResponse()
473 def valid_devicetype(type):
474 for t
in DEVICE_TYPES
:
479 def valid_episodeaction(type):
480 for t
in EPISODE_ACTION_TYPES
:
490 @allowed_methods(['GET'])
491 def devices(request
, username
):
492 devices
= filter(lambda d
: not d
.deleted
, request
.user
.devices
)
493 devices
= map(device_data
, devices
)
494 return JsonResponse(devices
)
497 def device_data(device
):
500 caption
= device
.name
,
502 subscriptions
= len(subscribed_podcast_ids_by_device(device
)),
507 def get_podcast_data(podcasts
, domain
, url
):
508 """ Gets podcast data for a URL from a dict of podcasts """
509 podcast
= podcasts
.get(url
)
510 return podcast_data(podcast
, domain
)
513 def get_episode_data(podcasts
, domain
, clean_action_data
, include_actions
, episode_status
):
514 """ Get episode data for an episode status object """
515 podcast_id
= episode_status
.episode
.podcast
516 podcast
= podcasts
.get(podcast_id
, None)
517 t
= episode_data(episode_status
.episode
, domain
, podcast
)
518 t
['status'] = episode_status
.status
520 # include latest action (bug 1419)
521 if include_actions
and episode_status
.action
:
522 t
['action'] = clean_action_data(episode_status
.action
)
528 class DeviceUpdates(View
):
530 @method_decorator(csrf_exempt
)
531 @method_decorator(require_valid_user
)
532 @method_decorator(check_username
)
533 @method_decorator(never_cache
)
534 def get(self
, request
, username
, device_uid
):
536 now_
= get_timestamp(now
)
539 device
= request
.user
.get_device_by_uid(device_uid
)
540 except DeviceDoesNotExist
as e
:
541 return HttpResponseNotFound(str(e
))
543 since_
= request
.GET
.get('since', None)
545 return HttpResponseBadRequest('parameter since missing')
547 since
= datetime
.fromtimestamp(float(since_
))
549 return HttpResponseBadRequest("'since' is not a valid timestamp")
551 include_actions
= parse_bool(request
.GET
.get('include_actions', False))
553 ret
= get_subscription_changes(request
.user
, device
, since
, now
)
554 domain
= RequestSite(request
).domain
556 subscriptions
= list(device
.get_subscribed_podcasts())
558 podcasts
= dict( (p
.url
, p
) for p
in subscriptions
)
559 prepare_podcast_data
= partial(get_podcast_data
, podcasts
, domain
)
561 ret
['add'] = map(prepare_podcast_data
, ret
['add'])
563 devices
= dict( (dev
.id, dev
.uid
) for dev
in request
.user
.devices
)
564 clean_action_data
= partial(clean_episode_action_data
,
565 user
=request
.user
, devices
=devices
)
567 # index subscribed podcasts by their Id for fast access
568 podcasts
= dict( (p
.get_id(), p
) for p
in subscriptions
)
569 prepare_episode_data
= partial(get_episode_data
, podcasts
, domain
,
570 clean_action_data
, include_actions
)
572 episode_updates
= self
.get_episode_updates(request
.user
,
573 subscriptions
, since
)
574 ret
['updates'] = map(prepare_episode_data
, episode_updates
)
576 return JsonResponse(ret
)
579 def get_episode_updates(self
, user
, subscribed_podcasts
, since
,
581 """ Returns the episode updates since the timestamp """
583 EpisodeStatus
= namedtuple('EpisodeStatus', 'episode status action')
589 episode_jobs
= [gevent
.spawn(episodes_for_podcast
, p
, since
,
590 limit
=max_per_podcast
) for p
in subscribed_podcasts
]
591 gevent
.joinall(episode_jobs
)
592 episodes
= chain
.from_iterable(job
.get() for job
in episode_jobs
)
595 episodes
= chain
.from_iterable(episodes_for_podcast(p
, since
,
596 limit
=max_per_podcast
) for p
in subscribed_podcasts
)
599 for episode
in episodes
:
600 episode_status
[episode
._id
] = EpisodeStatus(episode
, 'new', None)
605 e_action_jobs
= [gevent
.spawn(get_podcasts_episode_states
, p
,
606 user
._id
) for p
in subscribed_podcasts
]
607 gevent
.joinall(e_action_jobs
)
608 e_actions
= chain
.from_iterable(job
.get() for job
in e_action_jobs
)
611 e_actions
= chain
.from_iterable(get_podcasts_episode_states(p
,
612 user
._id
) for p
in subscribed_podcasts
)
615 for action
in e_actions
:
616 e_id
= action
['episode_id']
618 if e_id
in episode_status
:
619 episode
= episode_status
[e_id
].episode
621 episode
= episode_by_id(e_id
)
623 episode_status
[e_id
] = EpisodeStatus(episode
, action
['action'],
626 return episode_status
.itervalues()
632 def favorites(request
, username
):
633 favorites
= favorite_episodes_for_user(request
.user
)
634 domain
= RequestSite(request
).domain
635 e_data
= lambda e
: episode_data(e
, domain
)
636 ret
= map(e_data
, favorites
)
637 return JsonResponse(ret
)
640 def sanitize_append(url
, obj_type
, sanitized_list
):
641 urls
= sanitize_url(url
, obj_type
)
643 sanitized_list
.append( (url
, urls
) )