2 # This file is part of my.gpodder.org.
4 # my.gpodder.org is free software: you can redistribute it and/or modify it
5 # under the terms of the GNU Affero General Public License as published by
6 # the Free Software Foundation, either version 3 of the License, or (at your
7 # option) any later version.
9 # my.gpodder.org is distributed in the hope that it will be useful, but
10 # WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
11 # or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public
12 # License for more details.
14 # You should have received a copy of the GNU Affero General Public License
15 # along with my.gpodder.org. If not, see <http://www.gnu.org/licenses/>.
18 from functools
import partial
19 from itertools
import imap
, chain
20 from collections
import defaultdict
, namedtuple
21 from datetime
import datetime
23 import dateutil
.parser
26 from django
.http
import HttpResponse
, HttpResponseBadRequest
, Http404
, HttpResponseNotFound
27 from django
.contrib
.sites
.models
import RequestSite
28 from django
.db
import IntegrityError
29 from django
.views
.decorators
.csrf
import csrf_exempt
30 from django
.views
.decorators
.cache
import never_cache
32 from mygpo
.api
.constants
import EPISODE_ACTION_TYPES
, DEVICE_TYPES
33 from mygpo
.api
.httpresponse
import JsonResponse
34 from mygpo
.api
.sanitizing
import sanitize_url
, sanitize_urls
35 from mygpo
.api
.advanced
.directory
import episode_data
, podcast_data
36 from mygpo
.api
.backend
import get_device
, get_favorites
37 from mygpo
.log
import log
38 from mygpo
.utils
import parse_time
, format_time
, parse_bool
, get_to_dict
, get_timestamp
39 from mygpo
.decorators
import allowed_methods
, repeat_on_conflict
40 from mygpo
.core
import models
41 from mygpo
.core
.models
import SanitizingRule
, Podcast
42 from mygpo
.users
.models
import PodcastUserState
, EpisodeAction
, EpisodeUserState
, DeviceDoesNotExist
43 from mygpo
.json
import json
, JSONDecodeError
44 from mygpo
.api
.basic_auth
import require_valid_user
, check_username
45 from mygpo
.couchdb
import bulk_save_retry
48 # keys that are allowed in episode actions
49 EPISODE_ACTION_KEYS
= ('position', 'episode', 'action', 'device', 'timestamp',
50 'started', 'total', 'podcast')
57 @allowed_methods(['GET', 'POST'])
58 def subscriptions(request
, username
, device_uid
):
61 now_
= get_timestamp(now
)
63 if request
.method
== 'GET':
66 device
= request
.user
.get_device_by_uid(device_uid
)
67 except DeviceDoesNotExist
as e
:
68 return HttpResponseNotFound(str(e
))
70 since_
= request
.GET
.get('since', None)
72 return HttpResponseBadRequest('parameter since missing')
74 since
= datetime
.fromtimestamp(float(since_
))
76 return HttpResponseBadRequest('since-value is not a valid timestamp')
78 changes
= get_subscription_changes(request
.user
, device
, since
, now
)
80 return JsonResponse(changes
)
82 elif request
.method
== 'POST':
83 d
= get_device(request
.user
, device_uid
,
84 request
.META
.get('HTTP_USER_AGENT', ''))
86 if not request
.raw_post_data
:
87 return HttpResponseBadRequest('POST data must not be empty')
89 actions
= json
.loads(request
.raw_post_data
)
90 add
= actions
['add'] if 'add' in actions
else []
91 rem
= actions
['remove'] if 'remove' in actions
else []
93 add
= filter(None, add
)
94 rem
= filter(None, rem
)
97 update_urls
= update_subscriptions(request
.user
, d
, add
, rem
)
98 except IntegrityError
, e
:
99 return HttpResponseBadRequest(e
)
101 return JsonResponse({
103 'update_urls': update_urls
,
107 def update_subscriptions(user
, device
, add
, remove
):
111 raise IntegrityError('can not add and remove %s at the same time' % a
)
113 add_s
= list(sanitize_urls(add
, 'podcast'))
114 rem_s
= list(sanitize_urls(remove
, 'podcast'))
116 assert len(add
) == len(add_s
) and len(remove
) == len(rem_s
)
118 updated_urls
= filter(lambda (a
, b
): a
!= b
, zip(add
+ remove
, add_s
+ rem_s
))
120 add_s
= filter(None, add_s
)
121 rem_s
= filter(None, rem_s
)
123 # If two different URLs (in add and remove) have
124 # been sanitized to the same, we ignore the removal
125 rem_s
= filter(lambda x
: x
not in add_s
, rem_s
)
128 p
= Podcast
.for_url(a
, create
=True)
130 p
.subscribe(user
, device
)
131 except Exception as e
:
132 log('Advanced API: %(username)s: could not subscribe to podcast %(podcast_url)s on device %(device_id)s: %(exception)s' %
133 {'username': user
.username
, 'podcast_url': p
.url
, 'device_id': device
.id, 'exception': e
})
136 p
= Podcast
.for_url(r
, create
=True)
138 p
.unsubscribe(user
, device
)
139 except Exception as e
:
140 log('Advanced API: %(username)s: could not unsubscribe from podcast %(podcast_url)s on device %(device_id)s: %(exception)s' %
141 {'username': user
.username
, 'podcast_url': p
.url
, 'device_id': device
.id, 'exception': e
})
146 def get_subscription_changes(user
, device
, since
, until
):
147 add
, rem
= device
.get_subscription_changes(since
, until
)
149 podcast_ids
= add
+ rem
150 podcasts
= get_to_dict(Podcast
, podcast_ids
, get_id
=models
.Podcast
.get_id
)
152 add_podcasts
= filter(None, (podcasts
.get(i
, None) for i
in add
))
153 rem_podcasts
= filter(None, (podcasts
.get(i
, None) for i
in rem
))
154 add_urls
= [ podcast
.url
for podcast
in add_podcasts
]
155 rem_urls
= [ podcast
.url
for podcast
in rem_podcasts
]
157 until_
= get_timestamp(until
)
158 return {'add': add_urls
, 'remove': rem_urls
, 'timestamp': until_
}
165 @allowed_methods(['GET', 'POST'])
166 def episodes(request
, username
, version
=1):
168 version
= int(version
)
170 now_
= get_timestamp(now
)
171 ua_string
= request
.META
.get('HTTP_USER_AGENT', '')
173 if request
.method
== 'POST':
175 actions
= json
.loads(request
.raw_post_data
)
176 except (JSONDecodeError
, UnicodeDecodeError) as e
:
177 log('Advanced API: could not decode episode update POST data for user %s: %s' % (username
, e
))
178 return HttpResponseBadRequest()
181 update_urls
= update_episodes(request
.user
, actions
, now
, ua_string
)
184 log('could not update episodes for user %s: %s %s: %s' % (username
, e
, traceback
.format_exc(), actions
))
185 return HttpResponseBadRequest(e
)
187 return JsonResponse({'timestamp': now_
, 'update_urls': update_urls
})
189 elif request
.method
== 'GET':
190 podcast_url
= request
.GET
.get('podcast', None)
191 device_uid
= request
.GET
.get('device', None)
192 since_
= request
.GET
.get('since', None)
193 aggregated
= parse_bool(request
.GET
.get('aggregated', False))
196 since
= datetime
.fromtimestamp(float(since_
)) if since_
else None
198 return HttpResponseBadRequest('since-value is not a valid timestamp')
201 podcast
= Podcast
.for_url(podcast_url
)
210 device
= request
.user
.get_device_by_uid(device_uid
)
211 except DeviceDoesNotExist
as e
:
212 return HttpResponseNotFound(str(e
))
217 changes
= get_episode_changes(request
.user
, podcast
, device
, since
,
218 now
, aggregated
, version
)
220 return JsonResponse(changes
)
224 def convert_position(action
):
225 """ convert position parameter for API 1 compatibility """
226 pos
= getattr(action
, 'position', None)
228 action
.position
= format_time(pos
)
233 def get_episode_changes(user
, podcast
, device
, since
, until
, aggregated
, version
):
235 devices
= dict( (dev
.id, dev
.uid
) for dev
in user
.devices
)
238 if podcast
is not None:
239 args
['podcast_id'] = podcast
.get_id()
241 if device
is not None:
242 args
['device_id'] = device
.id
244 actions
= EpisodeAction
.filter(user
._id
, since
, until
, **args
)
247 actions
= imap(convert_position
, actions
)
249 clean_data
= partial(clean_episode_action_data
,
250 user
=user
, devices
=devices
)
252 actions
= map(clean_data
, actions
)
253 actions
= filter(None, actions
)
256 actions
= dict( (a
['episode'], a
) for a
in actions
).values()
258 until_
= get_timestamp(until
)
260 return {'actions': actions
, 'timestamp': until_
}
265 def clean_episode_action_data(action
, user
, devices
):
267 if None in (action
.get('podcast', None), action
.get('episode', None)):
270 if 'device_id' in action
:
271 device_id
= action
['device_id']
272 device_uid
= devices
.get(device_id
)
274 action
['device'] = device_uid
276 del action
['device_id']
278 # remove superfluous keys
279 for x
in action
.keys():
280 if x
not in EPISODE_ACTION_KEYS
:
283 # set missing keys to None
284 for x
in EPISODE_ACTION_KEYS
:
288 if action
['action'] != 'play':
289 if 'position' in action
:
290 del action
['position']
292 if 'total' in action
:
295 if 'started' in action
:
296 del action
['started']
298 if 'playmark' in action
:
299 del action
['playmark']
302 action
['position'] = action
.get('position', False) or 0
310 def update_episodes(user
, actions
, now
, ua_string
):
313 grouped_actions
= defaultdict(list)
315 # group all actions by their episode
316 for action
in actions
:
318 podcast_url
= action
['podcast']
319 podcast_url
= sanitize_append(podcast_url
, 'podcast', update_urls
)
320 if podcast_url
== '': continue
322 episode_url
= action
['episode']
323 episode_url
= sanitize_append(episode_url
, 'episode', update_urls
)
324 if episode_url
== '': continue
326 act
= parse_episode_action(action
, user
, update_urls
, now
, ua_string
)
327 grouped_actions
[ (podcast_url
, episode_url
) ].append(act
)
329 # Prepare the updates for each episode state
332 for (p_url
, e_url
), action_list
in grouped_actions
.iteritems():
333 episode_state
= EpisodeUserState
.for_ref_urls(user
, p_url
, e_url
)
335 fun
= partial(update_episode_actions
, action_list
=action_list
)
336 obj_funs
.append( (episode_state
, fun
) )
338 db
= EpisodeUserState
.get_db()
339 bulk_save_retry(db
, obj_funs
)
344 def update_episode_actions(episode_state
, action_list
):
345 """ Adds actions to the episode state and saves if necessary """
347 len1
= len(episode_state
.actions
)
348 episode_state
.add_actions(action_list
)
350 if len(episode_state
.actions
) == len1
:
357 def parse_episode_action(action
, user
, update_urls
, now
, ua_string
):
358 action_str
= action
.get('action', None)
359 if not valid_episodeaction(action_str
):
360 raise Exception('invalid action %s' % action_str
)
362 new_action
= EpisodeAction()
364 new_action
.action
= action
['action']
366 if action
.get('device', False):
367 device
= get_device(user
, action
['device'], ua_string
)
368 new_action
.device
= device
.id
370 if action
.get('timestamp', False):
371 new_action
.timestamp
= dateutil
.parser
.parse(action
['timestamp'])
373 new_action
.timestamp
= now
374 new_action
.timestamp
= new_action
.timestamp
.replace(microsecond
=0)
376 new_action
.started
= action
.get('started', None)
377 new_action
.playmark
= action
.get('position', None)
378 new_action
.total
= action
.get('total', None)
387 # Workaround for mygpoclient 1.0: It uses "PUT" requests
388 # instead of "POST" requests for uploading device settings
389 @allowed_methods(['POST', 'PUT'])
390 def device(request
, username
, device_uid
):
391 d
= get_device(request
.user
, device_uid
,
392 request
.META
.get('HTTP_USER_AGENT', ''))
394 data
= json
.loads(request
.raw_post_data
)
396 if 'caption' in data
:
397 if not data
['caption']:
398 return HttpResponseBadRequest('caption must not be empty')
399 d
.name
= data
['caption']
402 if not valid_devicetype(data
['type']):
403 return HttpResponseBadRequest('invalid device type %s' % data
['type'])
404 d
.type = data
['type']
407 request
.user
.update_device(d
)
409 return HttpResponse()
412 def valid_devicetype(type):
413 for t
in DEVICE_TYPES
:
418 def valid_episodeaction(type):
419 for t
in EPISODE_ACTION_TYPES
:
429 @allowed_methods(['GET'])
430 def devices(request
, username
):
431 devices
= filter(lambda d
: not d
.deleted
, request
.user
.devices
)
432 devices
= map(device_data
, devices
)
433 return JsonResponse(devices
)
436 def device_data(device
):
439 caption
= device
.name
,
441 subscriptions
= len(device
.get_subscribed_podcast_ids())
446 def get_podcast_data(podcasts
, domain
, url
):
447 """ Gets podcast data for a URL from a dict of podcasts """
448 podcast
= podcasts
.get(url
)
449 return podcast_data(podcast
, domain
)
452 def get_episode_data(podcasts
, domain
, clean_action_data
, include_actions
, episode_status
):
453 """ Get episode data for an episode status object """
454 podcast_id
= episode_status
.episode
.podcast
455 podcast
= podcasts
.get(podcast_id
, None)
456 t
= episode_data(episode_status
.episode
, domain
, podcast
)
457 t
['status'] = episode_status
.status
459 # include latest action (bug 1419)
460 if include_actions
and episode_status
.action
:
461 t
['action'] = clean_action_data(episode_status
.action
)
470 def updates(request
, username
, device_uid
):
472 now_
= get_timestamp(now
)
475 device
= request
.user
.get_device_by_uid(device_uid
)
476 except DeviceDoesNotExist
as e
:
477 return HttpResponseNotFound(str(e
))
479 since_
= request
.GET
.get('since', None)
481 return HttpResponseBadRequest('parameter since missing')
483 since
= datetime
.fromtimestamp(float(since_
))
485 return HttpResponseBadRequest('since-value is not a valid timestamp')
487 include_actions
= parse_bool(request
.GET
.get('include_actions', False))
489 ret
= get_subscription_changes(request
.user
, device
, since
, now
)
490 domain
= RequestSite(request
).domain
492 subscriptions
= list(device
.get_subscribed_podcasts())
494 podcasts
= dict( (p
.url
, p
) for p
in subscriptions
)
495 prepare_podcast_data
= partial(get_podcast_data
, podcasts
, domain
)
497 ret
['add'] = map(prepare_podcast_data
, ret
['add'])
499 devices
= dict( (dev
.id, dev
.uid
) for dev
in request
.user
.devices
)
500 clean_action_data
= partial(clean_episode_action_data
,
501 user
=request
.user
, devices
=devices
)
503 # index subscribed podcasts by their Id for fast access
504 podcasts
= dict( (p
.get_id(), p
) for p
in subscriptions
)
505 prepare_episode_data
= partial(get_episode_data
, podcasts
, domain
,
506 clean_action_data
, include_actions
)
508 episode_updates
= get_episode_updates(request
.user
, subscriptions
, since
)
509 ret
['updates'] = map(prepare_episode_data
, episode_updates
)
511 return JsonResponse(ret
)
514 def get_episode_updates(user
, subscribed_podcasts
, since
):
515 """ Returns the episode updates since the timestamp """
517 EpisodeStatus
= namedtuple('EpisodeStatus', 'episode status action')
522 episode_jobs
= [gevent
.spawn(p
.get_episodes
, since
) for p
in
524 gevent
.joinall(episode_jobs
)
525 episodes
= chain
.from_iterable(job
.get() for job
in episode_jobs
)
527 for episode
in episodes
:
528 episode_status
[episode
._id
] = EpisodeStatus(episode
, 'new', None)
531 e_action_jobs
= [gevent
.spawn(p
.get_episode_states
, user
._id
) for p
in
533 gevent
.joinall(e_action_jobs
)
534 e_actions
= chain
.from_iterable(job
.get() for job
in e_action_jobs
)
536 for action
in e_actions
:
537 e_id
= action
['episode_id']
539 if e_id
in episode_status
:
540 episode
= episode_status
[e_id
].episode
542 episode
= models
.Episode
.get(e_id
)
544 episode_status
[e_id
] = EpisodeStatus(episode
, action
['action'], action
)
546 return episode_status
.itervalues()
552 def favorites(request
, username
):
553 favorites
= get_favorites(request
.user
)
554 domain
= RequestSite(request
).domain
555 e_data
= lambda e
: episode_data(e
, domain
)
556 ret
= map(e_data
, favorites
)
557 return JsonResponse(ret
)
560 def sanitize_append(url
, obj_type
, sanitized_list
):
561 urls
= sanitize_url(url
, obj_type
)
563 sanitized_list
.append( (url
, urls
) )