handle large episode action uploads in background
authorStefan Kögl <stefan@skoegl.net>
Sat, 27 Apr 2013 13:57:07 +0000 (27 15:57 +0200)
committerStefan Kögl <stefan@skoegl.net>
Sat, 27 Apr 2013 13:59:09 +0000 (27 15:59 +0200)
mygpo/api/advanced/__init__.py
mygpo/api/tasks.py [new file with mode: 0644]
mygpo/cel.py
mygpo/settings.py

index 636f859..4e406a2 100644 (file)
@@ -19,6 +19,7 @@ from functools import partial
 from itertools import imap, chain
 from collections import defaultdict, namedtuple
 from datetime import datetime
+from importlib import import_module
 
 import dateutil.parser
 
@@ -33,6 +34,7 @@ from django.views.decorators.csrf import csrf_exempt
 from django.views.decorators.cache import never_cache
 from django.utils.decorators import method_decorator
 from django.views.generic.base import View
+from django.conf import settings
 
 from mygpo.api.constants import EPISODE_ACTION_TYPES, DEVICE_TYPES
 from mygpo.api.httpresponse import JsonResponse
@@ -187,6 +189,23 @@ def episodes(request, username, version=1):
             log(msg)
             return HttpResponseBadRequest(msg)
 
+        log('start: user %s: %d actions from %s' % (request.user._id, len(actions), ua_string))
+
+        # handle in background
+        if len(actions) > settings.API_ACTIONS_MAX_NONBG:
+            bg_handler = settings.API_ACTIONS_BG_HANDLER
+            if bg_handler is not None:
+
+                modname, funname = bg_handler.rsplit('.', 1)
+                mod = import_module(modname)
+                fun = getattr(mod, funname)
+
+                fun(request.user, actions, now, ua_string)
+
+                # TODO: return 202 Accepted
+                return JsonResponse({'timestamp': now_, 'update_urls': []})
+
+
         try:
             update_urls = update_episodes(request.user, actions, now, ua_string)
         except DeviceUIDException as e:
diff --git a/mygpo/api/tasks.py b/mygpo/api/tasks.py
new file mode 100644 (file)
index 0000000..3205883
--- /dev/null
@@ -0,0 +1,10 @@
+from mygpo.cel import celery
+from mygpo.api.advanced import update_episodes
+
+
+@celery.task(max_retries=5, default_retry_delay=60)
+def import_episode_actions(user, actions, upload_ts, ua_string):
+    update_episodes(user, actions, upload_ts, ua_string)
+
+# celery-based handler for episode-actions
+episode_actions_celery_handler = import_episode_actions.delay
index 8ee7bee..6a61b6a 100644 (file)
@@ -13,6 +13,7 @@ celery = Celery('mygpo.celery',
                 backend=settings.BACKEND_URL,
                 include=[
                     'mygpo.core.tasks',
+                    'mygpo.api.tasks',
                     'mygpo.users.tasks',
                     'mygpo.data.tasks',
                     'mygpo.admin.tasks',
index 59fbc94..520bc77 100644 (file)
@@ -193,6 +193,12 @@ PODCAST_SLUG_SUBSCRIBER_LIMIT = 10
 # categories to the top
 MIN_SUBSCRIBERS_CATEGORY=10
 
+# maximum number of episode actions that the API processes immediatelly before
+# returning the response. Larger requests will be handled in background.
+# Handler can be set to None to disable
+API_ACTIONS_MAX_NONBG=100
+API_ACTIONS_BG_HANDLER='mygpo.api.tasks.episode_actions_celery_handler'
+
 
 ADSENSE_CLIENT = ''
 ADSENSE_SLOT_BOTTOM = ''