indico/core/storage.py

   1 # This file is part of Indico.
   2 # Copyright (C) 2002 - 2015 European Organization for Nuclear Research (CERN).
   3 #
   4 # Indico is free software; you can redistribute it and/or
   5 # modify it under the terms of the GNU General Public License as
   6 # published by the Free Software Foundation; either version 3 of the
   7 # License, or (at your option) any later version.
   8 #
   9 # Indico is distributed in the hope that it will be useful, but
  10 # WITHOUT ANY WARRANTY; without even the implied warranty of
  11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  12 # General Public License for more details.
  13 #
  14 # You should have received a copy of the GNU General Public License
  15 # along with Indico; if not, see <http://www.gnu.org/licenses/>.
  16
  17 from __future__ import unicode_literals
  18
  19 import os
  20 import sys
  21 from contextlib import contextmanager
  22 from shutil import copyfileobj
  23 from tempfile import NamedTemporaryFile
  24
  25 from werkzeug.security import safe_join
  26
  27 from indico.core import signals
  28 from indico.core.config import Config
  29 from indico.util.signals import named_objects_from_signal
  30 from indico.util.string import return_ascii
  31 from indico.web.flask.util import send_file
  32
  33
  34 def get_storage(backend_name):
  35     """Returns an FS object for the given backend.
  36
  37     The backend must be defined in the StorageBackends dict in the
  38     indico config.  Once a backend has been used it is assumed to
  39     stay there forever or at least as long as it is referenced
  40     somewhere.
  41
  42     Each backend definition uses the ``name:data`` notation, e.g.
  43     ``fs:/some/folder/`` or ``foo:host=foo.host,token=secret``.
  44     """
  45     try:
  46         definition = Config.getInstance().getStorageBackends()[backend_name]
  47     except KeyError:
  48         raise RuntimeError('Storage backend does not exist: {}'.format(backend_name))
  49     name, data = definition.split(':', 1)
  50     try:
  51         backend = get_storage_backends()[name]
  52     except KeyError:
  53         raise RuntimeError('Storage backend {} has invalid type {}'.format(backend_name, name))
  54     return backend(data)
  55
  56
  57 def get_storage_backends():
  58     return named_objects_from_signal(signals.get_storage_backends.send(), plugin_attr='plugin')
  59
  60
  61 class StorageError(Exception):
  62     """Exception used when a storage operation fails for any reason"""
  63
  64
  65 class Storage(object):
  66     """Base class for storage backends
  67
  68     To create a new storage backend, subclass this class and register
  69     it using the `get_storage_backends` signal.
  70
  71     In case you wonder why both `save` and `send_file` require certain
  72     file metadata: Depending on the storage backend, the information
  73     needs to be set when saving the file (usually with external storage
  74     services such as S3) or provided when sending the file back to the
  75     client (for example if the file is accessed through the local file
  76     system).
  77
  78     :param data: A string of data used to initialize the backend.
  79                  This could be a path, an url, or any other kind of
  80                  information needed by the backend.  If `simple_data`
  81                  is set, it should be a plain string. Otherwise it is
  82                  expected to be a string containing comma-separatey
  83                  key-value pairs: ``key=value,key2=value2,..``
  84
  85     """
  86     #: unique name of the storage backend
  87     name = None
  88     #: plugin containing this backend - assigned automatically
  89     plugin = None
  90     #: if the backend uses a simple data string instead of key-value pairs
  91     simple_data = True
  92
  93     def __init__(self, data):  # pragma: no cover
  94         pass
  95
  96     def _parse_data(self, data):
  97         """Util to parse a key=value data string to a dict"""
  98         return dict((x.strip() for x in item.split('=', 1)) for item in data.split(',')) if data else {}
  99
 100     def open(self, file_id):  # pragma: no cover
 101         """Opens a file in the storage for reading.
 102
 103         This returns a file-like object which contains the content of
 104         the file.
 105
 106         :param file_id: The ID of the file within the storage backend.
 107         """
 108         raise NotImplementedError
 109
 110     @contextmanager
 111     def get_local_path(self, file_id):
 112         """Returns a local path for the file.
 113
 114         While this path MAY point to the permanent location of the
 115         stored file, it MUST NOT be used for anything but read
 116         operations and MUST NOT be used after existing this function's
 117         contextmanager.
 118
 119         :param file_id: The ID of the file within the storage backend.
 120         """
 121         with self.open(file_id) as fd:
 122             with NamedTemporaryFile(suffix='indico.tmp', dir=Config.getInstance().getUploadedFilesTempDir()) as tmpfile:
 123                 copyfileobj(fd, tmpfile, 1024 * 1024)
 124                 tmpfile.flush()
 125                 yield tmpfile.name
 126
 127     def save(self, name, content_type, filename, fileobj):  # pragma: no cover
 128         """Creates a new file in the storage.
 129
 130         This returns a a string identifier which can be used later to
 131         retrieve the file from the storage.
 132
 133         :param name: A unique name for the file.  This must be usable
 134                      as a filesystem path even though it depends on
 135                      the backend whether this name is used in such a
 136                      way or used at all.  It SHOULD not contain ``..``
 137                      as this could result in two apparently-different
 138                      names to actually end up being the same on
 139                      storage backends that use the regular file system.
 140                      Using slashes in the name is allowed, but when
 141                      doing so extra caution is needed to avoid cases
 142                      which fail on a filesystem backend such as trying
 143                      to save 'foo' and 'foo/bar.txt'
 144         :param content_type: The content-type of the file (may or may
 145                              not be used depending on the backend).
 146         :param filename: The original filename of the file, used e.g.
 147                          when sending the file to a client (may or may
 148                          not be used depending on the backend).
 149         :param fileobj: A file-like object containing the file data as
 150                         bytes.
 151         :return: unicode -- A unique identifier for the file.
 152         """
 153         raise NotImplementedError
 154
 155     def delete(self, file_id):  # pragma: no cover
 156         """Deletes a file from the storage.
 157
 158         :param file_id: The ID of the file within the storage backend.
 159         """
 160         raise NotImplementedError
 161
 162     def getsize(self, file_id):  # pragma: no cover
 163         """Gets the size in bytes of a file
 164
 165         :param file_id: The ID of the file within the storage backend.
 166         """
 167         raise NotImplementedError
 168
 169     def send_file(self, file_id, content_type, filename):  # pragma: no cover
 170         """Sends the file to the client.
 171
 172         This returns a flask response that will eventually result in
 173         the user being offered to download the file (or view it in the
 174         browser).  Depending on the storage backend it may actually
 175         send a redirect to an external URL where the file is available.
 176
 177         :param file_id: The ID of the file within the storage backend.
 178         :param content_type: The content-type of the file (may or may
 179                              not be used depending on the backend)
 180         :param filename: The file name to use when sending the file to
 181                          the client (may or may not be used depending
 182                          on the backend).
 183         """
 184         raise NotImplementedError
 185
 186     def __repr__(self):
 187         return '<{}()>'.format(type(self).__name__)
 188
 189
 190 class ReadOnlyStorageMixin(object):
 191     """Mixin that makes write operations fail with an error"""
 192
 193     def save(self, name, content_type, filename, fileobj):
 194         raise StorageError('Cannot write to read-only storage')
 195
 196     def delete(self, file_id):
 197         raise StorageError('Cannot delete from read-only storage')
 198
 199
 200 class FileSystemStorage(Storage):
 201     name = 'fs'
 202     simple_data = True
 203
 204     def __init__(self, data):
 205         self.path = data
 206
 207     def _resolve_path(self, path):
 208         full_path = safe_join(self.path, path)
 209         if full_path is None:
 210             raise ValueError('Invalid path: {}'.format(path))
 211         return full_path
 212
 213     def open(self, file_id):
 214         try:
 215             return open(self._resolve_path(file_id), 'rb')
 216         except Exception as e:
 217             raise StorageError('Could not open "{}": {}'.format(file_id, e)), None, sys.exc_info()[2]
 218
 219     @contextmanager
 220     def get_local_path(self, file_id):
 221         yield self._resolve_path(file_id)
 222
 223     def save(self, name, content_type, filename, fileobj):
 224         try:
 225             filepath = self._resolve_path(name)
 226             if os.path.exists(filepath):
 227                 raise ValueError('A file with this name already exists')
 228             basedir = os.path.dirname(filepath)
 229             if not os.path.isdir(basedir):
 230                 os.makedirs(basedir)
 231             with open(filepath, 'wb') as f:
 232                 copyfileobj(fileobj, f, 1024 * 1024)
 233             return name
 234         except Exception as e:
 235             raise StorageError('Could not save "{}": {}'.format(name, e)), None, sys.exc_info()[2]
 236
 237     def delete(self, file_id):
 238         try:
 239             os.remove(self._resolve_path(file_id))
 240         except Exception as e:
 241             raise StorageError('Could not delete "{}": {}'.format(file_id, e)), None, sys.exc_info()[2]
 242
 243     def getsize(self, file_id):
 244         try:
 245             return os.path.getsize(self._resolve_path(file_id))
 246         except Exception as e:
 247             raise StorageError('Could not get size of "{}": {}'.format(file_id, e)), None, sys.exc_info()[2]
 248
 249     def send_file(self, file_id, content_type, filename):
 250         try:
 251             return send_file(filename, self._resolve_path(file_id), content_type)
 252         except Exception as e:
 253             raise StorageError('Could not send "{}": {}'.format(file_id, e)), None, sys.exc_info()[2]
 254
 255     @return_ascii
 256     def __repr__(self):
 257         return '<FileSystemStorage: {}>'.format(self.path)
 258
 259
 260 class ReadOnlyFileSystemStorage(ReadOnlyStorageMixin, FileSystemStorage):
 261     name = 'fs-readonly'
 262
 263     @return_ascii
 264     def __repr__(self):
 265         return '<ReadOnlyFileSystemStorage: {}>'.format(self.path)
 266
 267
 268 @signals.get_storage_backends.connect
 269 def _get_storage_backends(sender, **kwargs):
 270     yield FileSystemStorage
 271     yield ReadOnlyFileSystemStorage
 272
 273
 274 @signals.app_created.connect
 275 def _check_storage_backends(app, **kwargs):
 276     # This will raise RuntimeError if the backend names are not unique
 277     get_storage_backends()