indico/core/storage.py

   1 # This file is part of Indico.
   2 # Copyright (C) 2002 - 2015 European Organization for Nuclear Research (CERN).
   3 #
   4 # Indico is free software; you can redistribute it and/or
   5 # modify it under the terms of the GNU General Public License as
   6 # published by the Free Software Foundation; either version 3 of the
   7 # License, or (at your option) any later version.
   8 #
   9 # Indico is distributed in the hope that it will be useful, but
  10 # WITHOUT ANY WARRANTY; without even the implied warranty of
  11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  12 # General Public License for more details.
  13 #
  14 # You should have received a copy of the GNU General Public License
  15 # along with Indico; if not, see <http://www.gnu.org/licenses/>.
  16
  17 from __future__ import unicode_literals
  18
  19 import os
  20 import sys
  21 from shutil import copyfileobj
  22
  23 from werkzeug.security import safe_join
  24
  25 from indico.core import signals
  26 from indico.core.config import Config
  27 from indico.util.signals import named_objects_from_signal
  28 from indico.util.string import return_ascii
  29 from indico.web.flask.util import send_file
  30
  31
  32 def get_storage(backend_name):
  33     """Returns an FS object for the given backend.
  34
  35     The backend must be defined in the StorageBackends dict in the
  36     indico config.  Once a backend has been used it is assumed to
  37     stay there forever or at least as long as it is referenced
  38     somewhere.
  39
  40     Each backend definition uses the ``name:data`` notation, e.g.
  41     ``fs:/some/folder/`` or ``foo:host=foo.host,token=secret``.
  42     """
  43     try:
  44         definition = Config.getInstance().getStorageBackends()[backend_name]
  45     except KeyError:
  46         raise RuntimeError('Storage backend does not exist: {}'.format(backend_name))
  47     name, data = definition.split(':', 1)
  48     try:
  49         backend = get_storage_backends()[name]
  50     except KeyError:
  51         raise RuntimeError('Storage backend does not exist: {}'.format(backend_name))
  52     return backend(data)
  53
  54
  55 def get_storage_backends():
  56     return named_objects_from_signal(signals.get_storage_backends.send(), plugin_attr='plugin')
  57
  58
  59 class StorageError(Exception):
  60     """Exception used when a storage operation fails for any reason"""
  61
  62
  63 class Storage(object):
  64     """Base class for storage backends
  65
  66     To create a new storage backend, subclass this class and register
  67     it using the `get_storage_backends` signal.
  68
  69     In case you wonder why both `save` and `send_file` require certain
  70     file metadata: Depending on the storage backend, the information
  71     needs to be set when saving the file (usually with external storage
  72     services such as S3) or provided when sending the file back to the
  73     client (for example if the file is accessed through the local file
  74     system).
  75
  76     :param data: A string of data used to initialize the backend.
  77                  This could be a path, an url, or any other kind of
  78                  information needed by the backend.  If `simple_data`
  79                  is set, it should be a plain string. Otherwise it is
  80                  expected to be a string containing comma-separatey
  81                  key-value pairs: ``key=value,key2=value2,..``
  82
  83     """
  84     #: unique name of the storage backend
  85     name = None
  86     #: plugin containing this backend - assigned automatically
  87     plugin = None
  88     #: if the backend uses a simple data string instead of key-value pairs
  89     simple_data = True
  90
  91     def __init__(self, data):  # pragma: no cover
  92         pass
  93
  94     def _parse_data(self, data):
  95         """Util to parse a key=value data string to a dict"""
  96         return dict((x.strip() for x in item.split('=', 1)) for item in data.split(',')) if data else {}
  97
  98     def open(self, file_id):  # pragma: no cover
  99         """Opens a file in the storage for reading.
 100
 101         This returns a file-like object which contains the content of
 102         the file.
 103
 104         :param file_id: The ID of the file within the storage backend.
 105         """
 106         raise NotImplementedError
 107
 108     def save(self, name, content_type, filename, fileobj):  # pragma: no cover
 109         """Creates a new file in the storage.
 110
 111         This returns a a string identifier which can be used later to
 112         retrieve the file from the storage.
 113
 114         :param name: A unique name for the file.  This must be usable
 115                      as a filesystem path even though it depends on
 116                      the backend whether this name is used in such a
 117                      way or used at all.  It SHOULD not contain ``..``
 118                      as this could result in two apparently-different
 119                      names to actually end up being the same on
 120                      storage backends that use the regular file system.
 121                      Using slashes in the name is allowed, but when
 122                      doing so extra caution is needed to avoid cases
 123                      which fail on a filesystem backend such as trying
 124                      to save 'foo' and 'foo/bar.txt'
 125         :param content_type: The content-type of the file (may or may
 126                              not be used depending on the backend).
 127         :param filename: The original filename of the file, used e.g.
 128                          when sending the file to a client (may or may
 129                          not be used depending on the backend).
 130         :param fileobj: A file-like object containing the file data as
 131                         bytes.
 132         :return: unicode -- A unique identifier for the file.
 133         """
 134         raise NotImplementedError
 135
 136     def delete(self, file_id):  # pragma: no cover
 137         """Deletes a file from the storage.
 138
 139         :param file_id: The ID of the file within the storage backend.
 140         """
 141         raise NotImplementedError
 142
 143     def getsize(self, file_id):  # pragma: no cover
 144         """Gets the size in bytes of a file
 145
 146         :param file_id: The ID of the file within the storage backend.
 147         """
 148         raise NotImplementedError
 149
 150     def send_file(self, file_id, content_type, filename):  # pragma: no cover
 151         """Sends the file to the client.
 152
 153         This returns a flask response that will eventually result in
 154         the user being offered to download the file (or view it in the
 155         browser).  Depending on the storage backend it may actually
 156         send a redirect to an external URL where the file is available.
 157
 158         :param file_id: The ID of the file within the storage backend.
 159         :param content_type: The content-type of the file (may or may
 160                              not be used depending on the backend)
 161         :param filename: The file name to use when sending the file to
 162                          the client (may or may not be used depending
 163                          on the backend).
 164         """
 165         raise NotImplementedError
 166
 167     def __repr__(self):
 168         return '<{}()>'.format(type(self).__name__)
 169
 170
 171 class FileSystemStorage(Storage):
 172     name = 'fs'
 173     simple_data = True
 174
 175     def __init__(self, data):
 176         self.path = data
 177
 178     def _resolve_path(self, path):
 179         full_path = safe_join(self.path, path)
 180         if full_path is None:
 181             raise ValueError('Invalid path: {}'.format(path))
 182         return full_path
 183
 184     def open(self, file_id):
 185         try:
 186             return open(self._resolve_path(file_id), 'rb')
 187         except Exception as e:
 188             raise StorageError('Could not open "{}": {}'.format(file_id, e)), None, sys.exc_info()[2]
 189
 190     def save(self, name, content_type, filename, fileobj):
 191         try:
 192             filepath = self._resolve_path(name)
 193             if os.path.exists(filepath):
 194                 raise ValueError('A file with this name already exists')
 195             basedir = os.path.dirname(filepath)
 196             if not os.path.isdir(basedir):
 197                 os.makedirs(basedir)
 198             with open(filepath, 'wb') as f:
 199                 copyfileobj(fileobj, f, 1024 * 1024)
 200             return name
 201         except Exception as e:
 202             raise StorageError('Could not save "{}": {}'.format(name, e)), None, sys.exc_info()[2]
 203
 204     def delete(self, file_id):
 205         try:
 206             os.remove(self._resolve_path(file_id))
 207         except Exception as e:
 208             raise StorageError('Could not delete "{}": {}'.format(file_id, e)), None, sys.exc_info()[2]
 209
 210     def getsize(self, file_id):
 211         try:
 212             return os.path.getsize(self._resolve_path(file_id))
 213         except Exception as e:
 214             raise StorageError('Could not get size of "{}": {}'.format(file_id, e)), None, sys.exc_info()[2]
 215
 216     def send_file(self, file_id, content_type, filename):
 217         try:
 218             return send_file(filename, self._resolve_path(file_id), content_type)
 219         except Exception as e:
 220             raise StorageError('Could not send "{}": {}'.format(file_id, e)), None, sys.exc_info()[2]
 221
 222     @return_ascii
 223     def __repr__(self):
 224         return '<FileSystemStorage: {}>'.format(self.path)
 225
 226
 227 @signals.get_storage_backends.connect
 228 def _get_storage_backends(sender, **kwargs):
 229     return FileSystemStorage
 230
 231
 232 @signals.app_created.connect
 233 def _check_storage_backends(app, **kwargs):
 234     # This will raise RuntimeError if the backend names are not unique
 235     get_storage_backends()