From fafb7d4c179c6311398b720b87c92a7449114cc0 Mon Sep 17 00:00:00 2001 From: Adrian Moennich Date: Thu, 2 Jul 2015 16:21:40 +0200 Subject: [PATCH] Create symlinks in separate storage backend --- indico_zodbimport/modules/attachments.py | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/indico_zodbimport/modules/attachments.py b/indico_zodbimport/modules/attachments.py index 9937e8df5..f47df639d 100644 --- a/indico_zodbimport/modules/attachments.py +++ b/indico_zodbimport/modules/attachments.py @@ -40,6 +40,7 @@ from indico_zodbimport.util import protection_from_ac, patch_default_group_provi class AttachmentImporter(Importer): def __init__(self, **kwargs): self.storage_backend = kwargs.pop('storage_backend') + self.symlink_backend = kwargs.pop('symlink_backend') self.archive_dirs = kwargs.pop('archive_dir') self.default_group_provider = kwargs.pop('default_group_provider') self.avoid_storage_check = kwargs.pop('avoid_storage_check') @@ -47,6 +48,8 @@ class AttachmentImporter(Importer): if (self.avoid_storage_check or self.symlink_target) and len(self.archive_dirs) != 1: raise click.exceptions.UsageError('Invalid number of archive-dirs for --no-storage-access or ' '--symlink-target') + if bool(self.symlink_target) != bool(self.symlink_backend): + raise click.exceptions.UsageError('Both or none of --symlink-target and --symlink-backend must be used.') super(AttachmentImporter, self).__init__(**kwargs) @staticmethod @@ -63,6 +66,8 @@ class AttachmentImporter(Importer): help="Avoid checking files in storage unless absolutely necessary due to encoding " "issues. This will migrate all files with size=0. When this option is specified, " "--archive-dir must be used exactly once.")(command) + command = click.option('--symlink-backend', + help="The name of the storage backend used for symlinks.")(command) command = click.option('--symlink-target', help="If set, any files with a non-UTF8 path will be symlinked in this location and " "store the path to the symlink instead (relative to the archive dir). " @@ -144,10 +149,10 @@ class AttachmentImporter(Importer): parent_path = os.path.dirname(path) candidates = os.listdir(parent_path) if len(candidates) != 1: - return None, 0 + return None, None, 0 path = os.path.join(parent_path, candidates[0]) if not os.path.exists(path): - return None, 0 + return None, None, 0 assert path size = 0 if self.avoid_storage_check else os.path.getsize(path) @@ -156,11 +161,13 @@ class AttachmentImporter(Importer): rel_path = rel_path.decode('utf-8') except UnicodeDecodeError: if not self.symlink_target: - return None, 0 - symlink = os.path.join(self.symlink_target, bytes(uuid4())) + return None, None, 0 + symlink_name = uuid4() + symlink = os.path.join(self.symlink_target, bytes(symlink_name)) os.symlink(path, symlink) - rel_path = unicode(os.path.relpath(symlink, archive_path)) - return rel_path, size + return self.symlink_backend, symlink_name, size + else: + return self.storage_backend, rel_path, size def _attachment_from_resource(self, folder, material, resource, event): data = {'folder': folder, @@ -173,7 +180,7 @@ class AttachmentImporter(Importer): data['link_url'] = resource.url else: data['type'] = AttachmentType.file - storage_path, size = self._get_file_info(resource) + storage_backend, storage_path, size = self._get_file_info(resource) if storage_path is None: self.print_error(cformat('%{red!}File {} not found on disk').format(resource._LocalFile__archivedId), event_id=event.id) @@ -181,7 +188,7 @@ class AttachmentImporter(Importer): filename = secure_filename(convert_to_unicode(resource.fileName)) or 'attachment' data['file'] = AttachmentFile(user=self.janitor_user, created_dt=data['modified_dt'], filename=filename, content_type='application/octet-stream', # TODO - size=size, storage_backend=self.storage_backend, storage_file_id=storage_path) + size=size, storage_backend=storage_backend, storage_file_id=storage_path) attachment = Attachment(**data) protection_from_ac(attachment, resource._Resource__ac) return attachment -- 2.11.4.GIT