From 5c3c133336ad32a4d47ad2b86b6b37c0037cf4c0 Mon Sep 17 00:00:00 2001 From: Tim Cuthbertson Date: Sun, 24 Jun 2012 17:44:21 +1000 Subject: [PATCH] Add recipe step --- tests/RecipeRename.xml | 14 +++++++ tests/testdownload.py | 18 ++++++++- tests/testrecipe.py | 69 +++++++++++++++++++++++++++++++++ zeroinstall/injector/fetch.py | 89 ++++++++++++++++++++++++++++++++++--------- zeroinstall/injector/model.py | 16 ++++++++ 5 files changed, 188 insertions(+), 18 deletions(-) create mode 100644 tests/RecipeRename.xml create mode 100755 tests/testrecipe.py diff --git a/tests/RecipeRename.xml b/tests/RecipeRename.xml new file mode 100644 index 0000000..2a03a57 --- /dev/null +++ b/tests/RecipeRename.xml @@ -0,0 +1,14 @@ + + + Recipe + Recipe + Recipe + + + + + + + + + diff --git a/tests/testdownload.py b/tests/testdownload.py index bfcca05..4e421dc 100755 --- a/tests/testdownload.py +++ b/tests/testdownload.py @@ -81,10 +81,13 @@ class Reply: return self.reply def download_and_execute(driver, prog_args, main = None): + driver_download(driver) + run.execute_selections(driver.solver.selections, prog_args, stores = driver.config.stores, main = main) + +def driver_download(driver): downloaded = driver.solve_and_download_impls() if downloaded: tasks.wait_for_blocker(downloaded) - run.execute_selections(driver.solver.selections, prog_args, stores = driver.config.stores, main = main) class NetworkManager: def state(self): @@ -344,6 +347,19 @@ class TestDownload(BaseTest): raise ex finally: sys.stdout = old_out + + def testRename(self): + with output_suppressed(): + run_server(('HelloWorld.tar.bz2',)) + requirements = Requirements(os.path.abspath('RecipeRename.xml')) + requirements.command = None + driver = Driver(requirements = requirements, config = self.config) + driver_download(driver) + digests = driver.solver.selections[requirements.interface_uri].digests + path = self.config.stores.lookup_any(digests) + assert os.path.exists(os.path.join(path, 'HelloUniverse', 'minor')) + assert not os.path.exists(os.path.join(path, 'HelloWorld')) + assert not os.path.exists(os.path.join(path, 'HelloUniverse', 'main')) def testSymlink(self): old_out = sys.stdout diff --git a/tests/testrecipe.py b/tests/testrecipe.py new file mode 100755 index 0000000..0521090 --- /dev/null +++ b/tests/testrecipe.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python +from __future__ import with_statement +import unittest +import sys +import os +import tempfile +import shutil +from basetest import BaseTest + +sys.path.insert(0, '..') + +from zeroinstall import SafeException +from zeroinstall.injector.fetch import StepRunner +from zeroinstall.injector.model import RenameStep + +class TestRecipe(BaseTest): + def setUp(self): + super(TestRecipe, self).setUp() + self.basedir = tempfile.mkdtemp() + self.join = lambda *a: os.path.join(self.basedir, *a) + os.makedirs(self.join("dir1")) + os.makedirs(self.join("level1", "level2")) + with open(self.join("level1", "level2", "level3"), 'w') as f: + f.write("level3 contents") + with open(self.join("rootfile"), 'w') as f: + f.write("rootfile contents") + + def tearDown(self): + shutil.rmtree(self.basedir) + super(TestRecipe, self).tearDown() + + def _apply_step(self, step, **k): + if not 'force' in k: k['force'] = False + if not 'impl_hint' in k: k['impl_hint'] = None + cls = StepRunner.class_for(step) + runner = cls(step, **k) + # NOTE: runner.prepare() is not performed in these tests, + # as they test local operations only that require no preparation + runner.apply(self.basedir) + + def _assert_denies_escape(self, step): + try: + self._apply_step(step) + assert False + except SafeException as e: + if not 'is not within the base directory' in str(e): raise e + + def testRenameDisallowsEscapingArchiveDirViaSrcSymlink(self): + os.symlink("/usr/bin", self.join("bin")) + self._assert_denies_escape(RenameStep(source="bin/gpg", dest="gpg")) + + def testRenameDisallowsEscapingArchiveDirViaDestSymlink(self): + os.symlink("/tmp", self.join("tmp")) + self._assert_denies_escape(RenameStep(source="rootfile", dest="tmp/surprise")) + + def testRenameDisallowsEscapingArchiveDirViaSrcRelativePath(self): + self._assert_denies_escape(RenameStep(source="../somefile", dest="somefile")) + + def testRenameDisallowsEscapingArchiveDirViaDestRelativePath(self): + self._assert_denies_escape(RenameStep(source="rootfile", dest="../somefile")) + + def testRenameDisallowsEscapingArchiveDirViaSrcAbsolutePath(self): + self._assert_denies_escape(RenameStep(source="/usr/bin/gpg", dest="gpg")) + + def testRenameDisallowsEscapingArchiveDirViaDestAbsolutePath(self): + self._assert_denies_escape(RenameStep(source="rootfile", dest="/tmp/rootfile")) + +if __name__ == '__main__': + unittest.main() diff --git a/zeroinstall/injector/fetch.py b/zeroinstall/injector/fetch.py index f46add8..9fc7ad7 100644 --- a/zeroinstall/injector/fetch.py +++ b/zeroinstall/injector/fetch.py @@ -9,8 +9,10 @@ from zeroinstall import _, NeedDownload import os from logging import info, debug, warn +from zeroinstall import support from zeroinstall.support import tasks, basedir, portable_rename from zeroinstall.injector.namespaces import XMLNS_IFACE, config_site +from zeroinstall.injector import model from zeroinstall.injector.model import DownloadSource, Recipe, SafeException, escape, DistributionSource from zeroinstall.injector.iface_cache import PendingFeed, ReplayAttack from zeroinstall.injector.handler import NoTrustedKeys @@ -112,25 +114,24 @@ class Fetcher(object): @see: L{download_impl} uses this method when appropriate""" # Maybe we're taking this metaphor too far? - # Start downloading all the ingredients. - streams = {} # Streams collected from successful downloads - # Start a download for each ingredient blockers = [] - for step in recipe.steps: - blocker, stream = self.download_archive(step, force = force, impl_hint = impl_hint) - assert stream - blockers.append(blocker) - streams[step] = stream + steps = [] + for stepdata in recipe.steps: + cls = StepRunner.class_for(stepdata) + step = cls(stepdata, force=force, impl_hint=impl_hint) + step.prepare(self, blockers) + steps.append(step) while blockers: yield blockers tasks.check(blockers) blockers = [b for b in blockers if not b.happened] - from zeroinstall.zerostore import unpack if self.external_store: + # Note: external_store will not yet work with non- steps. + streams = [step.stream for step in steps] self._add_to_external_store(required_digest, recipe.steps, streams) else: # Create an empty directory for the new implementation @@ -138,20 +139,14 @@ class Fetcher(object): tmpdir = store.get_tmp_dir_for(required_digest) try: # Unpack each of the downloaded archives into it in turn - for step in recipe.steps: - stream = streams[step] - stream.seek(0) - unpack.unpack_archive_over(step.url, stream, tmpdir, - extract = step.extract, - type = step.type, - start_offset = step.start_offset or 0) + for step in steps: + step.apply(tmpdir) # Check that the result is correct and store it in the cache store.check_manifest_and_rename(required_digest, tmpdir) tmpdir = None finally: # If unpacking fails, remove the temporary directory if tmpdir is not None: - from zeroinstall import support support.ro_rmtree(tmpdir) def get_feed_mirror(self, url): @@ -561,3 +556,63 @@ class Fetcher(object): self.handler.monitor_download(dl) dl.downloaded = self.scheduler.download(dl) return dl + +class StepRunner(object): + """The base class of all step runners""" + def __init__(self, stepdata, force, impl_hint): + self.stepdata = stepdata + self.force = force + self.impl_hint = impl_hint + + def prepare(self, fetcher, blockers): + pass + + @classmethod + def class_for(cls, model): + for subcls in cls.__subclasses__(): + if subcls.model_type == type(model): + return subcls + assert False, "Couldn't find step runner for %s" % (type(model),) + +class RenameStepRunner(StepRunner): + """A step runner for the step""" + + model_type = model.RenameStep + + def apply(self, basedir): + source = native_path_within_base(basedir, self.stepdata.source) + dest = native_path_within_base(basedir, self.stepdata.dest) + os.rename(source, dest) + +class DownloadStepRunner(StepRunner): + """A step runner for the step""" + + model_type = model.DownloadSource + + def prepare(self, fetcher, blockers): + self.blocker, self.stream = fetcher.download_archive(self.stepdata, force = self.force, impl_hint = self.impl_hint) + assert self.stream + blockers.append(self.blocker) + + def apply(self, basedir): + from zeroinstall.zerostore import unpack + assert self.blocker.happened + unpack.unpack_archive_over(self.stepdata.url, self.stream, basedir, + extract = self.stepdata.extract, + type=self.stepdata.type, + start_offset = self.stepdata.start_offset or 0) + +def native_path_within_base(base, crossplatform_path): + """Takes a cross-platform relative path (i.e using forward slashes, even on windows) + and returns the absolute, platform-native version of the path. + If the path does not resolve to a location within `base`, a SafeError is raised. + """ + assert os.path.isabs(base) + if crossplatform_path.startswith("/"): + raise SafeException("path %r is not within the base directory" % (crossplatform_path,)) + native_path = os.path.join(*crossplatform_path.split("/")) + fullpath = os.path.realpath(os.path.join(base, native_path)) + base = os.path.realpath(base) + if not fullpath.startswith(base + os.path.sep): + raise SafeException("path %r is not within the base directory" % (crossplatform_path,)) + return fullpath diff --git a/zeroinstall/injector/model.py b/zeroinstall/injector/model.py index a8bcb83..437faad 100644 --- a/zeroinstall/injector/model.py +++ b/zeroinstall/injector/model.py @@ -496,6 +496,14 @@ class DownloadSource(RetrievalMethod): self.start_offset = start_offset self.type = type # MIME type - see unpack.py +class RenameStep(RetrievalMethod): + """A Rename provides a way to rename / move a file within an implementation.""" + __slots__ = ['source', 'dest'] + + def __init__(self, source, dest): + self.source = source + self.dest = dest + class Recipe(RetrievalMethod): """Get an implementation by following a series of steps. @ivar size: the combined download sizes from all the steps @@ -1128,6 +1136,14 @@ class ZeroInstallFeed(object): extract = recipe_step.getAttribute('extract'), start_offset = _get_long(recipe_step, 'start-offset'), type = recipe_step.getAttribute('type'))) + elif recipe_step.uri == XMLNS_IFACE and recipe_step.name == 'rename': + source = recipe_step.getAttribute('source') + if not source: + raise InvalidInterface(_("Missing source attribute on ")) + dest = recipe_step.getAttribute('dest') + if not dest: + raise InvalidInterface(_("Missing dest attribute on ")) + recipe.steps.append(RenameStep(source=source, dest=dest)) else: info(_("Unknown step '%s' in recipe; skipping recipe"), recipe_step.name) break -- 2.11.4.GIT