org.spearce.jgit/src/org/spearce/jgit/transport/TransportAmazonS3.java

   1 /*
   2  * Copyright (C) 2008, Shawn O. Pearce <spearce@spearce.org>
   3  *
   4  * All rights reserved.
   5  *
   6  * Redistribution and use in source and binary forms, with or
   7  * without modification, are permitted provided that the following
   8  * conditions are met:
   9  *
  10  * - Redistributions of source code must retain the above copyright
  11  *   notice, this list of conditions and the following disclaimer.
  12  *
  13  * - Redistributions in binary form must reproduce the above
  14  *   copyright notice, this list of conditions and the following
  15  *   disclaimer in the documentation and/or other materials provided
  16  *   with the distribution.
  17  *
  18  * - Neither the name of the Git Development Community nor the
  19  *   names of its contributors may be used to endorse or promote
  20  *   products derived from this software without specific prior
  21  *   written permission.
  22  *
  23  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
  24  * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
  25  * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  26  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  27  * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
  28  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  29  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  30  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  31  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
  32  * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
  33  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  34  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
  35  * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  36  */
  37
  38 package org.spearce.jgit.transport;
  39
  40 import java.io.BufferedReader;
  41 import java.io.File;
  42 import java.io.FileNotFoundException;
  43 import java.io.IOException;
  44 import java.io.InputStream;
  45 import java.io.OutputStream;
  46 import java.net.URLConnection;
  47 import java.util.ArrayList;
  48 import java.util.Collection;
  49 import java.util.HashSet;
  50 import java.util.Map;
  51 import java.util.Properties;
  52 import java.util.TreeMap;
  53
  54 import org.spearce.jgit.errors.NotSupportedException;
  55 import org.spearce.jgit.errors.TransportException;
  56 import org.spearce.jgit.lib.ObjectId;
  57 import org.spearce.jgit.lib.ProgressMonitor;
  58 import org.spearce.jgit.lib.Ref;
  59 import org.spearce.jgit.lib.Repository;
  60 import org.spearce.jgit.lib.Ref.Storage;
  61 import org.spearce.jgit.util.FS;
  62
  63 /**
  64  * Transport over the non-Git aware Amazon S3 protocol.
  65  * <p>
  66  * This transport communicates with the Amazon S3 servers (a non-free commercial
  67  * hosting service that users must subscribe to). Some users may find transport
  68  * to and from S3 to be a useful backup service.
  69  * <p>
  70  * The transport does not require any specialized Git support on the remote
  71  * (server side) repository, as Amazon does not provide any such support.
  72  * Repository files are retrieved directly through the S3 API, which uses
  73  * extended HTTP/1.1 semantics. This make it possible to read or write Git data
  74  * from a remote repository that is stored on S3.
  75  * <p>
  76  * Unlike the HTTP variant (see {@link TransportHttp}) we rely upon being able
  77  * to list objects in a bucket, as the S3 API supports this function. By listing
  78  * the bucket contents we can avoid relying on <code>objects/info/packs</code>
  79  * or <code>info/refs</code> in the remote repository.
  80  * <p>
  81  * Concurrent pushing over this transport is not supported. Multiple concurrent
  82  * push operations may cause confusion in the repository state.
  83  *
  84  * @see WalkFetchConnection
  85  * @see WalkPushConnection
  86  */
  87 class TransportAmazonS3 extends WalkTransport {
  88         static final String S3_SCHEME = "amazon-s3";
  89
  90         static boolean canHandle(final URIish uri) {
  91                 if (!uri.isRemote())
  92                         return false;
  93                 return S3_SCHEME.equals(uri.getScheme());
  94         }
  95
  96         /** User information necessary to connect to S3. */
  97         private final AmazonS3 s3;
  98
  99         /** Bucket the remote repository is stored in. */
 100         private final String bucket;
 101
 102         /**
 103          * Key prefix which all objects related to the repository start with.
 104          * <p>
 105          * The prefix does not start with "/".
 106          * <p>
 107          * The prefix does not end with "/". The trailing slash is stripped during
 108          * the constructor if a trailing slash was supplied in the URIish.
 109          * <p>
 110          * All files within the remote repository start with
 111          * <code>keyPrefix + "/"</code>.
 112          */
 113         private final String keyPrefix;
 114
 115         TransportAmazonS3(final Repository local, final URIish uri)
 116                         throws NotSupportedException {
 117                 super(local, uri);
 118
 119                 Properties props = null;
 120                 File propsFile = new File(local.getDirectory(), uri.getUser());
 121                 if (!propsFile.isFile())
 122                         propsFile = new File(FS.userHome(), uri.getUser());
 123                 if (propsFile.isFile()) {
 124                         try {
 125                                 props = AmazonS3.properties(propsFile);
 126                         } catch (IOException e) {
 127                                 throw new NotSupportedException("cannot read " + propsFile, e);
 128                         }
 129                 } else {
 130                         props = new Properties();
 131                         props.setProperty("accesskey", uri.getUser());
 132                         props.setProperty("secretkey", uri.getPass());
 133                 }
 134
 135                 s3 = new AmazonS3(props);
 136                 bucket = uri.getHost();
 137
 138                 String p = uri.getPath();
 139                 if (p.startsWith("/"))
 140                         p = p.substring(1);
 141                 if (p.endsWith("/"))
 142                         p = p.substring(0, p.length() - 1);
 143                 keyPrefix = p;
 144         }
 145
 146         @Override
 147         public FetchConnection openFetch() throws TransportException {
 148                 final DatabaseS3 c = new DatabaseS3(bucket, keyPrefix + "/objects");
 149                 final WalkFetchConnection r = new WalkFetchConnection(this, c);
 150                 r.available(c.readAdvertisedRefs());
 151                 return r;
 152         }
 153
 154         @Override
 155         public PushConnection openPush() throws TransportException {
 156                 final DatabaseS3 c = new DatabaseS3(bucket, keyPrefix + "/objects");
 157                 final WalkPushConnection r = new WalkPushConnection(this, c);
 158                 r.available(c.readAdvertisedRefs());
 159                 return r;
 160         }
 161
 162         @Override
 163         public void close() {
 164                 // No explicit connections are maintained.
 165         }
 166
 167         class DatabaseS3 extends WalkRemoteObjectDatabase {
 168                 private final String bucketName;
 169
 170                 private final String objectsKey;
 171
 172                 DatabaseS3(final String b, final String o) {
 173                         bucketName = b;
 174                         objectsKey = o;
 175                 }
 176
 177                 private String resolveKey(String subpath) {
 178                         if (subpath.endsWith("/"))
 179                                 subpath = subpath.substring(0, subpath.length() - 1);
 180                         String k = objectsKey;
 181                         while (subpath.startsWith(ROOT_DIR)) {
 182                                 k = k.substring(0, k.lastIndexOf('/'));
 183                                 subpath = subpath.substring(3);
 184                         }
 185                         return k + "/" + subpath;
 186                 }
 187
 188                 @Override
 189                 URIish getURI() {
 190                         URIish u = new URIish();
 191                         u = u.setScheme(S3_SCHEME);
 192                         u = u.setHost(bucketName);
 193                         u = u.setPath("/" + objectsKey);
 194                         return u;
 195                 }
 196
 197                 @Override
 198                 Collection<WalkRemoteObjectDatabase> getAlternates() throws IOException {
 199                         try {
 200                                 return readAlternates(INFO_ALTERNATES);
 201                         } catch (FileNotFoundException err) {
 202                                 // Fall through.
 203                         }
 204                         return null;
 205                 }
 206
 207                 @Override
 208                 WalkRemoteObjectDatabase openAlternate(final String location)
 209                                 throws IOException {
 210                         return new DatabaseS3(bucketName, resolveKey(location));
 211                 }
 212
 213                 @Override
 214                 Collection<String> getPackNames() throws IOException {
 215                         final HashSet<String> have = new HashSet<String>();
 216                         have.addAll(s3.list(bucket, resolveKey("pack")));
 217
 218                         final Collection<String> packs = new ArrayList<String>();
 219                         for (final String n : have) {
 220                                 if (!n.startsWith("pack-") || !n.endsWith(".pack"))
 221                                         continue;
 222
 223                                 final String in = n.substring(0, n.length() - 5) + ".idx";
 224                                 if (have.contains(in))
 225                                         packs.add(n);
 226                         }
 227                         return packs;
 228                 }
 229
 230                 @Override
 231                 FileStream open(final String path) throws IOException {
 232                         final URLConnection c = s3.get(bucket, resolveKey(path));
 233                         final InputStream raw = c.getInputStream();
 234                         final InputStream in = s3.decrypt(c);
 235                         final int len = c.getContentLength();
 236                         return new FileStream(in, raw == in ? len : -1);
 237                 }
 238
 239                 @Override
 240                 void deleteFile(final String path) throws IOException {
 241                         s3.delete(bucket, resolveKey(path));
 242                 }
 243
 244                 @Override
 245                 OutputStream writeFile(final String path,
 246                                 final ProgressMonitor monitor, final String monitorTask)
 247                                 throws IOException {
 248                         return s3.beginPut(bucket, resolveKey(path), monitor, monitorTask);
 249                 }
 250
 251                 @Override
 252                 void writeFile(final String path, final byte[] data) throws IOException {
 253                         s3.put(bucket, resolveKey(path), data);
 254                 }
 255
 256                 Map<String, Ref> readAdvertisedRefs() throws TransportException {
 257                         final TreeMap<String, Ref> avail = new TreeMap<String, Ref>();
 258                         readPackedRefs(avail);
 259                         readLooseRefs(avail);
 260                         readRef(avail, "HEAD");
 261                         return avail;
 262                 }
 263
 264                 private void readLooseRefs(final TreeMap<String, Ref> avail)
 265                                 throws TransportException {
 266                         try {
 267                                 for (final String n : s3.list(bucket, resolveKey(ROOT_DIR
 268                                                 + "refs")))
 269                                         readRef(avail, "refs/" + n);
 270                         } catch (IOException e) {
 271                                 throw new TransportException(getURI(), "cannot list refs", e);
 272                         }
 273                 }
 274
 275                 private Ref readRef(final TreeMap<String, Ref> avail, final String rn)
 276                                 throws TransportException {
 277                         final String s;
 278                         String ref = ROOT_DIR + rn;
 279                         try {
 280                                 final BufferedReader br = openReader(ref);
 281                                 try {
 282                                         s = br.readLine();
 283                                 } finally {
 284                                         br.close();
 285                                 }
 286                         } catch (FileNotFoundException noRef) {
 287                                 return null;
 288                         } catch (IOException err) {
 289                                 throw new TransportException(getURI(), "read " + ref, err);
 290                         }
 291
 292                         if (s == null)
 293                                 throw new TransportException(getURI(), "Empty ref: " + rn);
 294
 295                         if (s.startsWith("ref: ")) {
 296                                 final String target = s.substring("ref: ".length());
 297                                 Ref r = avail.get(target);
 298                                 if (r == null)
 299                                         r = readRef(avail, target);
 300                                 if (r == null)
 301                                         return null;
 302                                 r = new Ref(r.getStorage(), rn, r.getObjectId(), r
 303                                                 .getPeeledObjectId());
 304                                 avail.put(r.getName(), r);
 305                                 return r;
 306                         }
 307
 308                         if (ObjectId.isId(s)) {
 309                                 final Ref r = new Ref(loose(avail.get(rn)), rn, ObjectId
 310                                                 .fromString(s));
 311                                 avail.put(r.getName(), r);
 312                                 return r;
 313                         }
 314
 315                         throw new TransportException(getURI(), "Bad ref: " + rn + ": " + s);
 316                 }
 317
 318                 private Storage loose(final Ref r) {
 319                         if (r != null && r.getStorage() == Storage.PACKED)
 320                                 return Storage.LOOSE_PACKED;
 321                         return Storage.LOOSE;
 322                 }
 323
 324                 @Override
 325                 void close() {
 326                         // We do not maintain persistent connections.
 327                 }
 328         }
 329 }