2 * Copyright (C) 2009-2010, Google Inc.
3 * Copyright (C) 2008, Shawn O. Pearce <spearce@spearce.org>
4 * and other copyright owners as documented in the project's IP log.
6 * This program and the accompanying materials are made available
7 * under the terms of the Eclipse Distribution License v1.0 which
8 * accompanies this distribution, is reproduced below, and is
9 * available at http://www.eclipse.org/org/documents/edl-v10.php
11 * All rights reserved.
13 * Redistribution and use in source and binary forms, with or
14 * without modification, are permitted provided that the following
17 * - Redistributions of source code must retain the above copyright
18 * notice, this list of conditions and the following disclaimer.
20 * - Redistributions in binary form must reproduce the above
21 * copyright notice, this list of conditions and the following
22 * disclaimer in the documentation and/or other materials provided
23 * with the distribution.
25 * - Neither the name of the Eclipse Foundation, Inc. nor the
26 * names of its contributors may be used to endorse or promote
27 * products derived from this software without specific prior
30 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
31 * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
32 * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
33 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
34 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
35 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
36 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
37 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
38 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
39 * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
40 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
41 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
42 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
45 package org
.eclipse
.jgit
.transport
;
47 import static org
.eclipse
.jgit
.util
.HttpSupport
.ENCODING_GZIP
;
48 import static org
.eclipse
.jgit
.util
.HttpSupport
.HDR_ACCEPT
;
49 import static org
.eclipse
.jgit
.util
.HttpSupport
.HDR_ACCEPT_ENCODING
;
50 import static org
.eclipse
.jgit
.util
.HttpSupport
.HDR_CONTENT_ENCODING
;
51 import static org
.eclipse
.jgit
.util
.HttpSupport
.HDR_CONTENT_TYPE
;
52 import static org
.eclipse
.jgit
.util
.HttpSupport
.HDR_PRAGMA
;
53 import static org
.eclipse
.jgit
.util
.HttpSupport
.HDR_USER_AGENT
;
54 import static org
.eclipse
.jgit
.util
.HttpSupport
.METHOD_POST
;
56 import java
.io
.BufferedReader
;
57 import java
.io
.ByteArrayInputStream
;
58 import java
.io
.FileNotFoundException
;
59 import java
.io
.IOException
;
60 import java
.io
.InputStream
;
61 import java
.io
.InputStreamReader
;
62 import java
.io
.OutputStream
;
63 import java
.net
.HttpURLConnection
;
64 import java
.net
.MalformedURLException
;
65 import java
.net
.Proxy
;
66 import java
.net
.ProxySelector
;
68 import java
.util
.ArrayList
;
69 import java
.util
.Collection
;
72 import java
.util
.TreeMap
;
73 import java
.util
.zip
.GZIPInputStream
;
74 import java
.util
.zip
.GZIPOutputStream
;
76 import org
.eclipse
.jgit
.errors
.NoRemoteRepositoryException
;
77 import org
.eclipse
.jgit
.errors
.NotSupportedException
;
78 import org
.eclipse
.jgit
.errors
.PackProtocolException
;
79 import org
.eclipse
.jgit
.errors
.TransportException
;
80 import org
.eclipse
.jgit
.lib
.Config
;
81 import org
.eclipse
.jgit
.lib
.Constants
;
82 import org
.eclipse
.jgit
.lib
.ObjectId
;
83 import org
.eclipse
.jgit
.lib
.ObjectIdRef
;
84 import org
.eclipse
.jgit
.lib
.ProgressMonitor
;
85 import org
.eclipse
.jgit
.lib
.Ref
;
86 import org
.eclipse
.jgit
.lib
.RefDirectory
;
87 import org
.eclipse
.jgit
.lib
.Repository
;
88 import org
.eclipse
.jgit
.lib
.SymbolicRef
;
89 import org
.eclipse
.jgit
.lib
.Config
.SectionParser
;
90 import org
.eclipse
.jgit
.util
.HttpSupport
;
91 import org
.eclipse
.jgit
.util
.IO
;
92 import org
.eclipse
.jgit
.util
.RawParseUtils
;
93 import org
.eclipse
.jgit
.util
.TemporaryBuffer
;
94 import org
.eclipse
.jgit
.util
.io
.DisabledOutputStream
;
95 import org
.eclipse
.jgit
.util
.io
.UnionInputStream
;
98 * Transport over HTTP and FTP protocols.
100 * If the transport is using HTTP and the remote HTTP service is Git-aware
101 * (speaks the "smart-http protocol") this client will automatically take
102 * advantage of the additional Git-specific HTTP extensions. If the remote
103 * service does not support these extensions, the client will degrade to direct
106 * If the remote (server side) repository does not have the specialized Git
107 * support, object files are retrieved directly through standard HTTP GET (or
108 * binary FTP GET) requests. This make it easy to serve a Git repository through
109 * a standard web host provider that does not offer specific support for Git.
111 * @see WalkFetchConnection
113 public class TransportHttp
extends HttpTransport
implements WalkTransport
,
115 private static final String SVC_UPLOAD_PACK
= "git-upload-pack";
117 private static final String SVC_RECEIVE_PACK
= "git-receive-pack";
119 private static final String userAgent
= computeUserAgent();
121 static boolean canHandle(final URIish uri
) {
124 final String s
= uri
.getScheme();
125 return "http".equals(s
) || "https".equals(s
) || "ftp".equals(s
);
128 private static String
computeUserAgent() {
130 final Package pkg
= TransportHttp
.class.getPackage();
131 if (pkg
!= null && pkg
.getImplementationVersion() != null) {
132 version
= pkg
.getImplementationVersion();
134 version
= "unknown"; //$NON-NLS-1$
136 return "JGit/" + version
; //$NON-NLS-1$
139 private static final Config
.SectionParser
<HttpConfig
> HTTP_KEY
= new SectionParser
<HttpConfig
>() {
140 public HttpConfig
parse(final Config cfg
) {
141 return new HttpConfig(cfg
);
145 private static class HttpConfig
{
146 final int postBuffer
;
148 HttpConfig(final Config rc
) {
149 postBuffer
= rc
.getInt("http", "postbuffer", 1 * 1024 * 1024);
153 private final URL baseUrl
;
155 private final URL objectsUrl
;
157 private final HttpConfig http
;
159 private final ProxySelector proxySelector
;
161 private boolean useSmartHttp
= true;
163 TransportHttp(final Repository local
, final URIish uri
)
164 throws NotSupportedException
{
167 String uriString
= uri
.toString();
168 if (!uriString
.endsWith("/"))
170 baseUrl
= new URL(uriString
);
171 objectsUrl
= new URL(baseUrl
, "objects/");
172 } catch (MalformedURLException e
) {
173 throw new NotSupportedException("Invalid URL " + uri
, e
);
175 http
= local
.getConfig().get(HTTP_KEY
);
176 proxySelector
= ProxySelector
.getDefault();
180 * Toggle whether or not smart HTTP transport should be used.
182 * This flag exists primarily to support backwards compatibility testing
183 * within a testing framework, there is no need to modify it in most
187 * if {@code true} (default), smart HTTP is enabled.
189 public void setUseSmartHttp(final boolean on
) {
194 public FetchConnection
openFetch() throws TransportException
,
195 NotSupportedException
{
196 final String service
= SVC_UPLOAD_PACK
;
198 final HttpURLConnection c
= connect(service
);
199 final InputStream in
= openInputStream(c
);
201 if (isSmartHttp(c
, service
)) {
202 readSmartHeaders(in
, service
);
203 return new SmartHttpFetchConnection(in
);
206 // Assume this server doesn't support smart HTTP fetch
207 // and fall back on dumb object walking.
209 return newDumbConnection(in
);
214 } catch (NotSupportedException err
) {
216 } catch (TransportException err
) {
218 } catch (IOException err
) {
219 throw new TransportException(uri
, "error reading info/refs", err
);
223 private FetchConnection
newDumbConnection(InputStream in
)
224 throws IOException
, PackProtocolException
{
225 HttpObjectDB d
= new HttpObjectDB(objectsUrl
);
226 BufferedReader br
= toBufferedReader(in
);
227 Map
<String
, Ref
> refs
;
229 refs
= d
.readAdvertisedImpl(br
);
234 if (!refs
.containsKey(Constants
.HEAD
)) {
235 // If HEAD was not published in the info/refs file (it usually
236 // is not there) download HEAD by itself as a loose file and do
237 // the resolution by hand.
239 HttpURLConnection conn
= httpOpen(new URL(baseUrl
, Constants
.HEAD
));
240 int status
= HttpSupport
.response(conn
);
242 case HttpURLConnection
.HTTP_OK
: {
243 br
= toBufferedReader(openInputStream(conn
));
245 String line
= br
.readLine();
246 if (line
!= null && line
.startsWith(RefDirectory
.SYMREF
)) {
247 String target
= line
.substring(RefDirectory
.SYMREF
.length());
248 Ref r
= refs
.get(target
);
250 r
= new ObjectIdRef
.Unpeeled(Ref
.Storage
.NEW
, target
, null);
251 r
= new SymbolicRef(Constants
.HEAD
, r
);
252 refs
.put(r
.getName(), r
);
253 } else if (line
!= null && ObjectId
.isId(line
)) {
254 Ref r
= new ObjectIdRef
.Unpeeled(Ref
.Storage
.NETWORK
,
255 Constants
.HEAD
, ObjectId
.fromString(line
));
256 refs
.put(r
.getName(), r
);
264 case HttpURLConnection
.HTTP_NOT_FOUND
:
268 throw new TransportException(uri
, "cannot read HEAD: " + status
269 + " " + conn
.getResponseMessage());
273 WalkFetchConnection wfc
= new WalkFetchConnection(this, d
);
278 private BufferedReader
toBufferedReader(InputStream in
) {
279 return new BufferedReader(new InputStreamReader(in
, Constants
.CHARSET
));
283 public PushConnection
openPush() throws NotSupportedException
,
285 final String service
= SVC_RECEIVE_PACK
;
287 final HttpURLConnection c
= connect(service
);
288 final InputStream in
= openInputStream(c
);
290 if (isSmartHttp(c
, service
)) {
291 readSmartHeaders(in
, service
);
292 return new SmartHttpPushConnection(in
);
294 } else if (!useSmartHttp
) {
295 final String msg
= "smart HTTP push disabled";
296 throw new NotSupportedException(msg
);
299 final String msg
= "remote does not support smart HTTP push";
300 throw new NotSupportedException(msg
);
305 } catch (NotSupportedException err
) {
307 } catch (TransportException err
) {
309 } catch (IOException err
) {
310 throw new TransportException(uri
, "error reading info/refs", err
);
315 public void close() {
316 // No explicit connections are maintained.
319 private HttpURLConnection
connect(final String service
)
320 throws TransportException
, NotSupportedException
{
323 final StringBuilder b
= new StringBuilder();
326 if (b
.charAt(b
.length() - 1) != '/')
328 b
.append(Constants
.INFO_REFS
);
331 b
.append(b
.indexOf("?") < 0 ?
'?' : '&');
332 b
.append("service=");
336 u
= new URL(b
.toString());
337 } catch (MalformedURLException e
) {
338 throw new NotSupportedException("Invalid URL " + uri
, e
);
342 final HttpURLConnection conn
= httpOpen(u
);
344 String expType
= "application/x-" + service
+ "-advertisement";
345 conn
.setRequestProperty(HDR_ACCEPT
, expType
+ ", */*");
347 conn
.setRequestProperty(HDR_ACCEPT
, "*/*");
349 final int status
= HttpSupport
.response(conn
);
351 case HttpURLConnection
.HTTP_OK
:
354 case HttpURLConnection
.HTTP_NOT_FOUND
:
355 throw new NoRemoteRepositoryException(uri
, u
+ " not found");
357 case HttpURLConnection
.HTTP_FORBIDDEN
:
358 throw new TransportException(uri
, service
+ " not permitted");
361 String err
= status
+ " " + conn
.getResponseMessage();
362 throw new TransportException(uri
, err
);
364 } catch (NotSupportedException e
) {
366 } catch (TransportException e
) {
368 } catch (IOException e
) {
369 throw new TransportException(uri
, "cannot open " + service
, e
);
373 final HttpURLConnection
httpOpen(final URL u
) throws IOException
{
374 final Proxy proxy
= HttpSupport
.proxyFor(proxySelector
, u
);
375 HttpURLConnection conn
= (HttpURLConnection
) u
.openConnection(proxy
);
376 conn
.setUseCaches(false);
377 conn
.setRequestProperty(HDR_ACCEPT_ENCODING
, ENCODING_GZIP
);
378 conn
.setRequestProperty(HDR_PRAGMA
, "no-cache");//$NON-NLS-1$
379 conn
.setRequestProperty(HDR_USER_AGENT
, userAgent
);
383 final InputStream
openInputStream(HttpURLConnection conn
)
385 InputStream input
= conn
.getInputStream();
386 if (ENCODING_GZIP
.equals(conn
.getHeaderField(HDR_CONTENT_ENCODING
)))
387 input
= new GZIPInputStream(input
);
391 IOException
wrongContentType(String expType
, String actType
) {
392 final String why
= "expected Content-Type " + expType
393 + "; received Content-Type " + actType
;
394 return new TransportException(uri
, why
);
397 private boolean isSmartHttp(final HttpURLConnection c
, final String service
) {
398 final String expType
= "application/x-" + service
+ "-advertisement";
399 final String actType
= c
.getContentType();
400 return expType
.equals(actType
);
403 private void readSmartHeaders(final InputStream in
, final String service
)
405 // A smart reply will have a '#' after the first 4 bytes, but
406 // a dumb reply cannot contain a '#' until after byte 41. Do a
407 // quick check to make sure its a smart reply before we parse
408 // as a pkt-line stream.
410 final byte[] magic
= new byte[5];
411 IO
.readFully(in
, magic
, 0, magic
.length
);
412 if (magic
[4] != '#') {
413 throw new TransportException(uri
, "expected pkt-line with"
414 + " '# service=', got '" + RawParseUtils
.decode(magic
)
418 final PacketLineIn pckIn
= new PacketLineIn(new UnionInputStream(
419 new ByteArrayInputStream(magic
), in
));
420 final String exp
= "# service=" + service
;
421 final String act
= pckIn
.readString();
422 if (!exp
.equals(act
)) {
423 throw new TransportException(uri
, "expected '" + exp
+ "', got '"
427 while (pckIn
.readString() != PacketLineIn
.END
) {
428 // for now, ignore the remaining header lines
432 class HttpObjectDB
extends WalkRemoteObjectDatabase
{
433 private final URL objectsUrl
;
435 HttpObjectDB(final URL b
) {
441 return new URIish(objectsUrl
);
445 Collection
<WalkRemoteObjectDatabase
> getAlternates() throws IOException
{
447 return readAlternates(INFO_HTTP_ALTERNATES
);
448 } catch (FileNotFoundException err
) {
453 return readAlternates(INFO_ALTERNATES
);
454 } catch (FileNotFoundException err
) {
462 WalkRemoteObjectDatabase
openAlternate(final String location
)
464 return new HttpObjectDB(new URL(objectsUrl
, location
));
468 Collection
<String
> getPackNames() throws IOException
{
469 final Collection
<String
> packs
= new ArrayList
<String
>();
471 final BufferedReader br
= openReader(INFO_PACKS
);
474 final String s
= br
.readLine();
475 if (s
== null || s
.length() == 0)
477 if (!s
.startsWith("P pack-") || !s
.endsWith(".pack"))
478 throw invalidAdvertisement(s
);
479 packs
.add(s
.substring(2));
485 } catch (FileNotFoundException err
) {
491 FileStream
open(final String path
) throws IOException
{
492 final URL base
= objectsUrl
;
493 final URL u
= new URL(base
, path
);
494 final HttpURLConnection c
= httpOpen(u
);
495 switch (HttpSupport
.response(c
)) {
496 case HttpURLConnection
.HTTP_OK
:
497 final InputStream in
= openInputStream(c
);
498 final int len
= c
.getContentLength();
499 return new FileStream(in
, len
);
500 case HttpURLConnection
.HTTP_NOT_FOUND
:
501 throw new FileNotFoundException(u
.toString());
503 throw new IOException(u
.toString() + ": "
504 + HttpSupport
.response(c
) + " "
505 + c
.getResponseMessage());
509 Map
<String
, Ref
> readAdvertisedImpl(final BufferedReader br
)
510 throws IOException
, PackProtocolException
{
511 final TreeMap
<String
, Ref
> avail
= new TreeMap
<String
, Ref
>();
513 String line
= br
.readLine();
517 final int tab
= line
.indexOf('\t');
519 throw invalidAdvertisement(line
);
524 name
= line
.substring(tab
+ 1);
525 id
= ObjectId
.fromString(line
.substring(0, tab
));
526 if (name
.endsWith("^{}")) {
527 name
= name
.substring(0, name
.length() - 3);
528 final Ref prior
= avail
.get(name
);
530 throw outOfOrderAdvertisement(name
);
532 if (prior
.getPeeledObjectId() != null)
533 throw duplicateAdvertisement(name
+ "^{}");
535 avail
.put(name
, new ObjectIdRef
.PeeledTag(
536 Ref
.Storage
.NETWORK
, name
,
537 prior
.getObjectId(), id
));
539 Ref prior
= avail
.put(name
, new ObjectIdRef
.PeeledNonTag(
540 Ref
.Storage
.NETWORK
, name
, id
));
542 throw duplicateAdvertisement(name
);
548 private PackProtocolException
outOfOrderAdvertisement(final String n
) {
549 return new PackProtocolException("advertisement of " + n
550 + "^{} came before " + n
);
553 private PackProtocolException
invalidAdvertisement(final String n
) {
554 return new PackProtocolException("invalid advertisement of " + n
);
557 private PackProtocolException
duplicateAdvertisement(final String n
) {
558 return new PackProtocolException("duplicate advertisements of " + n
);
563 // We do not maintain persistent connections.
567 class SmartHttpFetchConnection
extends BasePackFetchConnection
{
568 SmartHttpFetchConnection(final InputStream advertisement
)
569 throws TransportException
{
570 super(TransportHttp
.this);
573 init(advertisement
, DisabledOutputStream
.INSTANCE
);
576 readAdvertisedRefs();
577 } catch (IOException err
) {
579 throw new TransportException(uri
, "remote hung up", err
);
584 protected void doFetch(final ProgressMonitor monitor
,
585 final Collection
<Ref
> want
, final Set
<ObjectId
> have
)
586 throws TransportException
{
587 final Service svc
= new Service(SVC_UPLOAD_PACK
);
588 init(svc
.in
, svc
.out
);
589 super.doFetch(monitor
, want
, have
);
593 class SmartHttpPushConnection
extends BasePackPushConnection
{
594 SmartHttpPushConnection(final InputStream advertisement
)
595 throws TransportException
{
596 super(TransportHttp
.this);
599 init(advertisement
, DisabledOutputStream
.INSTANCE
);
602 readAdvertisedRefs();
603 } catch (IOException err
) {
605 throw new TransportException(uri
, "remote hung up", err
);
609 protected void doPush(final ProgressMonitor monitor
,
610 final Map
<String
, RemoteRefUpdate
> refUpdates
)
611 throws TransportException
{
612 final Service svc
= new Service(SVC_RECEIVE_PACK
);
613 init(svc
.in
, svc
.out
);
614 super.doPush(monitor
, refUpdates
);
619 * State required to speak multiple HTTP requests with the remote.
621 * A service wrapper provides a normal looking InputStream and OutputStream
622 * pair which are connected via HTTP to the named remote service. Writing to
623 * the OutputStream is buffered until either the buffer overflows, or
624 * reading from the InputStream occurs. If overflow occurs HTTP/1.1 and its
625 * chunked transfer encoding is used to stream the request data to the
626 * remote service. If the entire request fits in the memory buffer, the
627 * older HTTP/1.0 standard and a fixed content length is used instead.
629 * It is an error to attempt to read without there being outstanding data
630 * ready for transmission on the OutputStream.
632 * No state is preserved between write-read request pairs. The caller is
633 * responsible for replaying state vector information as part of the request
634 * data written to the OutputStream. Any session HTTP cookies may or may not
635 * be preserved between requests, it is left up to the JVM's implementation
636 * of the HTTP client.
639 private final String serviceName
;
641 private final String requestType
;
643 private final String responseType
;
645 private final UnionInputStream httpIn
;
647 final HttpInputStream in
;
649 final HttpOutputStream out
;
651 HttpURLConnection conn
;
653 Service(final String serviceName
) {
654 this.serviceName
= serviceName
;
655 this.requestType
= "application/x-" + serviceName
+ "-request";
656 this.responseType
= "application/x-" + serviceName
+ "-result";
658 this.httpIn
= new UnionInputStream();
659 this.in
= new HttpInputStream(httpIn
);
660 this.out
= new HttpOutputStream();
663 void openStream() throws IOException
{
664 conn
= httpOpen(new URL(baseUrl
, serviceName
));
665 conn
.setRequestMethod(METHOD_POST
);
666 conn
.setInstanceFollowRedirects(false);
667 conn
.setDoOutput(true);
668 conn
.setRequestProperty(HDR_CONTENT_TYPE
, requestType
);
669 conn
.setRequestProperty(HDR_ACCEPT
, responseType
);
672 void execute() throws IOException
{
676 // Output hasn't started yet, because everything fit into
677 // our request buffer. Send with a Content-Length header.
679 if (out
.length() == 0) {
680 throw new TransportException(uri
, "Starting read stage"
681 + " without written request data pending"
682 + " is not supported");
685 // Try to compress the content, but only if that is smaller.
686 TemporaryBuffer buf
= new TemporaryBuffer
.Heap(http
.postBuffer
);
688 GZIPOutputStream gzip
= new GZIPOutputStream(buf
);
689 out
.writeTo(gzip
, null);
691 if (out
.length() < buf
.length())
693 } catch (IOException err
) {
694 // Most likely caused by overflowing the buffer, meaning
695 // its larger if it were compressed. Don't compress.
701 conn
.setRequestProperty(HDR_CONTENT_ENCODING
, ENCODING_GZIP
);
702 conn
.setFixedLengthStreamingMode((int) buf
.length());
703 final OutputStream httpOut
= conn
.getOutputStream();
705 buf
.writeTo(httpOut
, null);
713 final int status
= HttpSupport
.response(conn
);
714 if (status
!= HttpURLConnection
.HTTP_OK
) {
715 throw new TransportException(uri
, status
+ " "
716 + conn
.getResponseMessage());
719 final String contentType
= conn
.getContentType();
720 if (!responseType
.equals(contentType
)) {
721 conn
.getInputStream().close();
722 throw wrongContentType(responseType
, contentType
);
725 httpIn
.add(openInputStream(conn
));
729 class HttpOutputStream
extends TemporaryBuffer
{
731 super(http
.postBuffer
);
735 protected OutputStream
overflow() throws IOException
{
737 conn
.setChunkedStreamingMode(0);
738 return conn
.getOutputStream();
742 class HttpInputStream
extends InputStream
{
743 private final UnionInputStream src
;
745 HttpInputStream(UnionInputStream httpIn
) {
749 private InputStream
self() throws IOException
{
751 // If we have no InputStreams available it means we must
752 // have written data previously to the service, but have
753 // not yet finished the HTTP request in order to get the
754 // response from the service. Ensure we get it now.
761 public int available() throws IOException
{
762 return self().available();
765 public int read() throws IOException
{
766 return self().read();
769 public int read(byte[] b
, int off
, int len
) throws IOException
{
770 return self().read(b
, off
, len
);
773 public long skip(long n
) throws IOException
{
774 return self().skip(n
);
777 public void close() throws IOException
{