org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportHttp.java

/*
 * Copyright (C) 2008-2010, Google Inc.
 * Copyright (C) 2008, Shawn O. Pearce <spearce@spearce.org>
 * Copyright (C) 2013, Matthias Sohn <matthias.sohn@sap.com>
 * Copyright (C) 2017, Thomas Wolf <thomas.wolf@paranor.ch>
 * and other copyright owners as documented in the project's IP log.
 *
 * This program and the accompanying materials are made available
 * under the terms of the Eclipse Distribution License v1.0 which
 * accompanies this distribution, is reproduced below, and is
 * available at http://www.eclipse.org/org/documents/edl-v10.php
 *
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or
 * without modification, are permitted provided that the following
 * conditions are met:
 *
 * - Redistributions of source code must retain the above copyright
 *   notice, this list of conditions and the following disclaimer.
 *
 * - Redistributions in binary form must reproduce the above
 *   copyright notice, this list of conditions and the following
 *   disclaimer in the documentation and/or other materials provided
 *   with the distribution.
 *
 * - Neither the name of the Eclipse Foundation, Inc. nor the
 *   names of its contributors may be used to endorse or promote
 *   products derived from this software without specific prior
 *   written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
 * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
 * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
 * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

package org.eclipse.jgit.transport;

import static org.eclipse.jgit.lib.Constants.HEAD;
import static org.eclipse.jgit.util.HttpSupport.ENCODING_GZIP;
import static org.eclipse.jgit.util.HttpSupport.ENCODING_X_GZIP;
import static org.eclipse.jgit.util.HttpSupport.HDR_ACCEPT;
import static org.eclipse.jgit.util.HttpSupport.HDR_ACCEPT_ENCODING;
import static org.eclipse.jgit.util.HttpSupport.HDR_CONTENT_ENCODING;
import static org.eclipse.jgit.util.HttpSupport.HDR_CONTENT_TYPE;
import static org.eclipse.jgit.util.HttpSupport.HDR_LOCATION;
import static org.eclipse.jgit.util.HttpSupport.HDR_PRAGMA;
import static org.eclipse.jgit.util.HttpSupport.HDR_USER_AGENT;
import static org.eclipse.jgit.util.HttpSupport.HDR_WWW_AUTHENTICATE;
import static org.eclipse.jgit.util.HttpSupport.METHOD_GET;
import static org.eclipse.jgit.util.HttpSupport.METHOD_POST;

import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.net.MalformedURLException;
import java.net.Proxy;
import java.net.ProxySelector;
import java.net.URISyntaxException;
import java.net.URL;
import java.text.MessageFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.EnumSet;
import java.util.HashSet;
import java.util.LinkedHashSet;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
import java.util.function.Supplier;
import java.util.zip.GZIPInputStream;
import java.util.zip.GZIPOutputStream;

import org.eclipse.jgit.errors.NoRemoteRepositoryException;
import org.eclipse.jgit.errors.NotSupportedException;
import org.eclipse.jgit.errors.PackProtocolException;
import org.eclipse.jgit.errors.TransportException;
import org.eclipse.jgit.internal.JGitText;
import org.eclipse.jgit.internal.storage.file.RefDirectory;
import org.eclipse.jgit.lib.Config;
import org.eclipse.jgit.lib.Constants;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.ObjectIdRef;
import org.eclipse.jgit.lib.ProgressMonitor;
import org.eclipse.jgit.lib.Ref;
import org.eclipse.jgit.lib.Repository;
import org.eclipse.jgit.lib.SymbolicRef;
import org.eclipse.jgit.transport.HttpAuthMethod.Type;
import org.eclipse.jgit.transport.http.HttpConnection;
import org.eclipse.jgit.util.HttpSupport;
import org.eclipse.jgit.util.IO;
import org.eclipse.jgit.util.RawParseUtils;
import org.eclipse.jgit.util.SystemReader;
import org.eclipse.jgit.util.TemporaryBuffer;
import org.eclipse.jgit.util.io.DisabledOutputStream;
import org.eclipse.jgit.util.io.UnionInputStream;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 * Transport over HTTP and FTP protocols.
 * <p>
 * If the transport is using HTTP and the remote HTTP service is Git-aware
 * (speaks the "smart-http protocol") this client will automatically take
 * advantage of the additional Git-specific HTTP extensions. If the remote
 * service does not support these extensions, the client will degrade to direct
 * file fetching.
 * <p>
 * If the remote (server side) repository does not have the specialized Git
 * support, object files are retrieved directly through standard HTTP GET (or
 * binary FTP GET) requests. This make it easy to serve a Git repository through
 * a standard web host provider that does not offer specific support for Git.
 *
 * @see WalkFetchConnection
 */
public class TransportHttp extends HttpTransport implements WalkTransport,
		PackTransport {

	private static final Logger LOG = LoggerFactory
			.getLogger(TransportHttp.class);

	private static final String SVC_UPLOAD_PACK = "git-upload-pack"; //$NON-NLS-1$

	private static final String SVC_RECEIVE_PACK = "git-receive-pack"; //$NON-NLS-1$

	private static final String MAX_REDIRECT_SYSTEM_PROPERTY = "http.maxRedirects"; //$NON-NLS-1$

	private static final int DEFAULT_MAX_REDIRECTS = 5;

	private static final int MAX_REDIRECTS = (new Supplier<Integer>() {

		@Override
		public Integer get() {
			String rawValue = SystemReader.getInstance()
					.getProperty(MAX_REDIRECT_SYSTEM_PROPERTY);
			Integer value = Integer.valueOf(DEFAULT_MAX_REDIRECTS);
			if (rawValue != null) {
				try {
					value = Integer.valueOf(Integer.parseUnsignedInt(rawValue));
				} catch (NumberFormatException e) {
					LOG.warn(MessageFormat.format(
							JGitText.get().invalidSystemProperty,
							MAX_REDIRECT_SYSTEM_PROPERTY, rawValue, value));
				}
			}
			return value;
		}
	}).get().intValue();

	/**
	 * Accept-Encoding header in the HTTP request
	 * (https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html).
	 *
	 * @since 4.6
	 */
	public enum AcceptEncoding {
		/**
		 * Do not specify an Accept-Encoding header. In most servers this
		 * results in the content being transmitted as-is.
		 */
		UNSPECIFIED,

		/**
		 * Accept gzip content encoding.
		 */
		GZIP
	}

	static final TransportProtocol PROTO_HTTP = new TransportProtocol() {
		private final String[] schemeNames = { "http", "https" }; //$NON-NLS-1$ //$NON-NLS-2$

		private final Set<String> schemeSet = Collections
				.unmodifiableSet(new LinkedHashSet<>(Arrays
						.asList(schemeNames)));

		@Override
		public String getName() {
			return JGitText.get().transportProtoHTTP;
		}

		@Override
		public Set<String> getSchemes() {
			return schemeSet;
		}

		@Override
		public Set<URIishField> getRequiredFields() {
			return Collections.unmodifiableSet(EnumSet.of(URIishField.HOST,
					URIishField.PATH));
		}

		@Override
		public Set<URIishField> getOptionalFields() {
			return Collections.unmodifiableSet(EnumSet.of(URIishField.USER,
					URIishField.PASS, URIishField.PORT));
		}

		@Override
		public int getDefaultPort() {
			return 80;
		}

		@Override
		public Transport open(URIish uri, Repository local, String remoteName)
				throws NotSupportedException {
			return new TransportHttp(local, uri);
		}

		@Override
		public Transport open(URIish uri) throws NotSupportedException {
			return new TransportHttp(uri);
		}
	};

	static final TransportProtocol PROTO_FTP = new TransportProtocol() {
		@Override
		public String getName() {
			return JGitText.get().transportProtoFTP;
		}

		@Override
		public Set<String> getSchemes() {
			return Collections.singleton("ftp"); //$NON-NLS-1$
		}

		@Override
		public Set<URIishField> getRequiredFields() {
			return Collections.unmodifiableSet(EnumSet.of(URIishField.HOST,
					URIishField.PATH));
		}

		@Override
		public Set<URIishField> getOptionalFields() {
			return Collections.unmodifiableSet(EnumSet.of(URIishField.USER,
					URIishField.PASS, URIishField.PORT));
		}

		@Override
		public int getDefaultPort() {
			return 21;
		}

		@Override
		public Transport open(URIish uri, Repository local, String remoteName)
				throws NotSupportedException {
			return new TransportHttp(local, uri);
		}
	};

	/**
	 * Config values for http.followRedirect
	 */
	private static enum HttpRedirectMode implements Config.ConfigEnum {

		/** Always follow redirects (up to the http.maxRedirects limit). */
		TRUE("true"), //$NON-NLS-1$
		/**
		 * Only follow redirects on the initial GET request. This is the
		 * default.
		 */
		INITIAL("initial"), //$NON-NLS-1$
		/** Never follow redirects. */
		FALSE("false"); //$NON-NLS-1$

		private final String configValue;

		private HttpRedirectMode(String configValue) {
			this.configValue = configValue;
		}

		@Override
		public String toConfigValue() {
			return configValue;
		}

		@Override
		public boolean matchConfigValue(String s) {
			return configValue.equals(s);
		}
	}

	private static class HttpConfig {
		final int postBuffer;

		final boolean sslVerify;

		final HttpRedirectMode followRedirects;

		final int maxRedirects;

		HttpConfig(final Config rc) {
			postBuffer = rc.getInt("http", "postbuffer", 1 * 1024 * 1024); //$NON-NLS-1$  //$NON-NLS-2$
			sslVerify = rc.getBoolean("http", "sslVerify", true); //$NON-NLS-1$ //$NON-NLS-2$
			followRedirects = rc.getEnum(HttpRedirectMode.values(), "http", //$NON-NLS-1$
					null, "followRedirects", HttpRedirectMode.INITIAL); //$NON-NLS-1$
			int redirectLimit = rc.getInt("http", "maxRedirects", //$NON-NLS-1$ //$NON-NLS-2$
					MAX_REDIRECTS);
			if (redirectLimit < 0) {
				redirectLimit = MAX_REDIRECTS;
			}
			maxRedirects = redirectLimit;
		}

		HttpConfig() {
			this(new Config());
		}
	}

	private URL baseUrl;

	private URL objectsUrl;

	final HttpConfig http;

	private final ProxySelector proxySelector;

	private boolean useSmartHttp = true;

	private HttpAuthMethod authMethod = HttpAuthMethod.Type.NONE.method(null);

	private Map<String, String> headers;

	TransportHttp(final Repository local, final URIish uri)
			throws NotSupportedException {
		super(local, uri);
		setURI(uri);
		http = local.getConfig().get(HttpConfig::new);
		proxySelector = ProxySelector.getDefault();
	}

	private URL toURL(URIish urish) throws MalformedURLException {
		String uriString = urish.toString();
		if (!uriString.endsWith("/")) { //$NON-NLS-1$
			uriString += '/';
		}
		return new URL(uriString);
	}

	/**
	 * @param uri
	 * @throws NotSupportedException
	 * @since 4.9
	 */
	protected void setURI(final URIish uri) throws NotSupportedException {
		try {
			baseUrl = toURL(uri);
			objectsUrl = new URL(baseUrl, "objects/"); //$NON-NLS-1$
		} catch (MalformedURLException e) {
			throw new NotSupportedException(MessageFormat.format(JGitText.get().invalidURL, uri), e);
		}
	}

	/**
	 * Create a minimal HTTP transport with default configuration values.
	 *
	 * @param uri
	 * @throws NotSupportedException
	 */
	TransportHttp(final URIish uri) throws NotSupportedException {
		super(uri);
		setURI(uri);
		http = new HttpConfig();
		proxySelector = ProxySelector.getDefault();
	}

	/**
	 * Toggle whether or not smart HTTP transport should be used.
	 * <p>
	 * This flag exists primarily to support backwards compatibility testing
	 * within a testing framework, there is no need to modify it in most
	 * applications.
	 *
	 * @param on
	 *            if {@code true} (default), smart HTTP is enabled.
	 */
	public void setUseSmartHttp(final boolean on) {
		useSmartHttp = on;
	}

	@Override
	public FetchConnection openFetch() throws TransportException,
			NotSupportedException {
		final String service = SVC_UPLOAD_PACK;
		try {
			final HttpConnection c = connect(service);
			final InputStream in = openInputStream(c);
			try {
				BaseConnection f;
				if (isSmartHttp(c, service)) {
					readSmartHeaders(in, service);
					f = new SmartHttpFetchConnection(in);
				} else {
					// Assume this server doesn't support smart HTTP fetch
					// and fall back on dumb object walking.
					f = newDumbConnection(in);
				}
				f.setPeerUserAgent(c.getHeaderField(HttpSupport.HDR_SERVER));
				return (FetchConnection) f;
			} finally {
				in.close();
			}
		} catch (NotSupportedException err) {
			throw err;
		} catch (TransportException err) {
			throw err;
		} catch (IOException err) {
			throw new TransportException(uri, JGitText.get().errorReadingInfoRefs, err);
		}
	}

	private WalkFetchConnection newDumbConnection(InputStream in)
			throws IOException, PackProtocolException {
		HttpObjectDB d = new HttpObjectDB(objectsUrl);
		BufferedReader br = toBufferedReader(in);
		Map<String, Ref> refs;
		try {
			refs = d.readAdvertisedImpl(br);
		} finally {
			br.close();
		}

		if (!refs.containsKey(HEAD)) {
			// If HEAD was not published in the info/refs file (it usually
			// is not there) download HEAD by itself as a loose file and do
			// the resolution by hand.
			//
			HttpConnection conn = httpOpen(
					METHOD_GET,
					new URL(baseUrl, HEAD),
					AcceptEncoding.GZIP);
			int status = HttpSupport.response(conn);
			switch (status) {
			case HttpConnection.HTTP_OK: {
				br = toBufferedReader(openInputStream(conn));
				try {
					String line = br.readLine();
					if (line != null && line.startsWith(RefDirectory.SYMREF)) {
						String target = line.substring(RefDirectory.SYMREF.length());
						Ref r = refs.get(target);
						if (r == null)
							r = new ObjectIdRef.Unpeeled(Ref.Storage.NEW, target, null);
						r = new SymbolicRef(HEAD, r);
						refs.put(r.getName(), r);
					} else if (line != null && ObjectId.isId(line)) {
						Ref r = new ObjectIdRef.Unpeeled(Ref.Storage.NETWORK,
								HEAD, ObjectId.fromString(line));
						refs.put(r.getName(), r);
					}
				} finally {
					br.close();
				}
				break;
			}

			case HttpConnection.HTTP_NOT_FOUND:
				break;

			default:
				throw new TransportException(uri, MessageFormat.format(
						JGitText.get().cannotReadHEAD, Integer.valueOf(status),
						conn.getResponseMessage()));
			}
		}

		WalkFetchConnection wfc = new WalkFetchConnection(this, d);
		wfc.available(refs);
		return wfc;
	}

	private BufferedReader toBufferedReader(InputStream in) {
		return new BufferedReader(new InputStreamReader(in, Constants.CHARSET));
	}

	@Override
	public PushConnection openPush() throws NotSupportedException,
			TransportException {
		final String service = SVC_RECEIVE_PACK;
		try {
			final HttpConnection c = connect(service);
			final InputStream in = openInputStream(c);
			try {
				if (isSmartHttp(c, service)) {
					return smartPush(service, c, in);
				} else if (!useSmartHttp) {
					final String msg = JGitText.get().smartHTTPPushDisabled;
					throw new NotSupportedException(msg);

				} else {
					final String msg = JGitText.get().remoteDoesNotSupportSmartHTTPPush;
					throw new NotSupportedException(msg);
				}
			} finally {
				in.close();
			}
		} catch (NotSupportedException err) {
			throw err;
		} catch (TransportException err) {
			throw err;
		} catch (IOException err) {
			throw new TransportException(uri, JGitText.get().errorReadingInfoRefs, err);
		}
	}

	private PushConnection smartPush(String service, HttpConnection c,
			InputStream in) throws IOException, TransportException {
		readSmartHeaders(in, service);
		SmartHttpPushConnection p = new SmartHttpPushConnection(in);
		p.setPeerUserAgent(c.getHeaderField(HttpSupport.HDR_SERVER));
		return p;
	}

	@Override
	public void close() {
		// No explicit connections are maintained.
	}

	/**
	 * Set additional headers on the HTTP connection
	 *
	 * @param headers
	 *            a map of name:values that are to be set as headers on the HTTP
	 *            connection
	 * @since 3.4
	 */
	public void setAdditionalHeaders(Map<String, String> headers) {
		this.headers = headers;
	}

	private HttpConnection connect(final String service)
			throws TransportException, NotSupportedException {
		URL u = getServiceURL(service);
		int authAttempts = 1;
		int redirects = 0;
		Collection<Type> ignoreTypes = null;
		for (;;) {
			try {
				final HttpConnection conn = httpOpen(METHOD_GET, u, AcceptEncoding.GZIP);
				if (useSmartHttp) {
					String exp = "application/x-" + service + "-advertisement"; //$NON-NLS-1$ //$NON-NLS-2$
					conn.setRequestProperty(HDR_ACCEPT, exp + ", */*"); //$NON-NLS-1$
				} else {
					conn.setRequestProperty(HDR_ACCEPT, "*/*"); //$NON-NLS-1$
				}
				final int status = HttpSupport.response(conn);
				switch (status) {
				case HttpConnection.HTTP_OK:
					// Check if HttpConnection did some authentication in the
					// background (e.g Kerberos/SPNEGO).
					// That may not work for streaming requests and jgit
					// explicit authentication would be required
					if (authMethod.getType() == HttpAuthMethod.Type.NONE
							&& conn.getHeaderField(HDR_WWW_AUTHENTICATE) != null)
						authMethod = HttpAuthMethod.scanResponse(conn, ignoreTypes);
					return conn;

				case HttpConnection.HTTP_NOT_FOUND:
					throw new NoRemoteRepositoryException(uri,
							MessageFormat.format(JGitText.get().uriNotFound, u));

				case HttpConnection.HTTP_UNAUTHORIZED:
					authMethod = HttpAuthMethod.scanResponse(conn, ignoreTypes);
					if (authMethod.getType() == HttpAuthMethod.Type.NONE)
						throw new TransportException(uri, MessageFormat.format(
								JGitText.get().authenticationNotSupported, uri));
					CredentialsProvider credentialsProvider = getCredentialsProvider();
					if (credentialsProvider == null)
						throw new TransportException(uri,
								JGitText.get().noCredentialsProvider);
					if (authAttempts > 1)
						credentialsProvider.reset(uri);
					if (3 < authAttempts
							|| !authMethod.authorize(uri, credentialsProvider)) {
						throw new TransportException(uri,
								JGitText.get().notAuthorized);
					}
					authAttempts++;
					continue;

				case HttpConnection.HTTP_FORBIDDEN:
					throw new TransportException(uri, MessageFormat.format(
							JGitText.get().serviceNotPermitted, service));

				case HttpConnection.HTTP_MOVED_PERM:
				case HttpConnection.HTTP_MOVED_TEMP:
				case HttpConnection.HTTP_SEE_OTHER:
				case HttpConnection.HTTP_11_MOVED_TEMP:
					// SEE_OTHER should actually never be sent by a git server,
					// and in general should occur only on POST requests. But it
					// doesn't hurt to accept it here as a redirect.
					if (http.followRedirects == HttpRedirectMode.FALSE) {
						throw new TransportException(MessageFormat.format(
								JGitText.get().redirectsOff, uri,
								Integer.valueOf(status)));
					}
					URIish newUri = redirect(conn.getHeaderField(HDR_LOCATION),
							Constants.INFO_REFS, redirects++);
					setURI(newUri);
					u = getServiceURL(service);
					authAttempts = 1;
					break;
				default:
					String err = status + " " + conn.getResponseMessage(); //$NON-NLS-1$
					throw new TransportException(uri, err);
				}
			} catch (NotSupportedException e) {
				throw e;
			} catch (TransportException e) {
				throw e;
			} catch (IOException e) {
				if (authMethod.getType() != HttpAuthMethod.Type.NONE) {
					if (ignoreTypes == null) {
						ignoreTypes = new HashSet<>();
					}

					ignoreTypes.add(authMethod.getType());

					// reset auth method & attempts for next authentication type
					authMethod = HttpAuthMethod.Type.NONE.method(null);
					authAttempts = 1;

					continue;
				}

				throw new TransportException(uri, MessageFormat.format(JGitText.get().cannotOpenService, service), e);
			}
		}
	}

	private URIish redirect(String location, String checkFor, int redirects)
			throws TransportException {
		if (location == null || location.isEmpty()) {
			throw new TransportException(MessageFormat.format(
					JGitText.get().redirectLocationMissing, uri, baseUrl));
		}
		if (redirects >= http.maxRedirects) {
			throw new TransportException(MessageFormat.format(
					JGitText.get().redirectLimitExceeded, uri,
					Integer.valueOf(http.maxRedirects), baseUrl, location));
		}
		try {
			if (!isValidRedirect(baseUrl, location, checkFor)) {
				throw new TransportException(
						MessageFormat.format(JGitText.get().redirectBlocked,
								uri, baseUrl, location));
			}
			location = location.substring(0, location.indexOf(checkFor));
			URIish result = new URIish(location);
			if (LOG.isInfoEnabled()) {
				LOG.info(MessageFormat.format(JGitText.get().redirectHttp, uri,
						Integer.valueOf(redirects), baseUrl, result));
			}
			return result;
		} catch (URISyntaxException e) {
			throw new TransportException(MessageFormat.format(
					JGitText.get().invalidRedirectLocation,
					uri, baseUrl, location), e);
		}
	}

	private boolean isValidRedirect(URL current, String next, String checkFor) {
		// Protocols must be the same, or current is "http" and next "https". We
		// do not follow redirects from https back to http.
		String oldProtocol = current.getProtocol().toLowerCase(Locale.ROOT);
		int schemeEnd = next.indexOf("://"); //$NON-NLS-1$
		if (schemeEnd < 0) {
			return false;
		}
		String newProtocol = next.substring(0, schemeEnd)
				.toLowerCase(Locale.ROOT);
		if (!oldProtocol.equals(newProtocol)) {
			if (!"https".equals(newProtocol)) { //$NON-NLS-1$
				return false;
			}
		}
		// git allows only rewriting the root, i.e., everything before INFO_REFS
		// or the service name
		if (next.indexOf(checkFor) < 0) {
			return false;
		}
		// Basically we should test here that whatever follows INFO_REFS is
		// unchanged. But since we re-construct the query part
		// anyway, it doesn't matter.
		return true;
	}

	private URL getServiceURL(final String service)
			throws NotSupportedException {
		try {
			final StringBuilder b = new StringBuilder();
			b.append(baseUrl);

			if (b.charAt(b.length() - 1) != '/') {
				b.append('/');
			}
			b.append(Constants.INFO_REFS);

			if (useSmartHttp) {
				b.append(b.indexOf("?") < 0 ? '?' : '&'); //$NON-NLS-1$
				b.append("service="); //$NON-NLS-1$
				b.append(service);
			}

			return new URL(b.toString());
		} catch (MalformedURLException e) {
			throw new NotSupportedException(MessageFormat.format(JGitText.get().invalidURL, uri), e);
		}
	}

	/**
	 * Open an HTTP connection, setting the accept-encoding request header to gzip.
	 *
	 * @param method HTTP request method
	 * @param u url of the HTTP connection
	 * @return the HTTP connection
	 * @throws IOException
	 * @since 3.3
	 * @deprecated use {@link #httpOpen(String, URL, AcceptEncoding)} instead.
	 */
	@Deprecated
	protected HttpConnection httpOpen(String method, URL u) throws IOException {
		return httpOpen(method, u, AcceptEncoding.GZIP);
	}

	/**
	 * Open an HTTP connection.
	 *
	 * @param method HTTP request method
	 * @param u url of the HTTP connection
	 * @param acceptEncoding accept-encoding header option
	 * @return the HTTP connection
	 * @throws IOException
	 * @since 4.6
	 */
	protected HttpConnection httpOpen(String method, URL u,
			AcceptEncoding acceptEncoding) throws IOException {
		if (method == null || u == null || acceptEncoding == null) {
			throw new NullPointerException();
		}

		final Proxy proxy = HttpSupport.proxyFor(proxySelector, u);
		HttpConnection conn = connectionFactory.create(u, proxy);

		if (!http.sslVerify && "https".equals(u.getProtocol())) { //$NON-NLS-1$
			HttpSupport.disableSslVerify(conn);
		}

		// We must do our own redirect handling to implement git rules and to
		// handle http->https redirects
		conn.setInstanceFollowRedirects(false);

		conn.setRequestMethod(method);
		conn.setUseCaches(false);
		if (acceptEncoding == AcceptEncoding.GZIP) {
			conn.setRequestProperty(HDR_ACCEPT_ENCODING, ENCODING_GZIP);
		}
		conn.setRequestProperty(HDR_PRAGMA, "no-cache"); //$NON-NLS-1$
		if (UserAgent.get() != null) {
			conn.setRequestProperty(HDR_USER_AGENT, UserAgent.get());
		}
		int timeOut = getTimeout();
		if (timeOut != -1) {
			int effTimeOut = timeOut * 1000;
			conn.setConnectTimeout(effTimeOut);
			conn.setReadTimeout(effTimeOut);
		}
		if (this.headers != null && !this.headers.isEmpty()) {
			for (Map.Entry<String, String> entry : this.headers.entrySet())
				conn.setRequestProperty(entry.getKey(), entry.getValue());
		}
		authMethod.configureRequest(conn);
		return conn;
	}

	final InputStream openInputStream(HttpConnection conn)
			throws IOException {
		InputStream input = conn.getInputStream();
		if (isGzipContent(conn))
			input = new GZIPInputStream(input);
		return input;
	}

	IOException wrongContentType(String expType, String actType) {
		final String why = MessageFormat.format(JGitText.get().expectedReceivedContentType, expType, actType);
		return new TransportException(uri, why);
	}

	private boolean isSmartHttp(final HttpConnection c, final String service) {
		final String expType = "application/x-" + service + "-advertisement"; //$NON-NLS-1$ //$NON-NLS-2$
		final String actType = c.getContentType();
		return expType.equals(actType);
	}

	private boolean isGzipContent(final HttpConnection c) {
		return ENCODING_GZIP.equals(c.getHeaderField(HDR_CONTENT_ENCODING))
				|| ENCODING_X_GZIP.equals(c.getHeaderField(HDR_CONTENT_ENCODING));
	}

	private void readSmartHeaders(final InputStream in, final String service)
			throws IOException {
		// A smart reply will have a '#' after the first 4 bytes, but
		// a dumb reply cannot contain a '#' until after byte 41. Do a
		// quick check to make sure its a smart reply before we parse
		// as a pkt-line stream.
		//
		final byte[] magic = new byte[5];
		IO.readFully(in, magic, 0, magic.length);
		if (magic[4] != '#') {
			throw new TransportException(uri, MessageFormat.format(
					JGitText.get().expectedPktLineWithService, RawParseUtils.decode(magic)));
		}

		final PacketLineIn pckIn = new PacketLineIn(new UnionInputStream(
				new ByteArrayInputStream(magic), in));
		final String exp = "# service=" + service; //$NON-NLS-1$
		final String act = pckIn.readString();
		if (!exp.equals(act)) {
			throw new TransportException(uri, MessageFormat.format(
					JGitText.get().expectedGot, exp, act));
		}

		while (pckIn.readString() != PacketLineIn.END) {
			// for now, ignore the remaining header lines
		}
	}

	class HttpObjectDB extends WalkRemoteObjectDatabase {
		private final URL httpObjectsUrl;

		HttpObjectDB(final URL b) {
			httpObjectsUrl = b;
		}

		@Override
		URIish getURI() {
			return new URIish(httpObjectsUrl);
		}

		@Override
		Collection<WalkRemoteObjectDatabase> getAlternates() throws IOException {
			try {
				return readAlternates(INFO_HTTP_ALTERNATES);
			} catch (FileNotFoundException err) {
				// Fall through.
			}

			try {
				return readAlternates(INFO_ALTERNATES);
			} catch (FileNotFoundException err) {
				// Fall through.
			}

			return null;
		}

		@Override
		WalkRemoteObjectDatabase openAlternate(final String location)
				throws IOException {
			return new HttpObjectDB(new URL(httpObjectsUrl, location));
		}

		@Override
		BufferedReader openReader(String path) throws IOException {
			// Line oriented readable content is likely to compress well.
			// Request gzip encoding.
			InputStream is = open(path, AcceptEncoding.GZIP).in;
			return new BufferedReader(new InputStreamReader(is, Constants.CHARSET));
		}

		@Override
		Collection<String> getPackNames() throws IOException {
			final Collection<String> packs = new ArrayList<>();
			try {
				final BufferedReader br = openReader(INFO_PACKS);
				try {
					for (;;) {
						final String s = br.readLine();
						if (s == null || s.length() == 0)
							break;
						if (!s.startsWith("P pack-") || !s.endsWith(".pack")) //$NON-NLS-1$ //$NON-NLS-2$
							throw invalidAdvertisement(s);
						packs.add(s.substring(2));
					}
					return packs;
				} finally {
					br.close();
				}
			} catch (FileNotFoundException err) {
				return packs;
			}
		}

		@Override
		FileStream open(final String path) throws IOException {
			return open(path, AcceptEncoding.UNSPECIFIED);
		}

		FileStream open(String path, AcceptEncoding acceptEncoding)
				throws IOException {
			final URL base = httpObjectsUrl;
			final URL u = new URL(base, path);
			final HttpConnection c = httpOpen(METHOD_GET, u, acceptEncoding);
			switch (HttpSupport.response(c)) {
			case HttpConnection.HTTP_OK:
				final InputStream in = openInputStream(c);
				// If content is being gzipped and then transferred, the content
				// length in the header is the zipped content length, not the
				// actual content length.
				if (!isGzipContent(c)) {
					final int len = c.getContentLength();
					return new FileStream(in, len);
				}
				return new FileStream(in);
			case HttpConnection.HTTP_NOT_FOUND:
				throw new FileNotFoundException(u.toString());
			default:
				throw new IOException(u.toString() + ": " //$NON-NLS-1$
						+ HttpSupport.response(c) + " " //$NON-NLS-1$
						+ c.getResponseMessage());
			}
		}

		Map<String, Ref> readAdvertisedImpl(final BufferedReader br)
				throws IOException, PackProtocolException {
			final TreeMap<String, Ref> avail = new TreeMap<>();
			for (;;) {
				String line = br.readLine();
				if (line == null)
					break;

				final int tab = line.indexOf('\t');
				if (tab < 0)
					throw invalidAdvertisement(line);

				String name;
				final ObjectId id;

				name = line.substring(tab + 1);
				id = ObjectId.fromString(line.substring(0, tab));
				if (name.endsWith("^{}")) { //$NON-NLS-1$
					name = name.substring(0, name.length() - 3);
					final Ref prior = avail.get(name);
					if (prior == null)
						throw outOfOrderAdvertisement(name);

					if (prior.getPeeledObjectId() != null)
						throw duplicateAdvertisement(name + "^{}"); //$NON-NLS-1$

					avail.put(name, new ObjectIdRef.PeeledTag(
							Ref.Storage.NETWORK, name,
							prior.getObjectId(), id));
				} else {
					Ref prior = avail.put(name, new ObjectIdRef.PeeledNonTag(
							Ref.Storage.NETWORK, name, id));
					if (prior != null)
						throw duplicateAdvertisement(name);
				}
			}
			return avail;
		}

		private PackProtocolException outOfOrderAdvertisement(final String n) {
			return new PackProtocolException(MessageFormat.format(JGitText.get().advertisementOfCameBefore, n, n));
		}

		private PackProtocolException invalidAdvertisement(final String n) {
			return new PackProtocolException(MessageFormat.format(JGitText.get().invalidAdvertisementOf, n));
		}

		private PackProtocolException duplicateAdvertisement(final String n) {
			return new PackProtocolException(MessageFormat.format(JGitText.get().duplicateAdvertisementsOf, n));
		}

		@Override
		void close() {
			// We do not maintain persistent connections.
		}
	}

	class SmartHttpFetchConnection extends BasePackFetchConnection {
		private MultiRequestService svc;

		SmartHttpFetchConnection(final InputStream advertisement)
				throws TransportException {
			super(TransportHttp.this);
			statelessRPC = true;

			init(advertisement, DisabledOutputStream.INSTANCE);
			outNeedsEnd = false;
			readAdvertisedRefs();
		}

		@Override
		protected void doFetch(final ProgressMonitor monitor,
				final Collection<Ref> want, final Set<ObjectId> have,
				final OutputStream outputStream) throws TransportException {
			try {
				svc = new MultiRequestService(SVC_UPLOAD_PACK);
				init(svc.getInputStream(), svc.getOutputStream());
				super.doFetch(monitor, want, have, outputStream);
			} finally {
				svc = null;
			}
		}

		@Override
		protected void onReceivePack() {
			svc.finalRequest = true;
		}
	}

	class SmartHttpPushConnection extends BasePackPushConnection {
		SmartHttpPushConnection(final InputStream advertisement)
				throws TransportException {
			super(TransportHttp.this);
			statelessRPC = true;

			init(advertisement, DisabledOutputStream.INSTANCE);
			outNeedsEnd = false;
			readAdvertisedRefs();
		}

		@Override
		protected void doPush(final ProgressMonitor monitor,
				final Map<String, RemoteRefUpdate> refUpdates,
				OutputStream outputStream) throws TransportException {
			final Service svc = new MultiRequestService(SVC_RECEIVE_PACK);
			init(svc.getInputStream(), svc.getOutputStream());
			super.doPush(monitor, refUpdates, outputStream);
		}
	}

	/** Basic service for sending and receiving HTTP requests. */
	abstract class Service {
		protected final String serviceName;

		protected final String requestType;

		protected final String responseType;

		protected HttpConnection conn;

		protected HttpOutputStream out;

		protected final HttpExecuteStream execute;

		final UnionInputStream in;

		Service(String serviceName) {
			this.serviceName = serviceName;
			this.requestType = "application/x-" + serviceName + "-request"; //$NON-NLS-1$ //$NON-NLS-2$
			this.responseType = "application/x-" + serviceName + "-result"; //$NON-NLS-1$ //$NON-NLS-2$

			this.out = new HttpOutputStream();
			this.execute = new HttpExecuteStream();
			this.in = new UnionInputStream(execute);
		}

		void openStream() throws IOException {
			conn = httpOpen(METHOD_POST, new URL(baseUrl, serviceName),
					AcceptEncoding.GZIP);
			conn.setInstanceFollowRedirects(false);
			conn.setDoOutput(true);
			conn.setRequestProperty(HDR_CONTENT_TYPE, requestType);
			conn.setRequestProperty(HDR_ACCEPT, responseType);
		}

		void sendRequest() throws IOException {
			// Try to compress the content, but only if that is smaller.
			TemporaryBuffer buf = new TemporaryBuffer.Heap(http.postBuffer);
			try {
				GZIPOutputStream gzip = new GZIPOutputStream(buf);
				out.writeTo(gzip, null);
				gzip.close();
				if (out.length() < buf.length())
					buf = out;
			} catch (IOException err) {
				// Most likely caused by overflowing the buffer, meaning
				// its larger if it were compressed. Don't compress.
				buf = out;
			}

			int redirects = 0;
			for (;;) {
				openStream();
				if (buf != out) {
					conn.setRequestProperty(HDR_CONTENT_ENCODING, ENCODING_GZIP);
				}
				conn.setFixedLengthStreamingMode((int) buf.length());
				try (OutputStream httpOut = conn.getOutputStream()) {
					buf.writeTo(httpOut, null);
				}

				if (http.followRedirects == HttpRedirectMode.TRUE) {
					final int status = HttpSupport.response(conn);
					switch (status) {
					case HttpConnection.HTTP_MOVED_PERM:
					case HttpConnection.HTTP_MOVED_TEMP:
					case HttpConnection.HTTP_11_MOVED_TEMP:
						// SEE_OTHER after a POST doesn't make sense for a git
						// server, so we don't handle it here and thus we'll
						// report an error in openResponse() later on.
						URIish newUri = redirect(
								conn.getHeaderField(HDR_LOCATION),
								'/' + serviceName, redirects++);
						try {
							baseUrl = toURL(newUri);
						} catch (MalformedURLException e) {
							throw new TransportException(MessageFormat.format(
									JGitText.get().invalidRedirectLocation,
									uri, baseUrl, newUri), e);
						}
						continue;
					default:
						break;
					}
				}
				break;
			}
		}

		void openResponse() throws IOException {
			final int status = HttpSupport.response(conn);
			if (status != HttpConnection.HTTP_OK) {
				throw new TransportException(uri, status + " " //$NON-NLS-1$
						+ conn.getResponseMessage());
			}

			final String contentType = conn.getContentType();
			if (!responseType.equals(contentType)) {
				conn.getInputStream().close();
				throw wrongContentType(responseType, contentType);
			}
		}

		HttpOutputStream getOutputStream() {
			return out;
		}

		InputStream getInputStream() {
			return in;
		}

		abstract void execute() throws IOException;

		class HttpExecuteStream extends InputStream {
			@Override
			public int read() throws IOException {
				execute();
				return -1;
			}

			@Override
			public int read(byte[] b, int off, int len) throws IOException {
				execute();
				return -1;
			}

			@Override
			public long skip(long n) throws IOException {
				execute();
				return 0;
			}
		}

		class HttpOutputStream extends TemporaryBuffer {
			HttpOutputStream() {
				super(http.postBuffer);
			}

			@Override
			protected OutputStream overflow() throws IOException {
				openStream();
				conn.setChunkedStreamingMode(0);
				return conn.getOutputStream();
			}
		}
	}

	/**
	 * State required to speak multiple HTTP requests with the remote.
	 * <p>
	 * A service wrapper provides a normal looking InputStream and OutputStream
	 * pair which are connected via HTTP to the named remote service. Writing to
	 * the OutputStream is buffered until either the buffer overflows, or
	 * reading from the InputStream occurs. If overflow occurs HTTP/1.1 and its
	 * chunked transfer encoding is used to stream the request data to the
	 * remote service. If the entire request fits in the memory buffer, the
	 * older HTTP/1.0 standard and a fixed content length is used instead.
	 * <p>
	 * It is an error to attempt to read without there being outstanding data
	 * ready for transmission on the OutputStream.
	 * <p>
	 * No state is preserved between write-read request pairs. The caller is
	 * responsible for replaying state vector information as part of the request
	 * data written to the OutputStream. Any session HTTP cookies may or may not
	 * be preserved between requests, it is left up to the JVM's implementation
	 * of the HTTP client.
	 */
	class MultiRequestService extends Service {
		boolean finalRequest;

		MultiRequestService(final String serviceName) {
			super(serviceName);
		}

		/** Keep opening send-receive pairs to the given URI. */
		@Override
		void execute() throws IOException {
			out.close();

			if (conn == null) {
				if (out.length() == 0) {
					// Request output hasn't started yet, but more data is being
					// requested. If there is no request data buffered and the
					// final request was already sent, do nothing to ensure the
					// caller is shown EOF on the InputStream; otherwise an
					// programming error has occurred within this module.
					if (finalRequest)
						return;
					throw new TransportException(uri,
							JGitText.get().startingReadStageWithoutWrittenRequestDataPendingIsNotSupported);
				}

				sendRequest();
			}

			out.reset();

			openResponse();

			in.add(openInputStream(conn));
			if (!finalRequest)
				in.add(execute);
			conn = null;
		}
	}

	/** Service for maintaining a single long-poll connection. */
	class LongPollService extends Service {
		/**
		 * @param serviceName
		 */
		LongPollService(String serviceName) {
			super(serviceName);
		}

		/** Only open one send-receive request. */
		@Override
		void execute() throws IOException {
			out.close();
			if (conn == null)
				sendRequest();
			openResponse();
			in.add(openInputStream(conn));
		}
	}
}