Support multiple mirror URLs for external repos This change improves upon 4c67807964e37cfd55bbcda4c6374fcc480bcecc. - A urls attribute has been added to the native workspace rules, with the exception of maven_jar and git_repository. The Skylark repository API also supports multiple URLs now. - The earlier mirrors in the list are preferred. Failover will happen automatically in parallel. - The first 32kB of data is checked before choosing a mirror in order to evade captive portals. - If one's Internet goes down or a download times out, then the download will resume automatically where it left off, provided the server supports RFC7233 for that particular file. Please note that GitHub does not support this for archive snapshots. Files should always be mirrored to a CDN, e.g. GCS, because they support this. - A semaphore is now used on downloads so only 8 can happen at once. Fixes #1814 Fixes #2131 Fixes #2008 Fixes #1968 Fixes #1717 Fixes #943 Wont fix #1194 Fixes tensorflow/tensorflow#5933 Fixes tensorflow/tensorflow#5924 Fixes tensorflow/tensorflow#5924 Fixes tensorflow/tensorflow#5432 See #1607 See #821 See tensorflow/tensorflow#5080 See tensorflow/tensorflow#5029 See tensorflow/tensorflow#4583 See tensorflow/tensorflow#4058 RELNOTES: A urls attribute has been added to repository rules to support multiple mirror URLs for reliably downloading files. -- MOS_MIGRATED_REVID=140495736
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/commands/FetchCommand.java b/src/main/java/com/google/devtools/build/lib/bazel/commands/FetchCommand.java index 0d78adc..47fa3f3 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/commands/FetchCommand.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/commands/FetchCommand.java
@@ -123,7 +123,9 @@ // Throw away the result. } }); - } catch (QueryException | InterruptedException e) { + } catch (InterruptedException e) { + return ExitCode.COMMAND_LINE_ERROR; + } catch (QueryException e) { // Keep consistent with reportBuildFileError() env.getReporter().handle(Event.error(e.getMessage())); return ExitCode.COMMAND_LINE_ERROR;
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/GitCloner.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/GitCloner.java index 12ff396..9b347ae 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/repository/GitCloner.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/GitCloner.java
@@ -14,6 +14,7 @@ package com.google.devtools.build.lib.bazel.repository; +import com.google.common.base.Ascii; import com.google.devtools.build.lib.bazel.repository.downloader.ProxyHelper; import com.google.devtools.build.lib.events.EventHandler; import com.google.devtools.build.lib.packages.Rule; @@ -25,7 +26,11 @@ import com.google.devtools.build.lib.vfs.Path; import com.google.devtools.build.skyframe.SkyFunctionException.Transience; import com.google.devtools.build.skyframe.SkyValue; - +import java.io.IOException; +import java.net.URL; +import java.util.Map; +import java.util.Objects; +import java.util.Set; import org.eclipse.jgit.api.Git; import org.eclipse.jgit.api.Status; import org.eclipse.jgit.api.errors.GitAPIException; @@ -39,11 +44,6 @@ import org.eclipse.jgit.storage.file.FileRepositoryBuilder; import org.eclipse.jgit.transport.NetRCCredentialsProvider; -import java.io.IOException; -import java.util.Map; -import java.util.Objects; -import java.util.Set; - /** * Clones a Git repository, checks out the provided branch, tag, or commit, and * clones submodules if specified. @@ -123,9 +123,9 @@ } // Setup proxy if remote is http or https - if (descriptor.remote != null && descriptor.remote.startsWith("http")) { + if (descriptor.remote != null && Ascii.toLowerCase(descriptor.remote).startsWith("http")) { try { - ProxyHelper.createProxyIfNeeded(descriptor.remote, clientEnvironment); + new ProxyHelper(clientEnvironment).createProxyIfNeeded(new URL(descriptor.remote)); } catch (IOException ie) { throw new RepositoryFunctionException(ie, Transience.TRANSIENT); }
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/cache/RepositoryCache.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/cache/RepositoryCache.java index 64cdfdf..bd743b3 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/repository/cache/RepositoryCache.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/cache/RepositoryCache.java
@@ -15,6 +15,7 @@ package com.google.devtools.build.lib.bazel.repository.cache; import com.google.common.base.Preconditions; +import com.google.common.base.Strings; import com.google.common.hash.HashFunction; import com.google.common.hash.Hasher; import com.google.common.hash.Hashing; @@ -48,7 +49,7 @@ } public boolean isValid(@Nullable String checksum) { - return checksum != null && checksum.matches(regexp); + return !Strings.isNullOrEmpty(checksum) && checksum.matches(regexp); } public Path getCachePath(Path parentDirectory) { @@ -58,7 +59,7 @@ public Hasher newHasher() { return hashFunction.newHasher(); } - + @Override public String toString() { return stringRepr;
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/BUILD b/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/BUILD index 82f39cc..c4ebcf0 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/BUILD +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/BUILD
@@ -12,9 +12,11 @@ srcs = glob(["*.java"]), deps = [ "//src/main/java/com/google/devtools/build/lib:build-base", + "//src/main/java/com/google/devtools/build/lib:concurrent", "//src/main/java/com/google/devtools/build/lib:events", "//src/main/java/com/google/devtools/build/lib:packages-internal", "//src/main/java/com/google/devtools/build/lib:syntax", + "//src/main/java/com/google/devtools/build/lib:util", "//src/main/java/com/google/devtools/build/lib:vfs", "//src/main/java/com/google/devtools/build/lib/bazel/repository/cache", "//src/main/java/com/google/devtools/build/skyframe",
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/HashInputStream.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/HashInputStream.java new file mode 100644 index 0000000..ee7b1bd --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/HashInputStream.java
@@ -0,0 +1,93 @@ +// Copyright 2016 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.repository.downloader; + +import com.google.common.hash.HashCode; +import com.google.common.hash.HashFunction; +import com.google.common.hash.Hasher; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadCompatible; +import java.io.IOException; +import java.io.InputStream; +import javax.annotation.Nullable; +import javax.annotation.WillCloseWhenClosed; + +/** + * Input stream that guarantees its contents matches a hash code. + * + * <p>The actual checksum is computed gradually as the input is read. If it doesn't match, then an + * {@link IOException} will be thrown when {@link #close()} is called, or when any read method is + * called that detects the end of stream. This error will be thrown multiple times if these methods + * are called again for some reason. + * + * <p>This class is not thread safe, but it is safe to message pass this object between threads. + */ +@ThreadCompatible +final class HashInputStream extends InputStream { + + private final InputStream delegate; + private final Hasher hasher; + private final HashCode code; + @Nullable private volatile HashCode actual; + + HashInputStream( + @WillCloseWhenClosed InputStream delegate, HashFunction function, HashCode code) { + this.delegate = delegate; + this.hasher = function.newHasher(); + this.code = code; + } + + @Override + public int read() throws IOException { + int result = delegate.read(); + if (result == -1) { + check(); + } else { + hasher.putByte((byte) result); + } + return result; + } + + @Override + public int read(byte[] buffer, int offset, int length) throws IOException { + int amount = delegate.read(buffer, offset, length); + if (amount == -1) { + check(); + } else { + hasher.putBytes(buffer, offset, amount); + } + return amount; + } + + @Override + public int available() throws IOException { + return delegate.available(); + } + + @Override + public void close() throws IOException { + delegate.close(); + check(); + } + + private void check() throws IOException { + if (actual == null) { + actual = hasher.hash(); + } + if (!code.equals(actual)) { + throw new UnrecoverableHttpException( + String.format("Checksum was %s but wanted %s", actual, code)); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpConnector.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpConnector.java index c055f6f..ba2c64f 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpConnector.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpConnector.java
@@ -14,89 +14,109 @@ package com.google.devtools.build.lib.bazel.repository.downloader; -import static com.google.common.base.MoreObjects.firstNonNull; -import static com.google.common.base.Preconditions.checkNotNull; -import static com.google.common.base.Strings.nullToEmpty; - -import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Ascii; +import com.google.common.base.Strings; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; +import com.google.common.io.ByteStreams; import com.google.common.math.IntMath; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe; import com.google.devtools.build.lib.events.Event; import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.util.Sleeper; import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; import java.io.InterruptedIOException; import java.net.HttpURLConnection; -import java.net.MalformedURLException; -import java.net.Proxy; import java.net.SocketTimeoutException; -import java.net.URI; -import java.net.URISyntaxException; import java.net.URL; +import java.net.URLConnection; import java.net.UnknownHostException; import java.util.ArrayList; import java.util.List; -import java.util.concurrent.TimeUnit; -import java.util.zip.GZIPInputStream; +import java.util.Locale; +import java.util.Map; import javax.annotation.Nullable; +import javax.annotation.WillClose; -/** Utility class for connecting to HTTP servers for downloading files. */ -final class HttpConnector { +/** + * Class for establishing connections to HTTP servers for downloading files. + * + * <p>This class must be used in conjunction with {@link HttpConnectorMultiplexer}. + * + * <p>Instances are thread safe and can be reused. + */ +@ThreadSafe +class HttpConnector { private static final int MAX_RETRIES = 8; - private static final int MAX_REDIRECTS = 20; + private static final int MAX_REDIRECTS = 40; private static final int MIN_RETRY_DELAY_MS = 100; - private static final int CONNECT_TIMEOUT_MS = 1000; + private static final int MIN_CONNECT_TIMEOUT_MS = 1000; private static final int MAX_CONNECT_TIMEOUT_MS = 10000; private static final int READ_TIMEOUT_MS = 20000; private static final ImmutableSet<String> COMPRESSED_EXTENSIONS = ImmutableSet.of("bz2", "gz", "jar", "tgz", "war", "xz", "zip"); - /** - * Connects to HTTP (or file) URL with GET request and lazily returns payload. - * - * <p>This routine supports gzip, redirects, retries, and exponential backoff. It's designed to - * recover fast from transient errors. However please note that this this reliability magic only - * applies to the connection and header reading phase. - * - * @param url URL to download, which can be file, http, or https - * @param proxy HTTP proxy to use or {@link Proxy#NO_PROXY} if none is desired - * @param eventHandler Bazel event handler for reporting real-time progress on retries - * @throws IOException if response returned ≥400 after max retries or ≥300 after max redirects - * @throws InterruptedException if thread is being cast into oblivion - */ - static InputStream connect( - URL url, Proxy proxy, EventHandler eventHandler) - throws IOException, InterruptedException { - checkNotNull(proxy); - checkNotNull(eventHandler); - if (isProtocol(url, "file")) { - return url.openConnection().getInputStream(); + private final Locale locale; + private final EventHandler eventHandler; + private final ProxyHelper proxyHelper; + private final Sleeper sleeper; + + HttpConnector( + Locale locale, EventHandler eventHandler, ProxyHelper proxyHelper, Sleeper sleeper) { + this.locale = locale; + this.eventHandler = eventHandler; + this.proxyHelper = proxyHelper; + this.sleeper = sleeper; + } + + URLConnection connect( + URL originalUrl, ImmutableMap<String, String> requestHeaders) + throws IOException { + if (Thread.interrupted()) { + throw new InterruptedIOException(); } - if (!isHttp(url)) { - throw new IOException("Protocol must be http, https, or file"); + URL url = originalUrl; + if (HttpUtils.isProtocol(url, "file")) { + return url.openConnection(); } List<Throwable> suppressions = new ArrayList<>(); int retries = 0; int redirects = 0; - int connectTimeout = CONNECT_TIMEOUT_MS; + int connectTimeout = MIN_CONNECT_TIMEOUT_MS; while (true) { HttpURLConnection connection = null; try { - connection = (HttpURLConnection) url.openConnection(proxy); - if (!COMPRESSED_EXTENSIONS.contains(getExtension(url.getPath()))) { - connection.setRequestProperty("Accept-Encoding", "gzip"); + connection = (HttpURLConnection) + url.openConnection(proxyHelper.createProxyIfNeeded(url)); + boolean isAlreadyCompressed = + COMPRESSED_EXTENSIONS.contains(HttpUtils.getExtension(url.getPath())) + || COMPRESSED_EXTENSIONS.contains(HttpUtils.getExtension(originalUrl.getPath())); + for (Map.Entry<String, String> entry : requestHeaders.entrySet()) { + if (isAlreadyCompressed && Ascii.equalsIgnoreCase(entry.getKey(), "Accept-Encoding")) { + // We're not going to ask for compression if we're downloading a file that already + // appears to be compressed. + continue; + } + connection.setRequestProperty(entry.getKey(), entry.getValue()); } connection.setConnectTimeout(connectTimeout); + // The read timeout is always large because it stays in effect after this method. connection.setReadTimeout(READ_TIMEOUT_MS); + // Java tries to abstract HTTP error responses for us. We don't want that. So we're going + // to try and undo any IOException that doesn't appepar to be a legitimate I/O exception. int code; try { connection.connect(); code = connection.getResponseCode(); } catch (FileNotFoundException ignored) { code = connection.getResponseCode(); + } catch (UnknownHostException e) { + String message = "Unknown host: " + e.getMessage(); + eventHandler.handle(Event.progress(message)); + throw new UnrecoverableHttpException(message); } catch (IllegalArgumentException e) { // This will happen if the user does something like specify a port greater than 2^16-1. throw new UnrecoverableHttpException(e.getMessage()); @@ -106,163 +126,115 @@ } code = connection.getResponseCode(); } - if (code == 200) { - return getInputStream(connection); - } else if (code == 301 || code == 302) { + // 206 means partial content and only happens if caller specified Range. See RFC7233 § 4.1. + if (code == 200 || code == 206) { + return connection; + } else if (code == 301 || code == 302 || code == 307) { readAllBytesAndClose(connection.getInputStream()); if (++redirects == MAX_REDIRECTS) { + eventHandler.handle(Event.progress("Redirect loop detected in " + originalUrl)); throw new UnrecoverableHttpException("Redirect loop detected"); } - url = getLocation(connection); - } else if (code < 500) { + url = HttpUtils.getLocation(connection); + if (code == 301) { + originalUrl = url; + } + } else if (code == 403) { + // jart@ has noticed BitBucket + Amazon AWS downloads frequently flake with this code. + throw new IOException(describeHttpResponse(connection)); + } else if (code == 408) { + // The 408 (Request Timeout) status code indicates that the server did not receive a + // complete request message within the time that it was prepared to wait. Server SHOULD + // send the "close" connection option (Section 6.1 of [RFC7230]) in the response, since + // 408 implies that the server has decided to close the connection rather than continue + // waiting. If the client has an outstanding request in transit, the client MAY repeat + // that request on a new connection. Quoth RFC7231 § 6.5.7 + throw new IOException(describeHttpResponse(connection)); + } else if (code < 500 // 4xx means client seems to have erred quoth RFC7231 § 6.5 + || code == 501 // Server doesn't support function quoth RFC7231 § 6.6.2 + || code == 502 // Host not configured on server cf. RFC7231 § 6.6.3 + || code == 505) { // Server refuses to support version quoth RFC7231 § 6.6.6 + // This is a permanent error so we're not going to retry. readAllBytesAndClose(connection.getErrorStream()); throw new UnrecoverableHttpException(describeHttpResponse(connection)); } else { + // However we will retry on some 5xx errors, particularly 500 and 503. throw new IOException(describeHttpResponse(connection)); } - } catch (InterruptedIOException e) { - throw new InterruptedException(); } catch (UnrecoverableHttpException e) { throw e; - } catch (UnknownHostException e) { - throw new IOException("Unknown host: " + e.getMessage()); } catch (IOException e) { if (connection != null) { + // If we got here, it means we might not have consumed the entire payload of the + // response, if any. So we're going to force this socket to disconnect and not be + // reused. This is particularly important if multiple threads end up establishing + // connections to multiple mirrors simultaneously for a large file. We don't want to + // download that large file twice. connection.disconnect(); } + // We don't respect the Retry-After header (RFC7231 § 7.1.3) because it's rarely used and + // tends to be too conservative when it is. We're already being good citizens by using + // exponential backoff. Furthermore RFC law didn't use the magic word "MUST". + int timeout = IntMath.pow(2, retries) * MIN_RETRY_DELAY_MS; if (e instanceof SocketTimeoutException) { + eventHandler.handle(Event.progress("Timeout connecting to " + url)); connectTimeout = Math.min(connectTimeout * 2, MAX_CONNECT_TIMEOUT_MS); + // If we got connect timeout, we're already doing exponential backoff, so no point + // in sleeping too. + timeout = 1; + } else if (e instanceof InterruptedIOException) { + // Please note that SocketTimeoutException is a subtype of InterruptedIOException. + throw e; } if (++retries == MAX_RETRIES) { + if (!(e instanceof SocketTimeoutException)) { + eventHandler + .handle(Event.progress(format("Error connecting to %s: %s", url, e.getMessage()))); + } for (Throwable suppressed : suppressions) { e.addSuppressed(suppressed); } throw e; } + // Java 7 allows us to create a tree of all errors that led to the ultimate failure. suppressions.add(e); - int timeout = IntMath.pow(2, retries) * MIN_RETRY_DELAY_MS; - eventHandler.handle(Event.progress( - String.format("Failed to connect to %s trying again in %,dms: %s", - url, timeout, e))); - TimeUnit.MILLISECONDS.sleep(timeout); + eventHandler.handle( + Event.progress(format("Failed to connect to %s trying again in %,dms", url, timeout))); + url = originalUrl; + try { + sleeper.sleepMillis(timeout); + } catch (InterruptedException translated) { + throw new InterruptedIOException(); + } } catch (RuntimeException e) { if (connection != null) { connection.disconnect(); } + eventHandler.handle(Event.progress(format("Unknown error connecting to %s: %s", url, e))); throw e; } } } - private static String describeHttpResponse(HttpURLConnection connection) throws IOException { - return String.format( - "%s returned %s %s", + private String describeHttpResponse(HttpURLConnection connection) throws IOException { + return format( + "%s returned %d %s", connection.getRequestMethod(), connection.getResponseCode(), - nullToEmpty(connection.getResponseMessage())); + Strings.nullToEmpty(connection.getResponseMessage())); } - private static void readAllBytesAndClose(@Nullable InputStream stream) throws IOException { + private String format(String format, Object... args) { + return String.format(locale, format, args); + } + + // Exhausts all bytes in an HTTP to make it easier for Java infrastructure to reuse sockets. + private static void readAllBytesAndClose( + @WillClose @Nullable InputStream stream) + throws IOException { if (stream != null) { - // TODO: Replace with ByteStreams#exhaust when Guava 20 comes out. - byte[] buf = new byte[8192]; - while (stream.read(buf) != -1) {} + ByteStreams.exhaust(stream); stream.close(); } } - - private static InputStream getInputStream(HttpURLConnection connection) throws IOException { - // See RFC2616 § 3.5 and § 14.11 - switch (firstNonNull(connection.getContentEncoding(), "identity")) { - case "identity": - return connection.getInputStream(); - case "gzip": - case "x-gzip": - // Some web servers will send Content-Encoding: gzip even when we didn't request it, iff - // the file is a .gz file. - if (connection.getURL().getPath().endsWith(".gz")) { - return connection.getInputStream(); - } else { - return new GZIPInputStream(connection.getInputStream()); - } - default: - throw new UnrecoverableHttpException( - "Unsupported and unrequested Content-Encoding: " + connection.getContentEncoding()); - } - } - - @VisibleForTesting - static URL getLocation(HttpURLConnection connection) throws IOException { - String newLocation = connection.getHeaderField("Location"); - if (newLocation == null) { - throw new IOException("Remote redirect missing Location."); - } - URL result = mergeUrls(URI.create(newLocation), connection.getURL()); - if (!isHttp(result)) { - throw new IOException("Bad Location: " + newLocation); - } - return result; - } - - private static URL mergeUrls(URI preferred, URL original) throws IOException { - // If the Location value provided in a 3xx (Redirection) response does not have a fragment - // component, a user agent MUST process the redirection as if the value inherits the fragment - // component of the URI reference used to generate the request target (i.e., the redirection - // inherits the original reference's fragment, if any). Quoth RFC7231 § 7.1.2 - String protocol = firstNonNull(preferred.getScheme(), original.getProtocol()); - String userInfo = preferred.getUserInfo(); - String host = preferred.getHost(); - int port; - if (host == null) { - host = original.getHost(); - port = original.getPort(); - userInfo = original.getUserInfo(); - } else { - port = preferred.getPort(); - if (userInfo == null - && host.equals(original.getHost()) - && port == original.getPort()) { - userInfo = original.getUserInfo(); - } - } - String path = preferred.getPath(); - String query = preferred.getQuery(); - String fragment = preferred.getFragment(); - if (fragment == null) { - fragment = original.getRef(); - } - URL result; - try { - result = new URI(protocol, userInfo, host, port, path, query, fragment).toURL(); - } catch (URISyntaxException | MalformedURLException e) { - throw new IOException("Could not merge " + preferred + " into " + original); - } - return result; - } - - private static boolean isHttp(URL url) { - return isProtocol(url, "http") || isProtocol(url, "https"); - } - - private static boolean isProtocol(URL url, String protocol) { - // An implementation should accept uppercase letters as equivalent to lowercase in scheme names - // (e.g., allow "HTTP" as well as "http") for the sake of robustness. Quoth RFC3986 § 3.1 - return Ascii.equalsIgnoreCase(protocol, url.getProtocol()); - } - - private static String getExtension(String path) { - int index = path.lastIndexOf('.'); - if (index == -1) { - return ""; - } - return path.substring(index + 1); - } - - private static final class UnrecoverableHttpException extends IOException { - UnrecoverableHttpException(String message) { - super(message); - } - } - - private HttpConnector() {} }
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpConnectorMultiplexer.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpConnectorMultiplexer.java new file mode 100644 index 0000000..ac0c080 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpConnectorMultiplexer.java
@@ -0,0 +1,332 @@ +// Copyright 2016 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.repository.downloader; + +import com.google.common.base.Function; +import com.google.common.base.Preconditions; +import com.google.common.base.Predicates; +import com.google.common.collect.FluentIterable; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Ordering; +import com.google.devtools.build.lib.bazel.repository.downloader.RetryingInputStream.Reconnector; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.util.Clock; +import com.google.devtools.build.lib.util.Sleeper; +import java.io.IOException; +import java.io.InputStream; +import java.io.InterruptedIOException; +import java.net.URL; +import java.net.URLConnection; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Deque; +import java.util.LinkedList; +import java.util.List; +import java.util.logging.Level; +import java.util.logging.Logger; +import javax.annotation.Nullable; +import javax.annotation.concurrent.GuardedBy; + +/** + * Class for establishing HTTP connections. + * + * <p>This is the most amazing way to download files ever. It makes Bazel builds as reliable as + * Blaze builds in Google's internal hermettically sealed repository. But this class isn't just + * reliable. It's also fast. It even works on the worst Internet connections in the farthest corners + * of the Earth. You are just not going to believe how fast and reliable this design is. It’s + * incredible. Your builds are never going to break again due to downloads. You’re going to be so + * happy. Your developer community is going to be happy. Mr. Jenkins will be happy too. Everyone is + * going to have such a magnificent developer experience due to the product excellence of this + * class. + */ +@ThreadSafe +final class HttpConnectorMultiplexer { + + private static final Logger logger = Logger.getLogger(HttpConnectorMultiplexer.class.getName()); + + private static final int MAX_THREADS_PER_CONNECT = 2; + private static final long FAILOVER_DELAY_MS = 2000; + private static final ImmutableMap<String, String> REQUEST_HEADERS = + ImmutableMap.of("Accept-Encoding", "gzip"); + + private final EventHandler eventHandler; + private final HttpConnector connector; + private final HttpStream.Factory httpStreamFactory; + private final Clock clock; + private final Sleeper sleeper; + + /** + * Creates a new instance. + * + * <p>Instances are thread safe and can be reused. + */ + HttpConnectorMultiplexer( + EventHandler eventHandler, + HttpConnector connector, + HttpStream.Factory httpStreamFactory, + Clock clock, + Sleeper sleeper) { + this.eventHandler = eventHandler; + this.connector = connector; + this.httpStreamFactory = httpStreamFactory; + this.clock = clock; + this.sleeper = sleeper; + } + + /** + * Establishes reliable HTTP connection to a good mirror URL. + * + * <p>This routine supports HTTP redirects in an RFC compliant manner. It requests gzip content + * encoding when appropriate in order to minimize bandwidth consumption when downloading + * uncompressed files. It reports download progress. It enforces a SHA-256 checksum which + * continues to be enforced even after this method returns. + * + * <p>This routine spawns {@value #MAX_THREADS_PER_CONNECT} threads that initiate connections in + * parallel to {@code urls} with a {@value #FAILOVER_DELAY_MS} millisecond failover waterfall so + * earlier mirrors are preferred. Each connector thread retries automatically on transient errors + * with exponential backoff. It vets the first 32kB of any payload before selecting a mirror in + * order to evade captive portals and avoid ultra-low-bandwidth servers. Even after this method + * returns the reliability doesn't stop. Each read operation wiil intercept timeouts and errors + * and block until the connection can be renegotiated transparently right where it left off. + * + * @param urls mirrors by preference; each URL can be: file, http, or https + * @param sha256 hex checksum lazily checked on entire payload, or empty to disable + * @return an {@link InputStream} of response payload + * @throws IOException if all mirrors are down and contains suppressed exception of each attempt + * @throws InterruptedIOException if current thread is being cast into oblivion + * @throws IllegalArgumentException if {@code urls} is empty or has an unsupported protocol + */ + public HttpStream connect(List<URL> urls, String sha256) throws IOException { + Preconditions.checkNotNull(sha256); + HttpUtils.checkUrlsArgument(urls); + if (Thread.interrupted()) { + throw new InterruptedIOException(); + } + // If there's only one URL then there's no need for us to run all our fancy thread stuff. + if (urls.size() == 1) { + return establishConnection(urls.get(0), sha256); + } + MutexConditionSharedMemory context = new MutexConditionSharedMemory(); + // The parent thread always holds the lock except when released by wait(). + synchronized (context) { + // Create the jobs for workers to do. + long now = clock.currentTimeMillis(); + long startAtTime = now; + for (URL url : urls) { + context.jobs.add(new WorkItem(url, sha256, startAtTime)); + startAtTime += FAILOVER_DELAY_MS; + } + // Create the worker thread pool. + for (int i = 0; i < Math.min(urls.size(), MAX_THREADS_PER_CONNECT); i++) { + Thread thread = new Thread(new Worker(context)); + thread.setName("HttpConnector"); + // These threads will not start doing anything until we release the lock below. + thread.start(); + context.threads.add(thread); + } + // Wait for the first worker to compute a result, or for all workers to fail. + boolean interrupted = false; + while (context.result == null && !context.threads.isEmpty()) { + try { + // Please note that waiting on a conndition releases the mutex. It also throws + // InterruptedException if the thread is *already* interrupted. + context.wait(); + } catch (InterruptedException e) { + // The interrupted state of this thread is now cleared, so we can call wait() again. + interrupted = true; + // We need to terminate the workers before rethrowing InterruptedException. + break; + } + } + // Now that we have the answer or are interrupted, we need to terminate any remaining workers. + for (Thread thread : context.threads) { + thread.interrupt(); + } + // Now wait for all threads to exit. We technically don't need to do this, but it helps with + // the regression testing of this implementation. + while (!context.threads.isEmpty()) { + try { + context.wait(); + } catch (InterruptedException e) { + // We don't care right now. Leave us alone. + interrupted = true; + } + } + // Now that the workers are terminated, we can safely propagate interruptions. + if (interrupted) { + throw new InterruptedIOException(); + } + // Please do not modify this code to call join() because the way we've implemented this + // routine is much better and faster. join() is basically a sleep loop when multiple threads + // exist. By sharing our mutex condition across threads, we were able to make things go + // lightning fast. If the child threads have not terminated by now, they are guaranteed to do + // so very soon. + if (context.result != null) { + return context.result; + } else { + IOException error = + new IOException("All mirrors are down: " + describeErrors(context.errors)); + // By this point, we probably have a very complex tree of exceptions. Beware! + for (Throwable workerError : context.errors) { + error.addSuppressed(workerError); + } + throw error; + } + } + } + + private static class MutexConditionSharedMemory { + @GuardedBy("this") @Nullable HttpStream result; + @GuardedBy("this") final List<Thread> threads = new ArrayList<>(); + @GuardedBy("this") final Deque<WorkItem> jobs = new LinkedList<>(); + @GuardedBy("this") final List<Throwable> errors = new ArrayList<>(); + } + + private static class WorkItem { + final URL url; + final String sha256; + final long startAtTime; + + WorkItem(URL url, String sha256, long startAtTime) { + this.url = url; + this.sha256 = sha256; + this.startAtTime = startAtTime; + } + } + + private class Worker implements Runnable { + private final MutexConditionSharedMemory context; + + Worker(MutexConditionSharedMemory context) { + this.context = context; + } + + @Override + public void run() { + while (true) { + WorkItem work; + synchronized (context) { + // A lot could have happened while we were waiting for this lock. Let's check. + if (context.result != null + || context.jobs.isEmpty() + || Thread.currentThread().isInterrupted()) { + tellParentThreadWeAreDone(); + return; + } + // Now remove a the first job from the fifo. + work = context.jobs.pop(); + } + // Wait if necessary before starting this thread. + long now = clock.currentTimeMillis(); + // Java does not have a true monotonic clock; but since currentTimeMillis returns UTC, it's + // monotonic enough for our purposes. This routine will not be pwnd by DST or JVM freezes. + // However it may be trivially impacted by system clock skew correction that go backwards. + if (now < work.startAtTime) { + try { + sleeper.sleepMillis(work.startAtTime - now); + } catch (InterruptedException e) { + // The parent thread or JVM has asked us to terminate this thread. + synchronized (context) { + tellParentThreadWeAreDone(); + return; + } + } + } + // Now we're actually going to attempt to connect to the remote server. + HttpStream result; + try { + result = establishConnection(work.url, work.sha256); + } catch (InterruptedIOException e) { + // The parent thread got its result from another thread and killed this one. + synchronized (context) { + tellParentThreadWeAreDone(); + return; + } + } catch (Throwable e) { + // Oh no the connector failed for some reason. We won't let that interfere with our plans. + synchronized (context) { + context.errors.add(e); + continue; + } + } + // Our connection attempt succeeded! Let's inform the parent thread of this joyous occasion. + synchronized (context) { + if (context.result == null) { + context.result = result; + result = null; + } + tellParentThreadWeAreDone(); + } + // We created a connection but we lost the race. Now we need to close it outside the mutex. + // We're not going to slow the parent thread down waiting for this operation to complete. + if (result != null) { + try { + result.close(); + } catch (IOException | RuntimeException e) { + logger.log(Level.WARNING, "close() failed in loser zombie thread", e); + } + } + } + } + + @GuardedBy("context") + private void tellParentThreadWeAreDone() { + // Remove this thread from the list of threads so parent thread knows when all have exited. + context.threads.remove(Thread.currentThread()); + // Wake up parent thread so it can check if that list is empty. + context.notify(); + } + } + + private HttpStream establishConnection(final URL url, String sha256) throws IOException { + final URLConnection connection = connector.connect(url, REQUEST_HEADERS); + return httpStreamFactory.create( + connection, url, sha256, + new Reconnector() { + @Override + public URLConnection connect( + Throwable cause, ImmutableMap<String, String> extraHeaders) + throws IOException { + eventHandler.handle( + Event.progress(String.format("Lost connection for %s due to %s", url, cause))); + return connector.connect( + connection.getURL(), + new ImmutableMap.Builder<String, String>() + .putAll(REQUEST_HEADERS) + .putAll(extraHeaders) + .build()); + } + }); + } + + private static String describeErrors(Collection<Throwable> errors) { + return + FluentIterable + .from(errors) + .transform( + new Function<Throwable, String>() { + @Nullable + @Override + public String apply(Throwable workerError) { + return workerError.getMessage(); + } + }) + .filter(Predicates.notNull()) + .toSortedSet(Ordering.natural()) + .toString(); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpDownloader.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpDownloader.java index ea4cc81..7ed6a92 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpDownloader.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpDownloader.java
@@ -1,4 +1,4 @@ -// Copyright 2014 The Bazel Authors. All rights reserved. +// Copyright 2016 The Bazel Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -14,98 +14,153 @@ package com.google.devtools.build.lib.bazel.repository.downloader; +import com.google.common.base.MoreObjects; +import com.google.common.base.Optional; +import com.google.common.base.Strings; +import com.google.common.collect.ImmutableList; +import com.google.common.io.ByteStreams; import com.google.devtools.build.lib.bazel.repository.cache.RepositoryCache; import com.google.devtools.build.lib.bazel.repository.cache.RepositoryCache.KeyType; -import com.google.devtools.build.lib.events.Event; import com.google.devtools.build.lib.events.EventHandler; -import com.google.devtools.build.lib.events.Location; import com.google.devtools.build.lib.packages.Rule; import com.google.devtools.build.lib.rules.repository.RepositoryFunction.RepositoryFunctionException; import com.google.devtools.build.lib.rules.repository.WorkspaceAttributeMapper; import com.google.devtools.build.lib.syntax.EvalException; import com.google.devtools.build.lib.syntax.Type; +import com.google.devtools.build.lib.util.Clock; +import com.google.devtools.build.lib.util.JavaClock; +import com.google.devtools.build.lib.util.JavaSleeper; +import com.google.devtools.build.lib.util.Sleeper; import com.google.devtools.build.lib.vfs.Path; import com.google.devtools.build.lib.vfs.PathFragment; -import com.google.devtools.build.skyframe.SkyFunctionException; import com.google.devtools.build.skyframe.SkyFunctionException.Transience; import java.io.IOException; -import java.io.InputStream; +import java.io.InterruptedIOException; import java.io.OutputStream; -import java.net.Proxy; -import java.net.URI; -import java.net.URISyntaxException; +import java.net.MalformedURLException; import java.net.URL; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; import java.util.Map; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ScheduledFuture; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.Semaphore; /** - * Helper class for downloading a file from a URL. + * Bazel file downloader. + * + * <p>This class uses {@link HttpConnectorMultiplexer} to connect to HTTP mirrors and then reads the + * file to disk. */ public class HttpDownloader { - private static final int BUFFER_SIZE = 32 * 1024; - private static final int KB = 1024; - private static final String UNITS = " KMGTPEY"; - private static final double LOG_OF_KB = Math.log(1024); - private final ScheduledExecutorService scheduler; - private Location ruleUrlAttributeLocation; + private static final int MAX_PARALLEL_DOWNLOADS = 8; + private static final Semaphore semaphore = new Semaphore(MAX_PARALLEL_DOWNLOADS, true); protected final RepositoryCache repositoryCache; public HttpDownloader(RepositoryCache repositoryCache) { - this.scheduler = Executors.newScheduledThreadPool(1); - this.ruleUrlAttributeLocation = null; this.repositoryCache = repositoryCache; } + /** Validates native repository rule attributes and calls the other download method. */ public Path download( Rule rule, Path outputDirectory, EventHandler eventHandler, Map<String, String> clientEnv) - throws RepositoryFunctionException, InterruptedException { + throws RepositoryFunctionException, InterruptedException { WorkspaceAttributeMapper mapper = WorkspaceAttributeMapper.of(rule); - String url; + List<URL> urls = new ArrayList<>(); String sha256; String type; try { - ruleUrlAttributeLocation = rule.getAttributeLocation("url"); - - url = mapper.get("url", Type.STRING); - sha256 = mapper.get("sha256", Type.STRING); - type = mapper.isAttributeValueExplicitlySpecified("type") - ? mapper.get("type", Type.STRING) : ""; + String urlString = Strings.nullToEmpty(mapper.get("url", Type.STRING)); + if (!urlString.isEmpty()) { + try { + URL url = new URL(urlString); + if (!HttpUtils.isUrlSupportedByDownloader(url)) { + throw new EvalException( + rule.getAttributeLocation("url"), "Unsupported protocol: " + url.getProtocol()); + } + urls.add(url); + } catch (MalformedURLException e) { + throw new EvalException(rule.getAttributeLocation("url"), e.toString()); + } + } + List<String> urlStrings = + MoreObjects.firstNonNull( + mapper.get("urls", Type.STRING_LIST), + ImmutableList.<String>of()); + if (!urlStrings.isEmpty()) { + if (!urls.isEmpty()) { + throw new EvalException(rule.getAttributeLocation("url"), "Don't set url if urls is set"); + } + try { + for (String urlString2 : urlStrings) { + URL url = new URL(urlString2); + if (!HttpUtils.isUrlSupportedByDownloader(url)) { + throw new EvalException( + rule.getAttributeLocation("urls"), "Unsupported protocol: " + url.getProtocol()); + } + urls.add(url); + } + } catch (MalformedURLException e) { + throw new EvalException(rule.getAttributeLocation("urls"), e.toString()); + } + } + if (urls.isEmpty()) { + throw new EvalException(rule.getLocation(), "urls attribute not set"); + } + sha256 = Strings.nullToEmpty(mapper.get("sha256", Type.STRING)); + if (!sha256.isEmpty() && !RepositoryCache.KeyType.SHA256.isValid(sha256)) { + throw new EvalException(rule.getAttributeLocation("sha256"), "Invalid SHA256 checksum"); + } + type = Strings.nullToEmpty(mapper.get("type", Type.STRING)); } catch (EvalException e) { throw new RepositoryFunctionException(e, Transience.PERSISTENT); } - try { - return download(url, sha256, type, outputDirectory, eventHandler, clientEnv); + return download(urls, sha256, Optional.of(type), outputDirectory, eventHandler, clientEnv); } catch (IOException e) { - throw new RepositoryFunctionException(e, SkyFunctionException.Transience.TRANSIENT); + throw new RepositoryFunctionException(e, Transience.TRANSIENT); } } /** - * Attempt to download a file from the repository's URL. Returns the path to the file downloaded. + * Downloads file to disk and returns path. * - * If the SHA256 checksum and path to the repository cache is specified, attempt - * to load the file from the RepositoryCache. If it doesn't exist, proceed to - * download the file and load it into the cache prior to returning the value. + * <p>If the SHA256 checksum and path to the repository cache is specified, attempt to load the + * file from the {@link RepositoryCache}. If it doesn't exist, proceed to download the file and + * load it into the cache prior to returning the value. + * + * @param urls list of mirror URLs with identical content + * @param sha256 valid SHA256 hex checksum string which is checked, or empty to disable + * @param type extension, e.g. "tar.gz" to force on downloaded filename, or empty to not do this + * @param output destination filename if {@code type} is <i>absent</i>, otherwise output directory + * @param eventHandler CLI progress reporter + * @param clientEnv environment variables in shell issuing this command + * @throws IllegalArgumentException on parameter badness, which should be checked beforehand + * @throws IOException if download was attempted and ended up failing + * @throws InterruptedException if this thread is being cast into oblivion */ public Path download( - String urlString, String sha256, String type, Path outputDirectory, - EventHandler eventHandler, Map<String, String> clientEnv) - throws IOException, InterruptedException, RepositoryFunctionException { - Path destination = getDownloadDestination(urlString, type, outputDirectory); + List<URL> urls, + String sha256, + Optional<String> type, + Path output, + EventHandler eventHandler, + Map<String, String> clientEnv) + throws IOException, InterruptedException { + if (Thread.interrupted()) { + throw new InterruptedException(); + } + + Path destination = getDownloadDestination(urls.get(0), type, output); // Used to decide whether to cache the download at the end of this method. boolean isCaching = false; - if (RepositoryCache.KeyType.SHA256.isValid(sha256)) { + if (!sha256.isEmpty()) { try { - String currentSha256 = RepositoryCache.getChecksum(KeyType.SHA256, destination); + String currentSha256 = + RepositoryCache.getChecksum(KeyType.SHA256, destination); if (currentSha256.equals(sha256)) { // No need to download. return destination; @@ -125,36 +180,30 @@ } } - AtomicInteger totalBytes = new AtomicInteger(0); - final ScheduledFuture<?> loggerHandle = getLoggerHandle(totalBytes, eventHandler, urlString); - final URL url = new URL(urlString); - Proxy proxy = ProxyHelper.createProxyIfNeeded(url.toString(), clientEnv); + // TODO: Consider using Dagger2 to automate this. + Clock clock = new JavaClock(); + Sleeper sleeper = new JavaSleeper(); + Locale locale = Locale.getDefault(); + ProxyHelper proxyHelper = new ProxyHelper(clientEnv); + HttpConnector connector = new HttpConnector(locale, eventHandler, proxyHelper, sleeper); + ProgressInputStream.Factory progressInputStreamFactory = + new ProgressInputStream.Factory(locale, clock, eventHandler); + HttpStream.Factory httpStreamFactory = new HttpStream.Factory(progressInputStreamFactory); + HttpConnectorMultiplexer multiplexer = + new HttpConnectorMultiplexer(eventHandler, connector, httpStreamFactory, clock, sleeper); - try (OutputStream out = destination.getOutputStream(); - InputStream inputStream = HttpConnector.connect(url, proxy, eventHandler)) { - int read; - byte[] buf = new byte[BUFFER_SIZE]; - while ((read = inputStream.read(buf)) > 0) { - totalBytes.addAndGet(read); - out.write(buf, 0, read); - if (Thread.interrupted()) { - throw new InterruptedException("Download interrupted"); - } - } + // Connect to the best mirror and download the file, while reporting progress to the CLI. + semaphore.acquire(); + try (HttpStream payload = multiplexer.connect(urls, sha256); + OutputStream out = destination.getOutputStream()) { + ByteStreams.copy(payload, out); + } catch (InterruptedIOException e) { + throw new InterruptedException(); } catch (IOException e) { throw new IOException( - "Error downloading " + url + " to " + destination + ": " + e.getMessage()); + "Error downloading " + urls + " to " + destination + ": " + e.getMessage()); } finally { - scheduler.schedule(new Runnable() { - @Override - public void run() { - loggerHandle.cancel(true); - } - }, 0, TimeUnit.SECONDS); - } - - if (!sha256.isEmpty()) { - RepositoryCache.assertFileChecksum(sha256, destination, KeyType.SHA256); + semaphore.release(); } if (isCaching) { @@ -164,55 +213,20 @@ return destination; } - private Path getDownloadDestination(String urlString, String type, Path outputDirectory) - throws RepositoryFunctionException { - URI uri = null; - try { - uri = new URI(urlString); - } catch (URISyntaxException e) { - throw new RepositoryFunctionException( - new EvalException(ruleUrlAttributeLocation, e), Transience.PERSISTENT); + private Path getDownloadDestination(URL url, Optional<String> type, Path output) { + if (!type.isPresent()) { + return output; } - if (type == null) { - return outputDirectory; - } else { - String filename = new PathFragment(uri.getPath()).getBaseName(); - if (filename.isEmpty()) { - filename = "temp"; - } else if (!type.isEmpty()) { - filename += "." + type; + String basename = + MoreObjects.firstNonNull( + Strings.emptyToNull(new PathFragment(url.getPath()).getBaseName()), + "temp"); + if (!type.get().isEmpty()) { + String suffix = "." + type.get(); + if (!basename.endsWith(suffix)) { + basename += suffix; } - return outputDirectory.getRelative(filename); } + return output.getRelative(basename); } - - private ScheduledFuture<?> getLoggerHandle( - final AtomicInteger totalBytes, final EventHandler eventHandler, final String urlString) { - final Runnable logger = new Runnable() { - @Override - public void run() { - try { - eventHandler.handle(Event.progress( - "Downloading from " + urlString + ": " + formatSize(totalBytes.get()))); - } catch (Exception e) { - eventHandler.handle(Event.error( - "Error generating download progress: " + e.getMessage())); - } - } - }; - return scheduler.scheduleAtFixedRate(logger, 0, 1, TimeUnit.SECONDS); - } - - private String formatSize(int bytes) { - if (bytes < KB) { - return bytes + "B"; - } - int logBaseUnitOfBytes = (int) (Math.log(bytes) / LOG_OF_KB); - if (logBaseUnitOfBytes < 0 || logBaseUnitOfBytes >= UNITS.length()) { - return bytes + "B"; - } - return (int) (bytes / Math.pow(KB, logBaseUnitOfBytes)) - + (UNITS.charAt(logBaseUnitOfBytes) + "B"); - } - }
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpStream.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpStream.java new file mode 100644 index 0000000..4921b15 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpStream.java
@@ -0,0 +1,140 @@ +// Copyright 2016 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.repository.downloader; + +import com.google.common.base.Splitter; +import com.google.common.base.Strings; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; +import com.google.common.hash.HashCode; +import com.google.common.hash.Hashing; +import com.google.common.io.ByteStreams; +import com.google.devtools.build.lib.bazel.repository.downloader.RetryingInputStream.Reconnector; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadCompatible; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe; +import java.io.ByteArrayInputStream; +import java.io.FilterInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.SequenceInputStream; +import java.net.URL; +import java.net.URLConnection; +import java.util.zip.GZIPInputStream; +import javax.annotation.WillCloseWhenClosed; + +/** + * Input stream that validates checksum resumes downloads on error. + * + * <p>This class is not thread safe, but it is safe to message pass its objects between threads. + */ +@ThreadCompatible +final class HttpStream extends FilterInputStream { + + static final int PRECHECK_BYTES = 32 * 1024; + private static final int GZIP_BUFFER_BYTES = 8192; // same as ByteStreams#copy + private static final ImmutableSet<String> GZIPPED_EXTENSIONS = ImmutableSet.of("gz", "tgz"); + private static final ImmutableSet<String> GZIP_CONTENT_ENCODING = + ImmutableSet.of("gzip", "x-gzip"); + + /** Factory for {@link HttpStream}. */ + @ThreadSafe + static class Factory { + + private final ProgressInputStream.Factory progressInputStreamFactory; + + Factory(ProgressInputStream.Factory progressInputStreamFactory) { + this.progressInputStreamFactory = progressInputStreamFactory; + } + + @SuppressWarnings("resource") + HttpStream create( + @WillCloseWhenClosed URLConnection connection, + URL originalUrl, + String sha256, + Reconnector reconnector) + throws IOException { + InputStream stream = new InterruptibleInputStream(connection.getInputStream()); + try { + // If server supports range requests, we can retry on read errors. See RFC7233 § 2.3. + RetryingInputStream retrier = null; + if (Iterables.contains( + Splitter.on(',') + .trimResults() + .split(Strings.nullToEmpty(connection.getHeaderField("Accept-Ranges"))), + "bytes")) { + retrier = new RetryingInputStream(stream, reconnector); + stream = retrier; + } + + stream = progressInputStreamFactory.create(stream, connection.getURL(), originalUrl); + + // Determine if we need to transparently gunzip. See RFC2616 § 3.5 and § 14.11. Please note + // that some web servers will send Content-Encoding: gzip even when we didn't request it if + // the file is a .gz file. + if (GZIP_CONTENT_ENCODING.contains(Strings.nullToEmpty(connection.getContentEncoding())) + && !GZIPPED_EXTENSIONS.contains(HttpUtils.getExtension(connection.getURL().getPath())) + && !GZIPPED_EXTENSIONS.contains(HttpUtils.getExtension(originalUrl.getPath()))) { + stream = new GZIPInputStream(stream, GZIP_BUFFER_BYTES); + } + + if (!sha256.isEmpty()) { + stream = new HashInputStream(stream, Hashing.sha256(), HashCode.fromString(sha256)); + if (retrier != null) { + retrier.disabled = true; + } + byte[] buffer = new byte[PRECHECK_BYTES]; + int read = 0; + while (read < PRECHECK_BYTES) { + int amount; + amount = stream.read(buffer, read, PRECHECK_BYTES - read); + if (amount == -1) { + break; + } + read += amount; + } + if (read < PRECHECK_BYTES) { + stream.close(); + stream = ByteStreams.limit(new ByteArrayInputStream(buffer), read); + } else { + stream = new SequenceInputStream(new ByteArrayInputStream(buffer), stream); + if (retrier != null) { + retrier.disabled = false; + } + } + } + } catch (Exception e) { + try { + stream.close(); + } catch (IOException e2) { + e.addSuppressed(e2); + } + throw e; + } + return new HttpStream(stream, connection.getURL()); + } + } + + private final URL url; + + HttpStream(@WillCloseWhenClosed InputStream delegate, URL url) { + super(delegate); + this.url = url; + } + + /** Returns final redirected URL. */ + URL getUrl() { + return url; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpUtils.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpUtils.java new file mode 100644 index 0000000..5c558d6 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpUtils.java
@@ -0,0 +1,110 @@ +// Copyright 2016 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.repository.downloader; + +import com.google.common.base.Ascii; +import com.google.common.base.MoreObjects; +import com.google.common.base.Preconditions; +import java.io.IOException; +import java.net.HttpURLConnection; +import java.net.MalformedURLException; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URL; +import java.util.Collection; + +/** HTTP utilities. */ +public final class HttpUtils { + + /** Returns {@code true} if {@code url} is supported by {@link HttpDownloader}. */ + public static boolean isUrlSupportedByDownloader(URL url) { + return isHttp(url) || isProtocol(url, "file"); + } + + static boolean isHttp(URL url) { + return isProtocol(url, "http") || isProtocol(url, "https"); + } + + static boolean isProtocol(URL url, String protocol) { + // An implementation should accept uppercase letters as equivalent to lowercase in scheme names + // (e.g., allow "HTTP" as well as "http") for the sake of robustness. Quoth RFC3986 § 3.1 + return Ascii.equalsIgnoreCase(protocol, url.getProtocol()); + } + + static void checkUrlsArgument(Collection<URL> urls) { + Preconditions.checkArgument(!urls.isEmpty(), "urls list empty"); + for (URL url : urls) { + Preconditions.checkArgument(isUrlSupportedByDownloader(url), "unsupported protocol: %s", url); + } + } + + static String getExtension(String path) { + int index = path.lastIndexOf('.'); + if (index == -1) { + return ""; + } + return Ascii.toLowerCase(path.substring(index + 1)); + } + + static URL getLocation(HttpURLConnection connection) throws IOException { + String newLocation = connection.getHeaderField("Location"); + if (newLocation == null) { + throw new IOException("Remote redirect missing Location."); + } + URL result = mergeUrls(URI.create(newLocation), connection.getURL()); + if (!isHttp(result)) { + throw new IOException("Bad Location: " + newLocation); + } + return result; + } + + private static URL mergeUrls(URI preferred, URL original) throws IOException { + // If the Location value provided in a 3xx (Redirection) response does not have a fragment + // component, a user agent MUST process the redirection as if the value inherits the fragment + // component of the URI reference used to generate the request target (i.e., the redirection + // inherits the original reference's fragment, if any). Quoth RFC7231 § 7.1.2 + String protocol = MoreObjects.firstNonNull(preferred.getScheme(), original.getProtocol()); + String userInfo = preferred.getUserInfo(); + String host = preferred.getHost(); + int port; + if (host == null) { + host = original.getHost(); + port = original.getPort(); + userInfo = original.getUserInfo(); + } else { + port = preferred.getPort(); + if (userInfo == null + && host.equals(original.getHost()) + && port == original.getPort()) { + userInfo = original.getUserInfo(); + } + } + String path = preferred.getPath(); + String query = preferred.getQuery(); + String fragment = preferred.getFragment(); + if (fragment == null) { + fragment = original.getRef(); + } + URL result; + try { + result = new URI(protocol, userInfo, host, port, path, query, fragment).toURL(); + } catch (URISyntaxException | MalformedURLException e) { + throw new IOException("Could not merge " + preferred + " into " + original); + } + return result; + } + + private HttpUtils() {} +}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/InterruptibleInputStream.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/InterruptibleInputStream.java new file mode 100644 index 0000000..81158e3 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/InterruptibleInputStream.java
@@ -0,0 +1,88 @@ +// Copyright 2016 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.repository.downloader; + +import com.google.devtools.build.lib.concurrent.ThreadSafety.ConditionallyThreadSafe; +import java.io.IOException; +import java.io.InputStream; +import java.io.InterruptedIOException; +import javax.annotation.WillCloseWhenClosed; + +/** + * Input stream that guarantees {@link InterruptedIOException}. + * + * <p>This class exists to hedge against the possibility that the JVM might not implement this + * functionality. See <a href="http://bugs.java.com/view_bug.do?bug_id=4385444">bug 4385444</a>. + */ +@ConditionallyThreadSafe +final class InterruptibleInputStream extends InputStream { + + private final InputStream delegate; + + InterruptibleInputStream(@WillCloseWhenClosed InputStream delegate) { + this.delegate = delegate; + } + + @Override + public int read() throws IOException { + check(); + return delegate.read(); + } + + @Override + public int read(byte[] buffer) throws IOException { + check(); + return delegate.read(buffer); + } + + @Override + public int read(byte[] buffer, int offset, int length) throws IOException { + check(); + return delegate.read(buffer, offset, length); + } + + @Override + public int available() throws IOException { + return delegate.available(); + } + + @Override + public boolean markSupported() { + return delegate.markSupported(); + } + + @Override + @SuppressWarnings("sync-override") + public void mark(int readlimit) { + delegate.mark(readlimit); + } + + @Override + @SuppressWarnings("sync-override") + public void reset() throws IOException { + delegate.reset(); + } + + @Override + public void close() throws IOException { + delegate.close(); + } + + private static void check() throws InterruptedIOException { + if (Thread.interrupted()) { + throw new InterruptedIOException(); + } + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/ProgressInputStream.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/ProgressInputStream.java new file mode 100644 index 0000000..acaf3e0 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/ProgressInputStream.java
@@ -0,0 +1,130 @@ +// Copyright 2016 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.repository.downloader; + +import com.google.common.base.Preconditions; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadCompatible; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.util.Clock; +import java.io.IOException; +import java.io.InputStream; +import java.net.URL; +import java.util.Locale; +import java.util.concurrent.atomic.AtomicLong; +import javax.annotation.WillCloseWhenClosed; + +/** + * Input stream that reports progress on total bytes read as the download progresses. + * + * <p>This class is not thread safe, but it is safe to message pass its objects between threads. + */ +@ThreadCompatible +final class ProgressInputStream extends InputStream { + + private static final long PROGRESS_INTERVAL_MS = 200; + + /** Factory for {@link ProgressInputStream}. */ + @ThreadSafe + static class Factory { + private final Locale locale; + private final Clock clock; + private final EventHandler eventHandler; + + Factory(Locale locale, Clock clock, EventHandler eventHandler) { + this.locale = locale; + this.clock = clock; + this.eventHandler = eventHandler; + } + + InputStream create(@WillCloseWhenClosed InputStream delegate, URL url, URL originalUrl) { + return new ProgressInputStream( + locale, clock, eventHandler, PROGRESS_INTERVAL_MS, delegate, url, originalUrl); + } + } + + private final Locale locale; + private final Clock clock; + private final EventHandler eventHandler; + private final InputStream delegate; + private final long intervalMs; + private final URL url; + private final URL originalUrl; + private final AtomicLong toto = new AtomicLong(); + private final AtomicLong nextEvent; + + ProgressInputStream( + Locale locale, + Clock clock, + EventHandler eventHandler, + long intervalMs, + InputStream delegate, + URL url, + URL originalUrl) { + Preconditions.checkArgument(intervalMs >= 0); + this.locale = locale; + this.clock = clock; + this.eventHandler = eventHandler; + this.intervalMs = intervalMs; + this.delegate = delegate; + this.url = url; + this.originalUrl = originalUrl; + this.nextEvent = new AtomicLong(clock.currentTimeMillis() + intervalMs); + } + + @Override + public int read() throws IOException { + int result = delegate.read(); + if (result != -1) { + reportProgress(toto.incrementAndGet()); + } + return result; + } + + @Override + public int read(byte[] buffer, int offset, int length) throws IOException { + int amount = delegate.read(buffer, offset, length); + if (amount > 0) { + reportProgress(toto.addAndGet(amount)); + } + return amount; + } + + @Override + public int available() throws IOException { + return delegate.available(); + } + + @Override + public void close() throws IOException { + delegate.close(); + } + + private void reportProgress(long bytesRead) { + long now = clock.currentTimeMillis(); + if (now < nextEvent.get()) { + return; + } + String via = ""; + if (!url.getHost().equals(originalUrl.getHost())) { + via = " via " + url.getHost(); + } + eventHandler.handle( + Event.progress( + String.format(locale, "Downloading %s%s: %,d bytes", originalUrl, via, bytesRead))); + nextEvent.set(now + intervalMs); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/ProxyHelper.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/ProxyHelper.java index fd09369..1ae265b 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/ProxyHelper.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/ProxyHelper.java
@@ -20,36 +20,43 @@ import java.net.InetSocketAddress; import java.net.PasswordAuthentication; import java.net.Proxy; +import java.net.URL; import java.net.URLDecoder; import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; +import javax.annotation.Nullable; /** * Helper class for setting up a proxy server for network communication - * */ public class ProxyHelper { + private final Map<String, String> env; + + /** + * Creates new instance. + * + * @param env client environment to check for proxy settings + */ + public ProxyHelper(Map<String, String> env) { + this.env = env; + } + /** * This method takes a String for the resource being requested and sets up a proxy to make * the request if HTTP_PROXY and/or HTTPS_PROXY environment variables are set. - * @param requestedUrl The url for the remote resource that may need to be retrieved through a - * proxy - * @param env The client environment to check for proxy settings. - * @return Proxy - * @throws IOException + * + * @param requestedUrl remote resource that may need to be retrieved through a proxy */ - public static Proxy createProxyIfNeeded(String requestedUrl, Map<String, String> env) - throws IOException { - String lcUrl = requestedUrl.toLowerCase(); + public Proxy createProxyIfNeeded(URL requestedUrl) throws IOException { String proxyAddress = null; - if (lcUrl.startsWith("https")) { + if (HttpUtils.isProtocol(requestedUrl, "https")) { proxyAddress = env.get("https_proxy"); if (Strings.isNullOrEmpty(proxyAddress)) { proxyAddress = env.get("HTTPS_PROXY"); } - } else if (lcUrl.startsWith("http")) { + } else if (HttpUtils.isProtocol(requestedUrl, "http")) { proxyAddress = env.get("http_proxy"); if (Strings.isNullOrEmpty(proxyAddress)) { proxyAddress = env.get("HTTP_PROXY"); @@ -67,7 +74,7 @@ * @return Proxy * @throws IOException */ - public static Proxy createProxy(String proxyAddress) throws IOException { + public static Proxy createProxy(@Nullable String proxyAddress) throws IOException { if (Strings.isNullOrEmpty(proxyAddress)) { return Proxy.NO_PROXY; }
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/RetryingInputStream.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/RetryingInputStream.java new file mode 100644 index 0000000..4263cc2 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/RetryingInputStream.java
@@ -0,0 +1,149 @@ +// Copyright 2016 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.repository.downloader; + +import com.google.common.base.Strings; +import com.google.common.collect.ImmutableMap; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadCompatible; +import java.io.IOException; +import java.io.InputStream; +import java.io.InterruptedIOException; +import java.net.SocketTimeoutException; +import java.net.URLConnection; +import java.util.Vector; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; + +/** + * Input stream that reconnects on read timeouts and errors. + * + * <p>This class is not thread safe, but it is safe to message pass between threads. + */ +@ThreadCompatible +class RetryingInputStream extends InputStream { + + private static final int MAX_RESUMES = 3; + + /** Lambda for establishing a connection. */ + interface Reconnector { + + /** Establishes a connection with the same parameters as what was passed to us initially. */ + URLConnection connect( + Throwable cause, ImmutableMap<String, String> extraHeaders) + throws IOException; + } + + volatile boolean disabled; + private volatile InputStream delegate; + private final Reconnector reconnector; + private final AtomicLong toto = new AtomicLong(); + private final AtomicInteger resumes = new AtomicInteger(); + private final Vector<Throwable> suppressed = new Vector<>(); + + RetryingInputStream(InputStream delegate, Reconnector reconnector) { + this.delegate = delegate; + this.reconnector = reconnector; + } + + @Override + public int read() throws IOException { + while (true) { + try { + int result = delegate.read(); + if (result != -1) { + toto.incrementAndGet(); + } + return result; + } catch (IOException e) { + tryAgainIfPossible(e); + } + } + } + + @Override + public int read(byte[] buffer, int offset, int length) throws IOException { + while (true) { + try { + int amount = delegate.read(buffer, offset, length); + if (amount != -1) { + toto.addAndGet(amount); + } + return amount; + } catch (IOException e) { + tryAgainIfPossible(e); + } + } + } + + @Override + public int available() throws IOException { + return delegate.available(); + } + + @Override + public void close() throws IOException { + delegate.close(); + } + + private void tryAgainIfPossible(IOException cause) throws IOException { + if (disabled) { + throw cause; + } + if (cause instanceof InterruptedIOException && !(cause instanceof SocketTimeoutException)) { + throw cause; + } + if (resumes.incrementAndGet() > MAX_RESUMES) { + propagate(cause); + } + try { + delegate.close(); + } catch (Exception ignored) { + // We know this connection failed so if it reminds us we're going to ignore it. + } + suppressed.add(cause); + reconnectWhereWeLeftOff(cause); + } + + private void reconnectWhereWeLeftOff(IOException cause) throws IOException { + try { + URLConnection connection; + long amountRead = toto.get(); + if (amountRead == 0) { + connection = reconnector.connect(cause, ImmutableMap.<String, String>of()); + } else { + connection = + reconnector.connect( + cause, ImmutableMap.of("Range", String.format("bytes %d-", amountRead))); + if (!Strings.nullToEmpty(connection.getHeaderField("Content-Range")) + .startsWith(String.format("bytes %d-", amountRead))) { + throw new IOException(String.format( + "Tried to reconnect at offset %,d but server didn't support it", amountRead)); + } + } + delegate = new InterruptibleInputStream(connection.getInputStream()); + } catch (InterruptedIOException e) { + throw e; + } catch (IOException e) { + propagate(e); + } + } + + private <T extends Throwable> void propagate(T error) throws T { + for (Throwable e : suppressed) { + error.addSuppressed(e); + } + throw error; + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/UnrecoverableHttpException.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/UnrecoverableHttpException.java new file mode 100644 index 0000000..3ccd2f4 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/UnrecoverableHttpException.java
@@ -0,0 +1,23 @@ +// Copyright 2016 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.repository.downloader; + +import java.io.IOException; + +final class UnrecoverableHttpException extends IOException { + UnrecoverableHttpException(String message) { + super(message); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/skylark/SkylarkRepositoryContext.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/skylark/SkylarkRepositoryContext.java index 97b295f..95e8532 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/repository/skylark/SkylarkRepositoryContext.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/skylark/SkylarkRepositoryContext.java
@@ -15,11 +15,14 @@ package com.google.devtools.build.lib.bazel.repository.skylark; import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Optional; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.devtools.build.lib.bazel.repository.DecompressorDescriptor; import com.google.devtools.build.lib.bazel.repository.DecompressorValue; +import com.google.devtools.build.lib.bazel.repository.cache.RepositoryCache.KeyType; import com.google.devtools.build.lib.bazel.repository.downloader.HttpDownloader; +import com.google.devtools.build.lib.bazel.repository.downloader.HttpUtils; import com.google.devtools.build.lib.cmdline.Label; import com.google.devtools.build.lib.cmdline.LabelSyntaxException; import com.google.devtools.build.lib.events.Location; @@ -53,7 +56,11 @@ import java.io.File; import java.io.IOException; import java.io.OutputStream; +import java.net.MalformedURLException; +import java.net.URL; import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; import java.util.Map; /** Skylark API for the repository_rule's context. */ @@ -447,10 +454,11 @@ parameters = { @Param( name = "url", - type = String.class, - doc = - "URL to the file to download. There is no authentication." - + " Redirection are followed." + allowedTypes = { + @ParamType(type = String.class), + @ParamType(type = SkylarkList.class, generic1 = String.class), + }, + doc = "List of mirror URLs referencing the same file." ), @Param( name = "output", @@ -482,18 +490,28 @@ ), } ) - public void download(String url, Object output, String sha256, Boolean executable) - throws RepositoryFunctionException, EvalException, InterruptedException { + public void download( + Object url, Object output, String sha256, Boolean executable) + throws RepositoryFunctionException, EvalException, InterruptedException { + validateSha256(sha256); + List<URL> urls = getUrls(url); SkylarkPath outputPath = getPath("download()", output); try { checkInOutputDirectory(outputPath); makeDirectories(outputPath.getPath()); - - httpDownloader.download(url, sha256, null, outputPath.getPath(), env.getListener(), + httpDownloader.download( + urls, + sha256, + Optional.<String>absent(), + outputPath.getPath(), + env.getListener(), osObject.getEnvironmentVariables()); if (executable) { outputPath.getPath().setExecutable(true); } + } catch (InterruptedException e) { + throw new RepositoryFunctionException( + new IOException("thread interrupted"), Transience.TRANSIENT); } catch (IOException e) { throw new RepositoryFunctionException(e, Transience.TRANSIENT); } @@ -505,11 +523,11 @@ parameters = { @Param( name = "url", - type = String.class, - doc = - "a URL referencing an archive file containing a Bazel repository." - + " Archives of type .zip, .jar, .war, .tar.gz or .tgz are supported." - + " There is no support for authentication. Redirections are followed." + allowedTypes = { + @ParamType(type = String.class), + @ParamType(type = SkylarkList.class, generic1 = String.class), + }, + doc = "List of mirror URLs referencing the same file." ), @Param( name = "output", @@ -542,8 +560,8 @@ doc = "the archive type of the downloaded file." + " By default, the archive type is determined from the file extension of the URL." - + " If the file has no extension, you can explicitly specify either" - + "\"zip\", \"jar\", \"tar.gz\", or \"tgz\" here." + + " If the file has no extension, you can explicitly specify either \"zip\"," + + " \"jar\", \"war\", \"tar.gz\", \"tgz\", \"tar.bz2\", or \"tar.xz\" here." ), @Param( name = "stripPrefix", @@ -560,8 +578,11 @@ } ) public void downloadAndExtract( - String url, Object output, String sha256, String type, String stripPrefix) - throws RepositoryFunctionException, InterruptedException, EvalException { + Object url, Object output, String sha256, String type, String stripPrefix) + throws RepositoryFunctionException, InterruptedException, EvalException { + validateSha256(sha256); + List<URL> urls = getUrls(url); + // Download to outputDirectory and delete it after extraction SkylarkPath outputPath = getPath("download_and_extract()", output); checkInOutputDirectory(outputPath); @@ -569,8 +590,14 @@ Path downloadedPath; try { - downloadedPath = httpDownloader.download(url, sha256, type, outputPath.getPath(), - env.getListener(), osObject.getEnvironmentVariables()); + downloadedPath = + httpDownloader.download( + urls, + sha256, + Optional.of(type), + outputPath.getPath(), + env.getListener(), + osObject.getEnvironmentVariables()); } catch (IOException e) { throw new RepositoryFunctionException(e, Transience.TRANSIENT); } @@ -594,6 +621,43 @@ } } + private static void validateSha256(String sha256) throws RepositoryFunctionException { + if (!sha256.isEmpty() && !KeyType.SHA256.isValid(sha256)) { + throw new RepositoryFunctionException( + new IOException("Invalid SHA256 checksum"), Transience.TRANSIENT); + } + } + + private static List<URL> getUrls(Object urlOrList) throws RepositoryFunctionException { + List<String> urlStrings; + if (urlOrList instanceof String) { + urlStrings = ImmutableList.of((String) urlOrList); + } else { + @SuppressWarnings("unchecked") + List<String> list = (List<String>) urlOrList; + urlStrings = list; + } + if (urlStrings.isEmpty()) { + throw new RepositoryFunctionException(new IOException("urls not set"), Transience.PERSISTENT); + } + List<URL> urls = new ArrayList<>(); + for (String urlString : urlStrings) { + URL url; + try { + url = new URL(urlString); + } catch (MalformedURLException e) { + throw new RepositoryFunctionException( + new IOException("Bad URL: " + urlString), Transience.PERSISTENT); + } + if (!HttpUtils.isUrlSupportedByDownloader(url)) { + throw new RepositoryFunctionException( + new IOException("Unsupported protocol: " + url.getProtocol()), Transience.PERSISTENT); + } + urls.add(url); + } + return urls; + } + // This is just for test to overwrite the path environment private static ImmutableList<String> pathEnv = null;
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/workspace/HttpArchiveRule.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/workspace/HttpArchiveRule.java index 0406fab..3734725 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/rules/workspace/HttpArchiveRule.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/workspace/HttpArchiveRule.java
@@ -16,6 +16,7 @@ import static com.google.devtools.build.lib.packages.Attribute.attr; import static com.google.devtools.build.lib.syntax.Type.STRING; +import static com.google.devtools.build.lib.syntax.Type.STRING_LIST; import com.google.devtools.build.lib.analysis.RuleDefinition; import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment; @@ -36,12 +37,19 @@ public RuleClass build(Builder builder, RuleDefinitionEnvironment environment) { return builder /* <!-- #BLAZE_RULE(http_archive).ATTRIBUTE(url) --> - A URL referencing an archive file containing a Bazel repository. + (Deprecated) A URL referencing an archive file containing a Bazel repository. - <p>Archives of type .zip, .jar, .war, .tar.gz or .tgz are supported. There is no support - for authentication.</p> + <p>This value has the same meaning as a <code>urls</code> list with a single item. This + must not be specified if <code>urls</code> is also specified.</p> <!-- #END_BLAZE_RULE.ATTRIBUTE --> */ - .add(attr("url", STRING).mandatory()) + .add(attr("url", STRING)) + /* <!-- #BLAZE_RULE(http_archive).ATTRIBUTE(urls) --> + List of mirror URLs referencing the same archive file containing a Bazel repository. + + <p>This must be an http, https, or file URL. Archives of type .zip, .jar, .war, .tar.gz, + .tgz, tar.bz2, or tar.xz are supported. There is no support for authentication.</p> + <!-- #END_BLAZE_RULE.ATTRIBUTE --> */ + .add(attr("urls", STRING_LIST)) /* <!-- #BLAZE_RULE(http_archive).ATTRIBUTE(sha256) --> The expected SHA-256 hash of the file downloaded.
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/workspace/HttpFileRule.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/workspace/HttpFileRule.java index 7c96c4c..9ef7b0d 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/rules/workspace/HttpFileRule.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/workspace/HttpFileRule.java
@@ -17,6 +17,7 @@ import static com.google.devtools.build.lib.packages.Attribute.attr; import static com.google.devtools.build.lib.syntax.Type.BOOLEAN; import static com.google.devtools.build.lib.syntax.Type.STRING; +import static com.google.devtools.build.lib.syntax.Type.STRING_LIST; import com.google.devtools.build.lib.analysis.RuleDefinition; import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment; @@ -37,24 +38,31 @@ public RuleClass build(Builder builder, RuleDefinitionEnvironment environment) { return builder /* <!-- #BLAZE_RULE(http_file).ATTRIBUTE(url) --> - A URL to a file that will be made available to Bazel. + (Deprecated) A URL to a file that will be made available to Bazel. - <p>This must be an http or https URL. Authentication is not supported.</p> - <!-- #END_BLAZE_RULE.ATTRIBUTE --> */ - .add(attr("url", STRING).mandatory()) + <p>This value has the same meaning as a <code>urls</code> list with a single item. This + must not be specified if <code>urls</code> is also specified.</p> + <!-- #END_BLAZE_RULE.ATTRIBUTE --> */ + .add(attr("url", STRING)) + /* <!-- #BLAZE_RULE(http_file).ATTRIBUTE(urls) --> + List of mirror URLs referencing the same file that will be made available to Bazel. + + <p>This must be an http, https, or file URL. Authentication is not supported.</p> + <!-- #END_BLAZE_RULE.ATTRIBUTE --> */ + .add(attr("urls", STRING_LIST)) /* <!-- #BLAZE_RULE(http_file).ATTRIBUTE(sha256) --> - The expected SHA-256 of the file downloaded. + The expected SHA-256 of the file downloaded. - <p>This must match the SHA-256 of the file downloaded. <em>It is a security risk to - omit the SHA-256 as remote files can change.</em> At best omitting this field will make - your build non-hermetic. It is optional to make development easier but should be set - before shipping.</p> - <!-- #END_BLAZE_RULE.ATTRIBUTE --> */ + <p>This must match the SHA-256 of the file downloaded. <em>It is a security risk to + omit the SHA-256 as remote files can change.</em> At best omitting this field will make + your build non-hermetic. It is optional to make development easier but should be set + before shipping.</p> + <!-- #END_BLAZE_RULE.ATTRIBUTE --> */ .add(attr("sha256", STRING)) /* <!-- #BLAZE_RULE(http_file).ATTRIBUTE(executable) --> - If the downloaded file should be made executable. Defaults to False. + If the downloaded file should be made executable. Defaults to False. - <!-- #END_BLAZE_RULE.ATTRIBUTE --> */ + <!-- #END_BLAZE_RULE.ATTRIBUTE --> */ .add(attr("executable", BOOLEAN)) .setWorkspaceOnly() .build();
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/workspace/NewHttpArchiveRule.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/workspace/NewHttpArchiveRule.java index 30a5cd8..9f29fd6 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/rules/workspace/NewHttpArchiveRule.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/workspace/NewHttpArchiveRule.java
@@ -16,6 +16,7 @@ import static com.google.devtools.build.lib.packages.Attribute.attr; import static com.google.devtools.build.lib.syntax.Type.STRING; +import static com.google.devtools.build.lib.syntax.Type.STRING_LIST; import com.google.devtools.build.lib.analysis.RuleDefinition; import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment; @@ -33,12 +34,19 @@ public RuleClass build(RuleClass.Builder builder, RuleDefinitionEnvironment environment) { return builder /* <!-- #BLAZE_RULE(new_http_archive).ATTRIBUTE(url) --> - A URL referencing an archive file containing a Bazel repository. + (Deprecated) A URL referencing an archive file. - <p>Archives of type .zip, .jar, .war, .tar.gz or .tgz are supported. There is no support - for authentication.</p> - <!-- #END_BLAZE_RULE.ATTRIBUTE --> */ - .add(attr("url", STRING).mandatory()) + <p>This value has the same meaning as a <code>urls</code> list with a single item. This + must not be specified if <code>urls</code> is also specified.</p> + <!-- #END_BLAZE_RULE.ATTRIBUTE --> */ + .add(attr("url", STRING)) + /* <!-- #BLAZE_RULE(new_http_archive).ATTRIBUTE(urls) --> + List of mirror URLs referencing the same archive file containing a Bazel repository. + + <p>This must be an http, https, or file URL. Archives of type .zip, .jar, .war, .tar.gz, + .tgz, tar.bz2, or tar.xz are supported. There is no support for authentication.</p> + <!-- #END_BLAZE_RULE.ATTRIBUTE --> */ + .add(attr("urls", STRING_LIST)) /* <!-- #BLAZE_RULE(new_http_archive).ATTRIBUTE(sha256) --> The expected SHA-256 hash of the file downloaded.
diff --git a/src/main/java/com/google/devtools/build/lib/packages/AttributeContainer.java b/src/main/java/com/google/devtools/build/lib/packages/AttributeContainer.java index 65b890d..d7856e8 100644 --- a/src/main/java/com/google/devtools/build/lib/packages/AttributeContainer.java +++ b/src/main/java/com/google/devtools/build/lib/packages/AttributeContainer.java
@@ -17,6 +17,7 @@ import com.google.common.base.Function; import com.google.devtools.build.lib.events.Location; import java.util.Arrays; +import javax.annotation.Nullable; /** * Provides attribute setting and retrieval for a Rule. Encapsulating attribute access @@ -77,6 +78,7 @@ /** * Returns an attribute value by name, or null on no match. */ + @Nullable public Object getAttr(String attrName) { Integer idx = ruleClass.getAttributeIndex(attrName); return idx != null ? attributeValues[idx] : null;
diff --git a/src/main/java/com/google/devtools/build/lib/rules/repository/WorkspaceAttributeMapper.java b/src/main/java/com/google/devtools/build/lib/rules/repository/WorkspaceAttributeMapper.java index b2edf3d..99cc363 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/repository/WorkspaceAttributeMapper.java +++ b/src/main/java/com/google/devtools/build/lib/rules/repository/WorkspaceAttributeMapper.java
@@ -14,12 +14,16 @@ package com.google.devtools.build.lib.rules.repository; +import static com.google.common.base.Preconditions.checkNotNull; + +import com.google.common.base.Preconditions; import com.google.devtools.build.lib.cmdline.Label; import com.google.devtools.build.lib.packages.AggregatingAttributeMapper; import com.google.devtools.build.lib.packages.BuildType.SelectorList; import com.google.devtools.build.lib.packages.Rule; import com.google.devtools.build.lib.syntax.EvalException; import com.google.devtools.build.lib.syntax.Type; +import javax.annotation.Nullable; /** * An attribute mapper for workspace rules. Similar to NonconfigurableAttributeWrapper, but throws @@ -37,7 +41,12 @@ this.rule = rule; } + /** + * Returns typecasted value for attribute or {@code null} on no match. + */ + @Nullable public <T> T get(String attributeName, Type<T> type) throws EvalException { + Preconditions.checkNotNull(type); Object value = getObject(attributeName); try { return type.cast(value); @@ -48,10 +57,11 @@ } /** - * Returns the value for an attribute without casting it to any particular type. + * Returns value for attribute without casting it to any particular type, or null on no match. */ + @Nullable public Object getObject(String attributeName) throws EvalException { - Object value = rule.getAttributeContainer().getAttr(attributeName); + Object value = rule.getAttributeContainer().getAttr(checkNotNull(attributeName)); if (value instanceof SelectorList) { String message; if (rule.getLocation().getPath().getBaseName().equals(
diff --git a/src/main/java/com/google/devtools/build/lib/util/JavaSleeper.java b/src/main/java/com/google/devtools/build/lib/util/JavaSleeper.java new file mode 100644 index 0000000..d31c617 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/util/JavaSleeper.java
@@ -0,0 +1,27 @@ +// Copyright 2016 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.util; + +import java.util.concurrent.TimeUnit; + +/** Production implementation of {@link Sleeper} */ +public final class JavaSleeper implements Sleeper { + + @Override + public void sleepMillis(long milliseconds) throws InterruptedException { + Preconditions.checkArgument(milliseconds >= 0, "sleeper can't time travel"); + TimeUnit.MILLISECONDS.sleep(milliseconds); + } +}
diff --git a/src/main/java/com/google/devtools/build/lib/util/Sleeper.java b/src/main/java/com/google/devtools/build/lib/util/Sleeper.java new file mode 100644 index 0000000..53b29d2 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/util/Sleeper.java
@@ -0,0 +1,32 @@ +// Copyright 2016 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.util; + +/** + * Interface accepting requests to put current thread to sleep. + * + * <p>The only implementation of this interface intended for production use is {@link JavaSleeper}. + * Use {@link com.google.devtools.build.lib.testutil.ManualSleeper ManualSleeper} for testing. + */ +public interface Sleeper { + + /** + * Puts current thread to sleep for given duration. + * + * @throws InterruptedException if current thread is being cast into oblivion + * @throws IllegalArgumentException if {@code milliseconds} is negative + */ + void sleepMillis(long milliseconds) throws InterruptedException; +}
diff --git a/src/test/java/com/google/devtools/build/lib/bazel/repository/downloader/BUILD b/src/test/java/com/google/devtools/build/lib/bazel/repository/downloader/BUILD index 6d9253a..ceaaf30 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/repository/downloader/BUILD +++ b/src/test/java/com/google/devtools/build/lib/bazel/repository/downloader/BUILD
@@ -13,9 +13,11 @@ ], deps = [ "//src/main/java/com/google/devtools/build/lib:events", + "//src/main/java/com/google/devtools/build/lib:util", "//src/main/java/com/google/devtools/build/lib/bazel/repository/downloader", "//src/test/java/com/google/devtools/build/lib:foundations_testutil", "//src/test/java/com/google/devtools/build/lib:test_runner", + "//src/test/java/com/google/devtools/build/lib:testutil", "//third_party:guava", "//third_party:junit4", "//third_party:mockito",
diff --git a/src/test/java/com/google/devtools/build/lib/bazel/repository/downloader/DownloaderTestSuite.java b/src/test/java/com/google/devtools/build/lib/bazel/repository/downloader/DownloaderTestSuite.java index 1a48a1c..1c2477e 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/repository/downloader/DownloaderTestSuite.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/repository/downloader/DownloaderTestSuite.java
@@ -21,7 +21,14 @@ /** Test suite for downloader package. */ @RunWith(Suite.class) @SuiteClasses({ + HashInputStreamTest.class, + HttpConnectorMultiplexerIntegrationTest.class, + HttpConnectorMultiplexerTest.class, HttpConnectorTest.class, + HttpStreamTest.class, + HttpUtilsTest.class, + ProgressInputStreamTest.class, ProxyHelperTest.class, + RetryingInputStreamTest.class, }) class DownloaderTestSuite {}
diff --git a/src/test/java/com/google/devtools/build/lib/bazel/repository/downloader/DownloaderTestUtils.java b/src/test/java/com/google/devtools/build/lib/bazel/repository/downloader/DownloaderTestUtils.java new file mode 100644 index 0000000..1f55667 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/bazel/repository/downloader/DownloaderTestUtils.java
@@ -0,0 +1,45 @@ +// Copyright 2016 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.repository.downloader; + +import static java.nio.charset.StandardCharsets.ISO_8859_1; + +import com.google.common.base.Joiner; +import com.google.common.io.ByteStreams; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.Socket; +import java.net.URL; +import javax.annotation.WillNotClose; + +final class DownloaderTestUtils { + + static URL makeUrl(String url) { + try { + return new URL(url); + } catch (MalformedURLException e) { + throw new RuntimeException(e); + } + } + + static void sendLines(@WillNotClose Socket socket, String... data) throws IOException { + ByteStreams.copy( + new ByteArrayInputStream(Joiner.on("\r\n").join(data).getBytes(ISO_8859_1)), + socket.getOutputStream()); + } + + private DownloaderTestUtils() {} +}
diff --git a/src/test/java/com/google/devtools/build/lib/bazel/repository/downloader/HashInputStreamTest.java b/src/test/java/com/google/devtools/build/lib/bazel/repository/downloader/HashInputStreamTest.java new file mode 100644 index 0000000..c1e14df --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/bazel/repository/downloader/HashInputStreamTest.java
@@ -0,0 +1,67 @@ +// Copyright 2016 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.repository.downloader; + +import static com.google.common.truth.Truth.assertThat; +import static java.nio.charset.StandardCharsets.UTF_8; + +import com.google.common.hash.HashCode; +import com.google.common.hash.Hashing; +import com.google.common.io.CharStreams; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStreamReader; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Unit tests for {@link HashInputStream}. */ +@RunWith(JUnit4.class) +@SuppressWarnings("resource") +public class HashInputStreamTest { + + @Rule + public final ExpectedException thrown = ExpectedException.none(); + + @Test + public void validChecksum_readsOk() throws Exception { + assertThat( + CharStreams.toString( + new InputStreamReader( + new HashInputStream( + new ByteArrayInputStream("hello".getBytes(UTF_8)), + Hashing.sha1(), + HashCode.fromString("aaf4c61ddcc5e8a2dabede0f3b482cd9aea9434d")), + UTF_8))) + .isEqualTo("hello"); + } + + @Test + public void badChecksum_throwsIOException() throws Exception { + thrown.expect(IOException.class); + thrown.expectMessage("Checksum"); + assertThat( + CharStreams.toString( + new InputStreamReader( + new HashInputStream( + new ByteArrayInputStream("hello".getBytes(UTF_8)), + Hashing.sha1(), + HashCode.fromString("0000000000000000000000000000000000000000")), + UTF_8))) + .isNull(); // Only here to make @CheckReturnValue happy. + } +}
diff --git a/src/test/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpConnectorMultiplexerIntegrationTest.java b/src/test/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpConnectorMultiplexerIntegrationTest.java new file mode 100644 index 0000000..b5b77a2 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpConnectorMultiplexerIntegrationTest.java
@@ -0,0 +1,227 @@ +// Copyright 2016 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.repository.downloader; + +import static com.google.common.io.ByteStreams.toByteArray; +import static com.google.common.truth.Truth.assertThat; +import static com.google.devtools.build.lib.bazel.repository.downloader.DownloaderTestUtils.sendLines; +import static com.google.devtools.build.lib.bazel.repository.downloader.HttpParser.readHttpRequest; +import static java.nio.charset.StandardCharsets.US_ASCII; +import static java.util.Arrays.asList; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyLong; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.testutil.ManualClock; +import com.google.devtools.build.lib.util.Sleeper; +import java.io.IOException; +import java.net.InetAddress; +import java.net.Proxy; +import java.net.ServerSocket; +import java.net.Socket; +import java.net.URL; +import java.util.Locale; +import java.util.concurrent.Callable; +import java.util.concurrent.CyclicBarrier; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Phaser; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.junit.rules.Timeout; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; + +/** Black box integration tests for {@link HttpConnectorMultiplexer}. */ +@RunWith(JUnit4.class) +public class HttpConnectorMultiplexerIntegrationTest { + + @Rule + public final ExpectedException thrown = ExpectedException.none(); + + @Rule + public final Timeout globalTimeout = new Timeout(10000); + + private final ExecutorService executor = Executors.newFixedThreadPool(3); + private final ProxyHelper proxyHelper = mock(ProxyHelper.class); + private final EventHandler eventHandler = mock(EventHandler.class); + private final ManualClock clock = new ManualClock(); + private final Sleeper sleeper = mock(Sleeper.class); + private final Locale locale = Locale.US; + private final HttpConnector connector = + new HttpConnector(locale, eventHandler, proxyHelper, sleeper); + private final ProgressInputStream.Factory progressInputStreamFactory = + new ProgressInputStream.Factory(locale, clock, eventHandler); + private final HttpStream.Factory httpStreamFactory = + new HttpStream.Factory(progressInputStreamFactory); + private final HttpConnectorMultiplexer multiplexer = + new HttpConnectorMultiplexer(eventHandler, connector, httpStreamFactory, clock, sleeper); + + @Before + public void before() throws Exception { + when(proxyHelper.createProxyIfNeeded(any(URL.class))).thenReturn(Proxy.NO_PROXY); + } + + @After + public void after() throws Exception { + executor.shutdown(); + } + + @Test + public void normalRequest() throws Exception { + final Phaser phaser = new Phaser(3); + try (ServerSocket server1 = new ServerSocket(0, 1, InetAddress.getByName("127.0.0.1")); + ServerSocket server2 = new ServerSocket(0, 1, InetAddress.getByName("127.0.0.1"))) { + for (final ServerSocket server : asList(server1, server2)) { + executor.submit( + new Callable<Object>() { + @Override + public Object call() throws Exception { + for (String status : asList("503 MELTDOWN", "500 ERROR", "200 OK")) { + phaser.arriveAndAwaitAdvance(); + try (Socket socket = server.accept()) { + readHttpRequest(socket.getInputStream()); + sendLines(socket, + "HTTP/1.1 " + status, + "Date: Fri, 31 Dec 1999 23:59:59 GMT", + "Connection: close", + "", + "hello"); + } + } + return null; + } + }); + } + phaser.arriveAndAwaitAdvance(); + phaser.arriveAndDeregister(); + try (HttpStream stream = + multiplexer.connect( + ImmutableList.of( + new URL(String.format("http://127.0.0.1:%d", server1.getLocalPort())), + new URL(String.format("http://127.0.0.1:%d", server2.getLocalPort()))), + "2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824")) { + assertThat(toByteArray(stream)).isEqualTo("hello".getBytes(US_ASCII)); + } + } + } + + @Test + public void captivePortal_isAvoided() throws Exception { + final CyclicBarrier barrier = new CyclicBarrier(2); + doAnswer( + new Answer<Void>() { + @Override + public Void answer(InvocationOnMock invocation) throws Throwable { + barrier.await(); + return null; + } + }).when(sleeper).sleepMillis(anyLong()); + try (final ServerSocket server1 = new ServerSocket(0, 1, InetAddress.getByName("127.0.0.1")); + final ServerSocket server2 = new ServerSocket(0, 1, InetAddress.getByName("127.0.0.1"))) { + executor.submit( + new Callable<Object>() { + @Override + public Object call() throws Exception { + try (Socket socket = server1.accept()) { + readHttpRequest(socket.getInputStream()); + sendLines(socket, + "HTTP/1.1 200 OK", + "Date: Fri, 31 Dec 1999 23:59:59 GMT", + "Warning: https://youtu.be/rJ6O5sTPn1k", + "Connection: close", + "", + "Und wird die Welt auch in Flammen stehen", + "Wir werden wieder auferstehen"); + } + barrier.await(); + return null; + } + }); + executor.submit( + new Callable<Object>() { + @Override + public Object call() throws Exception { + try (Socket socket = server2.accept()) { + readHttpRequest(socket.getInputStream()); + sendLines(socket, + "HTTP/1.1 200 OK", + "Date: Fri, 31 Dec 1999 23:59:59 GMT", + "Connection: close", + "", + "hello"); + } + return null; + } + }); + try (HttpStream stream = + multiplexer.connect( + ImmutableList.of( + new URL(String.format("http://127.0.0.1:%d", server1.getLocalPort())), + new URL(String.format("http://127.0.0.1:%d", server2.getLocalPort()))), + "2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824")) { + assertThat(toByteArray(stream)).isEqualTo("hello".getBytes(US_ASCII)); + } + } + } + + @Test + public void allMirrorsDown_throwsIOException() throws Exception { + final CyclicBarrier barrier = new CyclicBarrier(4); + try (ServerSocket server1 = new ServerSocket(0, 1, InetAddress.getByName("127.0.0.1")); + ServerSocket server2 = new ServerSocket(0, 1, InetAddress.getByName("127.0.0.1")); + ServerSocket server3 = new ServerSocket(0, 1, InetAddress.getByName("127.0.0.1"))) { + for (final ServerSocket server : asList(server1, server2, server3)) { + executor.submit( + new Callable<Object>() { + @Override + public Object call() throws Exception { + barrier.await(); + while (true) { + try (Socket socket = server.accept()) { + readHttpRequest(socket.getInputStream()); + sendLines(socket, + "HTTP/1.1 503 MELTDOWN", + "Date: Fri, 31 Dec 1999 23:59:59 GMT", + "Warning: https://youtu.be/6M6samPEMpM", + "Connection: close", + "", + ""); + } + } + } + }); + } + barrier.await(); + thrown.expect(IOException.class); + thrown.expectMessage("All mirrors are down: [GET returned 503 MELTDOWN]"); + multiplexer.connect( + ImmutableList.of( + new URL(String.format("http://127.0.0.1:%d", server1.getLocalPort())), + new URL(String.format("http://127.0.0.1:%d", server2.getLocalPort())), + new URL(String.format("http://127.0.0.1:%d", server3.getLocalPort()))), + "2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9825"); + } + } +}
diff --git a/src/test/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpConnectorMultiplexerTest.java b/src/test/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpConnectorMultiplexerTest.java new file mode 100644 index 0000000..dedf316 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpConnectorMultiplexerTest.java
@@ -0,0 +1,266 @@ +// Copyright 2016 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.repository.downloader; + +import static com.google.common.io.ByteStreams.toByteArray; +import static com.google.common.truth.Truth.assertThat; +import static com.google.devtools.build.lib.bazel.repository.downloader.DownloaderTestUtils.makeUrl; +import static java.nio.charset.StandardCharsets.UTF_8; +import static java.util.Arrays.asList; +import static org.junit.Assert.fail; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyLong; +import static org.mockito.Matchers.anyString; +import static org.mockito.Matchers.eq; +import static org.mockito.Matchers.same; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.verifyZeroInteractions; +import static org.mockito.Mockito.when; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.devtools.build.lib.bazel.repository.downloader.RetryingInputStream.Reconnector; +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.testutil.ManualClock; +import com.google.devtools.build.lib.util.Sleeper; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InterruptedIOException; +import java.net.URL; +import java.net.URLConnection; +import java.util.concurrent.CyclicBarrier; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.junit.rules.Timeout; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; + +/** Unit tests for {@link HttpConnectorMultiplexer}. */ +@RunWith(JUnit4.class) +@SuppressWarnings("unchecked") +public class HttpConnectorMultiplexerTest { + + private static final URL URL1 = makeUrl("http://first.example"); + private static final URL URL2 = makeUrl("http://second.example"); + private static final URL URL3 = makeUrl("http://third.example"); + private static final byte[] data1 = "first".getBytes(UTF_8); + private static final byte[] data2 = "second".getBytes(UTF_8); + private static final byte[] data3 = "third".getBytes(UTF_8); + + @Rule + public final ExpectedException thrown = ExpectedException.none(); + + @Rule + public final Timeout globalTimeout = new Timeout(10000); + + private final HttpStream stream1 = fakeStream(URL1, data1); + private final HttpStream stream2 = fakeStream(URL2, data2); + private final HttpStream stream3 = fakeStream(URL3, data3); + private final ManualClock clock = new ManualClock(); + private final Sleeper sleeper = mock(Sleeper.class); + private final HttpConnector connector = mock(HttpConnector.class); + private final URLConnection connection1 = mock(URLConnection.class); + private final URLConnection connection2 = mock(URLConnection.class); + private final URLConnection connection3 = mock(URLConnection.class); + private final EventHandler eventHandler = mock(EventHandler.class); + private final HttpStream.Factory streamFactory = mock(HttpStream.Factory.class); + private final HttpConnectorMultiplexer multiplexer = + new HttpConnectorMultiplexer(eventHandler, connector, streamFactory, clock, sleeper); + + @Before + public void before() throws Exception { + when(connector.connect(eq(URL1), any(ImmutableMap.class))).thenReturn(connection1); + when(connector.connect(eq(URL2), any(ImmutableMap.class))).thenReturn(connection2); + when(connector.connect(eq(URL3), any(ImmutableMap.class))).thenReturn(connection3); + when(streamFactory + .create(same(connection1), any(URL.class), anyString(), any(Reconnector.class))) + .thenReturn(stream1); + when(streamFactory + .create(same(connection2), any(URL.class), anyString(), any(Reconnector.class))) + .thenReturn(stream2); + when(streamFactory + .create(same(connection3), any(URL.class), anyString(), any(Reconnector.class))) + .thenReturn(stream3); + } + + @Test + public void emptyList_throwsIae() throws Exception { + thrown.expect(IllegalArgumentException.class); + multiplexer.connect(ImmutableList.<URL>of(), ""); + } + + @Test + public void ftpUrl_throwsIae() throws Exception { + thrown.expect(IllegalArgumentException.class); + multiplexer.connect(asList(new URL("ftp://lol.example")), ""); + } + + @Test + public void threadIsInterrupted_throwsIeProntoAndDoesNothingElse() throws Exception { + final AtomicBoolean wasInterrupted = new AtomicBoolean(true); + Thread task = new Thread( + new Runnable() { + @Override + public void run() { + Thread.currentThread().interrupt(); + try { + multiplexer.connect(asList(new URL("http://lol.example")), ""); + } catch (InterruptedIOException ignored) { + return; + } catch (Exception ignored) { + // ignored + } + wasInterrupted.set(false); + } + }); + task.start(); + task.join(); + assertThat(wasInterrupted.get()).isTrue(); + verifyZeroInteractions(connector); + } + + @Test + public void singleUrl_justCallsConnector() throws Exception { + assertThat(toByteArray(multiplexer.connect(asList(URL1), "abc"))).isEqualTo(data1); + verify(connector).connect(eq(URL1), any(ImmutableMap.class)); + verify(streamFactory) + .create(any(URLConnection.class), any(URL.class), eq("abc"), any(Reconnector.class)); + verifyNoMoreInteractions(sleeper, connector, streamFactory); + } + + @Test + public void multipleUrlsFail_throwsIOException() throws Exception { + when(connector.connect(any(URL.class), any(ImmutableMap.class))).thenThrow(new IOException()); + try { + multiplexer.connect(asList(URL1, URL2, URL3), ""); + fail("Expected IOException"); + } catch (IOException e) { + assertThat(e.getMessage()).contains("All mirrors are down"); + } + verify(connector, times(3)).connect(any(URL.class), any(ImmutableMap.class)); + verify(sleeper, times(2)).sleepMillis(anyLong()); + verifyNoMoreInteractions(sleeper, connector, streamFactory); + } + + @Test + public void firstUrlFails_returnsSecond() throws Exception { + doAnswer( + new Answer<Void>() { + @Override + public Void answer(InvocationOnMock invocation) throws Throwable { + clock.advanceMillis(1000); + return null; + } + }).when(sleeper).sleepMillis(anyLong()); + when(connector.connect(eq(URL1), any(ImmutableMap.class))).thenThrow(new IOException()); + assertThat(toByteArray(multiplexer.connect(asList(URL1, URL2), "abc"))).isEqualTo(data2); + assertThat(clock.currentTimeMillis()).isEqualTo(1000L); + verify(connector).connect(eq(URL1), any(ImmutableMap.class)); + verify(connector).connect(eq(URL2), any(ImmutableMap.class)); + verify(streamFactory) + .create(any(URLConnection.class), any(URL.class), eq("abc"), any(Reconnector.class)); + verify(sleeper).sleepMillis(anyLong()); + verifyNoMoreInteractions(sleeper, connector, streamFactory); + } + + @Test + public void twoSuccessfulUrlsAndFirstWins_returnsFirstAndInterruptsSecond() throws Exception { + final CyclicBarrier barrier = new CyclicBarrier(2); + final AtomicBoolean wasInterrupted = new AtomicBoolean(true); + when(connector.connect(eq(URL1), any(ImmutableMap.class))).thenAnswer( + new Answer<URLConnection>() { + @Override + public URLConnection answer(InvocationOnMock invocation) throws Throwable { + barrier.await(); + return connection1; + } + }); + doAnswer( + new Answer<Void>() { + @Override + public Void answer(InvocationOnMock invocation) throws Throwable { + barrier.await(); + TimeUnit.MILLISECONDS.sleep(10000); + wasInterrupted.set(false); + return null; + } + }).when(sleeper).sleepMillis(anyLong()); + assertThat(toByteArray(multiplexer.connect(asList(URL1, URL2), "abc"))).isEqualTo(data1); + assertThat(wasInterrupted.get()).isTrue(); + } + + @Test + public void parentThreadGetsInterrupted_interruptsChildrenThenThrowsIe() throws Exception { + final CyclicBarrier barrier = new CyclicBarrier(3); + final AtomicBoolean wasInterrupted1 = new AtomicBoolean(true); + final AtomicBoolean wasInterrupted2 = new AtomicBoolean(true); + final AtomicBoolean wasInterrupted3 = new AtomicBoolean(true); + when(connector.connect(eq(URL1), any(ImmutableMap.class))).thenAnswer( + new Answer<URLConnection>() { + @Override + public URLConnection answer(InvocationOnMock invocation) throws Throwable { + barrier.await(); + TimeUnit.MILLISECONDS.sleep(10000); + wasInterrupted1.set(false); + throw new RuntimeException(); + } + }); + when(connector.connect(eq(URL2), any(ImmutableMap.class))).thenAnswer( + new Answer<URLConnection>() { + @Override + public URLConnection answer(InvocationOnMock invocation) throws Throwable { + barrier.await(); + TimeUnit.MILLISECONDS.sleep(10000); + wasInterrupted2.set(false); + throw new RuntimeException(); + } + }); + Thread task = new Thread( + new Runnable() { + @Override + public void run() { + try { + multiplexer.connect(asList(URL1, URL2), ""); + } catch (InterruptedIOException ignored) { + return; + } catch (Exception ignored) { + // ignored + } + wasInterrupted3.set(false); + } + }); + task.start(); + barrier.await(); + task.interrupt(); + task.join(); + assertThat(wasInterrupted1.get()).isTrue(); + assertThat(wasInterrupted2.get()).isTrue(); + assertThat(wasInterrupted3.get()).isTrue(); + } + + private static HttpStream fakeStream(URL url, byte[] data) { + return new HttpStream(new ByteArrayInputStream(data), url); + } +}
diff --git a/src/test/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpConnectorTest.java b/src/test/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpConnectorTest.java index fa5a1b6..ccc5019 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpConnectorTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpConnectorTest.java
@@ -15,41 +15,54 @@ package com.google.devtools.build.lib.bazel.repository.downloader; import static com.google.common.truth.Truth.assertThat; +import static com.google.devtools.build.lib.bazel.repository.downloader.DownloaderTestUtils.sendLines; +import static com.google.devtools.build.lib.bazel.repository.downloader.HttpParser.readHttpRequest; import static java.nio.charset.StandardCharsets.ISO_8859_1; +import static java.nio.charset.StandardCharsets.US_ASCII; import static java.nio.charset.StandardCharsets.UTF_8; +import static org.junit.Assert.fail; +import static org.mockito.Matchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -import com.google.common.io.ByteSource; +import com.google.common.collect.ImmutableMap; import com.google.common.io.ByteStreams; import com.google.common.io.CharStreams; import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.testutil.ManualClock; +import com.google.devtools.build.lib.testutil.ManualSleeper; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; +import java.io.InputStream; import java.io.InputStreamReader; import java.io.Reader; -import java.net.HttpURLConnection; import java.net.InetAddress; import java.net.Proxy; import java.net.ServerSocket; import java.net.Socket; import java.net.URL; +import java.net.URLConnection; +import java.util.Locale; +import java.util.Map; import java.util.concurrent.Callable; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CyclicBarrier; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; -import java.util.concurrent.Future; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; import org.junit.After; +import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; import org.junit.rules.TemporaryFolder; +import org.junit.rules.Timeout; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; -/** - * Unit tests for {@link HttpConnector}. - */ +/** Unit tests for {@link HttpConnector}. */ @RunWith(JUnit4.class) public class HttpConnectorTest { @@ -57,191 +70,368 @@ public final ExpectedException thrown = ExpectedException.none(); @Rule - public TemporaryFolder testFolder = new TemporaryFolder(); + public final TemporaryFolder testFolder = new TemporaryFolder(); - private final ExecutorService executor = Executors.newSingleThreadExecutor(); - private final HttpURLConnection connection = mock(HttpURLConnection.class); + @Rule + public final Timeout globalTimeout = new Timeout(10000); + + private final ExecutorService executor = Executors.newFixedThreadPool(2); + private final ManualClock clock = new ManualClock(); + private final ManualSleeper sleeper = new ManualSleeper(clock); private final EventHandler eventHandler = mock(EventHandler.class); + private final ProxyHelper proxyHelper = mock(ProxyHelper.class); + private final HttpConnector connector = + new HttpConnector(Locale.US, eventHandler, proxyHelper, sleeper); + + @Before + public void before() throws Exception { + when(proxyHelper.createProxyIfNeeded(any(URL.class))).thenReturn(Proxy.NO_PROXY); + } @After public void after() throws Exception { - executor.shutdownNow(); + executor.shutdown(); } @Test - public void testLocalFileDownload() throws Exception { + public void localFileDownload() throws Exception { byte[] fileContents = "this is a test".getBytes(UTF_8); assertThat( ByteStreams.toByteArray( - HttpConnector.connect( - createTempFile(fileContents).toURI().toURL(), - Proxy.NO_PROXY, - eventHandler))) + connector.connect( + createTempFile(fileContents).toURI().toURL(), + ImmutableMap.<String, String>of()) + .getInputStream())) .isEqualTo(fileContents); } @Test - public void missingLocationInRedirect_throwsIOException() throws Exception { + public void badHost_throwsIOException() throws Exception { thrown.expect(IOException.class); - when(connection.getURL()).thenReturn(new URL("http://lol.example")); - HttpConnector.getLocation(connection); + thrown.expectMessage("Unknown host: bad.example"); + connector.connect(new URL("http://bad.example"), ImmutableMap.<String, String>of()); } @Test - public void absoluteLocationInRedirect_returnsNewUrl() throws Exception { - when(connection.getURL()).thenReturn(new URL("http://lol.example")); - when(connection.getHeaderField("Location")).thenReturn("http://new.example/hi"); - assertThat(HttpConnector.getLocation(connection)).isEqualTo(new URL("http://new.example/hi")); - } - - @Test - public void redirectOnlyHasPath_mergesHostFromOriginalUrl() throws Exception { - when(connection.getURL()).thenReturn(new URL("http://lol.example")); - when(connection.getHeaderField("Location")).thenReturn("/hi"); - assertThat(HttpConnector.getLocation(connection)).isEqualTo(new URL("http://lol.example/hi")); - } - - @Test - public void locationOnlyHasPathWithoutSlash_failsToMerge() throws Exception { - thrown.expect(IOException.class); - thrown.expectMessage("Could not merge"); - when(connection.getURL()).thenReturn(new URL("http://lol.example")); - when(connection.getHeaderField("Location")).thenReturn("omg"); - HttpConnector.getLocation(connection); - } - - @Test - public void locationHasFragment_prefersNewFragment() throws Exception { - when(connection.getURL()).thenReturn(new URL("http://lol.example#a")); - when(connection.getHeaderField("Location")).thenReturn("http://new.example/hi#b"); - assertThat(HttpConnector.getLocation(connection)).isEqualTo(new URL("http://new.example/hi#b")); - } - - @Test - public void locationHasNoFragmentButOriginalDoes_mergesOldFragment() throws Exception { - when(connection.getURL()).thenReturn(new URL("http://lol.example#a")); - when(connection.getHeaderField("Location")).thenReturn("http://new.example/hi"); - assertThat(HttpConnector.getLocation(connection)).isEqualTo(new URL("http://new.example/hi#a")); - } - - @Test - public void oldUrlHasPasswordRedirectingToSameDomain_mergesPassword() throws Exception { - when(connection.getURL()).thenReturn(new URL("http://a:b@lol.example")); - when(connection.getHeaderField("Location")).thenReturn("http://lol.example/hi"); - assertThat(HttpConnector.getLocation(connection)) - .isEqualTo(new URL("http://a:b@lol.example/hi")); - when(connection.getURL()).thenReturn(new URL("http://a:b@lol.example")); - when(connection.getHeaderField("Location")).thenReturn("/hi"); - assertThat(HttpConnector.getLocation(connection)) - .isEqualTo(new URL("http://a:b@lol.example/hi")); - } - - @Test - public void oldUrlHasPasswordRedirectingToNewServer_doesntMergePassword() throws Exception { - when(connection.getURL()).thenReturn(new URL("http://a:b@lol.example")); - when(connection.getHeaderField("Location")).thenReturn("http://new.example/hi"); - assertThat(HttpConnector.getLocation(connection)).isEqualTo(new URL("http://new.example/hi")); - when(connection.getURL()).thenReturn(new URL("http://a:b@lol.example")); - when(connection.getHeaderField("Location")).thenReturn("http://lol.example:81/hi"); - assertThat(HttpConnector.getLocation(connection)) - .isEqualTo(new URL("http://lol.example:81/hi")); - } - - @Test - public void redirectToFtp_throwsIOException() throws Exception { - thrown.expect(IOException.class); - thrown.expectMessage("Bad Location"); - when(connection.getURL()).thenReturn(new URL("http://lol.example")); - when(connection.getHeaderField("Location")).thenReturn("ftp://lol.example"); - HttpConnector.getLocation(connection); - } - - @Test - public void redirectToHttps_works() throws Exception { - when(connection.getURL()).thenReturn(new URL("http://lol.example")); - when(connection.getHeaderField("Location")).thenReturn("https://lol.example"); - assertThat(HttpConnector.getLocation(connection)).isEqualTo(new URL("https://lol.example")); - } - - @Test - public void testNormalRequest() throws Exception { - try (final ServerSocket server = new ServerSocket(0, 1, InetAddress.getByName("127.0.0.1"))) { - Future<Void> thread = - executor.submit( - new Callable<Void>() { - @Override - public Void call() throws Exception { - try (Socket socket = server.accept()) { - send(socket, - "HTTP/1.1 200 OK\r\n" - + "Date: Fri, 31 Dec 1999 23:59:59 GMT\r\n" - + "Content-Type: text/plain\r\n" - + "Content-Length: 5\r\n" - + "\r\n" - + "hello"); - } - return null; - } - }); + public void normalRequest() throws Exception { + final Map<String, String> headers = new ConcurrentHashMap<>(); + try (ServerSocket server = new ServerSocket(0, 1, InetAddress.getByName("127.0.0.1"))) { + executor.submit( + new Callable<Object>() { + @Override + public Object call() throws Exception { + try (Socket socket = server.accept()) { + readHttpRequest(socket.getInputStream(), headers); + sendLines(socket, + "HTTP/1.1 200 OK", + "Date: Fri, 31 Dec 1999 23:59:59 GMT", + "Connection: close", + "Content-Type: text/plain", + "Content-Length: 5", + "", + "hello"); + } + return null; + } + }); try (Reader payload = new InputStreamReader( - HttpConnector.connect( - new URL(String.format("http://127.0.0.1:%d", server.getLocalPort())), - Proxy.NO_PROXY, - eventHandler), + connector.connect( + new URL(String.format("http://127.0.0.1:%d/boo", server.getLocalPort())), + ImmutableMap.of("Content-Encoding", "gzip")) + .getInputStream(), ISO_8859_1)) { assertThat(CharStreams.toString(payload)).isEqualTo("hello"); } - thread.get(); + } + assertThat(headers).containsEntry("x-method", "GET"); + assertThat(headers).containsEntry("x-request-uri", "/boo"); + assertThat(headers).containsEntry("content-encoding", "gzip"); + } + + @Test + public void serverError_retriesConnect() throws Exception { + try (ServerSocket server = new ServerSocket(0, 1, InetAddress.getByName("127.0.0.1"))) { + executor.submit( + new Callable<Object>() { + @Override + public Object call() throws Exception { + try (Socket socket = server.accept()) { + readHttpRequest(socket.getInputStream()); + sendLines(socket, + "HTTP/1.1 500 Incredible Catastrophe", + "Date: Fri, 31 Dec 1999 23:59:59 GMT", + "Connection: close", + "Content-Type: text/plain", + "Content-Length: 8", + "", + "nononono"); + } + try (Socket socket = server.accept()) { + readHttpRequest(socket.getInputStream()); + sendLines(socket, + "HTTP/1.1 200 OK", + "Date: Fri, 31 Dec 1999 23:59:59 GMT", + "Connection: close", + "Content-Type: text/plain", + "Content-Length: 5", + "", + "hello"); + } + return null; + } + }); + try (Reader payload = + new InputStreamReader( + connector.connect( + new URL(String.format("http://127.0.0.1:%d", server.getLocalPort())), + ImmutableMap.<String, String>of()) + .getInputStream(), + ISO_8859_1)) { + assertThat(CharStreams.toString(payload)).isEqualTo("hello"); + assertThat(clock.currentTimeMillis()).isEqualTo(100L); + } } } @Test - public void testRetry() throws Exception { - try (final ServerSocket server = new ServerSocket(0, 1, InetAddress.getByName("127.0.0.1"))) { - Future<Void> thread = - executor.submit( - new Callable<Void>() { - @Override - public Void call() throws Exception { - try (Socket socket = server.accept()) { - send(socket, - "HTTP/1.1 500 Incredible Catastrophe\r\n" - + "Date: Fri, 31 Dec 1999 23:59:59 GMT\r\n" - + "Content-Type: text/plain\r\n" - + "Content-Length: 8\r\n" - + "\r\n" - + "nononono"); - } - try (Socket socket = server.accept()) { - send(socket, - "HTTP/1.1 200 OK\r\n" - + "Date: Fri, 31 Dec 1999 23:59:59 GMT\r\n" - + "Content-Type: text/plain\r\n" - + "Content-Length: 5\r\n" - + "\r\n" - + "hello"); - } - return null; - } - }); - try (Reader payload = - new InputStreamReader( - HttpConnector.connect( - new URL(String.format("http://127.0.0.1:%d", server.getLocalPort())), - Proxy.NO_PROXY, - eventHandler), - ISO_8859_1)) { - assertThat(CharStreams.toString(payload)).isEqualTo("hello"); - } - thread.get(); + public void permanentError_doesNotRetryAndThrowsIOException() throws Exception { + try (ServerSocket server = new ServerSocket(0, 1, InetAddress.getByName("127.0.0.1"))) { + executor.submit( + new Callable<Object>() { + @Override + public Object call() throws Exception { + try (Socket socket = server.accept()) { + readHttpRequest(socket.getInputStream()); + sendLines(socket, + "HTTP/1.1 404 Not Here", + "Date: Fri, 31 Dec 1999 23:59:59 GMT", + "Connection: close", + "Content-Type: text/plain", + "Content-Length: 0", + "", + ""); + } + return null; + } + }); + thrown.expect(IOException.class); + thrown.expectMessage("404 Not Here"); + connector.connect( + new URL(String.format("http://127.0.0.1:%d", server.getLocalPort())), + ImmutableMap.<String, String>of()); } } - private static void send(Socket socket, String data) throws IOException { - ByteStreams.copy( - ByteSource.wrap(data.getBytes(ISO_8859_1)).openStream(), - socket.getOutputStream()); + @Test + public void permanentError_consumesPayloadBeforeReturningn() throws Exception { + final CyclicBarrier barrier = new CyclicBarrier(2); + final AtomicBoolean consumed = new AtomicBoolean(); + try (ServerSocket server = new ServerSocket(0, 1, InetAddress.getByName("127.0.0.1"))) { + executor.submit( + new Callable<Object>() { + @Override + public Object call() throws Exception { + try (Socket socket = server.accept()) { + readHttpRequest(socket.getInputStream()); + sendLines(socket, + "HTTP/1.1 501 Oh No", + "Date: Fri, 31 Dec 1999 23:59:59 GMT", + "Connection: close", + "Content-Type: text/plain", + "Content-Length: 1", + "", + "b"); + consumed.set(true); + } finally { + barrier.await(); + } + return null; + } + }); + connector.connect( + new URL(String.format("http://127.0.0.1:%d", server.getLocalPort())), + ImmutableMap.<String, String>of()); + fail(); + } catch (IOException ignored) { + // ignored + } finally { + barrier.await(); + } + assertThat(consumed.get()).isTrue(); + assertThat(clock.currentTimeMillis()).isEqualTo(0L); + } + + @Test + public void always500_givesUpEventually() throws Exception { + final AtomicInteger tries = new AtomicInteger(); + try (ServerSocket server = new ServerSocket(0, 1, InetAddress.getByName("127.0.0.1"))) { + executor.submit(new Callable<Object>() { + @Override + public Object call() throws Exception { + while (true) { + try (Socket socket = server.accept()) { + readHttpRequest(socket.getInputStream()); + sendLines(socket, + "HTTP/1.1 500 Oh My", + "Date: Fri, 31 Dec 1999 23:59:59 GMT", + "Connection: close", + "Content-Type: text/plain", + "Content-Length: 0", + "", + ""); + tries.incrementAndGet(); + } + } + } + }); + thrown.expect(IOException.class); + thrown.expectMessage("500 Oh My"); + try { + connector.connect( + new URL(String.format("http://127.0.0.1:%d", server.getLocalPort())), + ImmutableMap.<String, String>of()); + } finally { + assertThat(tries.get()).isGreaterThan(2); + } + } + } + + @Test + public void serverSays403_clientRetriesAnyway() throws Exception { + final AtomicInteger tries = new AtomicInteger(); + try (ServerSocket server = new ServerSocket(0, 1, InetAddress.getByName("127.0.0.1"))) { + executor.submit( + new Callable<Object>() { + @Override + public Object call() throws Exception { + while (true) { + try (Socket socket = server.accept()) { + readHttpRequest(socket.getInputStream()); + sendLines(socket, + "HTTP/1.1 403 Forbidden", + "Date: Fri, 31 Dec 1999 23:59:59 GMT", + "Connection: close", + "Content-Type: text/plain", + "Content-Length: 0", + "", + ""); + tries.incrementAndGet(); + } + } + } + }); + thrown.expect(IOException.class); + thrown.expectMessage("403 Forbidden"); + try { + connector.connect( + new URL(String.format("http://127.0.0.1:%d", server.getLocalPort())), + ImmutableMap.<String, String>of()); + } finally { + assertThat(tries.get()).isGreaterThan(2); + } + } + } + + @Test + public void redirectToDifferentPath_works() throws Exception { + final Map<String, String> headers1 = new ConcurrentHashMap<>(); + final Map<String, String> headers2 = new ConcurrentHashMap<>(); + try (ServerSocket server = new ServerSocket(0, 1, InetAddress.getByName("127.0.0.1"))) { + executor.submit( + new Callable<Object>() { + @Override + public Object call() throws Exception { + try (Socket socket = server.accept()) { + readHttpRequest(socket.getInputStream(), headers1); + sendLines(socket, + "HTTP/1.1 301 Redirect", + "Date: Fri, 31 Dec 1999 23:59:59 GMT", + "Connection: close", + "Location: /doodle.tar.gz", + "Content-Length: 0", + "", + ""); + } + try (Socket socket = server.accept()) { + readHttpRequest(socket.getInputStream(), headers2); + sendLines(socket, + "HTTP/1.1 200 OK", + "Date: Fri, 31 Dec 1999 23:59:59 GMT", + "Connection: close", + "Content-Type: text/plain", + "Content-Length: 0", + "", + ""); + } + return null; + } + }); + URLConnection connection = + connector.connect( + new URL(String.format("http://127.0.0.1:%d", server.getLocalPort())), + ImmutableMap.<String, String>of()); + assertThat(connection.getURL()).isEqualTo( + new URL(String.format("http://127.0.0.1:%d/doodle.tar.gz", server.getLocalPort()))); + try (InputStream input = connection.getInputStream()) { + assertThat(ByteStreams.toByteArray(input)).isEmpty(); + } + } + assertThat(headers1).containsEntry("x-request-uri", "/"); + assertThat(headers2).containsEntry("x-request-uri", "/doodle.tar.gz"); + } + + @Test + public void redirectToDifferentServer_works() throws Exception { + try (ServerSocket server1 = new ServerSocket(0, 1, InetAddress.getByName("127.0.0.1")); + ServerSocket server2 = new ServerSocket(0, 1, InetAddress.getByName("127.0.0.1"))) { + executor.submit( + new Callable<Object>() { + @Override + public Object call() throws Exception { + try (Socket socket = server1.accept()) { + readHttpRequest(socket.getInputStream()); + sendLines(socket, + "HTTP/1.1 301 Redirect", + "Date: Fri, 31 Dec 1999 23:59:59 GMT", + "Connection: close", + String.format("Location: http://127.0.0.1:%d/doodle.tar.gz", + server2.getLocalPort()), + "Content-Length: 0", + "", + ""); + } + return null; + } + }); + executor.submit( + new Callable<Object>() { + @Override + public Object call() throws Exception { + try (Socket socket = server2.accept()) { + readHttpRequest(socket.getInputStream()); + sendLines(socket, + "HTTP/1.1 200 OK", + "Date: Fri, 31 Dec 1999 23:59:59 GMT", + "Connection: close", + "Content-Type: text/plain", + "Content-Length: 5", + "", + "hello"); + } + return null; + } + }); + URLConnection connection = + connector.connect( + new URL(String.format("http://127.0.0.1:%d", server1.getLocalPort())), + ImmutableMap.<String, String>of()); + assertThat(connection.getURL()).isEqualTo( + new URL(String.format("http://127.0.0.1:%d/doodle.tar.gz", server2.getLocalPort()))); + try (InputStream input = connection.getInputStream()) { + assertThat(ByteStreams.toByteArray(input)).isEqualTo("hello".getBytes(US_ASCII)); + } + } } private File createTempFile(byte[] fileContents) throws IOException {
diff --git a/src/test/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpParser.java b/src/test/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpParser.java new file mode 100644 index 0000000..c901367 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpParser.java
@@ -0,0 +1,154 @@ +// Copyright 2016 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.repository.downloader; + +import com.google.common.base.Ascii; +import java.io.IOException; +import java.io.InputStream; +import java.util.HashMap; +import java.util.Map; + +/** Utility class for parsing HTTP messages. */ +final class HttpParser { + + /** Exhausts request line and headers of HTTP request. */ + static void readHttpRequest(InputStream stream) throws IOException { + readHttpRequest(stream, new HashMap<String, String>()); + } + + /** + * Parses request line and headers of HTTP request. + * + * <p>This parser is correct and extremely lax. This implementation is Θ(n) and the stream should + * be buffered. All decoding is ISO-8859-1. A 1mB upper bound on memory is enforced. + * + * @throws IOException if reading failed or premature end of stream encountered + * @throws HttpParserError if 400 error should be sent to client and connection must be closed + */ + static void readHttpRequest(InputStream stream, Map<String, String> output) throws IOException { + StringBuilder builder = new StringBuilder(256); + State state = State.METHOD; + String key = ""; + int toto = 0; + while (true) { + int c = stream.read(); + if (c == -1) { + throw new IOException(); // RFC7230 § 3.4 + } + if (++toto == 1024 * 1024) { + throw new HttpParserError(); // RFC7230 § 3.2.5 + } + switch (state) { + case METHOD: + if (c == ' ') { + if (builder.length() == 0) { + throw new HttpParserError(); + } + output.put("x-method", builder.toString()); + builder.setLength(0); + state = State.URI; + } else if (c == '\r' || c == '\n') { + break; // RFC7230 § 3.5 + } else { + builder.append(Ascii.toUpperCase((char) c)); + } + break; + case URI: + if (c == ' ') { + if (builder.length() == 0) { + throw new HttpParserError(); + } + output.put("x-request-uri", builder.toString()); + builder.setLength(0); + state = State.VERSION; + } else { + builder.append((char) c); + } + break; + case VERSION: + if (c == '\r' || c == '\n') { + output.put("x-version", builder.toString()); + builder.setLength(0); + state = c == '\r' ? State.CR1 : State.LF1; + } else { + builder.append(Ascii.toUpperCase((char) c)); + } + break; + case CR1: + if (c == '\n') { + state = State.LF1; + break; + } + throw new HttpParserError(); + case LF1: + if (c == '\r') { + state = State.LF2; + break; + } else if (c == '\n') { + return; + } else if (c == ' ' || c == '\t') { + throw new HttpParserError("Line folding unacceptable"); // RFC7230 § 3.2.4 + } + state = State.HKEY; + // epsilon transition + case HKEY: + if (c == ':') { + key = builder.toString(); + builder.setLength(0); + state = State.HSEP; + } else { + builder.append(Ascii.toLowerCase((char) c)); + } + break; + case HSEP: + if (c == ' ' || c == '\t') { + break; + } + state = State.HVAL; + // epsilon transition + case HVAL: + if (c == '\r' || c == '\n') { + output.put(key, builder.toString()); + builder.setLength(0); + state = c == '\r' ? State.CR1 : State.LF1; + } else { + builder.append((char) c); + } + break; + case LF2: + if (c == '\n') { + return; + } + throw new HttpParserError(); + default: + throw new AssertionError(); + } + } + } + + static final class HttpParserError extends IOException { + HttpParserError() { + this("Malformed Request"); + } + + HttpParserError(String messageForClient) { + super(messageForClient); + } + } + + private enum State { METHOD, URI, VERSION, HKEY, HSEP, HVAL, CR1, LF1, LF2 } + + private HttpParser() {} +}
diff --git a/src/test/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpStreamTest.java b/src/test/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpStreamTest.java new file mode 100644 index 0000000..ac0351e --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpStreamTest.java
@@ -0,0 +1,195 @@ +// Copyright 2016 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.repository.downloader; + +import static com.google.common.io.ByteStreams.toByteArray; +import static com.google.common.truth.Truth.assertThat; +import static com.google.devtools.build.lib.bazel.repository.downloader.DownloaderTestUtils.makeUrl; +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.junit.Assert.fail; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import com.google.common.hash.Hashing; +import com.google.common.io.ByteStreams; +import com.google.devtools.build.lib.bazel.repository.downloader.RetryingInputStream.Reconnector; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InterruptedIOException; +import java.io.OutputStream; +import java.net.HttpURLConnection; +import java.net.URL; +import java.util.Random; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.zip.GZIPOutputStream; +import java.util.zip.ZipException; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.junit.rules.Timeout; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; + +/** Integration tests for {@link HttpStream.Factory} and friends. */ +@RunWith(JUnit4.class) +public class HttpStreamTest { + + private static final Random randoCalrissian = new Random(); + private static final byte[] data = "hello".getBytes(UTF_8); + private static final String GOOD_CHECKSUM = + "2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824"; + private static final String BAD_CHECKSUM = + "0000000000000000000000000000000000000000000000000000000000000000"; + private static final URL AURL = makeUrl("http://doodle.example"); + + @Rule + public final ExpectedException thrown = ExpectedException.none(); + + @Rule + public final Timeout globalTimeout = new Timeout(10000); + + private final HttpURLConnection connection = mock(HttpURLConnection.class); + private final Reconnector reconnector = mock(Reconnector.class); + private final ProgressInputStream.Factory progress = mock(ProgressInputStream.Factory.class); + private final HttpStream.Factory streamFactory = new HttpStream.Factory(progress); + + @Before + public void before() throws Exception { + when(connection.getInputStream()).thenReturn(new ByteArrayInputStream(data)); + when(progress.create(any(InputStream.class), any(URL.class), any(URL.class))).thenAnswer( + new Answer<InputStream>() { + @Override + public InputStream answer(InvocationOnMock invocation) throws Throwable { + return (InputStream) invocation.getArguments()[0]; + } + }); + } + + @Test + public void noChecksum_readsOk() throws Exception { + try (HttpStream stream = streamFactory.create(connection, AURL, "", reconnector)) { + assertThat(toByteArray(stream)).isEqualTo(data); + } + } + + @Test + public void smallDataWithValidChecksum_readsOk() throws Exception { + try (HttpStream stream = streamFactory.create(connection, AURL, GOOD_CHECKSUM, reconnector)) { + assertThat(toByteArray(stream)).isEqualTo(data); + } + } + + @Test + public void smallDataWithInvalidChecksum_throwsIOExceptionInCreatePhase() throws Exception { + thrown.expect(IOException.class); + thrown.expectMessage("Checksum"); + streamFactory.create(connection, AURL, BAD_CHECKSUM, reconnector); + } + + @Test + public void bigDataWithValidChecksum_readsOk() throws Exception { + // at google, we know big data + byte[] bigData = new byte[HttpStream.PRECHECK_BYTES + 70001]; + randoCalrissian.nextBytes(bigData); + when(connection.getInputStream()).thenReturn(new ByteArrayInputStream(bigData)); + try (HttpStream stream = + streamFactory.create( + connection, AURL, Hashing.sha256().hashBytes(bigData).toString(), reconnector)) { + assertThat(toByteArray(stream)).isEqualTo(bigData); + } + } + + @Test + public void bigDataWithInvalidChecksum_throwsIOExceptionAfterCreateOnEof() throws Exception { + // the probability of this test flaking is 8.6361686e-78 + byte[] bigData = new byte[HttpStream.PRECHECK_BYTES + 70001]; + randoCalrissian.nextBytes(bigData); + when(connection.getInputStream()).thenReturn(new ByteArrayInputStream(bigData)); + try (HttpStream stream = streamFactory.create(connection, AURL, BAD_CHECKSUM, reconnector)) { + thrown.expect(IOException.class); + thrown.expectMessage("Checksum"); + toByteArray(stream); + fail("Should have thrown error before close()"); + } + } + + @Test + public void httpServerSaidGzippedButNotGzipped_throwsZipExceptionInCreate() throws Exception { + when(connection.getURL()).thenReturn(AURL); + when(connection.getContentEncoding()).thenReturn("gzip"); + thrown.expect(ZipException.class); + streamFactory.create(connection, AURL, "", reconnector); + } + + @Test + public void javascriptGzippedInTransit_automaticallyGunzips() throws Exception { + when(connection.getURL()).thenReturn(AURL); + when(connection.getContentEncoding()).thenReturn("x-gzip"); + when(connection.getInputStream()).thenReturn(new ByteArrayInputStream(gzipData(data))); + try (HttpStream stream = streamFactory.create(connection, AURL, "", reconnector)) { + assertThat(toByteArray(stream)).isEqualTo(data); + } + } + + @Test + public void serverSaysTarballPathIsGzipped_doesntAutomaticallyGunzip() throws Exception { + byte[] gzData = gzipData(data); + when(connection.getURL()).thenReturn(new URL("http://doodle.example/foo.tar.gz")); + when(connection.getContentEncoding()).thenReturn("gzip"); + when(connection.getInputStream()).thenReturn(new ByteArrayInputStream(gzData)); + try (HttpStream stream = streamFactory.create(connection, AURL, "", reconnector)) { + assertThat(toByteArray(stream)).isEqualTo(gzData); + } + } + + @Test + public void threadInterrupted_haltsReadingAndThrowsInterrupt() throws Exception { + final AtomicBoolean wasInterrupted = new AtomicBoolean(); + Thread thread = new Thread( + new Runnable() { + @Override + public void run() { + try (HttpStream stream = streamFactory.create(connection, AURL, "", reconnector)) { + stream.read(); + Thread.currentThread().interrupt(); + stream.read(); + fail(); + } catch (InterruptedIOException expected) { + wasInterrupted.set(true); + } catch (IOException ignored) { + // ignored + } + } + }); + thread.start(); + thread.join(); + assertThat(wasInterrupted.get()).isTrue(); + } + + private static byte[] gzipData(byte[] bytes) throws IOException { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + try (InputStream input = new ByteArrayInputStream(bytes); + OutputStream output = new GZIPOutputStream(baos)) { + ByteStreams.copy(input, output); + } + return baos.toByteArray(); + } +}
diff --git a/src/test/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpUtilsTest.java b/src/test/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpUtilsTest.java new file mode 100644 index 0000000..b79cbc5 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpUtilsTest.java
@@ -0,0 +1,131 @@ +// Copyright 2016 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.repository.downloader; + +import static com.google.common.truth.Truth.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.io.IOException; +import java.net.HttpURLConnection; +import java.net.URL; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Unit tests for {@link HttpUtils}. */ +@RunWith(JUnit4.class) +public class HttpUtilsTest { + + @Rule + public final ExpectedException thrown = ExpectedException.none(); + + private final HttpURLConnection connection = mock(HttpURLConnection.class); + + @Test + public void getExtension_twoExtensions_returnsLast() throws Exception { + assertThat(HttpUtils.getExtension("doodle.tar.gz")).isEqualTo("gz"); + } + + @Test + public void getExtension_isUppercase_returnsLowered() throws Exception { + assertThat(HttpUtils.getExtension("DOODLE.TXT")).isEqualTo("txt"); + } + + @Test + public void getLocation_missingInRedirect_throwsIOException() throws Exception { + thrown.expect(IOException.class); + when(connection.getURL()).thenReturn(new URL("http://lol.example")); + HttpUtils.getLocation(connection); + } + + @Test + public void getLocation_absoluteInRedirect_returnsNewUrl() throws Exception { + when(connection.getURL()).thenReturn(new URL("http://lol.example")); + when(connection.getHeaderField("Location")).thenReturn("http://new.example/hi"); + assertThat(HttpUtils.getLocation(connection)).isEqualTo(new URL("http://new.example/hi")); + } + + @Test + public void getLocation_redirectOnlyHasPath_mergesHostFromOriginalUrl() throws Exception { + when(connection.getURL()).thenReturn(new URL("http://lol.example")); + when(connection.getHeaderField("Location")).thenReturn("/hi"); + assertThat(HttpUtils.getLocation(connection)).isEqualTo(new URL("http://lol.example/hi")); + } + + @Test + public void getLocation_onlyHasPathWithoutSlash_failsToMerge() throws Exception { + thrown.expect(IOException.class); + thrown.expectMessage("Could not merge"); + when(connection.getURL()).thenReturn(new URL("http://lol.example")); + when(connection.getHeaderField("Location")).thenReturn("omg"); + HttpUtils.getLocation(connection); + } + + @Test + public void getLocation_hasFragment_prefersNewFragment() throws Exception { + when(connection.getURL()).thenReturn(new URL("http://lol.example#a")); + when(connection.getHeaderField("Location")).thenReturn("http://new.example/hi#b"); + assertThat(HttpUtils.getLocation(connection)).isEqualTo(new URL("http://new.example/hi#b")); + } + + @Test + public void getLocation_hasNoFragmentButOriginalDoes_mergesOldFragment() throws Exception { + when(connection.getURL()).thenReturn(new URL("http://lol.example#a")); + when(connection.getHeaderField("Location")).thenReturn("http://new.example/hi"); + assertThat(HttpUtils.getLocation(connection)).isEqualTo(new URL("http://new.example/hi#a")); + } + + @Test + public void getLocation_oldUrlHasPassRedirectingToSameDomain_mergesPassword() throws Exception { + when(connection.getURL()).thenReturn(new URL("http://a:b@lol.example")); + when(connection.getHeaderField("Location")).thenReturn("http://lol.example/hi"); + assertThat(HttpUtils.getLocation(connection)) + .isEqualTo(new URL("http://a:b@lol.example/hi")); + when(connection.getURL()).thenReturn(new URL("http://a:b@lol.example")); + when(connection.getHeaderField("Location")).thenReturn("/hi"); + assertThat(HttpUtils.getLocation(connection)) + .isEqualTo(new URL("http://a:b@lol.example/hi")); + } + + @Test + public void getLocation_oldUrlHasPasswordRedirectingToNewServer_doesntMerge() throws Exception { + when(connection.getURL()).thenReturn(new URL("http://a:b@lol.example")); + when(connection.getHeaderField("Location")).thenReturn("http://new.example/hi"); + assertThat(HttpUtils.getLocation(connection)).isEqualTo(new URL("http://new.example/hi")); + when(connection.getURL()).thenReturn(new URL("http://a:b@lol.example")); + when(connection.getHeaderField("Location")).thenReturn("http://lol.example:81/hi"); + assertThat(HttpUtils.getLocation(connection)) + .isEqualTo(new URL("http://lol.example:81/hi")); + } + + @Test + public void getLocation_redirectToFtp_throwsIOException() throws Exception { + thrown.expect(IOException.class); + thrown.expectMessage("Bad Location"); + when(connection.getURL()).thenReturn(new URL("http://lol.example")); + when(connection.getHeaderField("Location")).thenReturn("ftp://lol.example"); + HttpUtils.getLocation(connection); + } + + @Test + public void getLocation_redirectToHttps_works() throws Exception { + when(connection.getURL()).thenReturn(new URL("http://lol.example")); + when(connection.getHeaderField("Location")).thenReturn("https://lol.example"); + assertThat(HttpUtils.getLocation(connection)).isEqualTo(new URL("https://lol.example")); + } +}
diff --git a/src/test/java/com/google/devtools/build/lib/bazel/repository/downloader/ProgressInputStreamTest.java b/src/test/java/com/google/devtools/build/lib/bazel/repository/downloader/ProgressInputStreamTest.java new file mode 100644 index 0000000..2e047f7 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/bazel/repository/downloader/ProgressInputStreamTest.java
@@ -0,0 +1,149 @@ +// Copyright 2016 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.repository.downloader; + +import static com.google.common.truth.Truth.assertThat; +import static com.google.devtools.build.lib.bazel.repository.downloader.DownloaderTestUtils.makeUrl; +import static org.junit.Assert.fail; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyInt; +import static org.mockito.Matchers.eq; +import static org.mockito.Matchers.same; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.when; + +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.testutil.ManualClock; +import java.io.IOException; +import java.io.InputStream; +import java.net.URL; +import java.util.Locale; +import org.junit.After; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Unit tests for {@link ProgressInputStream}. */ +@RunWith(JUnit4.class) +public class ProgressInputStreamTest { + + private final ManualClock clock = new ManualClock(); + private final EventHandler eventHandler = mock(EventHandler.class); + private final InputStream delegate = mock(InputStream.class); + private final URL url = makeUrl("http://lol.example"); + private ProgressInputStream stream = + new ProgressInputStream(Locale.US, clock, eventHandler, 1, delegate, url, url); + + @After + public void after() throws Exception { + verifyNoMoreInteractions(eventHandler, delegate); + } + + @Test + public void close_callsDelegate() throws Exception { + stream.close(); + verify(delegate).close(); + } + + @Test + public void available_callsDelegate() throws Exception { + stream.available(); + verify(delegate).available(); + } + + @Test + public void read_callsdelegate() throws Exception { + stream.read(); + verify(delegate).read(); + } + + @Test + public void readThrowsException_passesThrough() throws Exception { + when(delegate.read()).thenThrow(new IOException()); + try { + stream.read(); + fail("Expected IOException"); + } catch (IOException expected) { + verify(delegate).read(); + } + } + + @Test + public void readsAfterInterval_emitsProgressOnce() throws Exception { + when(delegate.read()).thenReturn(42); + assertThat(stream.read()).isEqualTo(42); + clock.advanceMillis(1); + assertThat(stream.read()).isEqualTo(42); + assertThat(stream.read()).isEqualTo(42); + verify(delegate, times(3)).read(); + verify(eventHandler).handle(Event.progress("Downloading http://lol.example: 2 bytes")); + } + + @Test + public void multipleIntervalsElapsed_showsMultipleProgress() throws Exception { + stream.read(); + stream.read(); + clock.advanceMillis(1); + stream.read(); + stream.read(); + clock.advanceMillis(1); + stream.read(); + stream.read(); + verify(delegate, times(6)).read(); + verify(eventHandler).handle(Event.progress("Downloading http://lol.example: 3 bytes")); + verify(eventHandler).handle(Event.progress("Downloading http://lol.example: 5 bytes")); + } + + @Test + public void bufferReadsAfterInterval_emitsProgressOnce() throws Exception { + byte[] buffer = new byte[1024]; + when(delegate.read(any(byte[].class), anyInt(), anyInt())).thenReturn(1024); + assertThat(stream.read(buffer)).isEqualTo(1024); + clock.advanceMillis(1); + assertThat(stream.read(buffer)).isEqualTo(1024); + assertThat(stream.read(buffer)).isEqualTo(1024); + verify(delegate, times(3)).read(same(buffer), eq(0), eq(1024)); + verify(eventHandler).handle(Event.progress("Downloading http://lol.example: 2,048 bytes")); + } + + @Test + public void bufferReadsAfterIntervalInGermany_usesPeriodAsSeparator() throws Exception { + stream = new ProgressInputStream(Locale.GERMANY, clock, eventHandler, 1, delegate, url, url); + byte[] buffer = new byte[1024]; + when(delegate.read(any(byte[].class), anyInt(), anyInt())).thenReturn(1024); + clock.advanceMillis(1); + stream.read(buffer); + verify(delegate).read(same(buffer), eq(0), eq(1024)); + verify(eventHandler).handle(Event.progress("Downloading http://lol.example: 1.024 bytes")); + } + + @Test + public void redirectedToDifferentServer_showsOriginalUrlWithVia() throws Exception { + stream = new ProgressInputStream( + Locale.US, clock, eventHandler, 1, delegate, new URL("http://cdn.example/foo"), url); + when(delegate.read()).thenReturn(42); + assertThat(stream.read()).isEqualTo(42); + clock.advanceMillis(1); + assertThat(stream.read()).isEqualTo(42); + assertThat(stream.read()).isEqualTo(42); + verify(delegate, times(3)).read(); + verify(eventHandler).handle( + Event.progress("Downloading http://lol.example via cdn.example: 2 bytes")); + } +}
diff --git a/src/test/java/com/google/devtools/build/lib/bazel/repository/downloader/ProxyHelperTest.java b/src/test/java/com/google/devtools/build/lib/bazel/repository/downloader/ProxyHelperTest.java index 2a725c1..6063ae5 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/repository/downloader/ProxyHelperTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/repository/downloader/ProxyHelperTest.java
@@ -21,46 +21,43 @@ import com.google.common.collect.ImmutableMap; import java.io.IOException; import java.net.Proxy; +import java.net.URL; import java.util.Map; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; /** - * Tests for @{link ProxyHelper}. + * Tests for {@link ProxyHelper}. */ @RunWith(JUnit4.class) public class ProxyHelperTest { @Test public void testCreateIfNeededHttpLowerCase() throws Exception { - Map<String, String> env = ImmutableMap.<String, String>builder() - .put("http_proxy", "http://my.example.com").build(); - Proxy proxy = ProxyHelper.createProxyIfNeeded("http://www.something.com", env); + ProxyHelper helper = new ProxyHelper(ImmutableMap.of("http_proxy", "http://my.example.com")); + Proxy proxy = helper.createProxyIfNeeded(new URL("http://www.something.com")); assertThat(proxy.toString()).endsWith("my.example.com:80"); } @Test public void testCreateIfNeededHttpUpperCase() throws Exception { - Map<String, String> env = ImmutableMap.<String, String>builder() - .put("HTTP_PROXY", "http://my.example.com").build(); - Proxy proxy = ProxyHelper.createProxyIfNeeded("http://www.something.com", env); + ProxyHelper helper = new ProxyHelper(ImmutableMap.of("HTTP_PROXY", "http://my.example.com")); + Proxy proxy = helper.createProxyIfNeeded(new URL("http://www.something.com")); assertThat(proxy.toString()).endsWith("my.example.com:80"); } @Test public void testCreateIfNeededHttpsLowerCase() throws Exception { - Map<String, String> env = ImmutableMap.<String, String>builder() - .put("https_proxy", "https://my.example.com").build(); - Proxy proxy = ProxyHelper.createProxyIfNeeded("https://www.something.com", env); + ProxyHelper helper = new ProxyHelper(ImmutableMap.of("https_proxy", "https://my.example.com")); + Proxy proxy = helper.createProxyIfNeeded(new URL("https://www.something.com")); assertThat(proxy.toString()).endsWith("my.example.com:443"); } @Test public void testCreateIfNeededHttpsUpperCase() throws Exception { - Map<String, String> env = ImmutableMap.<String, String>builder() - .put("HTTPS_PROXY", "https://my.example.com").build(); - Proxy proxy = ProxyHelper.createProxyIfNeeded("https://www.something.com", env); + ProxyHelper helper = new ProxyHelper(ImmutableMap.of("HTTPS_PROXY", "https://my.example.com")); + Proxy proxy = helper.createProxyIfNeeded(new URL("https://www.something.com")); assertThat(proxy.toString()).endsWith("my.example.com:443"); } @@ -72,7 +69,8 @@ proxy = ProxyHelper.createProxy(""); assertEquals(Proxy.NO_PROXY, proxy); Map<String, String> env = ImmutableMap.of(); - proxy = ProxyHelper.createProxyIfNeeded("https://www.something.com", env); + ProxyHelper helper = new ProxyHelper(env); + proxy = helper.createProxyIfNeeded(new URL("https://www.something.com")); assertEquals(Proxy.NO_PROXY, proxy); }
diff --git a/src/test/java/com/google/devtools/build/lib/bazel/repository/downloader/RetryingInputStreamTest.java b/src/test/java/com/google/devtools/build/lib/bazel/repository/downloader/RetryingInputStreamTest.java new file mode 100644 index 0000000..10f0923 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/bazel/repository/downloader/RetryingInputStreamTest.java
@@ -0,0 +1,173 @@ +// Copyright 2016 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.repository.downloader; + +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.fail; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.eq; +import static org.mockito.Matchers.same; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.when; + +import com.google.common.collect.ImmutableMap; +import com.google.devtools.build.lib.bazel.repository.downloader.RetryingInputStream.Reconnector; +import java.io.IOException; +import java.io.InputStream; +import java.io.InterruptedIOException; +import java.net.SocketTimeoutException; +import java.net.URLConnection; +import org.junit.After; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Unit tests for {@link RetryingInputStream}. */ +@RunWith(JUnit4.class) +public class RetryingInputStreamTest { + + private final InputStream delegate = mock(InputStream.class); + private final InputStream newDelegate = mock(InputStream.class); + private final Reconnector reconnector = mock(Reconnector.class); + private final URLConnection connection = mock(URLConnection.class); + private final RetryingInputStream stream = new RetryingInputStream(delegate, reconnector); + + @After + public void after() throws Exception { + verifyNoMoreInteractions(delegate, newDelegate, reconnector); + } + + @Test + public void close_callsDelegate() throws Exception { + stream.close(); + verify(delegate).close(); + } + + @Test + public void available_callsDelegate() throws Exception { + stream.available(); + verify(delegate).available(); + } + + @Test + public void read_callsdelegate() throws Exception { + stream.read(); + verify(delegate).read(); + } + + @Test + public void bufferRead_callsdelegate() throws Exception { + byte[] buffer = new byte[1024]; + stream.read(buffer); + verify(delegate).read(same(buffer), eq(0), eq(1024)); + } + + @Test + public void readThrowsExceptionWhenDisabled_passesThrough() throws Exception { + stream.disabled = true; + when(delegate.read()).thenThrow(new IOException()); + try { + stream.read(); + fail("Expected IOException"); + } catch (IOException expected) { + verify(delegate).read(); + } + } + + @Test + public void readInterrupted_alwaysPassesThrough() throws Exception { + when(delegate.read()).thenThrow(new InterruptedIOException()); + try { + stream.read(); + fail("Expected InterruptedIOException"); + } catch (InterruptedIOException expected) { + verify(delegate).read(); + } + } + + @Test + @SuppressWarnings("unchecked") + public void readTimesOut_retries() throws Exception { + when(delegate.read()).thenReturn(1).thenThrow(new SocketTimeoutException()); + when(reconnector.connect(any(Throwable.class), any(ImmutableMap.class))).thenReturn(connection); + when(connection.getInputStream()).thenReturn(newDelegate); + when(newDelegate.read()).thenReturn(2); + when(connection.getHeaderField("Content-Range")).thenReturn("bytes 1-42/42"); + assertThat(stream.read()).isEqualTo(1); + assertThat(stream.read()).isEqualTo(2); + verify(reconnector).connect(any(Throwable.class), eq(ImmutableMap.of("Range", "bytes 1-"))); + verify(delegate, times(2)).read(); + verify(delegate).close(); + verify(newDelegate).read(); + } + + @Test + @SuppressWarnings("unchecked") + public void failureWhenNoBytesAreRead_doesntUseRange() throws Exception { + when(delegate.read()).thenThrow(new SocketTimeoutException()); + when(newDelegate.read()).thenReturn(1); + when(reconnector.connect(any(Throwable.class), any(ImmutableMap.class))).thenReturn(connection); + when(connection.getInputStream()).thenReturn(newDelegate); + assertThat(stream.read()).isEqualTo(1); + verify(reconnector).connect(any(Throwable.class), eq(ImmutableMap.<String, String>of())); + verify(delegate).read(); + verify(delegate).close(); + verify(newDelegate).read(); + } + + @Test + @SuppressWarnings("unchecked") + public void reconnectFails_alwaysPassesThrough() throws Exception { + when(delegate.read()).thenThrow(new IOException()); + when(reconnector.connect(any(Throwable.class), any(ImmutableMap.class))) + .thenThrow(new IOException()); + try { + stream.read(); + fail("Expected IOException"); + } catch (IOException expected) { + verify(delegate).read(); + verify(delegate).close(); + verify(reconnector).connect(any(Throwable.class), any(ImmutableMap.class)); + } + } + + @Test + @SuppressWarnings("unchecked") + public void maxRetries_givesUp() throws Exception { + when(delegate.read()) + .thenReturn(1) + .thenThrow(new IOException()) + .thenThrow(new IOException()) + .thenThrow(new IOException()) + .thenThrow(new SocketTimeoutException()); + when(reconnector.connect(any(Throwable.class), any(ImmutableMap.class))).thenReturn(connection); + when(connection.getInputStream()).thenReturn(delegate); + when(connection.getHeaderField("Content-Range")).thenReturn("bytes 1-42/42"); + stream.read(); + try { + stream.read(); + fail("Expected SocketTimeoutException"); + } catch (SocketTimeoutException e) { + assertThat(e.getSuppressed()).hasLength(3); + verify(reconnector, times(3)) + .connect(any(Throwable.class), eq(ImmutableMap.of("Range", "bytes 1-"))); + verify(delegate, times(5)).read(); + verify(delegate, times(3)).close(); + } + } +}
diff --git a/src/test/java/com/google/devtools/build/lib/testutil/ManualClock.java b/src/test/java/com/google/devtools/build/lib/testutil/ManualClock.java index 15752cc..fe500d0 100644 --- a/src/test/java/com/google/devtools/build/lib/testutil/ManualClock.java +++ b/src/test/java/com/google/devtools/build/lib/testutil/ManualClock.java
@@ -15,18 +15,18 @@ package com.google.devtools.build.lib.testutil; import com.google.devtools.build.lib.util.Clock; - import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; /** * A fake clock for testing. */ public final class ManualClock implements Clock { - private long currentTimeMillis = 0L; + private final AtomicLong currentTimeMillis = new AtomicLong(); @Override public long currentTimeMillis() { - return currentTimeMillis; + return currentTimeMillis.get(); } /** @@ -36,11 +36,11 @@ */ @Override public long nanoTime() { - return TimeUnit.MILLISECONDS.toNanos(currentTimeMillis) + return TimeUnit.MILLISECONDS.toNanos(currentTimeMillis.get()) + TimeUnit.SECONDS.toNanos(1000); } - public void advanceMillis(long time) { - currentTimeMillis += time; + public long advanceMillis(long time) { + return currentTimeMillis.addAndGet(time); } }
diff --git a/src/test/java/com/google/devtools/build/lib/testutil/ManualSleeper.java b/src/test/java/com/google/devtools/build/lib/testutil/ManualSleeper.java new file mode 100644 index 0000000..6e16253 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/testutil/ManualSleeper.java
@@ -0,0 +1,36 @@ +// Copyright 2014 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.testutil; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkNotNull; + +import com.google.devtools.build.lib.util.Sleeper; + +/** Fake sleeper for testing. */ +public final class ManualSleeper implements Sleeper { + + private final ManualClock clock; + + public ManualSleeper(ManualClock clock) { + this.clock = checkNotNull(clock); + } + + @Override + public void sleepMillis(long milliseconds) throws InterruptedException { + checkArgument(milliseconds >= 0, "sleeper can't time travel"); + clock.advanceMillis(milliseconds); + } +}
diff --git a/src/test/shell/bazel/external_integration_test.sh b/src/test/shell/bazel/external_integration_test.sh index 0f610e0..52b01a1 100755 --- a/src/test/shell/bazel/external_integration_test.sh +++ b/src/test/shell/bazel/external_integration_test.sh
@@ -191,10 +191,9 @@ } function test_http_archive_no_server() { - nc_port=$(pick_random_unused_tcp_port) || exit 1 cat > WORKSPACE <<EOF -http_archive(name = 'endangered', url = 'http://localhost:$nc_port/repo.zip', - sha256 = 'dummy') +http_archive(name = 'endangered', url = 'http://bad.example/repo.zip', + sha256 = '2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9826') EOF cat > zoo/BUILD <<EOF @@ -212,7 +211,7 @@ chmod +x zoo/female.sh bazel fetch //zoo:breeding-program >& $TEST_log && fail "Expected fetch to fail" - expect_log "Connection refused" + expect_log "Unknown host: bad.example" } function test_http_archive_mismatched_sha256() { @@ -229,8 +228,11 @@ cd ${WORKSPACE_DIR} cat > WORKSPACE <<EOF -http_archive(name = 'endangered', url = 'http://localhost:$nc_port/repo.zip', - sha256 = '$wrong_sha256') +http_archive( + name = 'endangered', + url = 'http://localhost:$nc_port/repo.zip', + sha256 = '2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9826', +) EOF cat > zoo/BUILD <<EOF @@ -249,7 +251,7 @@ bazel fetch //zoo:breeding-program >& $TEST_log && echo "Expected fetch to fail" kill_nc - expect_log "does not match expected SHA-256" + expect_log "Checksum" } # Bazel should not re-download the .zip unless the user requests it or the @@ -338,7 +340,7 @@ http_file( name = 'toto', url = 'http://localhost:$nc_port/toto', - sha256 = 'whatever' + sha256 = '2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9826' ) EOF bazel build @toto//file &> $TEST_log && fail "Expected run to fail" @@ -356,7 +358,7 @@ http_file( name = 'toto', url = 'http://localhost:$nc_port/toto', - sha256 = 'whatever' + sha256 = '2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9826' ) EOF bazel build @toto//file &> $TEST_log && fail "Expected run to fail" @@ -476,7 +478,7 @@ function test_invalid_rule() { # http_jar with missing URL field. cat > WORKSPACE <<EOF -http_jar(name = 'endangered', sha256 = 'dummy') +http_jar(name = 'endangered', sha256 = '2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9826') EOF bazel fetch //external:endangered >& $TEST_log && fail "Expected fetch to fail"