Be smarter about Content-Encoding: gzip downloads
Fixes #2136
RELNOTES: n/a
--
MOS_MIGRATED_REVID=140190547
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpConnector.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpConnector.java
index 3f8f716..c055f6f 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpConnector.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpConnector.java
@@ -20,6 +20,7 @@
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Ascii;
+import com.google.common.collect.ImmutableSet;
import com.google.common.math.IntMath;
import com.google.devtools.build.lib.events.Event;
import com.google.devtools.build.lib.events.EventHandler;
@@ -50,6 +51,8 @@
private static final int CONNECT_TIMEOUT_MS = 1000;
private static final int MAX_CONNECT_TIMEOUT_MS = 10000;
private static final int READ_TIMEOUT_MS = 20000;
+ private static final ImmutableSet<String> COMPRESSED_EXTENSIONS =
+ ImmutableSet.of("bz2", "gz", "jar", "tgz", "war", "xz", "zip");
/**
* Connects to HTTP (or file) URL with GET request and lazily returns payload.
@@ -83,7 +86,9 @@
HttpURLConnection connection = null;
try {
connection = (HttpURLConnection) url.openConnection(proxy);
- connection.setRequestProperty("Accept-Encoding", "gzip");
+ if (!COMPRESSED_EXTENSIONS.contains(getExtension(url.getPath()))) {
+ connection.setRequestProperty("Accept-Encoding", "gzip");
+ }
connection.setConnectTimeout(connectTimeout);
connection.setReadTimeout(READ_TIMEOUT_MS);
int code;
@@ -173,7 +178,13 @@
return connection.getInputStream();
case "gzip":
case "x-gzip":
- return new GZIPInputStream(connection.getInputStream());
+ // Some web servers will send Content-Encoding: gzip even when we didn't request it, iff
+ // the file is a .gz file.
+ if (connection.getURL().getPath().endsWith(".gz")) {
+ return connection.getInputStream();
+ } else {
+ return new GZIPInputStream(connection.getInputStream());
+ }
default:
throw new UnrecoverableHttpException(
"Unsupported and unrequested Content-Encoding: " + connection.getContentEncoding());
@@ -239,6 +250,14 @@
return Ascii.equalsIgnoreCase(protocol, url.getProtocol());
}
+ private static String getExtension(String path) {
+ int index = path.lastIndexOf('.');
+ if (index == -1) {
+ return "";
+ }
+ return path.substring(index + 1);
+ }
+
private static final class UnrecoverableHttpException extends IOException {
UnrecoverableHttpException(String message) {
super(message);
