Fix problems with retrieving external deps behind a proxy
Fixes #587 by properly setting up the Proxy object for HttpDownloader and by setting JVM arguments for GitCloner.
--
Reviewed-on: https://github.com/bazelbuild/bazel/pull/795
MOS_MIGRATED_REVID=115857070
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/GitCloner.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/GitCloner.java
index ecc7f8d..b862cf5 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/repository/GitCloner.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/GitCloner.java
@@ -114,6 +114,15 @@
mapper.get("init_submodules", Type.BOOLEAN),
outputDirectory);
+ // Setup proxy if remote is http or https
+ if (descriptor.remote != null && descriptor.remote.startsWith("http")) {
+ try {
+ ProxyHelper.createProxyIfNeeded(descriptor.remote);
+ } catch (IOException ie) {
+ throw new RepositoryFunctionException(ie, Transience.TRANSIENT);
+ }
+ }
+
Git git = null;
try {
if (descriptor.directory.exists()) {
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/HttpDownloader.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/HttpDownloader.java
index 3594767a..147bb45 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/repository/HttpDownloader.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/HttpDownloader.java
@@ -14,8 +14,6 @@
package com.google.devtools.build.lib.bazel.repository;
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Strings;
import com.google.common.hash.Hasher;
import com.google.common.hash.Hashing;
import com.google.common.io.ByteStreams;
@@ -33,11 +31,8 @@
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
-import java.net.Authenticator;
import java.net.HttpURLConnection;
-import java.net.InetSocketAddress;
import java.net.MalformedURLException;
-import java.net.PasswordAuthentication;
import java.net.Proxy;
import java.net.URL;
import java.nio.charset.StandardCharsets;
@@ -46,8 +41,6 @@
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
import javax.annotation.Nullable;
@@ -241,7 +234,7 @@
public static HttpConnection createAndConnect(URL url) throws IOException {
int retries = MAX_REDIRECTS;
- Proxy proxy = createProxyIfNeeded(url.getProtocol());
+ Proxy proxy = ProxyHelper.createProxyIfNeeded(url.toString());
do {
HttpURLConnection connection = (HttpURLConnection) url.openConnection(proxy);
try {
@@ -310,76 +303,6 @@
}
}
- private static Proxy createProxyIfNeeded(String protocol) throws IOException {
- if (protocol.equals("https")) {
- return createProxy(System.getenv("HTTPS_PROXY"));
- } else if (protocol.equals("http")) {
- return createProxy(System.getenv("HTTP_PROXY"));
- }
- return Proxy.NO_PROXY;
- }
-
- @VisibleForTesting
- static Proxy createProxy(String proxyAddress) throws IOException {
- if (Strings.isNullOrEmpty(proxyAddress)) {
- return Proxy.NO_PROXY;
- }
-
- // Here there be dragons.
- Pattern urlPattern =
- Pattern.compile("^(https?)://(?:([^:@]+?)(?::([^@]+?))?@)?(?:[^:]+)(?::(\\d+))?$");
- Matcher matcher = urlPattern.matcher(proxyAddress);
- if (!matcher.matches()) {
- throw new IOException("Proxy address " + proxyAddress + " is not a valid URL");
- }
-
- String protocol = matcher.group(1);
- final String username = matcher.group(2);
- final String password = matcher.group(3);
- String port = matcher.group(4);
-
- boolean https;
- switch (protocol) {
- case "https":
- https = true;
- break;
- case "http":
- https = false;
- break;
- default:
- throw new IOException("Invalid proxy protocol for " + proxyAddress);
- }
-
- if (username != null) {
- if (password == null) {
- throw new IOException("No password given for proxy " + proxyAddress);
- }
- System.setProperty(protocol + ".proxyUser", username);
- System.setProperty(protocol + ".proxyPassword", password);
-
- Authenticator.setDefault(
- new Authenticator() {
- @Override
- public PasswordAuthentication getPasswordAuthentication() {
- return new PasswordAuthentication(username, password.toCharArray());
- }
- });
- }
-
- if (port == null) {
- return new Proxy(Proxy.Type.HTTP, new InetSocketAddress(proxyAddress, https ? 443 : 80));
- }
-
- try {
- return new Proxy(
- Proxy.Type.HTTP,
- new InetSocketAddress(
- proxyAddress.substring(0, proxyAddress.lastIndexOf(':')), Integer.parseInt(port)));
- } catch (NumberFormatException e) {
- throw new IOException("Error parsing proxy port: " + proxyAddress);
- }
- }
-
public static String getHash(Hasher hasher, Path path) throws IOException {
byte byteBuffer[] = new byte[BUFFER_SIZE];
try (InputStream stream = path.getInputStream()) {
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/ProxyHelper.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/ProxyHelper.java
new file mode 100644
index 0000000..0cb3772
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/ProxyHelper.java
@@ -0,0 +1,139 @@
+// Copyright 2016 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.bazel.repository;
+
+import com.google.common.base.Strings;
+
+import java.io.IOException;
+import java.net.Authenticator;
+import java.net.InetSocketAddress;
+import java.net.PasswordAuthentication;
+import java.net.Proxy;
+import java.net.URLDecoder;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * Helper class for setting up a proxy server for network communication
+ *
+ */
+public class ProxyHelper {
+
+ /**
+ * This method takes a String for the resource being requested and sets up a proxy to make
+ * the request if HTTP_PROXY and/or HTTPS_PROXY environment variables are set.
+ * @param requestedUrl The url for the remote resource that may need to be retrieved through a
+ * proxy
+ * @return Proxy
+ * @throws IOException
+ */
+ public static Proxy createProxyIfNeeded(String requestedUrl) throws IOException {
+ String lcUrl = requestedUrl.toLowerCase();
+ if (lcUrl.startsWith("https")) {
+ return createProxy(System.getenv("HTTPS_PROXY"));
+ } else if (lcUrl.startsWith("http")) {
+ return createProxy(System.getenv("HTTP_PROXY"));
+ }
+ return Proxy.NO_PROXY;
+ }
+
+ /**
+ * This method takes a proxyAddress as a String (ex.
+ * http://userId:password@proxyhost.domain.com:8000) and sets JVM arguments for http and https
+ * proxy as well as returns a java.net.Proxy object for optional use.
+ *
+ * @param proxyAddress The fully qualified address of the proxy server
+ * @return Proxy
+ * @throws IOException
+ */
+ public static Proxy createProxy(String proxyAddress) throws IOException {
+ if (Strings.isNullOrEmpty(proxyAddress)) {
+ return Proxy.NO_PROXY;
+ }
+
+ // Here there be dragons.
+ Pattern urlPattern =
+ Pattern.compile("^(https?)://(([^:@]+?)(?::([^@]+?))?@)?([^:]+)(?::(\\d+))?$");
+ Matcher matcher = urlPattern.matcher(proxyAddress);
+ if (!matcher.matches()) {
+ throw new IOException("Proxy address " + proxyAddress + " is not a valid URL");
+ }
+
+ final String protocol = matcher.group(1);
+ final String idAndPassword = matcher.group(2);
+ final String username = matcher.group(3);
+ final String password = matcher.group(4);
+ final String hostname = matcher.group(5);
+ final String portRaw = matcher.group(6);
+
+ String cleanProxyAddress = proxyAddress;
+ if (idAndPassword != null) {
+ cleanProxyAddress =
+ proxyAddress.replace(idAndPassword, ""); // Used to remove id+pwd from logging
+ }
+
+ boolean https;
+ switch (protocol) {
+ case "https":
+ https = true;
+ break;
+ case "http":
+ https = false;
+ break;
+ default:
+ throw new IOException("Invalid proxy protocol for " + cleanProxyAddress);
+ }
+
+ int port = https ? 443 : 80; // Default port numbers
+
+ if (portRaw != null) {
+ try {
+ port = Integer.parseInt(portRaw);
+ } catch (NumberFormatException e) {
+ throw new IOException("Error parsing proxy port: " + cleanProxyAddress);
+ }
+ }
+
+ // We need to set both of these because jgit uses whichever the resource dictates
+ System.setProperty("https.proxyHost", hostname);
+ System.setProperty("https.proxyPort", Integer.toString(port));
+ System.setProperty("http.proxyHost", hostname);
+ System.setProperty("http.proxyPort", Integer.toString(port));
+
+ if (username != null) {
+ if (password == null) {
+ throw new IOException("No password given for proxy " + cleanProxyAddress);
+ }
+
+ // We need to make sure the proxy password is not url encoded; some special characters in
+ // proxy passwords require url encoding for shells and other tools to properly consume.
+ final String decodedPassword = URLDecoder.decode(password, "UTF-8");
+ System.setProperty("http.proxyUser", username);
+ System.setProperty("http.proxyPassword", decodedPassword);
+ System.setProperty("https.proxyUser", username);
+ System.setProperty("https.proxyPassword", decodedPassword);
+
+ Authenticator.setDefault(
+ new Authenticator() {
+ @Override
+ public PasswordAuthentication getPasswordAuthentication() {
+ return new PasswordAuthentication(username, decodedPassword.toCharArray());
+ }
+ });
+ }
+
+ return new Proxy(Proxy.Type.HTTP, new InetSocketAddress(hostname, port));
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/bazel/repository/HttpDownloaderTest.java b/src/test/java/com/google/devtools/build/lib/bazel/repository/HttpDownloaderTest.java
deleted file mode 100644
index c30ec58..0000000
--- a/src/test/java/com/google/devtools/build/lib/bazel/repository/HttpDownloaderTest.java
+++ /dev/null
@@ -1,116 +0,0 @@
-// Copyright 2015 The Bazel Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package com.google.devtools.build.lib.bazel.repository;
-
-import static com.google.common.truth.Truth.assertThat;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
-
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.JUnit4;
-
-import java.io.IOException;
-import java.net.Proxy;
-
-/**
- * Tests for @{link HttpDownloader}.
- */
-@RunWith(JUnit4.class)
-public class HttpDownloaderTest {
- @Test
- public void testNoProxy() throws Exception {
- // Empty address.
- Proxy proxy = HttpDownloader.createProxy(null);
- assertEquals(Proxy.NO_PROXY, proxy);
- proxy = HttpDownloader.createProxy("");
- assertEquals(Proxy.NO_PROXY, proxy);
- }
-
- @Test
- public void testProxyDefaultPort() throws Exception {
- Proxy proxy = HttpDownloader.createProxy("http://my.example.com");
- assertEquals(Proxy.Type.HTTP, proxy.type());
- assertThat(proxy.toString()).endsWith(":80");
-
- proxy = HttpDownloader.createProxy("https://my.example.com");
- assertThat(proxy.toString()).endsWith(":443");
- }
-
- @Test
- public void testProxyExplicitPort() throws Exception {
- Proxy proxy = HttpDownloader.createProxy("http://my.example.com:12345");
- assertThat(proxy.toString()).endsWith(":12345");
-
- proxy = HttpDownloader.createProxy("https://my.example.com:12345");
- assertThat(proxy.toString()).endsWith(":12345");
- }
-
- @Test
- public void testProxyNoProtocol() throws Exception {
- try {
- HttpDownloader.createProxy("my.example.com");
- fail("Expected protocol error");
- } catch (IOException e) {
- assertThat(e.getMessage()).contains("Proxy address my.example.com is not a valid URL");
- }
- }
-
- @Test
- public void testProxyNoProtocolWithPort() throws Exception {
- try {
- HttpDownloader.createProxy("my.example.com:12345");
- fail("Expected protocol error");
- } catch (IOException e) {
- assertThat(e.getMessage()).contains("Proxy address my.example.com:12345 is not a valid URL");
- }
- }
-
- @Test
- public void testProxyPortParsingError() throws Exception {
- try {
- HttpDownloader.createProxy("http://my.example.com:foo");
- fail("Should have thrown an error for invalid port");
- } catch (IOException e) {
- assertThat(e.getMessage())
- .contains("Proxy address http://my.example.com:foo is not a valid URL");
- }
- }
-
- @Test
- public void testProxyAuth() throws Exception {
- Proxy proxy = HttpDownloader.createProxy("http://foo:barbaz@my.example.com");
- assertEquals(Proxy.Type.HTTP, proxy.type());
- assertThat(proxy.toString()).endsWith(":80");
- assertEquals(System.getProperty("http.proxyUser"), "foo");
- assertEquals(System.getProperty("http.proxyPassword"), "barbaz");
-
- proxy = HttpDownloader.createProxy("https://biz:bat@my.example.com");
- assertThat(proxy.toString()).endsWith(":443");
- assertEquals(System.getProperty("https.proxyUser"), "biz");
- assertEquals(System.getProperty("https.proxyPassword"), "bat");
- }
-
- @Test
- public void testInvalidAuth() throws Exception {
- try {
- HttpDownloader.createProxy("http://foo@my.example.com");
- fail("Should have thrown an error for invalid auth");
- } catch (IOException e) {
- assertThat(e.getMessage()).contains("No password given for proxy");
- }
- }
-
-}
diff --git a/src/test/java/com/google/devtools/build/lib/bazel/repository/ProxyHelperTest.java b/src/test/java/com/google/devtools/build/lib/bazel/repository/ProxyHelperTest.java
new file mode 100644
index 0000000..a1a9142
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/bazel/repository/ProxyHelperTest.java
@@ -0,0 +1,139 @@
+// Copyright 2015 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.bazel.repository;
+
+import static com.google.common.truth.Truth.assertThat;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+import java.io.IOException;
+import java.net.Proxy;
+
+/**
+ * Tests for @{link ProxyHelper}.
+ */
+@RunWith(JUnit4.class)
+public class ProxyHelperTest {
+
+ @Test
+ public void testNoProxy() throws Exception {
+ // Empty address.
+ Proxy proxy = ProxyHelper.createProxy(null);
+ assertEquals(Proxy.NO_PROXY, proxy);
+ proxy = ProxyHelper.createProxy("");
+ assertEquals(Proxy.NO_PROXY, proxy);
+ }
+
+ @Test
+ public void testProxyDefaultPort() throws Exception {
+ Proxy proxy = ProxyHelper.createProxy("http://my.example.com");
+ assertEquals(Proxy.Type.HTTP, proxy.type());
+ assertThat(proxy.toString()).endsWith(":80");
+ assertEquals(System.getProperty("http.proxyHost"), "my.example.com");
+ assertEquals(System.getProperty("http.proxyPort"), "80");
+
+ proxy = ProxyHelper.createProxy("https://my.example.com");
+ assertThat(proxy.toString()).endsWith(":443");
+ assertEquals(System.getProperty("https.proxyHost"), "my.example.com");
+ assertEquals(System.getProperty("https.proxyPort"), "443");
+ }
+
+ @Test
+ public void testProxyExplicitPort() throws Exception {
+ Proxy proxy = ProxyHelper.createProxy("http://my.example.com:12345");
+ assertThat(proxy.toString()).endsWith(":12345");
+ assertEquals(System.getProperty("http.proxyHost"), "my.example.com");
+ assertEquals(System.getProperty("http.proxyPort"), "12345");
+
+ proxy = ProxyHelper.createProxy("https://my.example.com:12345");
+ assertThat(proxy.toString()).endsWith(":12345");
+ assertEquals(System.getProperty("https.proxyHost"), "my.example.com");
+ assertEquals(System.getProperty("https.proxyPort"), "12345");
+ }
+
+ @Test
+ public void testProxyNoProtocol() throws Exception {
+ try {
+ ProxyHelper.createProxy("my.example.com");
+ fail("Expected protocol error");
+ } catch (IOException e) {
+ assertThat(e.getMessage()).contains("Proxy address my.example.com is not a valid URL");
+ }
+ }
+
+ @Test
+ public void testProxyNoProtocolWithPort() throws Exception {
+ try {
+ ProxyHelper.createProxy("my.example.com:12345");
+ fail("Expected protocol error");
+ } catch (IOException e) {
+ assertThat(e.getMessage()).contains("Proxy address my.example.com:12345 is not a valid URL");
+ }
+ }
+
+ @Test
+ public void testProxyPortParsingError() throws Exception {
+ try {
+ ProxyHelper.createProxy("http://my.example.com:foo");
+ fail("Should have thrown an error for invalid port");
+ } catch (IOException e) {
+ assertThat(e.getMessage())
+ .contains("Proxy address http://my.example.com:foo is not a valid URL");
+ }
+ }
+
+ @Test
+ public void testProxyAuth() throws Exception {
+ Proxy proxy = ProxyHelper.createProxy("http://foo:barbaz@my.example.com");
+ assertEquals(Proxy.Type.HTTP, proxy.type());
+ assertThat(proxy.toString()).endsWith(":80");
+ assertEquals(System.getProperty("http.proxyHost"), "my.example.com");
+ assertEquals(System.getProperty("http.proxyPort"), "80");
+ assertEquals(System.getProperty("http.proxyUser"), "foo");
+ assertEquals(System.getProperty("http.proxyPassword"), "barbaz");
+
+ proxy = ProxyHelper.createProxy("https://biz:bat@my.example.com");
+ assertThat(proxy.toString()).endsWith(":443");
+ assertEquals(System.getProperty("https.proxyHost"), "my.example.com");
+ assertEquals(System.getProperty("https.proxyPort"), "443");
+ assertEquals(System.getProperty("https.proxyUser"), "biz");
+ assertEquals(System.getProperty("https.proxyPassword"), "bat");
+ }
+
+ @Test
+ public void testEncodedProxyAuth() throws Exception {
+ Proxy proxy = ProxyHelper.createProxy("http://foo:b%40rb%40z@my.example.com");
+ assertEquals(Proxy.Type.HTTP, proxy.type());
+ assertThat(proxy.toString()).endsWith(":80");
+ assertEquals(System.getProperty("http.proxyHost"), "my.example.com");
+ assertEquals(System.getProperty("http.proxyPort"), "80");
+ assertEquals(System.getProperty("http.proxyUser"), "foo");
+ assertEquals(System.getProperty("http.proxyPassword"), "b@rb@z");
+ }
+
+ @Test
+ public void testInvalidAuth() throws Exception {
+ try {
+ ProxyHelper.createProxy("http://foo@my.example.com");
+ fail("Should have thrown an error for invalid auth");
+ } catch (IOException e) {
+ assertThat(e.getMessage()).contains("No password given for proxy");
+ }
+ }
+}
