HttpDownloader: Support scaling of timputs
Currently, the HttpDownloader hard-codes the values for timeouts. However,
depending on network situation, different values might be appropriate.
So support scaling all timeouts by a configurable factor (defaulting
to 1.0).
Change-Id: I84dc64e92d83f4f129a29a5f79874a6d662f1f0b
PiperOrigin-RevId: 245374111
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java b/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java
index 4c3126d..3114cfd 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java
@@ -277,6 +277,8 @@
httpDownloader.setDistdir(ImmutableList.<Path>of());
}
+ httpDownloader.setTimeoutScaling((float) repoOptions.httpTimeoutScaling);
+
if (repoOptions.repositoryOverrides != null) {
ImmutableMap.Builder<RepositoryName, PathFragment> builder = ImmutableMap.builder();
for (RepositoryOverride override : repoOptions.repositoryOverrides) {
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryOptions.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryOptions.java
index 8e46218..b7e1fa4 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryOptions.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryOptions.java
@@ -70,6 +70,14 @@
public List<PathFragment> experimentalDistdir;
@Option(
+ name = "http_timeout_scaling",
+ defaultValue = "1.0",
+ documentationCategory = OptionDocumentationCategory.BAZEL_CLIENT_OPTIONS,
+ effectTags = {OptionEffectTag.BAZEL_INTERNAL_CONFIGURATION},
+ help = "Scale all timeouts related to http downloads by the given factor")
+ public double httpTimeoutScaling;
+
+ @Option(
name = "override_repository",
defaultValue = "null",
allowMultiple = true,
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpConnector.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpConnector.java
index 9ec4592..be29680 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpConnector.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpConnector.java
@@ -63,13 +63,28 @@
private final EventHandler eventHandler;
private final ProxyHelper proxyHelper;
private final Sleeper sleeper;
+ private final float timeoutScaling;
HttpConnector(
- Locale locale, EventHandler eventHandler, ProxyHelper proxyHelper, Sleeper sleeper) {
+ Locale locale,
+ EventHandler eventHandler,
+ ProxyHelper proxyHelper,
+ Sleeper sleeper,
+ float timeoutScaling) {
this.locale = locale;
this.eventHandler = eventHandler;
this.proxyHelper = proxyHelper;
this.sleeper = sleeper;
+ this.timeoutScaling = timeoutScaling;
+ }
+
+ HttpConnector(
+ Locale locale, EventHandler eventHandler, ProxyHelper proxyHelper, Sleeper sleeper) {
+ this(locale, eventHandler, proxyHelper, sleeper, 1.0f);
+ }
+
+ private int scale(int unscaled) {
+ return Math.round(unscaled * timeoutScaling);
}
URLConnection connect(
@@ -85,7 +100,7 @@
List<Throwable> suppressions = new ArrayList<>();
int retries = 0;
int redirects = 0;
- int connectTimeout = MIN_CONNECT_TIMEOUT_MS;
+ int connectTimeout = scale(MIN_CONNECT_TIMEOUT_MS);
while (true) {
HttpURLConnection connection = null;
try {
@@ -105,7 +120,7 @@
}
connection.setConnectTimeout(connectTimeout);
// The read timeout is always large because it stays in effect after this method.
- connection.setReadTimeout(READ_TIMEOUT_MS);
+ connection.setReadTimeout(scale(READ_TIMEOUT_MS));
// Java tries to abstract HTTP error responses for us. We don't want that. So we're going
// to try and undo any IOException that doesn't appear to be a legitimate I/O exception.
int code;
@@ -184,7 +199,7 @@
int timeout = IntMath.pow(2, retries) * MIN_RETRY_DELAY_MS;
if (e instanceof SocketTimeoutException) {
eventHandler.handle(Event.progress("Timeout connecting to " + url));
- connectTimeout = Math.min(connectTimeout * 2, MAX_CONNECT_TIMEOUT_MS);
+ connectTimeout = Math.min(connectTimeout * 2, scale(MAX_CONNECT_TIMEOUT_MS));
// If we got connect timeout, we're already doing exponential backoff, so no point
// in sleeping too.
timeout = 1;
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpDownloader.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpDownloader.java
index e0fd9b3..010249a 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpDownloader.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpDownloader.java
@@ -54,6 +54,7 @@
protected final RepositoryCache repositoryCache;
private List<Path> distdir = ImmutableList.of();
+ private float timeoutScaling = 1.0f;
public HttpDownloader(RepositoryCache repositoryCache) {
this.repositoryCache = repositoryCache;
@@ -63,6 +64,10 @@
this.distdir = ImmutableList.copyOf(distdir);
}
+ public void setTimeoutScaling(float timeoutScaling) {
+ this.timeoutScaling = timeoutScaling;
+ }
+
/**
* Downloads file to disk and returns path.
*
@@ -181,7 +186,8 @@
Sleeper sleeper = new JavaSleeper();
Locale locale = Locale.getDefault();
ProxyHelper proxyHelper = new ProxyHelper(clientEnv);
- HttpConnector connector = new HttpConnector(locale, eventHandler, proxyHelper, sleeper);
+ HttpConnector connector =
+ new HttpConnector(locale, eventHandler, proxyHelper, sleeper, timeoutScaling);
ProgressInputStream.Factory progressInputStreamFactory =
new ProgressInputStream.Factory(locale, clock, eventHandler);
HttpStream.Factory httpStreamFactory = new HttpStream.Factory(progressInputStreamFactory);
