Make environment dependency of actions factor through individual values
With actions depending on the (white-listed part) of the environment
as a whole, even though they are only re-executed if the used parts of
the environment change, each action has to be reconsidered on any change
of the environment. For large dependency graphs, this can be a considerable
amount of effort; therefore add intermediate values for the individual
variables and make actions only depend on those actually used.
--
Change-Id: I283d289da3e0782dc4f9ac084a41425166cfede0
Reviewed-on: https://bazel-review.googlesource.com/#/c/5494
MOS_MIGRATED_REVID=133255911
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/ActionExecutionFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/ActionExecutionFunction.java
index 0808b7c..d3ffe1a 100644
--- a/src/main/java/com/google/devtools/build/lib/skyframe/ActionExecutionFunction.java
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/ActionExecutionFunction.java
@@ -87,6 +87,14 @@
stateMap = Maps.newConcurrentMap();
}
+ private static final Function<String, SkyKey> VAR_TO_SKYKEY =
+ new Function<String, SkyKey>() {
+ @Override
+ public SkyKey apply(String var) {
+ return SkyKey.create(SkyFunctions.CLIENT_ENVIRONMENT_VARIABLE, var);
+ }
+ };
+
@Override
public SkyValue compute(SkyKey skyKey, Environment env) throws ActionExecutionFunctionException,
InterruptedException {
@@ -102,8 +110,20 @@
// Depending on the buildID ensure that these actions have a chance to execute.
PrecomputedValue.BUILD_ID.get(env);
}
- // The client environment might influence the action.
- Map<String, String> clientEnv = PrecomputedValue.CLIENT_ENV.get(env);
+
+ // Look up the parts of the environment that influence the action.
+ Map<SkyKey, SkyValue> clientEnvLookup =
+ env.getValues(Iterables.transform(action.getClientEnvironmentVariables(), VAR_TO_SKYKEY));
+ if (env.valuesMissing()) {
+ return null;
+ }
+ Map<String, String> clientEnv = new HashMap<>();
+ for (Entry<SkyKey, SkyValue> entry : clientEnvLookup.entrySet()) {
+ ClientEnvironmentValue envValue = (ClientEnvironmentValue) entry.getValue();
+ if (envValue.getValue() != null) {
+ clientEnv.put((String) entry.getKey().argument(), envValue.getValue());
+ }
+ }
// For restarts of this ActionExecutionFunction we use a ContinuationState variable, below, to
// avoid redoing work. However, if two actions are shared and the first one executes, when the
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/ClientEnvironmentFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/ClientEnvironmentFunction.java
new file mode 100644
index 0000000..cc2c37a
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/ClientEnvironmentFunction.java
@@ -0,0 +1,38 @@
+// Copyright 2016 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.devtools.build.skyframe.SkyFunction;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+import java.util.Map;
+import javax.annotation.Nullable;
+
+/** The Skyframe function that generates values for variables of the client environment. */
+public final class ClientEnvironmentFunction implements SkyFunction {
+
+ @Nullable
+ @Override
+ public String extractTag(SkyKey skyKey) {
+ return null;
+ }
+
+ @Nullable
+ @Override
+ public SkyValue compute(SkyKey skyKey, Environment env) throws InterruptedException {
+ Map<String, String> clientEnv = PrecomputedValue.CLIENT_ENV.get(env);
+ return new ClientEnvironmentValue(clientEnv.get((String) skyKey.argument()));
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/ClientEnvironmentValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/ClientEnvironmentValue.java
new file mode 100644
index 0000000..bb00533
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/ClientEnvironmentValue.java
@@ -0,0 +1,33 @@
+// Copyright 2016 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.devtools.build.skyframe.SkyValue;
+import javax.annotation.Nullable;
+
+/** An aspect in the context of the Skyframe graph. */
+public final class ClientEnvironmentValue implements SkyValue {
+ private final String value;
+
+ public ClientEnvironmentValue(@Nullable String value) {
+ this.value = value;
+ }
+
+ /** @return the value in the client environment or null if unset in the environment. */
+ @Nullable
+ public String getValue() {
+ return value;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SkyFunctions.java b/src/main/java/com/google/devtools/build/lib/skyframe/SkyFunctions.java
index 116d3ba..a7b7224 100644
--- a/src/main/java/com/google/devtools/build/lib/skyframe/SkyFunctions.java
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/SkyFunctions.java
@@ -22,6 +22,8 @@
*/
public final class SkyFunctions {
public static final SkyFunctionName PRECOMPUTED = SkyFunctionName.create("PRECOMPUTED");
+ public static final SkyFunctionName CLIENT_ENVIRONMENT_VARIABLE =
+ SkyFunctionName.create("CLIENT_ENVIRONMENT_VARIABLE");
public static final SkyFunctionName FILE_STATE = SkyFunctionName.create("FILE_STATE");
public static final SkyFunctionName DIRECTORY_LISTING_STATE =
SkyFunctionName.create("DIRECTORY_LISTING_STATE");
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java
index 65011b7..142dbc3 100644
--- a/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java
@@ -330,6 +330,7 @@
// is inserted.
ImmutableMap.Builder<SkyFunctionName, SkyFunction> map = ImmutableMap.builder();
map.put(SkyFunctions.PRECOMPUTED, new PrecomputedFunction());
+ map.put(SkyFunctions.CLIENT_ENVIRONMENT_VARIABLE, new ClientEnvironmentFunction());
map.put(SkyFunctions.FILE_STATE, new FileStateFunction(tsgm, externalFilesHelper));
map.put(SkyFunctions.DIRECTORY_LISTING_STATE,
new DirectoryListingStateFunction(externalFilesHelper));
