Use List instead of Map in Skyframe's intermediate eval results.
This CL is a no-op and only provides the necessary methods for Skyframe
evaluation with Lists instead of Maps. Actual application will be done in
subsequent CLs.
Constructing a Map<SkyKey, SkyValue> to store intermediate Skyframe evaluation result is wasteful since we don't make use of the look up capabilities of Maps, but rather just iterate over the key-value pairs. As an alternative, we can present the Skyframe evaluation result with a more space-efficient List<SkyValue>, then rely on the given order of the requested SkyKeys to iterate over it in various SkyFunctions.
Our benchmarks/profiles showed some reduction in GC and CPU time as a result of this switch.
RELNOTES: None
PiperOrigin-RevId: 342917514
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/ProgressEventSuppressingEnvironment.java b/src/main/java/com/google/devtools/build/lib/skyframe/ProgressEventSuppressingEnvironment.java
index 4d23fee..093fd4e 100644
--- a/src/main/java/com/google/devtools/build/lib/skyframe/ProgressEventSuppressingEnvironment.java
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/ProgressEventSuppressingEnvironment.java
@@ -26,6 +26,7 @@
import com.google.devtools.build.skyframe.ValueOrException4;
import com.google.devtools.build.skyframe.ValueOrException5;
import com.google.devtools.build.skyframe.Version;
+import java.util.List;
import java.util.Map;
import javax.annotation.Nullable;
@@ -197,6 +198,75 @@
}
@Override
+ public List<SkyValue> getOrderedValues(Iterable<? extends SkyKey> depKeys)
+ throws InterruptedException {
+ return delegate.getOrderedValues(depKeys);
+ }
+
+ @Override
+ public <E extends Exception> List<ValueOrException<E>> getOrderedValuesOrThrow(
+ Iterable<? extends SkyKey> depKeys, Class<E> exceptionClass) throws InterruptedException {
+ return delegate.getOrderedValuesOrThrow(depKeys, exceptionClass);
+ }
+
+ @Override
+ public <E1 extends Exception, E2 extends Exception>
+ List<ValueOrException2<E1, E2>> getOrderedValuesOrThrow(
+ Iterable<? extends SkyKey> depKeys, Class<E1> exceptionClass1, Class<E2> exceptionClass2)
+ throws InterruptedException {
+ return delegate.getOrderedValuesOrThrow(depKeys, exceptionClass1, exceptionClass2);
+ }
+
+ @Override
+ public <E1 extends Exception, E2 extends Exception, E3 extends Exception>
+ List<ValueOrException3<E1, E2, E3>> getOrderedValuesOrThrow(
+ Iterable<? extends SkyKey> depKeys,
+ Class<E1> exceptionClass1,
+ Class<E2> exceptionClass2,
+ Class<E3> exceptionClass3)
+ throws InterruptedException {
+ return delegate.getOrderedValuesOrThrow(
+ depKeys, exceptionClass1, exceptionClass2, exceptionClass3);
+ }
+
+ @Override
+ public <E1 extends Exception, E2 extends Exception, E3 extends Exception, E4 extends Exception>
+ List<ValueOrException4<E1, E2, E3, E4>> getOrderedValuesOrThrow(
+ Iterable<? extends SkyKey> depKeys,
+ Class<E1> exceptionClass1,
+ Class<E2> exceptionClass2,
+ Class<E3> exceptionClass3,
+ Class<E4> exceptionClass4)
+ throws InterruptedException {
+ return delegate.getOrderedValuesOrThrow(
+ depKeys, exceptionClass1, exceptionClass2, exceptionClass3, exceptionClass4);
+ }
+
+ @Override
+ public <
+ E1 extends Exception,
+ E2 extends Exception,
+ E3 extends Exception,
+ E4 extends Exception,
+ E5 extends Exception>
+ List<ValueOrException5<E1, E2, E3, E4, E5>> getOrderedValuesOrThrow(
+ Iterable<? extends SkyKey> depKeys,
+ Class<E1> exceptionClass1,
+ Class<E2> exceptionClass2,
+ Class<E3> exceptionClass3,
+ Class<E4> exceptionClass4,
+ Class<E5> exceptionClass5)
+ throws InterruptedException {
+ return delegate.getOrderedValuesOrThrow(
+ depKeys,
+ exceptionClass1,
+ exceptionClass2,
+ exceptionClass3,
+ exceptionClass4,
+ exceptionClass5);
+ }
+
+ @Override
@Nullable
public GroupedList<SkyKey> getTemporaryDirectDeps() {
return delegate.getTemporaryDirectDeps();
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SkyFunctionEnvironmentForTesting.java b/src/main/java/com/google/devtools/build/lib/skyframe/SkyFunctionEnvironmentForTesting.java
index 894c544..1f2fc19 100644
--- a/src/main/java/com/google/devtools/build/lib/skyframe/SkyFunctionEnvironmentForTesting.java
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/SkyFunctionEnvironmentForTesting.java
@@ -25,6 +25,7 @@
import com.google.devtools.build.skyframe.SkyKey;
import com.google.devtools.build.skyframe.SkyValue;
import com.google.devtools.build.skyframe.ValueOrUntypedException;
+import java.util.List;
import java.util.Map;
/**
@@ -62,6 +63,12 @@
}
@Override
+ protected List<ValueOrUntypedException> getOrderedValueOrUntypedExceptions(
+ Iterable<? extends SkyKey> depKeys) throws InterruptedException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
public boolean inErrorBubblingForTesting() {
return false;
}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/StateInformingSkyFunctionEnvironment.java b/src/main/java/com/google/devtools/build/lib/skyframe/StateInformingSkyFunctionEnvironment.java
index 59d8182..a1c8ce7 100644
--- a/src/main/java/com/google/devtools/build/lib/skyframe/StateInformingSkyFunctionEnvironment.java
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/StateInformingSkyFunctionEnvironment.java
@@ -25,6 +25,7 @@
import com.google.devtools.build.skyframe.ValueOrException4;
import com.google.devtools.build.skyframe.ValueOrException5;
import com.google.devtools.build.skyframe.Version;
+import java.util.List;
import java.util.Map;
import javax.annotation.Nullable;
@@ -247,6 +248,105 @@
}
@Override
+ public List<SkyValue> getOrderedValues(Iterable<? extends SkyKey> depKeys)
+ throws InterruptedException {
+ preFetch.inform();
+ try {
+ return delegate.getOrderedValues(depKeys);
+ } finally {
+ postFetch.inform();
+ }
+ }
+
+ @Override
+ public <E extends Exception> List<ValueOrException<E>> getOrderedValuesOrThrow(
+ Iterable<? extends SkyKey> depKeys, Class<E> exceptionClass) throws InterruptedException {
+ preFetch.inform();
+ try {
+ return delegate.getOrderedValuesOrThrow(depKeys, exceptionClass);
+ } finally {
+ postFetch.inform();
+ }
+ }
+
+ @Override
+ public <E1 extends Exception, E2 extends Exception>
+ List<ValueOrException2<E1, E2>> getOrderedValuesOrThrow(
+ Iterable<? extends SkyKey> depKeys, Class<E1> exceptionClass1, Class<E2> exceptionClass2)
+ throws InterruptedException {
+ preFetch.inform();
+ try {
+ return delegate.getOrderedValuesOrThrow(depKeys, exceptionClass1, exceptionClass2);
+ } finally {
+ postFetch.inform();
+ }
+ }
+
+ @Override
+ public <E1 extends Exception, E2 extends Exception, E3 extends Exception>
+ List<ValueOrException3<E1, E2, E3>> getOrderedValuesOrThrow(
+ Iterable<? extends SkyKey> depKeys,
+ Class<E1> exceptionClass1,
+ Class<E2> exceptionClass2,
+ Class<E3> exceptionClass3)
+ throws InterruptedException {
+ preFetch.inform();
+ try {
+ return delegate.getOrderedValuesOrThrow(
+ depKeys, exceptionClass1, exceptionClass2, exceptionClass3);
+ } finally {
+ postFetch.inform();
+ }
+ }
+
+ @Override
+ public <E1 extends Exception, E2 extends Exception, E3 extends Exception, E4 extends Exception>
+ List<ValueOrException4<E1, E2, E3, E4>> getOrderedValuesOrThrow(
+ Iterable<? extends SkyKey> depKeys,
+ Class<E1> exceptionClass1,
+ Class<E2> exceptionClass2,
+ Class<E3> exceptionClass3,
+ Class<E4> exceptionClass4)
+ throws InterruptedException {
+ preFetch.inform();
+ try {
+ return delegate.getOrderedValuesOrThrow(
+ depKeys, exceptionClass1, exceptionClass2, exceptionClass3, exceptionClass4);
+ } finally {
+ postFetch.inform();
+ }
+ }
+
+ @Override
+ public <
+ E1 extends Exception,
+ E2 extends Exception,
+ E3 extends Exception,
+ E4 extends Exception,
+ E5 extends Exception>
+ List<ValueOrException5<E1, E2, E3, E4, E5>> getOrderedValuesOrThrow(
+ Iterable<? extends SkyKey> depKeys,
+ Class<E1> exceptionClass1,
+ Class<E2> exceptionClass2,
+ Class<E3> exceptionClass3,
+ Class<E4> exceptionClass4,
+ Class<E5> exceptionClass5)
+ throws InterruptedException {
+ preFetch.inform();
+ try {
+ return delegate.getOrderedValuesOrThrow(
+ depKeys,
+ exceptionClass1,
+ exceptionClass2,
+ exceptionClass3,
+ exceptionClass4,
+ exceptionClass5);
+ } finally {
+ postFetch.inform();
+ }
+ }
+
+ @Override
public ExtendedEventHandler getListener() {
return delegate.getListener();
}
diff --git a/src/main/java/com/google/devtools/build/skyframe/AbstractSkyFunctionEnvironment.java b/src/main/java/com/google/devtools/build/skyframe/AbstractSkyFunctionEnvironment.java
index 4ec831a..3a29f71 100644
--- a/src/main/java/com/google/devtools/build/skyframe/AbstractSkyFunctionEnvironment.java
+++ b/src/main/java/com/google/devtools/build/skyframe/AbstractSkyFunctionEnvironment.java
@@ -15,10 +15,12 @@
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.devtools.build.lib.util.GroupedList;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Map;
@@ -57,6 +59,10 @@
protected abstract Map<SkyKey, ValueOrUntypedException> getValueOrUntypedExceptions(
Iterable<? extends SkyKey> depKeys) throws InterruptedException;
+ /** Implementations should set {@link #valuesMissing} as necessary. */
+ protected abstract List<ValueOrUntypedException> getOrderedValueOrUntypedExceptions(
+ Iterable<? extends SkyKey> depKeys) throws InterruptedException;
+
@Override
@Nullable
public SkyValue getValue(SkyKey depKey) throws InterruptedException {
@@ -282,11 +288,33 @@
@Nullable Class<E3> exceptionClass3,
@Nullable Class<E4> exceptionClass4,
@Nullable Class<E5> exceptionClass5) {
+ checkValuesMissingBecauseOfFilteredError(
+ voes.values(),
+ exceptionClass1,
+ exceptionClass2,
+ exceptionClass3,
+ exceptionClass4,
+ exceptionClass5);
+ }
+
+ private <
+ E1 extends Exception,
+ E2 extends Exception,
+ E3 extends Exception,
+ E4 extends Exception,
+ E5 extends Exception>
+ void checkValuesMissingBecauseOfFilteredError(
+ Collection<ValueOrUntypedException> voes,
+ @Nullable Class<E1> exceptionClass1,
+ @Nullable Class<E2> exceptionClass2,
+ @Nullable Class<E3> exceptionClass3,
+ @Nullable Class<E4> exceptionClass4,
+ @Nullable Class<E5> exceptionClass5) {
if (!errorMightHaveBeenFound) {
// Short-circuit in the common case of no errors.
return;
}
- for (ValueOrUntypedException voe : voes.values()) {
+ for (ValueOrUntypedException voe : voes) {
SkyValue value = voe.getValue();
if (value == null) {
Exception e = voe.getException();
@@ -304,6 +332,135 @@
}
@Override
+ public List<SkyValue> getOrderedValues(Iterable<? extends SkyKey> depKeys)
+ throws InterruptedException {
+ List<ValueOrUntypedException> valuesOrExceptions = getOrderedValueOrUntypedExceptions(depKeys);
+ checkValuesMissingBecauseOfFilteredError(valuesOrExceptions, null, null, null, null, null);
+ return Collections.unmodifiableList(
+ Lists.transform(valuesOrExceptions, ValueOrUntypedException::getValue));
+ }
+
+ @Override
+ public <E extends Exception> List<ValueOrException<E>> getOrderedValuesOrThrow(
+ Iterable<? extends SkyKey> depKeys, Class<E> exceptionClass) throws InterruptedException {
+ SkyFunctionException.validateExceptionType(exceptionClass);
+ List<ValueOrUntypedException> valuesOrExceptions = getOrderedValueOrUntypedExceptions(depKeys);
+ checkValuesMissingBecauseOfFilteredError(
+ valuesOrExceptions, exceptionClass, null, null, null, null);
+ return Collections.unmodifiableList(
+ Lists.transform(
+ valuesOrExceptions, voe -> ValueOrException.fromUntypedException(voe, exceptionClass)));
+ }
+
+ @Override
+ public <E1 extends Exception, E2 extends Exception>
+ List<ValueOrException2<E1, E2>> getOrderedValuesOrThrow(
+ Iterable<? extends SkyKey> depKeys, Class<E1> exceptionClass1, Class<E2> exceptionClass2)
+ throws InterruptedException {
+ SkyFunctionException.validateExceptionType(exceptionClass1);
+ SkyFunctionException.validateExceptionType(exceptionClass2);
+ List<ValueOrUntypedException> valuesOrExceptions = getOrderedValueOrUntypedExceptions(depKeys);
+ checkValuesMissingBecauseOfFilteredError(
+ valuesOrExceptions, exceptionClass1, exceptionClass2, null, null, null);
+ return Collections.unmodifiableList(
+ Lists.transform(
+ valuesOrExceptions,
+ voe -> ValueOrException2.fromUntypedException(voe, exceptionClass1, exceptionClass2)));
+ }
+
+ @Override
+ public <E1 extends Exception, E2 extends Exception, E3 extends Exception>
+ List<ValueOrException3<E1, E2, E3>> getOrderedValuesOrThrow(
+ Iterable<? extends SkyKey> depKeys,
+ Class<E1> exceptionClass1,
+ Class<E2> exceptionClass2,
+ Class<E3> exceptionClass3)
+ throws InterruptedException {
+ SkyFunctionException.validateExceptionType(exceptionClass1);
+ SkyFunctionException.validateExceptionType(exceptionClass2);
+ SkyFunctionException.validateExceptionType(exceptionClass3);
+ List<ValueOrUntypedException> valuesOrExceptions = getOrderedValueOrUntypedExceptions(depKeys);
+ checkValuesMissingBecauseOfFilteredError(
+ valuesOrExceptions, exceptionClass1, exceptionClass2, exceptionClass3, null, null);
+ return Collections.unmodifiableList(
+ Lists.transform(
+ valuesOrExceptions,
+ voe ->
+ ValueOrException3.fromUntypedException(
+ voe, exceptionClass1, exceptionClass2, exceptionClass3)));
+ }
+
+ @Override
+ public <E1 extends Exception, E2 extends Exception, E3 extends Exception, E4 extends Exception>
+ List<ValueOrException4<E1, E2, E3, E4>> getOrderedValuesOrThrow(
+ Iterable<? extends SkyKey> depKeys,
+ Class<E1> exceptionClass1,
+ Class<E2> exceptionClass2,
+ Class<E3> exceptionClass3,
+ Class<E4> exceptionClass4)
+ throws InterruptedException {
+ SkyFunctionException.validateExceptionType(exceptionClass1);
+ SkyFunctionException.validateExceptionType(exceptionClass2);
+ SkyFunctionException.validateExceptionType(exceptionClass3);
+ SkyFunctionException.validateExceptionType(exceptionClass4);
+ List<ValueOrUntypedException> valuesOrExceptions = getOrderedValueOrUntypedExceptions(depKeys);
+ checkValuesMissingBecauseOfFilteredError(
+ valuesOrExceptions,
+ exceptionClass1,
+ exceptionClass2,
+ exceptionClass3,
+ exceptionClass4,
+ null);
+ return Collections.unmodifiableList(
+ Lists.transform(
+ valuesOrExceptions,
+ voe ->
+ ValueOrException4.fromUntypedException(
+ voe, exceptionClass1, exceptionClass2, exceptionClass3, exceptionClass4)));
+ }
+
+ @Override
+ public <
+ E1 extends Exception,
+ E2 extends Exception,
+ E3 extends Exception,
+ E4 extends Exception,
+ E5 extends Exception>
+ List<ValueOrException5<E1, E2, E3, E4, E5>> getOrderedValuesOrThrow(
+ Iterable<? extends SkyKey> depKeys,
+ Class<E1> exceptionClass1,
+ Class<E2> exceptionClass2,
+ Class<E3> exceptionClass3,
+ Class<E4> exceptionClass4,
+ Class<E5> exceptionClass5)
+ throws InterruptedException {
+ SkyFunctionException.validateExceptionType(exceptionClass1);
+ SkyFunctionException.validateExceptionType(exceptionClass2);
+ SkyFunctionException.validateExceptionType(exceptionClass3);
+ SkyFunctionException.validateExceptionType(exceptionClass4);
+ SkyFunctionException.validateExceptionType(exceptionClass5);
+ List<ValueOrUntypedException> valuesOrExceptions = getOrderedValueOrUntypedExceptions(depKeys);
+ checkValuesMissingBecauseOfFilteredError(
+ valuesOrExceptions,
+ exceptionClass1,
+ exceptionClass2,
+ exceptionClass3,
+ exceptionClass4,
+ exceptionClass5);
+ return Collections.unmodifiableList(
+ Lists.transform(
+ valuesOrExceptions,
+ voe ->
+ ValueOrException5.fromUntypedException(
+ voe,
+ exceptionClass1,
+ exceptionClass2,
+ exceptionClass3,
+ exceptionClass4,
+ exceptionClass5)));
+ }
+
+ @Override
public boolean valuesMissing() {
return valuesMissing || (externalDeps != null);
}
diff --git a/src/main/java/com/google/devtools/build/skyframe/QueryableGraphBackedSkyFunctionEnvironment.java b/src/main/java/com/google/devtools/build/skyframe/QueryableGraphBackedSkyFunctionEnvironment.java
index c07e510..d4bd5bd 100644
--- a/src/main/java/com/google/devtools/build/skyframe/QueryableGraphBackedSkyFunctionEnvironment.java
+++ b/src/main/java/com/google/devtools/build/skyframe/QueryableGraphBackedSkyFunctionEnvironment.java
@@ -17,6 +17,7 @@
import com.google.common.collect.Maps;
import com.google.devtools.build.lib.events.ExtendedEventHandler;
import com.google.devtools.build.skyframe.QueryableGraph.Reason;
+import java.util.List;
import java.util.Map;
/**
@@ -75,6 +76,12 @@
}
@Override
+ protected List<ValueOrUntypedException> getOrderedValueOrUntypedExceptions(
+ Iterable<? extends SkyKey> depKeys) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
public ExtendedEventHandler getListener() {
return eventHandler;
}
diff --git a/src/main/java/com/google/devtools/build/skyframe/RecordingSkyFunctionEnvironment.java b/src/main/java/com/google/devtools/build/skyframe/RecordingSkyFunctionEnvironment.java
index 4b55081..52e64f8 100644
--- a/src/main/java/com/google/devtools/build/skyframe/RecordingSkyFunctionEnvironment.java
+++ b/src/main/java/com/google/devtools/build/skyframe/RecordingSkyFunctionEnvironment.java
@@ -17,6 +17,7 @@
import com.google.devtools.build.lib.events.ExtendedEventHandler;
import com.google.devtools.build.lib.util.GroupedList;
import com.google.devtools.build.skyframe.SkyFunction.Environment;
+import java.util.List;
import java.util.Map;
import java.util.function.Consumer;
import javax.annotation.Nullable;
@@ -265,6 +266,115 @@
}
@Override
+ public List<SkyValue> getOrderedValues(Iterable<? extends SkyKey> depKeys)
+ throws InterruptedException {
+ return delegate.getOrderedValues(depKeys);
+ }
+
+ @Override
+ public <E extends Exception> List<ValueOrException<E>> getOrderedValuesOrThrow(
+ Iterable<? extends SkyKey> depKeys, Class<E> exceptionClass) throws InterruptedException {
+ recordDeps(depKeys);
+ try {
+ return delegate.getOrderedValuesOrThrow(depKeys, exceptionClass);
+ } catch (
+ @SuppressWarnings("InterruptedExceptionSwallowed")
+ Exception e) {
+ noteException(e);
+ throw e;
+ }
+ }
+
+ @Override
+ public <E1 extends Exception, E2 extends Exception>
+ List<ValueOrException2<E1, E2>> getOrderedValuesOrThrow(
+ Iterable<? extends SkyKey> depKeys, Class<E1> exceptionClass1, Class<E2> exceptionClass2)
+ throws InterruptedException {
+ recordDeps(depKeys);
+ try {
+ return delegate.getOrderedValuesOrThrow(depKeys, exceptionClass1, exceptionClass2);
+ } catch (
+ @SuppressWarnings("InterruptedExceptionSwallowed")
+ Exception e) {
+ noteException(e);
+ throw e;
+ }
+ }
+
+ @Override
+ public <E1 extends Exception, E2 extends Exception, E3 extends Exception>
+ List<ValueOrException3<E1, E2, E3>> getOrderedValuesOrThrow(
+ Iterable<? extends SkyKey> depKeys,
+ Class<E1> exceptionClass1,
+ Class<E2> exceptionClass2,
+ Class<E3> exceptionClass3)
+ throws InterruptedException {
+ recordDeps(depKeys);
+ try {
+ return delegate.getOrderedValuesOrThrow(
+ depKeys, exceptionClass1, exceptionClass2, exceptionClass3);
+ } catch (
+ @SuppressWarnings("InterruptedExceptionSwallowed")
+ Exception e) {
+ noteException(e);
+ throw e;
+ }
+ }
+
+ @Override
+ public <E1 extends Exception, E2 extends Exception, E3 extends Exception, E4 extends Exception>
+ List<ValueOrException4<E1, E2, E3, E4>> getOrderedValuesOrThrow(
+ Iterable<? extends SkyKey> depKeys,
+ Class<E1> exceptionClass1,
+ Class<E2> exceptionClass2,
+ Class<E3> exceptionClass3,
+ Class<E4> exceptionClass4)
+ throws InterruptedException {
+ recordDeps(depKeys);
+ try {
+ return delegate.getOrderedValuesOrThrow(
+ depKeys, exceptionClass1, exceptionClass2, exceptionClass3, exceptionClass4);
+ } catch (
+ @SuppressWarnings("InterruptedExceptionSwallowed")
+ Exception e) {
+ noteException(e);
+ throw e;
+ }
+ }
+
+ @Override
+ public <
+ E1 extends Exception,
+ E2 extends Exception,
+ E3 extends Exception,
+ E4 extends Exception,
+ E5 extends Exception>
+ List<ValueOrException5<E1, E2, E3, E4, E5>> getOrderedValuesOrThrow(
+ Iterable<? extends SkyKey> depKeys,
+ Class<E1> exceptionClass1,
+ Class<E2> exceptionClass2,
+ Class<E3> exceptionClass3,
+ Class<E4> exceptionClass4,
+ Class<E5> exceptionClass5)
+ throws InterruptedException {
+ recordDeps(depKeys);
+ try {
+ return delegate.getOrderedValuesOrThrow(
+ depKeys,
+ exceptionClass1,
+ exceptionClass2,
+ exceptionClass3,
+ exceptionClass4,
+ exceptionClass5);
+ } catch (
+ @SuppressWarnings("InterruptedExceptionSwallowed")
+ Exception e) {
+ noteException(e);
+ throw e;
+ }
+ }
+
+ @Override
public ExtendedEventHandler getListener() {
return delegate.getListener();
}
diff --git a/src/main/java/com/google/devtools/build/skyframe/SkyFunction.java b/src/main/java/com/google/devtools/build/skyframe/SkyFunction.java
index 1117bc8..0ba2590 100644
--- a/src/main/java/com/google/devtools/build/skyframe/SkyFunction.java
+++ b/src/main/java/com/google/devtools/build/skyframe/SkyFunction.java
@@ -21,6 +21,7 @@
import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe;
import com.google.devtools.build.lib.events.ExtendedEventHandler;
import com.google.devtools.build.lib.util.GroupedList;
+import java.util.List;
import java.util.Map;
import javax.annotation.Nullable;
@@ -245,6 +246,12 @@
Map<SkyKey, SkyValue> getValues(Iterable<? extends SkyKey> depKeys) throws InterruptedException;
/**
+ * Similar to getValues, but instead of returning a {@code Map<SkyKey, SkyValue>}, returns a
+ * {@code List<SkyValue>} in the order of the input {@code Iterable<SkyKey>}. b/172462551
+ */
+ List<SkyValue> getOrderedValues(Iterable<? extends SkyKey> depKeys) throws InterruptedException;
+
+ /**
* Similar to {@link #getValues} but allows the caller to specify a set of types that are proper
* subtypes of Exception (see {@link SkyFunctionException} for more details) to find out whether
* any of the dependencies' evaluations resulted in exceptions of those types. The returned
@@ -304,6 +311,53 @@
throws InterruptedException;
/**
+ * Similar to getValuesOrThrow, but instead of returning a {@code Map<SkyKey,
+ * ValueOrException>}, returns a {@code List<SkyValue>} in the order of the input {@code
+ * Iterable<SkyKey>}.
+ */
+ <E extends Exception> List<ValueOrException<E>> getOrderedValuesOrThrow(
+ Iterable<? extends SkyKey> depKeys, Class<E> exceptionClass) throws InterruptedException;
+
+ <E1 extends Exception, E2 extends Exception>
+ List<ValueOrException2<E1, E2>> getOrderedValuesOrThrow(
+ Iterable<? extends SkyKey> depKeys,
+ Class<E1> exceptionClass1,
+ Class<E2> exceptionClass2)
+ throws InterruptedException;
+
+ <E1 extends Exception, E2 extends Exception, E3 extends Exception>
+ List<ValueOrException3<E1, E2, E3>> getOrderedValuesOrThrow(
+ Iterable<? extends SkyKey> depKeys,
+ Class<E1> exceptionClass1,
+ Class<E2> exceptionClass2,
+ Class<E3> exceptionClass3)
+ throws InterruptedException;
+
+ <E1 extends Exception, E2 extends Exception, E3 extends Exception, E4 extends Exception>
+ List<ValueOrException4<E1, E2, E3, E4>> getOrderedValuesOrThrow(
+ Iterable<? extends SkyKey> depKeys,
+ Class<E1> exceptionClass1,
+ Class<E2> exceptionClass2,
+ Class<E3> exceptionClass3,
+ Class<E4> exceptionClass4)
+ throws InterruptedException;
+
+ <
+ E1 extends Exception,
+ E2 extends Exception,
+ E3 extends Exception,
+ E4 extends Exception,
+ E5 extends Exception>
+ List<ValueOrException5<E1, E2, E3, E4, E5>> getOrderedValuesOrThrow(
+ Iterable<? extends SkyKey> depKeys,
+ Class<E1> exceptionClass1,
+ Class<E2> exceptionClass2,
+ Class<E3> exceptionClass3,
+ Class<E4> exceptionClass4,
+ Class<E5> exceptionClass5)
+ throws InterruptedException;
+
+ /**
* Returns whether there was a previous getValue[s][OrThrow] that indicated a missing
* dependency. Formally, returns true iff at least one of the following occurred:
*
diff --git a/src/main/java/com/google/devtools/build/skyframe/SkyFunctionEnvironment.java b/src/main/java/com/google/devtools/build/skyframe/SkyFunctionEnvironment.java
index afe1492..ea24d9a 100644
--- a/src/main/java/com/google/devtools/build/skyframe/SkyFunctionEnvironment.java
+++ b/src/main/java/com/google/devtools/build/skyframe/SkyFunctionEnvironment.java
@@ -20,6 +20,7 @@
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import com.google.devtools.build.lib.collect.compacthashmap.CompactHashMap;
@@ -401,6 +402,58 @@
}
/**
+ * Similar to {@link #getValuesFromErrorOrDepsOrGraph}, but instead of a Map, return a List of
+ * SkyValue ordered by the given order of SkyKeys.
+ */
+ private List<SkyValue> getOrderedValuesFromErrorOrDepsOrGraph(Iterable<? extends SkyKey> keys)
+ throws InterruptedException {
+ int capacity = keys instanceof Collection ? ((Collection<?>) keys).size() : 16;
+ List<SkyValue> result = new ArrayList<>(capacity);
+
+ // Ignoring duplication check here since it's done in GroupedList.
+ List<SkyKey> missingKeys = new ArrayList<>();
+ newlyRequestedDeps.startGroup();
+ for (SkyKey key : keys) {
+ Preconditions.checkState(
+ !key.equals(ErrorTransienceValue.KEY),
+ "Error transience key cannot be in requested deps of %s",
+ skyKey);
+ SkyValue value = maybeGetValueFromErrorOrDeps(key);
+ if (value == null) {
+ missingKeys.add(key);
+ }
+ // To maintain the ordering.
+ result.add(value);
+ if (!previouslyRequestedDepsValues.containsKey(key)) {
+ newlyRequestedDeps.add(key);
+ }
+ }
+ newlyRequestedDeps.endGroup();
+
+ if (missingKeys.isEmpty()) {
+ return result;
+ }
+
+ Map<SkyKey, ? extends NodeEntry> missingEntries =
+ evaluatorContext.getBatchValues(skyKey, Reason.DEP_REQUESTED, missingKeys);
+ int i = 0;
+ for (SkyKey key : keys) {
+ if (result.get(i) != null) {
+ continue;
+ }
+ NodeEntry depEntry = missingEntries.get(key);
+ SkyValue valueOrNullMarker = getValueOrNullMarker(depEntry);
+ result.set(i, valueOrNullMarker);
+ newlyRequestedDepsValues.put(key, valueOrNullMarker);
+ if (valueOrNullMarker != NULL_MARKER) {
+ maybeUpdateMaxChildVersion(depEntry);
+ }
+ i++;
+ }
+ return result;
+ }
+
+ /**
* Returns the values of done deps in {@code depKeys}, by looking in order at:
*
* <ol>
@@ -571,46 +624,90 @@
}
}
- return Maps.transformValues(
- values,
- maybeWrappedValue -> {
- if (maybeWrappedValue == NULL_MARKER) {
- return ValueOrUntypedException.ofNull();
- }
- SkyValue justValue = ValueWithMetadata.justValue(maybeWrappedValue);
- ErrorInfo errorInfo = ValueWithMetadata.getMaybeErrorInfo(maybeWrappedValue);
+ return Maps.transformValues(values, this::transformToValueOrUntypedException);
+ }
- if (justValue != null && (evaluatorContext.keepGoing() || errorInfo == null)) {
- // If the dep did compute a value, it is given to the caller if we are in
- // keepGoing mode or if we are in noKeepGoingMode and there were no errors computing
- // it.
- return ValueOrUntypedException.ofValueUntyped(justValue);
- }
+ @Override
+ protected List<ValueOrUntypedException> getOrderedValueOrUntypedExceptions(
+ Iterable<? extends SkyKey> depKeys) throws InterruptedException {
+ checkActive();
+ List<SkyValue> values = getOrderedValuesFromErrorOrDepsOrGraph(depKeys);
+ int i = 0;
+ for (SkyKey depKey : depKeys) {
+ SkyValue depValue = values.get(i++);
- // There was an error building the value, which we will either report by throwing an
- // exception or insulate the caller from by returning null.
- Preconditions.checkNotNull(errorInfo, "%s %s", skyKey, maybeWrappedValue);
- Exception exception = errorInfo.getException();
-
- if (!evaluatorContext.keepGoing() && exception != null && bubbleErrorInfo == null) {
- // Child errors should not be propagated in noKeepGoing mode (except during error
- // bubbling). Instead we should fail fast.
- return ValueOrUntypedException.ofNull();
- }
-
- if (exception != null) {
- // Give builder a chance to handle this exception.
- return ValueOrUntypedException.ofExn(exception);
- }
- // In a cycle.
+ if (depValue == NULL_MARKER) {
+ valuesMissing = true;
+ if (previouslyRequestedDepsValues.containsKey(depKey)) {
Preconditions.checkState(
- !errorInfo.getCycleInfo().isEmpty(),
- "%s %s %s",
+ bubbleErrorInfo != null,
+ "Undone key %s was already in deps of %s( dep: %s, parent: %s )",
+ depKey,
skyKey,
- errorInfo,
- maybeWrappedValue);
- return ValueOrUntypedException.ofNull();
- });
+ evaluatorContext.getGraph().get(skyKey, Reason.OTHER, depKey),
+ evaluatorContext.getGraph().get(null, Reason.OTHER, skyKey));
+ }
+ continue;
+ }
+
+ ErrorInfo errorInfo = ValueWithMetadata.getMaybeErrorInfo(depValue);
+ if (errorInfo != null) {
+ errorMightHaveBeenFound = true;
+ childErrorInfos.add(errorInfo);
+ if (bubbleErrorInfo != null) {
+ // Set interrupted status, to try to prevent the calling SkyFunction from doing anything
+ // fancy after this. SkyFunctions executed during error bubbling are supposed to
+ // (quickly) rethrow errors or return a value/null (but there's currently no way to
+ // enforce this).
+ Thread.currentThread().interrupt();
+ }
+ if ((!evaluatorContext.keepGoing() && bubbleErrorInfo == null)
+ || errorInfo.getException() == null) {
+ valuesMissing = true;
+ // We arbitrarily record the first child error if we are about to abort.
+ if (!evaluatorContext.keepGoing() && depErrorKey == null) {
+ depErrorKey = depKey;
+ }
+ }
+ }
+ }
+
+ return Lists.transform(values, this::transformToValueOrUntypedException);
+ }
+
+ private ValueOrUntypedException transformToValueOrUntypedException(SkyValue maybeWrappedValue) {
+ if (maybeWrappedValue == NULL_MARKER) {
+ return ValueOrUntypedException.ofNull();
+ }
+ SkyValue justValue = ValueWithMetadata.justValue(maybeWrappedValue);
+ ErrorInfo errorInfo = ValueWithMetadata.getMaybeErrorInfo(maybeWrappedValue);
+
+ if (justValue != null && (evaluatorContext.keepGoing() || errorInfo == null)) {
+ // If the dep did compute a value, it is given to the caller if we are in
+ // keepGoing mode or if we are in noKeepGoingMode and there were no errors computing
+ // it.
+ return ValueOrUntypedException.ofValueUntyped(justValue);
+ }
+
+ // There was an error building the value, which we will either report by throwing an
+ // exception or insulate the caller from by returning null.
+ Preconditions.checkNotNull(errorInfo, "%s %s", skyKey, maybeWrappedValue);
+ Exception exception = errorInfo.getException();
+
+ if (!evaluatorContext.keepGoing() && exception != null && bubbleErrorInfo == null) {
+ // Child errors should not be propagated in noKeepGoing mode (except during error
+ // bubbling). Instead we should fail fast.
+ return ValueOrUntypedException.ofNull();
+ }
+
+ if (exception != null) {
+ // Give builder a chance to handle this exception.
+ return ValueOrUntypedException.ofExn(exception);
+ }
+ // In a cycle.
+ Preconditions.checkState(
+ !errorInfo.getCycleInfo().isEmpty(), "%s %s %s", skyKey, errorInfo, maybeWrappedValue);
+ return ValueOrUntypedException.ofNull();
}
/**
diff --git a/src/test/java/com/google/devtools/build/lib/actions/util/ActionsTestUtil.java b/src/test/java/com/google/devtools/build/lib/actions/util/ActionsTestUtil.java
index 39b4389..fda987b 100644
--- a/src/test/java/com/google/devtools/build/lib/actions/util/ActionsTestUtil.java
+++ b/src/test/java/com/google/devtools/build/lib/actions/util/ActionsTestUtil.java
@@ -355,6 +355,12 @@
}
@Override
+ protected List<ValueOrUntypedException> getOrderedValueOrUntypedExceptions(
+ Iterable<? extends SkyKey> depKeys) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
public ExtendedEventHandler getListener() {
return null;
}
