Split resolved hashes computation
...into a computation of the Skylark value contained in a resolved
file, and the computation of the expected hashes from this value.
In this way, we can reuse the code for computing the resolved value
for the (experimental) feature of using a resolved file instead of
a workspace file.
PiperOrigin-RevId: 209150452
Change-Id: Ib1a77128100d821c4cc3e0a0ba57f65d44e07e97
PiperOrigin-RevId: 210085624
diff --git a/src/main/java/com/google/devtools/build/lib/rules/repository/ResolvedFileFunction.java b/src/main/java/com/google/devtools/build/lib/rules/repository/ResolvedFileFunction.java
new file mode 100644
index 0000000..d9a135f
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/repository/ResolvedFileFunction.java
@@ -0,0 +1,150 @@
+// Copyright 2018 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.repository;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.devtools.build.lib.actions.FileValue;
+import com.google.devtools.build.lib.cmdline.Label;
+import com.google.devtools.build.lib.packages.BazelLibrary;
+import com.google.devtools.build.lib.packages.BuildFileContainsErrorsException;
+import com.google.devtools.build.lib.packages.NoSuchThingException;
+import com.google.devtools.build.lib.rules.repository.ResolvedFileValue.ResolvedFileKey;
+import com.google.devtools.build.lib.skyframe.PrecomputedValue;
+import com.google.devtools.build.lib.syntax.BuildFileAST;
+import com.google.devtools.build.lib.syntax.Environment;
+import com.google.devtools.build.lib.syntax.Mutability;
+import com.google.devtools.build.lib.syntax.ParserInputSource;
+import com.google.devtools.build.lib.syntax.SkylarkSemantics;
+import com.google.devtools.build.lib.vfs.FileSystemUtils;
+import com.google.devtools.build.skyframe.SkyFunction;
+import com.google.devtools.build.skyframe.SkyFunctionException;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import javax.annotation.Nullable;
+
+/** Function computing the Starlark value of 'resolved' in a file. */
+public class ResolvedFileFunction implements SkyFunction {
+
+ @Override
+ @Nullable
+ public SkyValue compute(SkyKey skyKey, Environment env)
+ throws InterruptedException, SkyFunctionException {
+
+ ResolvedFileKey key = (ResolvedFileKey) skyKey;
+ SkylarkSemantics skylarkSemantics = PrecomputedValue.SKYLARK_SEMANTICS.get(env);
+ if (skylarkSemantics == null) {
+ return null;
+ }
+ FileValue fileValue = (FileValue) env.getValue(FileValue.key(key.getPath()));
+ if (fileValue == null) {
+ return null;
+ }
+ try {
+ if (!fileValue.exists()) {
+ throw new ResolvedFileFunctionException(
+ new NoSuchThingException("Specified resolved file '" + key.getPath() + "' not found."));
+ } else {
+ byte[] bytes =
+ FileSystemUtils.readWithKnownFileSize(
+ key.getPath().asPath(), key.getPath().asPath().getFileSize());
+ BuildFileAST ast =
+ BuildFileAST.parseSkylarkFile(
+ ParserInputSource.create(bytes, key.getPath().asPath().asFragment()),
+ env.getListener());
+ if (ast.containsErrors()) {
+ throw new ResolvedFileFunctionException(
+ new BuildFileContainsErrorsException(
+ Label.EXTERNAL_PACKAGE_IDENTIFIER,
+ "Failed to parse file resolved file " + key.getPath()));
+ }
+ com.google.devtools.build.lib.syntax.Environment resolvedEnvironment;
+ try (Mutability mutability = Mutability.create("resolved file %s", key.getPath())) {
+ resolvedEnvironment =
+ com.google.devtools.build.lib.syntax.Environment.builder(mutability)
+ .setSemantics(skylarkSemantics)
+ .setGlobals(BazelLibrary.GLOBALS)
+ .build();
+ if (!ast.exec(resolvedEnvironment, env.getListener())) {
+ throw new ResolvedFileFunctionException(
+ new BuildFileContainsErrorsException(
+ Label.EXTERNAL_PACKAGE_IDENTIFIER,
+ "Failed to evaluate resolved file " + key.getPath()));
+ }
+ }
+ Object resolved = resolvedEnvironment.lookup("resolved");
+ if (resolved == null) {
+ throw new ResolvedFileFunctionException(
+ new BuildFileContainsErrorsException(
+ Label.EXTERNAL_PACKAGE_IDENTIFIER,
+ "Symbol 'resolved' not exported in resolved file " + key.getPath()));
+ }
+ if (!(resolved instanceof List)) {
+ throw new ResolvedFileFunctionException(
+ new BuildFileContainsErrorsException(
+ Label.EXTERNAL_PACKAGE_IDENTIFIER,
+ "Symbol 'resolved' in resolved file " + key.getPath() + " not a list"));
+ }
+ ImmutableList.Builder<Map<String, Object>> result
+ = new ImmutableList.Builder<Map<String, Object>>();
+ for (Object entry : (List) resolved) {
+ if (!(entry instanceof Map)) {
+ throw new ResolvedFileFunctionException(
+ new BuildFileContainsErrorsException(
+ Label.EXTERNAL_PACKAGE_IDENTIFIER,
+ "Symbol 'resolved' in resolved file " + key.getPath()
+ + " contains a non-map entry"));
+ }
+ ImmutableMap.Builder<String, Object> entryBuilder
+ = new ImmutableMap.Builder<String, Object>();
+ for (Map.Entry<Object, Object> keyValue : ((Map<Object, Object>) entry).entrySet()) {
+ Object attribute = keyValue.getKey();
+ if (!(attribute instanceof String)) {
+ throw new ResolvedFileFunctionException(
+ new BuildFileContainsErrorsException(
+ Label.EXTERNAL_PACKAGE_IDENTIFIER,
+ "Symbol 'resolved' in resolved file " + key.getPath()
+ + " contains a non-string key in one of its entries"));
+ }
+ entryBuilder.put((String) attribute, keyValue.getValue());
+ }
+ result.add(entryBuilder.build());
+ }
+ return new ResolvedFileValue(result.build());
+ }
+ } catch (IOException e) {
+ throw new ResolvedFileFunctionException(e);
+ }
+ }
+
+ @Override
+ @Nullable
+ public String extractTag(SkyKey skyKey) {
+ return null;
+ }
+
+ private static final class ResolvedFileFunctionException extends SkyFunctionException {
+ ResolvedFileFunctionException(IOException e) {
+ super(e, SkyFunctionException.Transience.PERSISTENT);
+ }
+
+ ResolvedFileFunctionException(NoSuchThingException e) {
+ super(e, SkyFunctionException.Transience.PERSISTENT);
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/repository/ResolvedFileValue.java b/src/main/java/com/google/devtools/build/lib/rules/repository/ResolvedFileValue.java
new file mode 100644
index 0000000..d38bcf4
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/repository/ResolvedFileValue.java
@@ -0,0 +1,107 @@
+// Copyright 2018 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.rules.repository;
+
+import com.google.common.collect.Interner;
+import com.google.devtools.build.lib.concurrent.BlazeInterners;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+import com.google.devtools.build.lib.skyframe.SkyFunctions;
+import com.google.devtools.build.lib.skyframe.serialization.autocodec.AutoCodec;
+import com.google.devtools.build.lib.vfs.RootedPath;
+import com.google.devtools.build.skyframe.SkyFunctionName;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+/** The value of the binding of "resolved" in a given file. */
+public class ResolvedFileValue implements SkyValue {
+
+ /** Argument for the SkyKey to request the resolved value of a file. */
+ @Immutable
+ @AutoCodec
+ public static class ResolvedFileKey implements SkyKey {
+ private static final Interner<ResolvedFileKey> interner = BlazeInterners.newWeakInterner();
+
+ private final RootedPath path;
+
+ private ResolvedFileKey(RootedPath path) {
+ this.path = path;
+ }
+
+ @AutoCodec.VisibleForSerialization
+ @AutoCodec.Instantiator
+ static ResolvedFileKey create(RootedPath path) {
+ return interner.intern(new ResolvedFileKey(path));
+ }
+
+ public RootedPath getPath() {
+ return path;
+ }
+
+ @Override
+ public SkyFunctionName functionName() {
+ return SkyFunctions.RESOLVED_FILE;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (!(obj instanceof ResolvedFileKey)) {
+ return false;
+ }
+ ResolvedFileKey other = (ResolvedFileKey) obj;
+ return Objects.equals(path, other.path);
+ }
+
+ @Override
+ public int hashCode() {
+ return path.hashCode();
+ }
+ }
+
+ public static SkyKey key(RootedPath path) {
+ return ResolvedFileKey.create(path);
+ }
+
+ private final List<Map<String, Object>> resolvedValue;
+
+ ResolvedFileValue(List<Map<String, Object>> resolvedValue) {
+ this.resolvedValue = resolvedValue;
+ }
+
+ public List<Map<String, Object>> getResolvedValue() {
+ return resolvedValue;
+ }
+
+ @Override
+ public int hashCode() {
+ return resolvedValue.hashCode();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (this == other) {
+ return true;
+ }
+ if (!(other instanceof ResolvedFileValue)) {
+ return false;
+ }
+ return this.resolvedValue.equals(((ResolvedFileValue) other).getResolvedValue());
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/repository/ResolvedHashesFunction.java b/src/main/java/com/google/devtools/build/lib/rules/repository/ResolvedHashesFunction.java
index 71d086a..0cfbf1c 100644
--- a/src/main/java/com/google/devtools/build/lib/rules/repository/ResolvedHashesFunction.java
+++ b/src/main/java/com/google/devtools/build/lib/rules/repository/ResolvedHashesFunction.java
@@ -16,32 +16,21 @@
import com.google.common.base.Optional;
import com.google.common.collect.ImmutableMap;
-import com.google.devtools.build.lib.actions.FileValue;
-import com.google.devtools.build.lib.cmdline.Label;
-import com.google.devtools.build.lib.packages.BazelLibrary;
import com.google.devtools.build.lib.packages.BuildFileContainsErrorsException;
-import com.google.devtools.build.lib.packages.NoSuchThingException;
-import com.google.devtools.build.lib.skyframe.PrecomputedValue;
-import com.google.devtools.build.lib.syntax.BuildFileAST;
import com.google.devtools.build.lib.syntax.Environment;
-import com.google.devtools.build.lib.syntax.Mutability;
-import com.google.devtools.build.lib.syntax.ParserInputSource;
-import com.google.devtools.build.lib.syntax.SkylarkSemantics;
-import com.google.devtools.build.lib.vfs.FileSystemUtils;
import com.google.devtools.build.lib.vfs.RootedPath;
import com.google.devtools.build.skyframe.SkyFunction;
import com.google.devtools.build.skyframe.SkyFunctionException;
import com.google.devtools.build.skyframe.SkyKey;
import com.google.devtools.build.skyframe.SkyValue;
-import java.io.IOException;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import javax.annotation.Nullable;
/**
- * Value of output hashes for the repositories specified in the resolved file designated for this
- * purpose.
+ * Computes the value of output hashes for the repositories specified in the resolved file
+ * designated for this purpose.
*/
public class ResolvedHashesFunction implements SkyFunction {
public static final String ORIGINAL_RULE_CLASS = "original_rule_class";
@@ -64,90 +53,37 @@
if (!resolvedFile.isPresent()) {
return new ResolvedHashesValue(ImmutableMap.<String, String>of());
}
- SkylarkSemantics skylarkSemantics = PrecomputedValue.SKYLARK_SEMANTICS.get(env);
- if (skylarkSemantics == null) {
+ ResolvedFileValue resolvedValue =
+ (ResolvedFileValue) env.getValue(ResolvedFileValue.key(resolvedFile.get()));
+ if (resolvedValue == null) {
return null;
}
- FileValue resolvedFileValue = (FileValue) env.getValue(FileValue.key(resolvedFile.get()));
- if (resolvedFileValue == null) {
- return null;
- }
- try {
- if (!resolvedFileValue.exists()) {
- throw new ResolvedHashesFunctionException(
- new NoSuchThingException(
- "Specified file for resolved hashes '" + resolvedFile.get() + "' not found."));
- } else {
- byte[] bytes =
- FileSystemUtils.readWithKnownFileSize(
- resolvedFile.get().asPath(), resolvedFile.get().asPath().getFileSize());
- BuildFileAST ast =
- BuildFileAST.parseSkylarkFile(
- ParserInputSource.create(bytes, resolvedFile.get().asPath().asFragment()),
- env.getListener());
- if (ast.containsErrors()) {
- throw new ResolvedHashesFunctionException(
- new BuildFileContainsErrorsException(
- Label.EXTERNAL_PACKAGE_IDENTIFIER,
- "Failed to parse file resolved file for hash verification"));
- }
- com.google.devtools.build.lib.syntax.Environment resolvedEnvironment;
- try (Mutability mutability = Mutability.create("resolved hashes %s", resolvedFile.get())) {
- resolvedEnvironment =
- com.google.devtools.build.lib.syntax.Environment.builder(mutability)
- .setSemantics(skylarkSemantics)
- .setGlobals(BazelLibrary.GLOBALS)
- .build();
- if (!ast.exec(resolvedEnvironment, env.getListener())) {
- throw new ResolvedHashesFunctionException(
- new BuildFileContainsErrorsException(
- Label.EXTERNAL_PACKAGE_IDENTIFIER,
- "Failed to evaluate resolved file for hash verification"));
- }
- }
- Object resolved = resolvedEnvironment.lookup("resolved");
- if (resolved == null) {
- throw new ResolvedHashesFunctionException(
- new BuildFileContainsErrorsException(
- Label.EXTERNAL_PACKAGE_IDENTIFIER,
- "Symbol 'resolved' not exported in file for hash verification"));
- }
- if (!(resolved instanceof List)) {
- throw new ResolvedHashesFunctionException(
- new BuildFileContainsErrorsException(
- Label.EXTERNAL_PACKAGE_IDENTIFIER,
- "Symbol 'resolved' not a list in file for hash verification"));
- }
- // Collect the hases in a mutable map, to be able to detect duplicates and
- // only take the first entry, following the "maybe pattern" of external repositories,
- // adding a repository only if not already present.
- Map<String, String> hashes = new LinkedHashMap<String, String>();
- for (Object entry : (List) resolved) {
- if (entry instanceof Map) {
- Object repositories = ((Map) entry).get(REPOSITORIES);
- if (repositories instanceof List) {
- for (Object repo : (List) repositories) {
- if (repo instanceof Map) {
- Object hash = ((Map) repo).get(OUTPUT_TREE_HASH);
- Object attributes = ((Map) repo).get(ATTRIBUTES);
- if (attributes instanceof Map) {
- Object name = ((Map) attributes).get("name");
- if ((name instanceof String) && (hash instanceof String)) {
- if (!hashes.containsKey((String) name)) {
- hashes.put((String) name, (String) hash);
- }
- }
- }
+ List<Map<String, Object>> resolved = resolvedValue.getResolvedValue();
+
+ // Collect the hashes in a mutable map, to be able to detect duplicates and
+ // only take the first entry, following the "maybe pattern" of external repositories,
+ // adding a repository only if not already present.
+ Map<String, String> hashes = new LinkedHashMap<String, String>();
+ for (Map<String, Object> entry : resolved) {
+ Object repositories = entry.get(REPOSITORIES);
+ if (repositories instanceof List) {
+ for (Object repo : (List) repositories) {
+ if (repo instanceof Map) {
+ Object hash = ((Map) repo).get(OUTPUT_TREE_HASH);
+ Object attributes = ((Map) repo).get(ATTRIBUTES);
+ if (attributes instanceof Map) {
+ Object name = ((Map) attributes).get("name");
+ if ((name instanceof String) && (hash instanceof String)) {
+ if (!hashes.containsKey((String) name)) {
+ hashes.put((String) name, (String) hash);
}
}
}
}
}
- return new ResolvedHashesValue(ImmutableMap.copyOf(hashes));
}
- } catch (IOException e) {
- throw new ResolvedHashesFunctionException(e);
}
+ return new ResolvedHashesValue(ImmutableMap.copyOf(hashes));
}
@Override
@@ -157,11 +93,7 @@
}
private static final class ResolvedHashesFunctionException extends SkyFunctionException {
- ResolvedHashesFunctionException(IOException e) {
- super(e, SkyFunctionException.Transience.PERSISTENT);
- }
-
- ResolvedHashesFunctionException(NoSuchThingException e) {
+ ResolvedHashesFunctionException(BuildFileContainsErrorsException e) {
super(e, SkyFunctionException.Transience.PERSISTENT);
}
}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SkyFunctions.java b/src/main/java/com/google/devtools/build/lib/skyframe/SkyFunctions.java
index 9673a02..f075c3d 100644
--- a/src/main/java/com/google/devtools/build/lib/skyframe/SkyFunctions.java
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/SkyFunctions.java
@@ -138,6 +138,8 @@
SkyFunctionName.createHermetic("TOOLCHAIN_RESOLUTION");
public static final SkyFunctionName REPOSITORY_MAPPING =
SkyFunctionName.createHermetic("REPOSITORY_MAPPING");
+ public static final SkyFunctionName RESOLVED_FILE =
+ SkyFunctionName.createHermetic("RESOLVED_FILE");
public static final SkyFunctionName RESOLVED_HASH_VALUES =
SkyFunctionName.createHermetic("RESOLVED_HASH_VALUES");
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java
index b06815b..22f3d66 100644
--- a/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java
@@ -115,6 +115,7 @@
import com.google.devtools.build.lib.pkgcache.TransitivePackageLoader;
import com.google.devtools.build.lib.profiler.Profiler;
import com.google.devtools.build.lib.profiler.SilentCloseable;
+import com.google.devtools.build.lib.rules.repository.ResolvedFileFunction;
import com.google.devtools.build.lib.rules.repository.ResolvedHashesFunction;
import com.google.devtools.build.lib.skyframe.AspectValue.AspectValueKey;
import com.google.devtools.build.lib.skyframe.CompletionFunction.PathResolverFactory;
@@ -549,6 +550,7 @@
map.put(SkyFunctions.TOOLCHAIN_RESOLUTION, new ToolchainResolutionFunction());
map.put(SkyFunctions.REPOSITORY_MAPPING, new RepositoryMappingFunction());
map.put(SkyFunctions.RESOLVED_HASH_VALUES, new ResolvedHashesFunction());
+ map.put(SkyFunctions.RESOLVED_FILE, new ResolvedFileFunction());
map.putAll(extraSkyFunctions);
return map.build();
}
