Add a module collecting the results of repository rules
We allow repository rules to return a reproducible version of themselves,
or a list of fully reproducible rules they expand to. Add a module collecting
all those answers, logging them at each invocation, if requested; this
collection can also be used by the upcoming 'sync' command to generate
a WORKSPACE.resolved file.
Change-Id: Iac1358de1b74633810d300ba2bf45bba8b3992dc
PiperOrigin-RevId: 195427096
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/Bazel.java b/src/main/java/com/google/devtools/build/lib/bazel/Bazel.java
index fde7dc7..8490efe 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/Bazel.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/Bazel.java
@@ -42,6 +42,7 @@
com.google.devtools.build.lib.bazel.BazelWorkspaceStatusModule.class,
com.google.devtools.build.lib.bazel.BazelDiffAwarenessModule.class,
com.google.devtools.build.lib.bazel.BazelRepositoryModule.class,
+ com.google.devtools.build.lib.bazel.repository.RepositoryResolvedModule.class,
com.google.devtools.build.lib.bazel.SpawnLogModule.class,
com.google.devtools.build.lib.ssd.SsdModule.class,
com.google.devtools.build.lib.worker.WorkerModule.class,
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryResolvedEvent.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryResolvedEvent.java
new file mode 100644
index 0000000..6eb3960
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryResolvedEvent.java
@@ -0,0 +1,109 @@
+// Copyright 2018 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.bazel.repository;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.devtools.build.lib.events.ExtendedEventHandler.Postable;
+import com.google.devtools.build.lib.packages.Attribute;
+import com.google.devtools.build.lib.packages.Info;
+import com.google.devtools.build.lib.packages.Rule;
+import com.google.devtools.build.lib.syntax.EvalException;
+import com.google.devtools.build.lib.syntax.Runtime;
+import java.util.Map;
+
+/**
+ * Event indicating that a repository rule was executed, together with the return value of the rule.
+ */
+public class RepositoryResolvedEvent implements Postable {
+ public static final String ORIGINAL_RULE_CLASS = "original_rule_class";
+ public static final String ORIGINAL_ATTRIBUTES = "original_attributes";
+ public static final String RULE_CLASS = "rule_class";
+ public static final String ATTRIBUTES = "attributes";
+ public static final String REPOSITORIES = "repositories";
+
+ /**
+ * The entry for WORSPACE.resolved corresponding to that rule invocation.
+ *
+ * <p>It will always be a dict with three entries <ul>
+ * <li> the original rule class (as String, e.g., "@bazel_tools//:git.bzl%git_repository")
+ * <li> the original attributes (as dict, e.g., mapping "name" to "build_bazel"
+ * and "remote" to "https://github.com/bazelbuild/bazel.git"), and
+ * <li> a "repositories" entry; this is a list, often a single entry, of fully resolved
+ * repositories the rule call expanded to (in the above example, the attributes entry
+ * would have an additional "commit" and "shallow-since" entry).
+ * </ul>
+ */
+ private final Object resolvedInformation;
+
+ public RepositoryResolvedEvent(Rule rule, Info attrs, Object result) {
+ ImmutableMap.Builder<String, Object> builder = ImmutableMap.builder();
+
+ String originalClass =
+ rule.getRuleClassObject().getRuleDefinitionEnvironmentLabel() + "%" + rule.getRuleClass();
+ builder.put(ORIGINAL_RULE_CLASS, originalClass);
+
+ ImmutableMap.Builder<String, Object> origAttrBuilder = ImmutableMap.builder();
+ for (Attribute attr : rule.getAttributes()) {
+ String name = attr.getPublicName();
+ if (!name.startsWith("_")) {
+ // TODO(aehlig): filter out remaining attributes that cannot be set in a
+ // WORKSPACE file.
+ try {
+ Object value = attrs.getValue(name, Object.class);
+ // Only record explicit values, skip computed defaults
+ if (!(value instanceof Attribute.ComputedDefault)) {
+ origAttrBuilder.put(name, value);
+ }
+ } catch (EvalException e) {
+ // Do nothing, just ignore the value.
+ }
+ }
+ }
+ ImmutableMap<String, Object> origAttr = origAttrBuilder.build();
+ builder.put(ORIGINAL_ATTRIBUTES, origAttr);
+
+ if (result == Runtime.NONE) {
+ // Rule claims to be already reproducible, so wants to be called as is.
+ builder.put(
+ REPOSITORIES,
+ ImmutableList.<Object>of(
+ ImmutableMap.<String, Object>builder()
+ .put(RULE_CLASS, originalClass)
+ .put(ATTRIBUTES, origAttr)
+ .build()));
+ } else if (result instanceof Map) {
+ // Rule claims that the returned (probably changed) arguments are a reproducible
+ // version of itself.
+ builder.put(
+ REPOSITORIES,
+ ImmutableList.<Object>of(
+ ImmutableMap.<String, Object>builder()
+ .put(RULE_CLASS, originalClass)
+ .put(ATTRIBUTES, result)
+ .build()));
+ } else {
+ // TODO(aehlig): handle strings specially to allow encodings of the former
+ // values to be accepted as well.
+ builder.put(REPOSITORIES, result);
+ }
+
+ this.resolvedInformation = builder.build();
+ }
+
+ /** Return the entry for the given rule invocation in a format suitable for WORKSPACE.resolved. */
+ public Object getResolvedInformation() {
+ return resolvedInformation;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryResolvedModule.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryResolvedModule.java
new file mode 100644
index 0000000..7f0e721
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryResolvedModule.java
@@ -0,0 +1,82 @@
+// Copyright 2018 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.bazel.repository;
+
+import com.google.common.base.Strings;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.eventbus.Subscribe;
+import com.google.common.io.Files;
+import com.google.devtools.build.lib.runtime.BlazeModule;
+import com.google.devtools.build.lib.runtime.Command;
+import com.google.devtools.build.lib.runtime.CommandEnvironment;
+import com.google.devtools.build.lib.syntax.Printer;
+import com.google.devtools.common.options.OptionsBase;
+import java.io.File;
+import java.io.IOException;
+import java.io.Writer;
+import java.nio.charset.StandardCharsets;
+import java.util.logging.Logger;
+
+/** Module providing the collection of the resolved values for the repository rules executed. */
+public final class RepositoryResolvedModule extends BlazeModule {
+ public static final String EXPORTED_NAME = "resolved";
+
+ private static final Logger logger = Logger.getLogger(RepositoryResolvedModule.class.getName());
+ private ImmutableList.Builder<Object> resultBuilder;
+ private String resolvedFile;
+
+ @Override
+ public Iterable<Class<? extends OptionsBase>> getCommandOptions(Command command) {
+ return ImmutableSet.of("fetch", "build", "query").contains(command.name())
+ ? ImmutableList.<Class<? extends OptionsBase>>of(RepositoryResolvedOptions.class)
+ : ImmutableList.<Class<? extends OptionsBase>>of();
+ }
+
+ @Override
+ public void beforeCommand(CommandEnvironment env) {
+ RepositoryResolvedOptions options =
+ env.getOptions().getOptions(RepositoryResolvedOptions.class);
+ if (options != null && !Strings.isNullOrEmpty(options.repositoryResolvedFile)) {
+ this.resolvedFile = options.repositoryResolvedFile;
+ env.getEventBus().register(this);
+ this.resultBuilder = new ImmutableList.Builder<>();
+ } else {
+ this.resolvedFile = null;
+ }
+ }
+
+ @Override
+ public void afterCommand() {
+ if (resolvedFile != null) {
+ try {
+ Writer writer = Files.newWriter(new File(resolvedFile), StandardCharsets.UTF_8);
+ // TODO(aehlig): pretty print
+ writer.write(EXPORTED_NAME + " = " + Printer.repr(resultBuilder.build()));
+ writer.close();
+ } catch (IOException e) {
+ logger.warning("IO Error writing to file " + resolvedFile + ": " + e);
+ }
+ }
+
+ this.resultBuilder = null;
+ }
+
+ @Subscribe
+ public void repositoryResolved(RepositoryResolvedEvent event) {
+ if (resultBuilder != null) {
+ resultBuilder.add(event.getResolvedInformation());
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryResolvedOptions.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryResolvedOptions.java
new file mode 100644
index 0000000..3ed520a
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryResolvedOptions.java
@@ -0,0 +1,35 @@
+// Copyright 2018 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.bazel.repository;
+
+import com.google.devtools.common.options.Option;
+import com.google.devtools.common.options.OptionDocumentationCategory;
+import com.google.devtools.common.options.OptionEffectTag;
+import com.google.devtools.common.options.OptionsBase;
+
+/** Options for handling the repository resolution */
+public class RepositoryResolvedOptions extends OptionsBase {
+
+ @Option(
+ name = "experimental_repository_resolved_file",
+ defaultValue = "",
+ documentationCategory = OptionDocumentationCategory.LOGGING,
+ effectTags = {OptionEffectTag.AFFECTS_OUTPUTS},
+ help =
+ "If non-empty, write a Skylark value with the resolved information of all Skylark"
+ + " respository rules that were executed."
+ )
+ public String repositoryResolvedFile;
+}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/skylark/SkylarkRepositoryFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/skylark/SkylarkRepositoryFunction.java
index 685a651..4c3611b 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/repository/skylark/SkylarkRepositoryFunction.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/skylark/SkylarkRepositoryFunction.java
@@ -18,6 +18,7 @@
import com.google.common.collect.ImmutableMap;
import com.google.devtools.build.lib.analysis.BlazeDirectories;
import com.google.devtools.build.lib.analysis.RuleDefinition;
+import com.google.devtools.build.lib.bazel.repository.RepositoryResolvedEvent;
import com.google.devtools.build.lib.bazel.repository.downloader.HttpDownloader;
import com.google.devtools.build.lib.cmdline.Label;
import com.google.devtools.build.lib.events.Event;
@@ -87,7 +88,10 @@
// means the rule might get restarted for legitimate reasons.
}
- // This has side-effect, we don't care about the output.
+ // This rule is mainly executed for its side effect. Nevertheless, the return value is
+ // of importance, as it provides information on how the call has to be modified to be a
+ // reproducible rule.
+ //
// Also we do a lot of stuff in there, maybe blocking operations and we should certainly make
// it possible to return null and not block but it doesn't seem to be easy with Skylark
// structure as it is.
@@ -101,6 +105,8 @@
env.getListener()
.handle(Event.info("Repository rule '" + rule.getName() + "' returned: " + retValue));
}
+ env.getListener()
+ .post(new RepositoryResolvedEvent(rule, skylarkRepositoryContext.getAttr(), retValue));
} catch (EvalException e) {
if (e.getCause() instanceof RepositoryMissingDependencyException) {
// A dependency is missing, cleanup and returns null
diff --git a/src/test/shell/bazel/BUILD b/src/test/shell/bazel/BUILD
index bfa6d6c..3e3eea9 100644
--- a/src/test/shell/bazel/BUILD
+++ b/src/test/shell/bazel/BUILD
@@ -386,6 +386,13 @@
)
sh_test(
+ name = "workspace_resolved_test",
+ size = "medium",
+ srcs = ["workspace_resolved_test.sh"],
+ data = [":test-deps"],
+)
+
+sh_test(
name = "cc_integration_test",
size = "medium",
srcs = ["cc_integration_test.sh"],
diff --git a/src/test/shell/bazel/workspace_resolved_test.sh b/src/test/shell/bazel/workspace_resolved_test.sh
new file mode 100755
index 0000000..01a33dd
--- /dev/null
+++ b/src/test/shell/bazel/workspace_resolved_test.sh
@@ -0,0 +1,80 @@
+#!/bin/bash
+#
+# Copyright 2018 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Load the test setup defined in the parent directory
+CURRENT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+source "${CURRENT_DIR}/../integration_test_setup.sh" \
+ || { echo "integration_test_setup.sh not found!" >&2; exit 1; }
+
+test_result_recorded() {
+ mkdir fetchrepo
+ cd fetchrepo
+ cat > rule.bzl <<'EOF'
+def _rule_impl(ctx):
+ ctx.symlink(ctx.attr.build_file, "BUILD")
+ return {"build_file": ctx.attr.build_file, "extra_arg": "foobar"}
+
+trivial_rule = repository_rule(
+ implementation = _rule_impl,
+ attrs = { "build_file" : attr.label() },
+)
+
+EOF
+ cat > ext.BUILD <<'EOF'
+genrule(
+ name = "foo",
+ outs = ["foo.txt"],
+ cmd = "echo bar > $@",
+)
+EOF
+ touch BUILD
+ cat > WORKSPACE <<'EOF'
+load("//:rule.bzl", "trivial_rule")
+trivial_rule(
+ name = "ext",
+ build_file = "//:ext.BUILD",
+)
+EOF
+
+ bazel clean --expunge
+ bazel build --experimental_repository_resolved_file=../repo.bzl @ext//... \
+ || fail "Expected success"
+
+ # Verify that bazel can read the generated repo.bzl file and that it contains
+ # the expected information
+ cd ..
+ mkdir analysisrepo
+ mv repo.bzl analysisrepo
+ cd analysisrepo
+ touch WORKSPACE
+ cat > BUILD <<'EOF'
+load("//:repo.bzl", "resolved")
+
+[ genrule(
+ name = "out",
+ outs = ["out.txt"],
+ cmd = "echo %s > $@" % entry["repositories"][0]["attributes"]["extra_arg"],
+ ) for entry in resolved if entry["original_rule_class"] == "//:rule.bzl%trivial_rule"
+]
+EOF
+ cat BUILD
+ bazel build //:out || fail "Expected success"
+ grep "foobar" `bazel info bazel-genfiles`/out.txt \
+ || fail "Did not find the expected value"
+
+}
+
+run_suite "workspace_resolved_test tests"
