Add a `sync` command
Add a command that ensures that all skylark workspace rules get called.
In follow-up changes the semantics of the sync command will be extended
to call all those rules unconditionally. This, together with recoding
of the return values of the repository rules, as currently provided by
--experimental_repository_resolved_file provides the framework for
resolving underspecified rules, e.g., rules following head of an external
repository.
Change-Id: I11061ec138a9ba7a7b61a431eeb1b8667dfabb95
PiperOrigin-RevId: 199792026
diff --git a/src/main/java/com/google/devtools/build/lib/BUILD b/src/main/java/com/google/devtools/build/lib/BUILD
index fbfcff8..3e3c1b7 100644
--- a/src/main/java/com/google/devtools/build/lib/BUILD
+++ b/src/main/java/com/google/devtools/build/lib/BUILD
@@ -757,7 +757,10 @@
java_library(
name = "bazel-commands",
srcs = glob(["bazel/commands/*.java"]),
- resources = ["bazel/commands/fetch.txt"],
+ resources = [
+ "bazel/commands/fetch.txt",
+ "bazel/commands/sync.txt",
+ ],
deps = [
":keep-going-option",
"//src/main/java/com/google/devtools/build/lib:build-base",
@@ -771,6 +774,10 @@
"//src/main/java/com/google/devtools/build/lib/query2",
"//src/main/java/com/google/devtools/build/lib/query2:abstract-blaze-query-env",
"//src/main/java/com/google/devtools/build/lib/query2:query-engine",
+ "//src/main/java/com/google/devtools/build/lib/vfs",
+ "//src/main/java/com/google/devtools/build/lib/vfs:pathfragment",
+ "//src/main/java/com/google/devtools/build/skyframe",
+ "//src/main/java/com/google/devtools/build/skyframe:skyframe-objects",
"//src/main/java/com/google/devtools/common/options",
"//third_party:guava",
],
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java b/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java
index 58441b2..742f35e 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java
@@ -24,6 +24,7 @@
import com.google.devtools.build.lib.analysis.RuleDefinition;
import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
import com.google.devtools.build.lib.bazel.commands.FetchCommand;
+import com.google.devtools.build.lib.bazel.commands.SyncCommand;
import com.google.devtools.build.lib.bazel.repository.GitRepositoryFunction;
import com.google.devtools.build.lib.bazel.repository.HttpArchiveFunction;
import com.google.devtools.build.lib.bazel.repository.HttpFileFunction;
@@ -181,6 +182,7 @@
@Override
public void serverInit(OptionsProvider startupOptions, ServerBuilder builder) {
builder.addCommands(new FetchCommand());
+ builder.addCommands(new SyncCommand());
builder.addInfoItems(new RepositoryCacheInfoItem(repositoryCache));
}
@@ -295,7 +297,7 @@
@Override
public Iterable<Class<? extends OptionsBase>> getCommandOptions(Command command) {
- return ImmutableSet.of("fetch", "build", "query").contains(command.name())
+ return ImmutableSet.of("sync", "fetch", "build", "query").contains(command.name())
? ImmutableList.<Class<? extends OptionsBase>>of(RepositoryOptions.class)
: ImmutableList.<Class<? extends OptionsBase>>of();
}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/commands/SyncCommand.java b/src/main/java/com/google/devtools/build/lib/bazel/commands/SyncCommand.java
new file mode 100644
index 0000000..c5c3085
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/bazel/commands/SyncCommand.java
@@ -0,0 +1,119 @@
+// Copyright 2018 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.bazel.commands;
+
+import com.google.common.collect.ImmutableSet;
+import com.google.devtools.build.lib.cmdline.Label;
+import com.google.devtools.build.lib.cmdline.LabelSyntaxException;
+import com.google.devtools.build.lib.cmdline.RepositoryName;
+import com.google.devtools.build.lib.packages.Rule;
+import com.google.devtools.build.lib.pkgcache.PackageCacheOptions;
+import com.google.devtools.build.lib.rules.repository.RepositoryDirectoryValue;
+import com.google.devtools.build.lib.runtime.BlazeCommand;
+import com.google.devtools.build.lib.runtime.BlazeCommandResult;
+import com.google.devtools.build.lib.runtime.Command;
+import com.google.devtools.build.lib.runtime.CommandEnvironment;
+import com.google.devtools.build.lib.runtime.KeepGoingOption;
+import com.google.devtools.build.lib.skyframe.PackageLookupValue;
+import com.google.devtools.build.lib.skyframe.SkyframeExecutor;
+import com.google.devtools.build.lib.skyframe.WorkspaceFileValue;
+import com.google.devtools.build.lib.util.AbruptExitException;
+import com.google.devtools.build.lib.util.ExitCode;
+import com.google.devtools.build.lib.vfs.RootedPath;
+import com.google.devtools.build.skyframe.EvaluationResult;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+import com.google.devtools.common.options.OptionsParser;
+import com.google.devtools.common.options.OptionsProvider;
+
+/** Syncs all repositories specifed in the workspace file */
+@Command(
+ name = SyncCommand.NAME,
+ options = {PackageCacheOptions.class, KeepGoingOption.class},
+ help = "resource:sync.txt",
+ shortDescription = "Syncs all repositories specifed in the workspace file",
+ allowResidue = false)
+public final class SyncCommand implements BlazeCommand {
+ public static final String NAME = "sync";
+
+ @Override
+ public void editOptions(OptionsParser optionsParser) {}
+
+ @Override
+ public BlazeCommandResult exec(CommandEnvironment env, OptionsProvider options) {
+ try {
+ env.setupPackageCache(options, env.getRuntime().getDefaultsPackageContent());
+ SkyframeExecutor skyframeExecutor = env.getSkyframeExecutor();
+
+ // Obtain the key for the top-level WORKSPACE file
+ SkyKey packageLookupKey = PackageLookupValue.key(Label.EXTERNAL_PACKAGE_IDENTIFIER);
+ EvaluationResult<SkyValue> packageLookupValue =
+ skyframeExecutor.prepareAndGet(
+ ImmutableSet.of(packageLookupKey),
+ SkyframeExecutor.DEFAULT_THREAD_COUNT,
+ env.getReporter());
+ if (packageLookupValue.hasError()) {
+ return BlazeCommandResult.exitCode(ExitCode.ANALYSIS_FAILURE);
+ }
+ RootedPath workspacePath =
+ ((PackageLookupValue) packageLookupValue.get(packageLookupKey))
+ .getRootedPath(Label.EXTERNAL_PACKAGE_IDENTIFIER);
+ SkyKey workspace = WorkspaceFileValue.key(workspacePath);
+
+ // read and evaluate the WORKSPACE file to its end
+ WorkspaceFileValue fileValue = null;
+ while (workspace != null) {
+ EvaluationResult<SkyValue> value =
+ skyframeExecutor.prepareAndGet(
+ ImmutableSet.of(workspace),
+ SkyframeExecutor.DEFAULT_THREAD_COUNT,
+ env.getReporter());
+ if (value.hasError()) {
+ return BlazeCommandResult.exitCode(ExitCode.ANALYSIS_FAILURE);
+ }
+ fileValue = (WorkspaceFileValue) value.get(workspace);
+ workspace = fileValue.next();
+ }
+
+ // take all skylark workspace rules and get their values
+ ImmutableSet.Builder<SkyKey> repositoriesToFetch = new ImmutableSet.Builder<>();
+ for (Rule rule : fileValue.getPackage().getTargets(Rule.class)) {
+ if (rule.getRuleClassObject().getWorkspaceOnly() && rule.getRuleClassObject().isSkylark()) {
+ // TODO(aehlig): avoid the detour of serializing and then parsing the repository name
+ try {
+ repositoriesToFetch.add(
+ RepositoryDirectoryValue.key(RepositoryName.create("@" + rule.getName())));
+ } catch (LabelSyntaxException e) {
+ return BlazeCommandResult.exitCode(ExitCode.BLAZE_INTERNAL_ERROR);
+ }
+ }
+ }
+ EvaluationResult<SkyValue> fetchValue;
+ fetchValue =
+ skyframeExecutor.prepareAndGet(
+ repositoriesToFetch.build(),
+ SkyframeExecutor.DEFAULT_THREAD_COUNT,
+ env.getReporter());
+ if (fetchValue.hasError()) {
+ return BlazeCommandResult.exitCode(ExitCode.ANALYSIS_FAILURE);
+ }
+ } catch (InterruptedException e) {
+ return BlazeCommandResult.exitCode(ExitCode.INTERRUPTED);
+ } catch (AbruptExitException e) {
+ return BlazeCommandResult.exitCode(ExitCode.LOCAL_ENVIRONMENTAL_ERROR);
+ }
+
+ return BlazeCommandResult.exitCode(ExitCode.SUCCESS);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/commands/sync.txt b/src/main/java/com/google/devtools/build/lib/bazel/commands/sync.txt
new file mode 100644
index 0000000..0e651c5
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/bazel/commands/sync.txt
@@ -0,0 +1,10 @@
+
+Usage: %{product} %{command} [<option> ...]
+
+Ensures that all Skylark repository rules of the top-level WORKSPACE
+file are called.
+
+NOTE: This command is still very experimental and the precise semantics
+will change in the near future.
+
+%{options}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryResolvedModule.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryResolvedModule.java
index 1664957..d4d4849 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryResolvedModule.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryResolvedModule.java
@@ -39,7 +39,7 @@
@Override
public Iterable<Class<? extends OptionsBase>> getCommandOptions(Command command) {
- return ImmutableSet.of("fetch", "build", "query").contains(command.name())
+ return ImmutableSet.of("sync", "fetch", "build", "query").contains(command.name())
? ImmutableList.<Class<? extends OptionsBase>>of(RepositoryResolvedOptions.class)
: ImmutableList.<Class<? extends OptionsBase>>of();
}
diff --git a/src/test/shell/bazel/workspace_resolved_test.sh b/src/test/shell/bazel/workspace_resolved_test.sh
index d4de82e..ace762c 100755
--- a/src/test/shell/bazel/workspace_resolved_test.sh
+++ b/src/test/shell/bazel/workspace_resolved_test.sh
@@ -20,6 +20,7 @@
|| { echo "integration_test_setup.sh not found!" >&2; exit 1; }
test_result_recorded() {
+ rm -rf fetchrepo
mkdir fetchrepo
cd fetchrepo
cat > rule.bzl <<'EOF'
@@ -85,4 +86,56 @@
|| fail "Not the correct number of original attributes"
}
+test_sync_calls_all() {
+ rm -rf fetchrepo
+ mkdir fetchrepo
+ rm -f repo.bzl
+ cd fetchrepo
+ cat > rule.bzl <<'EOF'
+def _rule_impl(ctx):
+ ctx.file("foo.bzl", """
+it = "foo"
+other = "bar"
+""")
+ ctx.file("BUILD", "")
+ return {"comment" : ctx.attr.comment }
+
+trivial_rule = repository_rule(
+ implementation = _rule_impl,
+ attrs = { "comment" : attr.string() },
+)
+EOF
+ touch BUILD
+ cat > WORKSPACE <<'EOF'
+load("//:rule.bzl", "trivial_rule")
+trivial_rule(name = "a", comment = "bootstrap")
+load("@a//:foo.bzl", "it")
+trivial_rule(name = "b", comment = it)
+trivial_rule(name = "c", comment = it)
+load("@c//:foo.bzl", "other")
+trivial_rule(name = "d", comment = other)
+EOF
+
+ bazel clean --expunge
+ bazel sync --experimental_repository_resolved_file=../repo.bzl
+
+ cd ..
+ echo; cat repo.bzl; echo
+ touch WORKSPACE
+ cat > BUILD <<'EOF'
+load("//:repo.bzl", "resolved")
+
+names = [entry["original_attributes"]["name"] for entry in resolved]
+
+[
+ genrule(
+ name = name,
+ outs = [ "%s.txt" % (name,) ],
+ cmd = "echo %s > $@" % (name,),
+ ) for name in names
+]
+EOF
+ bazel build :a :b :c :d || fail "Expected all 4 repositories to be present"
+}
+
run_suite "workspace_resolved_test tests"
