Implement `bazel sync --configure`
...to only unconditionally fetch all configure-like repositories.
Change-Id: I333e998dba1f091e7f487ea21bee49ec5292a8c2
PiperOrigin-RevId: 258364090
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java b/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java
index d240b9e..c5685bc 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java
@@ -350,6 +350,9 @@
// Nevertheless, we need to provide a default value for other commands.
PrecomputedValue.injected(
RepositoryDelegatorFunction.DEPENDENCY_FOR_UNCONDITIONAL_FETCHING,
+ RepositoryDelegatorFunction.DONT_FETCH_UNCONDITIONALLY),
+ PrecomputedValue.injected(
+ RepositoryDelegatorFunction.DEPENDENCY_FOR_UNCONDITIONAL_CONFIGURING,
RepositoryDelegatorFunction.DONT_FETCH_UNCONDITIONALLY));
}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/commands/SyncCommand.java b/src/main/java/com/google/devtools/build/lib/bazel/commands/SyncCommand.java
index ecfa9b8..d942050 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/commands/SyncCommand.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/commands/SyncCommand.java
@@ -19,6 +19,7 @@
import com.google.devtools.build.lib.analysis.NoBuildEvent;
import com.google.devtools.build.lib.analysis.NoBuildRequestFinishedEvent;
import com.google.devtools.build.lib.bazel.repository.RepositoryOrderEvent;
+import com.google.devtools.build.lib.bazel.repository.skylark.SkylarkRepositoryFunction;
import com.google.devtools.build.lib.cmdline.LabelConstants;
import com.google.devtools.build.lib.cmdline.LabelSyntaxException;
import com.google.devtools.build.lib.cmdline.RepositoryName;
@@ -57,7 +58,12 @@
/** Syncs all repositories specified in the workspace file */
@Command(
name = SyncCommand.NAME,
- options = {PackageCacheOptions.class, KeepGoingOption.class, LoadingPhaseThreadsOption.class},
+ options = {
+ PackageCacheOptions.class,
+ KeepGoingOption.class,
+ LoadingPhaseThreadsOption.class,
+ SyncOptions.class
+ },
help = "resource:sync.txt",
shortDescription = "Syncs all repositories specified in the workspace file",
allowResidue = false)
@@ -93,11 +99,21 @@
env.getCommandId().toString()));
env.setupPackageCache(options);
SkyframeExecutor skyframeExecutor = env.getSkyframeExecutor();
- skyframeExecutor.injectExtraPrecomputedValues(
- ImmutableList.of(
- PrecomputedValue.injected(
- RepositoryDelegatorFunction.DEPENDENCY_FOR_UNCONDITIONAL_FETCHING,
- env.getCommandId().toString())));
+
+ SyncOptions syncOptions = options.getOptions(SyncOptions.class);
+ if (syncOptions.configure) {
+ skyframeExecutor.injectExtraPrecomputedValues(
+ ImmutableList.of(
+ PrecomputedValue.injected(
+ RepositoryDelegatorFunction.DEPENDENCY_FOR_UNCONDITIONAL_CONFIGURING,
+ env.getCommandId().toString())));
+ } else {
+ skyframeExecutor.injectExtraPrecomputedValues(
+ ImmutableList.of(
+ PrecomputedValue.injected(
+ RepositoryDelegatorFunction.DEPENDENCY_FOR_UNCONDITIONAL_FETCHING,
+ env.getCommandId().toString())));
+ }
// Obtain the key for the top-level WORKSPACE file
SkyKey packageLookupKey = PackageLookupValue.key(LabelConstants.EXTERNAL_PACKAGE_IDENTIFIER);
@@ -167,7 +183,7 @@
// fetch anyway. So the only task remaining is to record the use of "bind" for whoever
// collects resolved information.
env.getReporter().post(resolveBind(rule));
- } else if (shouldSync(rule)) {
+ } else if (shouldSync(rule, syncOptions.configure)) {
// TODO(aehlig): avoid the detour of serializing and then parsing the repository name
try {
repositoriesToFetch.add(
@@ -208,11 +224,16 @@
return BlazeCommandResult.exitCode(exitCode);
}
- private static boolean shouldSync(Rule rule) {
+ private static boolean shouldSync(Rule rule, boolean configure) {
if (!rule.getRuleClassObject().getWorkspaceOnly()) {
// We should only sync workspace rules
return false;
}
+ if (configure) {
+ // If this is only a configure run, only sync Starlark rules that
+ // declare themselves as configure-like.
+ return SkylarkRepositoryFunction.isConfigureRule(rule);
+ }
if (rule.getRuleClassObject().isSkylark()) {
// Skylark rules are all whitelisted
return true;
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/commands/SyncOptions.java b/src/main/java/com/google/devtools/build/lib/bazel/commands/SyncOptions.java
new file mode 100644
index 0000000..0397921
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/bazel/commands/SyncOptions.java
@@ -0,0 +1,30 @@
+// Copyright 2019 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.bazel.commands;
+
+import com.google.devtools.common.options.Option;
+import com.google.devtools.common.options.OptionDocumentationCategory;
+import com.google.devtools.common.options.OptionEffectTag;
+import com.google.devtools.common.options.OptionsBase;
+
+/** Defines the options specific to Bazel's sync command */
+public class SyncOptions extends OptionsBase {
+ @Option(
+ name = "configure",
+ defaultValue = "False",
+ documentationCategory = OptionDocumentationCategory.EXECUTION_STRATEGY,
+ effectTags = {OptionEffectTag.CHANGES_INPUTS},
+ help = "Only sync repositories marked as 'configure' for system-configuration purpose.")
+ public boolean configure;
+}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/skylark/SkylarkRepositoryFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/skylark/SkylarkRepositoryFunction.java
index 38f2a9c..4edfa05 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/repository/skylark/SkylarkRepositoryFunction.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/skylark/SkylarkRepositoryFunction.java
@@ -249,7 +249,13 @@
return (Boolean) rule.getAttributeContainer().getAttr("$configure");
}
- public static boolean isConfigurelikeRule(Rule rule) {
+ /**
+ * Static method to determine if for a starlark repository rule {@code isConfigure} holds true. It
+ * also checks that the rule is indeed a Starlark rule so that this class is the appropriate
+ * handler for the given rule. As, however, only Starklark rules can be configure rules, this
+ * method can also be used as a universal check.
+ */
+ public static boolean isConfigureRule(Rule rule) {
return rule.getRuleClassObject().isSkylark()
&& ((Boolean) rule.getAttributeContainer().getAttr("$configure"));
}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/repository/RepositoryDelegatorFunction.java b/src/main/java/com/google/devtools/build/lib/rules/repository/RepositoryDelegatorFunction.java
index 8e77200..cb4bbce 100644
--- a/src/main/java/com/google/devtools/build/lib/rules/repository/RepositoryDelegatorFunction.java
+++ b/src/main/java/com/google/devtools/build/lib/rules/repository/RepositoryDelegatorFunction.java
@@ -74,6 +74,9 @@
new Precomputed<>(
PrecomputedValue.Key.create("dependency_for_unconditional_repository_fetching"));
+ public static final Precomputed<String> DEPENDENCY_FOR_UNCONDITIONAL_CONFIGURING =
+ new Precomputed<>(PrecomputedValue.Key.create("dependency_for_unconditional_configuring"));
+
public static final Precomputed<Optional<RootedPath>> RESOLVED_FILE_FOR_VERIFICATION =
new Precomputed<>(
PrecomputedValue.Key.create("resolved_file_for_external_repository_verification"));
@@ -139,6 +142,7 @@
Map<RepositoryName, PathFragment> overrides = REPOSITORY_OVERRIDES.get(env);
boolean doNotFetchUnconditionally =
DONT_FETCH_UNCONDITIONALLY.equals(DEPENDENCY_FOR_UNCONDITIONAL_FETCHING.get(env));
+ boolean needsConfiguring = false;
Path repoRoot = RepositoryFunction.getExternalRepositoryDirectory(directories)
.getRelative(repositoryName.strippedName());
@@ -163,6 +167,11 @@
return RepositoryDirectoryValue.NO_SUCH_REPOSITORY_VALUE;
}
+ if (handler.isConfigure(rule)) {
+ needsConfiguring =
+ !DONT_FETCH_UNCONDITIONALLY.equals(DEPENDENCY_FOR_UNCONDITIONAL_CONFIGURING.get(env));
+ }
+
byte[] ruleSpecificData = handler.getRuleSpecificMarkerData(rule, env);
if (env.valuesMissing()) {
return null;
@@ -185,10 +194,11 @@
&& managedDirectoriesExist(directories.getWorkspace(), managedDirectories)) {
// For the non-local repositories, check if they are already up-to-date:
// 1) unconditional fetching is not enabled, AND
- // 2) repository directory exists, AND
- // 3) marker file correctly describes the current repository state, AND
- // 4) managed directories, mapped to the repository, exist
- if (doNotFetchUnconditionally && repoRoot.exists()) {
+ // 2) unconditional syncing is not enabled or the rule is not a configure rule, AND
+ // 3) repository directory exists, AND
+ // 4) marker file correctly describes the current repository state, AND
+ // 5) managed directories, mapped to the repository, exist
+ if (!needsConfiguring && doNotFetchUnconditionally && repoRoot.exists()) {
byte[] markerHash = digestWriter.areRepositoryAndMarkerFileConsistent(handler, env);
if (env.valuesMissing()) {
return null;
diff --git a/src/test/shell/bazel/skylark_repository_test.sh b/src/test/shell/bazel/skylark_repository_test.sh
index 84629e1..ce3d635 100755
--- a/src/test/shell/bazel/skylark_repository_test.sh
+++ b/src/test/shell/bazel/skylark_repository_test.sh
@@ -1233,6 +1233,67 @@
expect_log "non_existing = False,False"
}
+function test_configure_like_repos() {
+ cat > repos.bzl <<'EOF'
+def _impl(ctx):
+ print("Executing %s" % (ctx.attr.name,))
+ ref = ctx.path(ctx.attr.reference)
+ # Here we explicitly copy a file where we constructed the name
+ # completely outside any build interfaces, so it is not registered
+ # as a dependency of the external repository.
+ ctx.execute(["cp", "%s.shadow" % (ref,), ctx.path("it.txt")])
+ ctx.file("BUILD", "exports_files(['it.txt'])")
+
+source = repository_rule(
+ implementation = _impl,
+ attrs = {"reference" : attr.label()},
+)
+
+configure = repository_rule(
+ implementation = _impl,
+ attrs = {"reference" : attr.label()},
+ configure = True,
+)
+
+EOF
+ cat > WORKSPACE <<'EOF'
+load("//:repos.bzl", "configure", "source")
+
+configure(name="configure", reference="@//:reference.txt")
+source(name="source", reference="@//:reference.txt")
+EOF
+ cat > BUILD <<'EOF'
+[ genrule(
+ name = name,
+ srcs = ["@%s//:it.txt" % (name,)],
+ outs = ["%s.txt" % (name,)],
+ cmd = "cp $< $@",
+ ) for name in ["source", "configure"] ]
+EOF
+ echo "Just to get the path" > reference.txt
+ echo "initial" > reference.txt.shadow
+
+ bazel build //:source //:configure
+ grep 'initial' `bazel info bazel-genfiles`/source.txt \
+ || fail '//:source not generated properly'
+ grep 'initial' `bazel info bazel-genfiles`/configure.txt \
+ || fail '//:configure not generated properly'
+
+ echo "new value" > reference.txt.shadow
+ bazel sync --configure --experimental_repository_resolved_file=resolved.bzl \
+ 2>&1 || fail "Expected sync --configure to succeed"
+ grep -q 'name.*configure' resolved.bzl \
+ || fail "Expected 'configure' to be synced"
+ grep -q 'name.*source' resolved.bzl \
+ && fail "Expected 'source' not to be synced" || :
+
+ bazel build //:source //:configure
+ grep -q 'initial' `bazel info bazel-genfiles`/source.txt \
+ || fail '//:source did not keep its old value'
+ grep -q 'new value' `bazel info bazel-genfiles`/configure.txt \
+ || fail '//:configure not synced properly'
+}
+
function test_timeout_tunable() {
cat >> $(create_workspace_with_default_repos WORKSPACE) <<'EOF'
