Add managed_directories attribute to workspace() function. This is only a part of the incrementally updated user-owned directory feature; that is why the parsed value is not yet used in computations. - Under --experimental_allow_incremental_repository_updates flag. - Parse results are put into WorkspaceFileValue map field. PiperOrigin-RevId: 244819268
diff --git a/src/main/java/com/google/devtools/build/lib/packages/StarlarkSemanticsOptions.java b/src/main/java/com/google/devtools/build/lib/packages/StarlarkSemanticsOptions.java index 5ee13c1..24a1699 100644 --- a/src/main/java/com/google/devtools/build/lib/packages/StarlarkSemanticsOptions.java +++ b/src/main/java/com/google/devtools/build/lib/packages/StarlarkSemanticsOptions.java
@@ -61,6 +61,21 @@ // <== Add new options here in alphabetic order ==> @Option( + name = "experimental_allow_incremental_repository_updates", + defaultValue = "false", + documentationCategory = OptionDocumentationCategory.STARLARK_SEMANTICS, + effectTags = {OptionEffectTag.BAZEL_INTERNAL_CONFIGURATION}, + metadataTags = {OptionMetadataTag.EXPERIMENTAL}, + help = + "If used, it is possible to define a mapping between external repositories" + + " and some (mostly likely ignored by .bazelignore) directories." + + " The repository rule can read and update files in those directories," + + " and the changes will be visible in the same build." + + " Use attribute 'managed_directories' of the global workspace()" + + " function in WORKSPACE file to define the mapping.") + public boolean experimentalAllowIncrementalRepositoryUpdates; + + @Option( name = "experimental_build_setting_api", defaultValue = "false", documentationCategory = OptionDocumentationCategory.UNDOCUMENTED, @@ -581,6 +596,8 @@ public StarlarkSemantics toSkylarkSemantics() { return StarlarkSemantics.builder() // <== Add new options here in alphabetic order ==> + .experimentalAllowIncrementalRepositoryUpdates( + experimentalAllowIncrementalRepositoryUpdates) .experimentalBuildSettingApi(experimentalBuildSettingApi) .experimentalCcSkylarkApiEnabledPackages(experimentalCcSkylarkApiEnabledPackages) .experimentalEnableAndroidMigrationApis(experimentalEnableAndroidMigrationApis)
diff --git a/src/main/java/com/google/devtools/build/lib/packages/WorkspaceFactory.java b/src/main/java/com/google/devtools/build/lib/packages/WorkspaceFactory.java index 520d588..ee4ca66 100644 --- a/src/main/java/com/google/devtools/build/lib/packages/WorkspaceFactory.java +++ b/src/main/java/com/google/devtools/build/lib/packages/WorkspaceFactory.java
@@ -24,6 +24,7 @@ import com.google.devtools.build.lib.analysis.skylark.SymbolGenerator; import com.google.devtools.build.lib.cmdline.LabelConstants; import com.google.devtools.build.lib.cmdline.LabelSyntaxException; +import com.google.devtools.build.lib.cmdline.RepositoryName; import com.google.devtools.build.lib.events.Event; import com.google.devtools.build.lib.events.NullEventHandler; import com.google.devtools.build.lib.events.StoredEventHandler; @@ -47,6 +48,7 @@ import com.google.devtools.build.lib.syntax.StarlarkSemantics; import com.google.devtools.build.lib.syntax.ValidationEnvironment; import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; import com.google.devtools.build.lib.vfs.Root; import com.google.devtools.build.lib.vfs.RootedPath; import java.io.File; @@ -76,7 +78,6 @@ private final Path defaultSystemJavabaseDir; private final Mutability mutability; - private final boolean allowOverride; private final RuleFactory ruleFactory; private final WorkspaceGlobals workspaceGlobals; @@ -119,7 +120,6 @@ this.installDir = installDir; this.workspaceDir = workspaceDir; this.defaultSystemJavabaseDir = defaultSystemJavabaseDir; - this.allowOverride = allowOverride; this.environmentExtensions = environmentExtensions; this.ruleFactory = new RuleFactory(ruleClassProvider, AttributeContainer::new); this.workspaceGlobals = new WorkspaceGlobals(allowOverride, ruleFactory); @@ -421,4 +421,8 @@ public Map<String, Object> getVariableBindings() { return variableBindings; } + + public Map<PathFragment, RepositoryName> getManagedDirectories() { + return workspaceGlobals.getManagedDirectories(); + } }
diff --git a/src/main/java/com/google/devtools/build/lib/packages/WorkspaceFileValue.java b/src/main/java/com/google/devtools/build/lib/packages/WorkspaceFileValue.java index bee379e..f1a5101 100644 --- a/src/main/java/com/google/devtools/build/lib/packages/WorkspaceFileValue.java +++ b/src/main/java/com/google/devtools/build/lib/packages/WorkspaceFileValue.java
@@ -22,6 +22,7 @@ import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; import com.google.devtools.build.lib.skyframe.serialization.autocodec.AutoCodec; import com.google.devtools.build.lib.syntax.Environment.Extension; +import com.google.devtools.build.lib.vfs.PathFragment; import com.google.devtools.build.lib.vfs.RootedPath; import com.google.devtools.build.skyframe.SkyFunctionName; import com.google.devtools.build.skyframe.SkyKey; @@ -105,6 +106,9 @@ private final ImmutableMap<String, Integer> importToChunkMap; private final ImmutableMap<RepositoryName, ImmutableMap<RepositoryName, RepositoryName>> repositoryMapping; + // Mapping of the relative paths of the incrementally updated managed directories + // to the managing external repositories + private final ImmutableMap<PathFragment, RepositoryName> managedDirectories; /** * Create a WorkspaceFileValue containing the various values necessary to compute the split @@ -122,6 +126,8 @@ * @param idx The index of this part of the split WORKSPACE file (0 for the first one, 1 for the * second one and so on). * @param hasNext Is there a next part in the WORKSPACE file or this part the last one? + * @param managedDirectories Mapping of the relative paths of the incrementally updated managed + * directories to the managing external repositories. */ public WorkspaceFileValue( Package pkg, @@ -130,7 +136,8 @@ Map<String, Object> bindings, RootedPath path, int idx, - boolean hasNext) { + boolean hasNext, + ImmutableMap<PathFragment, RepositoryName> managedDirectories) { this.pkg = Preconditions.checkNotNull(pkg); this.idx = idx; this.path = path; @@ -139,6 +146,7 @@ this.importMap = ImmutableMap.copyOf(importMap); this.importToChunkMap = ImmutableMap.copyOf(importToChunkMap); this.repositoryMapping = pkg.getExternalPackageRepositoryMappings(); + this.managedDirectories = managedDirectories; } /** @@ -220,4 +228,8 @@ getRepositoryMapping() { return repositoryMapping; } + + public ImmutableMap<PathFragment, RepositoryName> getManagedDirectories() { + return managedDirectories; + } }
diff --git a/src/main/java/com/google/devtools/build/lib/packages/WorkspaceGlobals.java b/src/main/java/com/google/devtools/build/lib/packages/WorkspaceGlobals.java index 595bf49..f4e02f3 100644 --- a/src/main/java/com/google/devtools/build/lib/packages/WorkspaceGlobals.java +++ b/src/main/java/com/google/devtools/build/lib/packages/WorkspaceGlobals.java
@@ -17,6 +17,8 @@ import static com.google.devtools.build.lib.syntax.Runtime.NONE; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; import com.google.devtools.build.lib.cmdline.Label; import com.google.devtools.build.lib.cmdline.LabelSyntaxException; import com.google.devtools.build.lib.cmdline.LabelValidator; @@ -29,8 +31,12 @@ import com.google.devtools.build.lib.syntax.EvalException; import com.google.devtools.build.lib.syntax.FuncallExpression; import com.google.devtools.build.lib.syntax.Runtime.NoneType; +import com.google.devtools.build.lib.syntax.SkylarkDict; import com.google.devtools.build.lib.syntax.SkylarkList; +import com.google.devtools.build.lib.vfs.PathFragment; +import java.util.List; import java.util.Map; +import java.util.TreeMap; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -42,14 +48,22 @@ private final boolean allowOverride; private final RuleFactory ruleFactory; + // Mapping of the relative paths of the incrementally updated managed directories + // to the managing external repositories + private final TreeMap<PathFragment, RepositoryName> managedDirectoriesMap; public WorkspaceGlobals(boolean allowOverride, RuleFactory ruleFactory) { this.allowOverride = allowOverride; this.ruleFactory = ruleFactory; + this.managedDirectoriesMap = Maps.newTreeMap(); } @Override - public NoneType workspace(String name, FuncallExpression ast, Environment env) + public NoneType workspace( + String name, + SkylarkDict<String, Object> managedDirectories, + FuncallExpression ast, + Environment env) throws EvalException, InterruptedException { if (allowOverride) { if (!isLegalWorkspaceName(name)) { @@ -80,6 +94,7 @@ RepositoryName.createFromValidStrippedName(name), RepositoryName.MAIN); } + parseManagedDirectories(managedDirectories, ast); return NONE; } else { throw new EvalException( @@ -88,6 +103,98 @@ } } + private void parseManagedDirectories( + SkylarkDict<String, Object> managedDirectories, FuncallExpression ast) throws EvalException { + Map<PathFragment, String> nonNormalizedPathsMap = Maps.newHashMap(); + for (Map.Entry<String, Object> entry : managedDirectories.entrySet()) { + RepositoryName repositoryName = createRepositoryName(entry.getKey(), ast.getLocation()); + List<PathFragment> paths = + getManagedDirectoriesPaths(entry.getValue(), ast.getLocation(), nonNormalizedPathsMap); + for (PathFragment dir : paths) { + PathFragment floorKey = managedDirectoriesMap.floorKey(dir); + if (dir.equals(floorKey)) { + throw new EvalException( + ast.getLocation(), + String.format( + "managed_directories attribute should not contain multiple" + + " (or duplicate) repository mappings for the same directory ('%s').", + nonNormalizedPathsMap.get(dir))); + } + PathFragment ceilingKey = managedDirectoriesMap.ceilingKey(dir); + boolean isDescendant = floorKey != null && dir.startsWith(floorKey); + if (isDescendant || (ceilingKey != null && ceilingKey.startsWith(dir))) { + throw new EvalException( + ast.getLocation(), + String.format( + "managed_directories attribute value can not contain nested mappings." + + " '%s' is a descendant of '%s'.", + nonNormalizedPathsMap.get(isDescendant ? dir : ceilingKey), + nonNormalizedPathsMap.get(isDescendant ? floorKey : dir))); + } + managedDirectoriesMap.put(dir, repositoryName); + } + } + } + + private RepositoryName createRepositoryName(String key, Location location) throws EvalException { + if (!key.startsWith("@")) { + throw new EvalException( + location, + String.format( + "Cannot parse repository name '%s'. Repository name should start with '@'.", key)); + } + try { + return RepositoryName.create(key); + } catch (LabelSyntaxException e) { + throw new EvalException(location, e); + } + } + + private List<PathFragment> getManagedDirectoriesPaths( + Object directoriesList, Location location, Map<PathFragment, String> nonNormalizedPathsMap) + throws EvalException { + if (!(directoriesList instanceof SkylarkList)) { + throw new EvalException( + location, + "managed_directories attribute value should be of the type attr.string_list_dict()," + + " mapping repository name to the list of managed directories."); + } + List<PathFragment> result = Lists.newArrayList(); + for (Object obj : (SkylarkList) directoriesList) { + if (!(obj instanceof String)) { + throw new EvalException( + location, + String.format("Expected managed directory path (as string), but got '%s'.", obj)); + } + String path = ((String) obj).trim(); + if (path.isEmpty()) { + throw new EvalException( + location, "Expected managed directory path to be non-empty string."); + } + PathFragment pathFragment = PathFragment.create(path); + if (pathFragment.isAbsolute()) { + throw new EvalException( + location, + String.format( + "Expected managed directory path ('%s') to be relative to the workspace root.", + path)); + } + if (pathFragment.containsUplevelReferences()) { + throw new EvalException( + location, + String.format( + "Expected managed directory path ('%s') to be under the workspace root.", path)); + } + nonNormalizedPathsMap.put(pathFragment, path); + result.add(pathFragment); + } + return result; + } + + public Map<PathFragment, RepositoryName> getManagedDirectories() { + return managedDirectoriesMap; + } + @Override public NoneType registerExecutionPlatforms( SkylarkList<?> platformLabels, Location location, Environment env)
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/WorkspaceFileFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/WorkspaceFileFunction.java index 5672730..d830e21 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/WorkspaceFileFunction.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/WorkspaceFileFunction.java
@@ -92,7 +92,8 @@ /* bindings = */ ImmutableMap.<String, Object>of(), workspaceRoot, /* idx = */ 0, // first fragment - /* hasNext = */ false); + /* hasNext = */ false, + ImmutableMap.of()); } catch (NoSuchPackageException e) { throw new WorkspaceFileFunctionException(e, Transience.TRANSIENT); } @@ -149,7 +150,8 @@ parser.getVariableBindings(), workspaceRoot, key.getIndex(), - key.getIndex() < workspaceASTValue.getASTs().size() - 1); + key.getIndex() < workspaceASTValue.getASTs().size() - 1, + ImmutableMap.copyOf(parser.getManagedDirectories())); } catch (NoSuchPackageException e) { throw new WorkspaceFileFunctionException(e, Transience.TRANSIENT); }
diff --git a/src/main/java/com/google/devtools/build/lib/skylarkbuildapi/WorkspaceGlobalsApi.java b/src/main/java/com/google/devtools/build/lib/skylarkbuildapi/WorkspaceGlobalsApi.java index bd3df41..fd346d4 100644 --- a/src/main/java/com/google/devtools/build/lib/skylarkbuildapi/WorkspaceGlobalsApi.java +++ b/src/main/java/com/google/devtools/build/lib/skylarkbuildapi/WorkspaceGlobalsApi.java
@@ -22,7 +22,9 @@ import com.google.devtools.build.lib.syntax.EvalException; import com.google.devtools.build.lib.syntax.FuncallExpression; import com.google.devtools.build.lib.syntax.Runtime.NoneType; +import com.google.devtools.build.lib.syntax.SkylarkDict; import com.google.devtools.build.lib.syntax.SkylarkList; +import com.google.devtools.build.lib.syntax.StarlarkSemantics.FlagIdentifier; /** A collection of global skylark build API functions that apply to WORKSPACE files. */ @SkylarkGlobalLibrary @@ -42,11 +44,31 @@ type = String.class, doc = "the name of the workspace.", named = true, - positional = false) + positional = false), + @Param( + name = "managed_directories", + type = SkylarkDict.class, + generic1 = String.class, + noneable = true, + named = true, + positional = false, + enableOnlyWithFlag = FlagIdentifier.EXPERIMENTAL_ALLOW_INCREMENTAL_REPOSITORY_UPDATES, + defaultValue = "{}", + valueWhenDisabled = "{}", + doc = + "Dict (strings to list of strings) for defining the mappings between external" + + " repositories and relative (to the workspace root) paths to directories" + + " they incrementally update." + + "\nManaged directories must be excluded from the source tree by listing" + + " them (or their parent directories) in the .bazelignore file."), }, useAst = true, useEnvironment = true) - NoneType workspace(String name, FuncallExpression ast, Environment env) + NoneType workspace( + String name, + SkylarkDict<String, Object> managedDirectories, + FuncallExpression ast, + Environment env) throws EvalException, InterruptedException; @SkylarkCallable(
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/StarlarkSemantics.java b/src/main/java/com/google/devtools/build/lib/syntax/StarlarkSemantics.java index fc5ab5d..bc025a3 100644 --- a/src/main/java/com/google/devtools/build/lib/syntax/StarlarkSemantics.java +++ b/src/main/java/com/google/devtools/build/lib/syntax/StarlarkSemantics.java
@@ -39,6 +39,8 @@ * the exact name of the flag transformed to upper case (for error representation). */ public enum FlagIdentifier { + EXPERIMENTAL_ALLOW_INCREMENTAL_REPOSITORY_UPDATES( + StarlarkSemantics::experimentalAllowIncrementalRepositoryUpdates), EXPERIMENTAL_ENABLE_ANDROID_MIGRATION_APIS( StarlarkSemantics::experimentalEnableAndroidMigrationApis), EXPERIMENTAL_BUILD_SETTING_API(StarlarkSemantics::experimentalBuildSettingApi), @@ -114,6 +116,8 @@ AutoValue_StarlarkSemantics.class; // <== Add new options here in alphabetic order ==> + public abstract boolean experimentalAllowIncrementalRepositoryUpdates(); + public abstract boolean experimentalBuildSettingApi(); public abstract ImmutableList<String> experimentalCcSkylarkApiEnabledPackages(); @@ -209,6 +213,7 @@ // <== Add new options here in alphabetic order ==> .experimentalBuildSettingApi(false) .experimentalCcSkylarkApiEnabledPackages(ImmutableList.of()) + .experimentalAllowIncrementalRepositoryUpdates(false) .experimentalEnableAndroidMigrationApis(false) .experimentalGoogleLegacyApi(false) .experimentalJavaCommonCreateProviderEnabledPackages(ImmutableList.of()) @@ -253,6 +258,8 @@ public abstract static class Builder { // <== Add new options here in alphabetic order ==> + public abstract Builder experimentalAllowIncrementalRepositoryUpdates(boolean value); + public abstract Builder experimentalBuildSettingApi(boolean value); public abstract Builder experimentalCcSkylarkApiEnabledPackages(List<String> value);