| // Copyright 2019 The Bazel Authors. All rights reserved. |
| // |
| // Licensed under the Apache License, Version 2.0 (the "License"); |
| // you may not use this file except in compliance with the License. |
| // You may obtain a copy of the License at |
| // |
| // http://www.apache.org/licenses/LICENSE-2.0 |
| // |
| // Unless required by applicable law or agreed to in writing, software |
| // distributed under the License is distributed on an "AS IS" BASIS, |
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| // See the License for the specific language governing permissions and |
| // limitations under the License. |
| // |
| |
| package com.google.devtools.build.lib.rules.repository; |
| |
| import static com.google.common.collect.ImmutableSet.toImmutableSet; |
| import static com.google.common.truth.Truth.assertThat; |
| |
| import com.google.common.base.Optional; |
| import com.google.common.collect.ImmutableList; |
| import com.google.common.collect.ImmutableMap; |
| import com.google.common.collect.ImmutableSet; |
| import com.google.devtools.build.lib.actions.FileStateValue; |
| import com.google.devtools.build.lib.actions.FileValue; |
| import com.google.devtools.build.lib.analysis.BlazeDirectories; |
| import com.google.devtools.build.lib.analysis.ConfiguredRuleClassProvider; |
| import com.google.devtools.build.lib.analysis.ServerDirectories; |
| import com.google.devtools.build.lib.analysis.util.AnalysisMock; |
| import com.google.devtools.build.lib.bazel.repository.downloader.DownloadManager; |
| import com.google.devtools.build.lib.bazel.repository.skylark.SkylarkRepositoryFunction; |
| import com.google.devtools.build.lib.bazel.repository.skylark.SkylarkRepositoryModule; |
| import com.google.devtools.build.lib.cmdline.RepositoryName; |
| import com.google.devtools.build.lib.events.StoredEventHandler; |
| import com.google.devtools.build.lib.packages.PackageFactory; |
| import com.google.devtools.build.lib.packages.Rule; |
| import com.google.devtools.build.lib.packages.WorkspaceFileValue; |
| import com.google.devtools.build.lib.pkgcache.PathPackageLocator; |
| import com.google.devtools.build.lib.rules.repository.RepositoryDirectoryValue.SuccessfulRepositoryDirectoryValue; |
| import com.google.devtools.build.lib.skyframe.ASTFileLookupFunction; |
| import com.google.devtools.build.lib.skyframe.BazelSkyframeExecutorConstants; |
| import com.google.devtools.build.lib.skyframe.BlacklistedPackagePrefixesFunction; |
| import com.google.devtools.build.lib.skyframe.ContainingPackageLookupFunction; |
| import com.google.devtools.build.lib.skyframe.ExternalFilesHelper; |
| import com.google.devtools.build.lib.skyframe.ExternalFilesHelper.ExternalFileAction; |
| import com.google.devtools.build.lib.skyframe.ExternalPackageFunction; |
| import com.google.devtools.build.lib.skyframe.FileFunction; |
| import com.google.devtools.build.lib.skyframe.FileStateFunction; |
| import com.google.devtools.build.lib.skyframe.LocalRepositoryLookupFunction; |
| import com.google.devtools.build.lib.skyframe.PackageFunction; |
| import com.google.devtools.build.lib.skyframe.PackageLookupFunction; |
| import com.google.devtools.build.lib.skyframe.PackageLookupFunction.CrossRepositoryLabelViolationStrategy; |
| import com.google.devtools.build.lib.skyframe.PrecomputedFunction; |
| import com.google.devtools.build.lib.skyframe.PrecomputedValue; |
| import com.google.devtools.build.lib.skyframe.SkyFunctions; |
| import com.google.devtools.build.lib.skyframe.SkylarkImportLookupFunction; |
| import com.google.devtools.build.lib.skyframe.WorkspaceASTFunction; |
| import com.google.devtools.build.lib.skyframe.WorkspaceFileFunction; |
| import com.google.devtools.build.lib.skylarkbuildapi.repository.RepositoryBootstrap; |
| import com.google.devtools.build.lib.syntax.StarlarkSemantics; |
| import com.google.devtools.build.lib.testutil.FoundationTestCase; |
| import com.google.devtools.build.lib.testutil.ManualClock; |
| import com.google.devtools.build.lib.testutil.TestConstants; |
| import com.google.devtools.build.lib.testutil.TestRuleClassProvider; |
| import com.google.devtools.build.lib.util.io.TimestampGranularityMonitor; |
| import com.google.devtools.build.lib.vfs.Path; |
| import com.google.devtools.build.lib.vfs.PathFragment; |
| import com.google.devtools.build.lib.vfs.Root; |
| import com.google.devtools.build.lib.vfs.RootedPath; |
| import com.google.devtools.build.lib.vfs.UnixGlob; |
| import com.google.devtools.build.skyframe.EvaluationContext; |
| import com.google.devtools.build.skyframe.EvaluationResult; |
| import com.google.devtools.build.skyframe.InMemoryMemoizingEvaluator; |
| import com.google.devtools.build.skyframe.MemoizingEvaluator; |
| import com.google.devtools.build.skyframe.RecordingDifferencer; |
| import com.google.devtools.build.skyframe.SequencedRecordingDifferencer; |
| import com.google.devtools.build.skyframe.SequentialBuildDriver; |
| import com.google.devtools.build.skyframe.SkyFunction; |
| import com.google.devtools.build.skyframe.SkyFunction.Environment; |
| import com.google.devtools.build.skyframe.SkyFunctionException.Transience; |
| import com.google.devtools.build.skyframe.SkyFunctionName; |
| import com.google.devtools.build.skyframe.SkyKey; |
| import com.google.devtools.build.skyframe.SkyValue; |
| import java.io.IOException; |
| import java.util.Map; |
| import java.util.concurrent.atomic.AtomicBoolean; |
| import java.util.concurrent.atomic.AtomicReference; |
| import javax.annotation.Nullable; |
| import org.junit.Before; |
| import org.junit.Test; |
| import org.junit.runner.RunWith; |
| import org.junit.runners.JUnit4; |
| import org.mockito.Mockito; |
| |
| /** |
| * Tests for {@link RepositoryDelegatorFunction} |
| */ |
| @RunWith(JUnit4.class) |
| public class RepositoryDelegatorTest extends FoundationTestCase { |
| private RepositoryDelegatorFunction delegatorFunction; |
| private Path overrideDirectory; |
| private SequentialBuildDriver driver; |
| private TestManagedDirectoriesKnowledge managedDirectoriesKnowledge; |
| private RecordingDifferencer differencer; |
| private TestSkylarkRepositoryFunction testSkylarkRepositoryFunction; |
| private Path rootPath; |
| |
| @Before |
| public void setupDelegator() throws Exception { |
| rootPath = scratch.dir("/outputbase"); |
| BlazeDirectories directories = |
| new BlazeDirectories( |
| new ServerDirectories(rootPath, rootPath, rootPath), |
| rootPath, |
| /* defaultSystemJavabase= */ null, |
| TestConstants.PRODUCT_NAME); |
| managedDirectoriesKnowledge = new TestManagedDirectoriesKnowledge(); |
| DownloadManager downloader = Mockito.mock(DownloadManager.class); |
| RepositoryFunction localRepositoryFunction = new LocalRepositoryFunction(); |
| testSkylarkRepositoryFunction = |
| new TestSkylarkRepositoryFunction(rootPath, downloader, managedDirectoriesKnowledge); |
| ImmutableMap<String, RepositoryFunction> repositoryHandlers = |
| ImmutableMap.of(LocalRepositoryRule.NAME, localRepositoryFunction); |
| delegatorFunction = |
| new RepositoryDelegatorFunction( |
| repositoryHandlers, |
| testSkylarkRepositoryFunction, |
| new AtomicBoolean(true), |
| ImmutableMap::of, |
| directories, |
| managedDirectoriesKnowledge); |
| AtomicReference<PathPackageLocator> pkgLocator = |
| new AtomicReference<>( |
| new PathPackageLocator( |
| rootPath, |
| ImmutableList.of(Root.fromPath(rootPath)), |
| BazelSkyframeExecutorConstants.BUILD_FILES_BY_PRIORITY)); |
| ExternalFilesHelper externalFilesHelper = ExternalFilesHelper.createForTesting( |
| pkgLocator, |
| ExternalFileAction.DEPEND_ON_EXTERNAL_PKG_FOR_EXTERNAL_REPO_PATHS, |
| directories); |
| differencer = new SequencedRecordingDifferencer(); |
| |
| ConfiguredRuleClassProvider.Builder builder = new ConfiguredRuleClassProvider.Builder(); |
| TestRuleClassProvider.addStandardRules(builder); |
| builder |
| .clearWorkspaceFileSuffixForTesting() |
| .addSkylarkBootstrap(new RepositoryBootstrap(new SkylarkRepositoryModule())); |
| ConfiguredRuleClassProvider ruleClassProvider = builder.build(); |
| |
| PackageFactory.BuilderForTesting pkgFactoryBuilder = |
| AnalysisMock.get().getPackageFactoryBuilderForTesting(directories); |
| SkylarkImportLookupFunction skylarkImportLookupFunction = |
| new SkylarkImportLookupFunction( |
| ruleClassProvider, |
| pkgFactoryBuilder.build(ruleClassProvider, fileSystem), |
| /*starlarkImportLookupValueCacheSize=*/ 2); |
| skylarkImportLookupFunction.resetCache(); |
| |
| MemoizingEvaluator evaluator = |
| new InMemoryMemoizingEvaluator( |
| ImmutableMap.<SkyFunctionName, SkyFunction>builder() |
| .put( |
| FileStateValue.FILE_STATE, |
| new FileStateFunction( |
| new AtomicReference<TimestampGranularityMonitor>(), |
| new AtomicReference<>(UnixGlob.DEFAULT_SYSCALLS), |
| externalFilesHelper)) |
| .put(FileValue.FILE, new FileFunction(pkgLocator)) |
| .put(SkyFunctions.REPOSITORY_DIRECTORY, delegatorFunction) |
| .put( |
| SkyFunctions.PACKAGE, |
| new PackageFunction(null, null, null, null, null, null, null)) |
| .put( |
| SkyFunctions.PACKAGE_LOOKUP, |
| new PackageLookupFunction( |
| new AtomicReference<>(ImmutableSet.of()), |
| CrossRepositoryLabelViolationStrategy.ERROR, |
| BazelSkyframeExecutorConstants.BUILD_FILES_BY_PRIORITY)) |
| .put(SkyFunctions.WORKSPACE_AST, new WorkspaceASTFunction(ruleClassProvider)) |
| .put( |
| WorkspaceFileValue.WORKSPACE_FILE, |
| new WorkspaceFileFunction( |
| ruleClassProvider, |
| TestConstants.PACKAGE_FACTORY_BUILDER_FACTORY_FOR_TESTING |
| .builder(directories) |
| .build(ruleClassProvider, fileSystem), |
| directories, |
| skylarkImportLookupFunction)) |
| .put(SkyFunctions.REPOSITORY, new RepositoryLoaderFunction()) |
| .put(SkyFunctions.LOCAL_REPOSITORY_LOOKUP, new LocalRepositoryLookupFunction()) |
| .put(SkyFunctions.EXTERNAL_PACKAGE, new ExternalPackageFunction()) |
| .put(SkyFunctions.PRECOMPUTED, new PrecomputedFunction()) |
| .put(SkyFunctions.AST_FILE_LOOKUP, new ASTFileLookupFunction(ruleClassProvider)) |
| .put(SkyFunctions.CONTAINING_PACKAGE_LOOKUP, new ContainingPackageLookupFunction()) |
| .put( |
| SkyFunctions.BLACKLISTED_PACKAGE_PREFIXES, |
| new BlacklistedPackagePrefixesFunction( |
| /*blacklistedPackagePrefixesFile=*/ PathFragment.EMPTY_FRAGMENT)) |
| .put(SkyFunctions.RESOLVED_HASH_VALUES, new ResolvedHashesFunction()) |
| .build(), |
| differencer); |
| driver = new SequentialBuildDriver(evaluator); |
| overrideDirectory = scratch.dir("/foo"); |
| scratch.file("/foo/WORKSPACE"); |
| RepositoryDelegatorFunction.REPOSITORY_OVERRIDES.set(differencer, ImmutableMap.of()); |
| RepositoryDelegatorFunction.DEPENDENCY_FOR_UNCONDITIONAL_FETCHING.set( |
| differencer, RepositoryDelegatorFunction.DONT_FETCH_UNCONDITIONALLY); |
| PrecomputedValue.PATH_PACKAGE_LOCATOR.set(differencer, pkgLocator.get()); |
| PrecomputedValue.STARLARK_SEMANTICS.set(differencer, StarlarkSemantics.DEFAULT_SEMANTICS); |
| RepositoryDelegatorFunction.RESOLVED_FILE_INSTEAD_OF_WORKSPACE.set( |
| differencer, Optional.<RootedPath>absent()); |
| PrecomputedValue.REPO_ENV.set(differencer, ImmutableMap.of()); |
| RepositoryDelegatorFunction.OUTPUT_VERIFICATION_REPOSITORY_RULES.set( |
| differencer, ImmutableSet.of()); |
| RepositoryDelegatorFunction.RESOLVED_FILE_FOR_VERIFICATION.set(differencer, Optional.absent()); |
| } |
| |
| @Test |
| public void testOverride() throws Exception { |
| RepositoryDelegatorFunction.REPOSITORY_OVERRIDES.set( |
| differencer, |
| ImmutableMap.of( |
| RepositoryName.createFromValidStrippedName("foo"), overrideDirectory.asFragment())); |
| |
| StoredEventHandler eventHandler = new StoredEventHandler(); |
| SkyKey key = RepositoryDirectoryValue.key(RepositoryName.createFromValidStrippedName("foo")); |
| EvaluationContext evaluationContext = |
| EvaluationContext.newBuilder() |
| .setKeepGoing(false) |
| .setNumThreads(8) |
| .setEventHander(eventHandler) |
| .build(); |
| EvaluationResult<SkyValue> result = driver.evaluate(ImmutableList.of(key), evaluationContext); |
| assertThat(result.hasError()).isFalse(); |
| RepositoryDirectoryValue repositoryDirectoryValue = (RepositoryDirectoryValue) result.get(key); |
| Path expectedPath = scratch.dir("/outputbase/external/foo"); |
| Path actualPath = repositoryDirectoryValue.getPath(); |
| assertThat(actualPath).isEqualTo(expectedPath); |
| assertThat(actualPath.isSymbolicLink()).isTrue(); |
| assertThat(actualPath.readSymbolicLink()).isEqualTo(overrideDirectory.asFragment()); |
| } |
| |
| @Test |
| public void testRepositoryDirtinessChecker() throws Exception { |
| TimestampGranularityMonitor tsgm = new TimestampGranularityMonitor(new ManualClock()); |
| TestManagedDirectoriesKnowledge knowledge = new TestManagedDirectoriesKnowledge(); |
| |
| RepositoryDirectoryDirtinessChecker checker = |
| new RepositoryDirectoryDirtinessChecker(rootPath, knowledge); |
| RepositoryName repositoryName = RepositoryName.create("@repo"); |
| RepositoryDirectoryValue.Key key = RepositoryDirectoryValue.key(repositoryName); |
| |
| SuccessfulRepositoryDirectoryValue usual = |
| RepositoryDirectoryValue.builder() |
| .setPath(rootDirectory.getRelative("a")) |
| .setDigest(new byte[] {1}) |
| .build(); |
| |
| assertThat(checker.check(key, usual, tsgm).isDirty()).isFalse(); |
| |
| SuccessfulRepositoryDirectoryValue fetchDelayed = |
| RepositoryDirectoryValue.builder() |
| .setPath(rootDirectory.getRelative("b")) |
| .setFetchingDelayed() |
| .build(); |
| |
| assertThat(checker.check(key, fetchDelayed, tsgm).isDirty()).isTrue(); |
| |
| SuccessfulRepositoryDirectoryValue withManagedDirectories = |
| RepositoryDirectoryValue.builder() |
| .setPath(rootDirectory.getRelative("c")) |
| .setDigest(new byte[] {1}) |
| .setManagedDirectories(ImmutableSet.of(PathFragment.create("m"))) |
| .build(); |
| |
| assertThat(checker.check(key, withManagedDirectories, tsgm).isDirty()).isTrue(); |
| |
| Path managedDirectoryM = rootPath.getRelative("m"); |
| assertThat(managedDirectoryM.createDirectory()).isTrue(); |
| |
| knowledge.setManagedDirectories( |
| ImmutableMap.of(PathFragment.create("m"), RepositoryName.create("@other"))); |
| assertThat(checker.check(key, withManagedDirectories, tsgm).isDirty()).isTrue(); |
| |
| knowledge.setManagedDirectories(ImmutableMap.of(PathFragment.create("m"), repositoryName)); |
| assertThat(checker.check(key, withManagedDirectories, tsgm).isDirty()).isFalse(); |
| |
| managedDirectoryM.deleteTree(); |
| assertThat(checker.check(key, withManagedDirectories, tsgm).isDirty()).isTrue(); |
| } |
| |
| @Test |
| public void testManagedDirectoriesCauseRepositoryReFetches() throws Exception { |
| scratch.file(rootPath.getRelative("BUILD").getPathString()); |
| scratch.file( |
| rootPath.getRelative("repo_rule.bzl").getPathString(), |
| "def _impl(rctx):", |
| " rctx.file('BUILD', '')", |
| "fictive_repo_rule = repository_rule(implementation = _impl)"); |
| scratch.overwriteFile( |
| rootPath.getRelative("WORKSPACE").getPathString(), |
| "workspace(name = 'abc')", |
| "load(':repo_rule.bzl', 'fictive_repo_rule')", |
| "fictive_repo_rule(name = 'repo1')"); |
| |
| // Managed directories from workspace() attribute will not be parsed by this test, since |
| // we are not calling SequencedSkyframeExecutor. |
| // That's why we will directly fill managed directories value (the corresponding structure |
| // is passed to RepositoryDelegatorFunction during construction). |
| managedDirectoriesKnowledge.setManagedDirectories( |
| ImmutableMap.of(PathFragment.create("dir1"), RepositoryName.create("@repo1"))); |
| |
| StarlarkSemantics semantics = |
| StarlarkSemantics.builderWithDefaults() |
| .experimentalAllowIncrementalRepositoryUpdates(true) |
| .build(); |
| PrecomputedValue.STARLARK_SEMANTICS.set(differencer, semantics); |
| |
| loadRepo("repo1"); |
| |
| assertThat(testSkylarkRepositoryFunction.isFetchCalled()).isTrue(); |
| testSkylarkRepositoryFunction.reset(); |
| |
| loadRepo("repo1"); |
| // Nothing changed, fetch does not happen. |
| assertThat(testSkylarkRepositoryFunction.isFetchCalled()).isFalse(); |
| testSkylarkRepositoryFunction.reset(); |
| |
| // Delete managed directory, fetch should happen again. |
| Path managedDirectory = rootPath.getRelative("dir1"); |
| managedDirectory.deleteTree(); |
| loadRepo("repo1"); |
| assertThat(testSkylarkRepositoryFunction.isFetchCalled()).isTrue(); |
| testSkylarkRepositoryFunction.reset(); |
| |
| // Change managed directories declaration, fetch should happen. |
| // NB: we are making sure that managed directories exist to check only the declaration changes |
| // were percepted. |
| rootPath.getRelative("dir1").createDirectory(); |
| rootPath.getRelative("dir2").createDirectory(); |
| |
| managedDirectoriesKnowledge.setManagedDirectories( |
| ImmutableMap.of( |
| PathFragment.create("dir1"), |
| RepositoryName.create("@repo1"), |
| PathFragment.create("dir2"), |
| RepositoryName.create("@repo1"))); |
| loadRepo("repo1"); |
| |
| assertThat(testSkylarkRepositoryFunction.isFetchCalled()).isTrue(); |
| testSkylarkRepositoryFunction.reset(); |
| |
| managedDirectoriesKnowledge.setManagedDirectories(ImmutableMap.of()); |
| loadRepo("repo1"); |
| |
| assertThat(testSkylarkRepositoryFunction.isFetchCalled()).isTrue(); |
| testSkylarkRepositoryFunction.reset(); |
| } |
| |
| private void loadRepo(String strippedRepoName) throws InterruptedException { |
| StoredEventHandler eventHandler = new StoredEventHandler(); |
| SkyKey key = |
| RepositoryDirectoryValue.key(RepositoryName.createFromValidStrippedName(strippedRepoName)); |
| // Make it be evaluated every time, as we are testing evaluation. |
| differencer.invalidate(ImmutableSet.of(key)); |
| EvaluationContext evaluationContext = |
| EvaluationContext.newBuilder() |
| .setKeepGoing(false) |
| .setNumThreads(8) |
| .setEventHander(eventHandler) |
| .build(); |
| EvaluationResult<SkyValue> result = driver.evaluate(ImmutableList.of(key), evaluationContext); |
| assertThat(result.hasError()).isFalse(); |
| RepositoryDirectoryValue repositoryDirectoryValue = (RepositoryDirectoryValue) result.get(key); |
| assertThat(repositoryDirectoryValue.repositoryExists()).isTrue(); |
| } |
| |
| private static class TestSkylarkRepositoryFunction extends SkylarkRepositoryFunction { |
| private boolean fetchCalled = false; |
| private final Path workspaceRoot; |
| private final TestManagedDirectoriesKnowledge managedDirectoriesKnowledge; |
| |
| private TestSkylarkRepositoryFunction( |
| Path workspaceRoot, |
| DownloadManager downloader, |
| TestManagedDirectoriesKnowledge managedDirectoriesKnowledge) { |
| super(downloader); |
| this.workspaceRoot = workspaceRoot; |
| this.managedDirectoriesKnowledge = managedDirectoriesKnowledge; |
| } |
| |
| public void reset() { |
| fetchCalled = false; |
| } |
| |
| private boolean isFetchCalled() { |
| return fetchCalled; |
| } |
| |
| @Nullable |
| @Override |
| public RepositoryDirectoryValue.Builder fetch( |
| Rule rule, |
| Path outputDirectory, |
| BlazeDirectories directories, |
| Environment env, |
| Map<String, String> markerData, |
| SkyKey key) |
| throws RepositoryFunctionException, InterruptedException { |
| fetchCalled = true; |
| RepositoryDirectoryValue.Builder builder = |
| super.fetch(rule, outputDirectory, directories, env, markerData, key); |
| ImmutableSet<PathFragment> managedDirectories = |
| managedDirectoriesKnowledge.getManagedDirectories((RepositoryName) key.argument()); |
| try { |
| for (PathFragment managedDirectory : managedDirectories) { |
| workspaceRoot.getRelative(managedDirectory).createDirectory(); |
| } |
| } catch (IOException e) { |
| throw new RepositoryFunctionException(e, Transience.PERSISTENT); |
| } |
| return builder; |
| } |
| } |
| |
| private static class TestManagedDirectoriesKnowledge implements ManagedDirectoriesKnowledge { |
| |
| private ImmutableMap<PathFragment, RepositoryName> map = ImmutableMap.of(); |
| |
| public void setManagedDirectories(ImmutableMap<PathFragment, RepositoryName> map) { |
| this.map = map; |
| } |
| |
| @Nullable |
| @Override |
| public RepositoryName getOwnerRepository(PathFragment relativePathFragment) { |
| return map.get(relativePathFragment); |
| } |
| |
| @Override |
| public ImmutableSet<PathFragment> getManagedDirectories(RepositoryName repositoryName) { |
| return map.keySet().stream() |
| .filter(path -> repositoryName.equals(map.get(path))) |
| .collect(toImmutableSet()); |
| } |
| |
| @Override |
| public boolean workspaceHeaderReloaded( |
| @Nullable WorkspaceFileValue oldValue, @Nullable WorkspaceFileValue newValue) { |
| throw new IllegalStateException(); |
| } |
| } |
| } |