blob: cc0741b7e7e7749b976310d7f044f5a276c5e044 [file] [log] [blame]
Kristina Chodorow335f0672015-11-16 23:19:13 +00001// Copyright 2015 The Bazel Authors. All rights reserved.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15package com.google.devtools.build.lib.skyframe;
16
17import static com.google.common.truth.Truth.assertThat;
nharmata58613102018-10-11 15:10:04 -070018import static com.google.devtools.build.skyframe.EvaluationResultSubjectFactory.assertThatEvaluationResult;
michajlo660d17f2020-03-27 09:01:57 -070019import static org.junit.Assert.assertThrows;
Florian Weikert92b22362015-12-03 10:17:18 +000020import static org.junit.Assert.fail;
michajlod05a9232020-02-25 16:39:00 -080021import static org.mockito.ArgumentMatchers.any;
michajlof05777b2021-03-18 05:51:36 -070022import static org.mockito.ArgumentMatchers.eq;
michajlod05a9232020-02-25 16:39:00 -080023import static org.mockito.Mockito.doAnswer;
michajlof05777b2021-03-18 05:51:36 -070024import static org.mockito.Mockito.verify;
25import static org.mockito.Mockito.when;
Kristina Chodorow335f0672015-11-16 23:19:13 +000026
27import com.google.common.base.Predicates;
28import com.google.common.collect.ImmutableList;
29import com.google.common.collect.ImmutableMap;
30import com.google.common.collect.Maps;
Googlerc804c662016-12-01 16:53:28 +000031import com.google.common.collect.Sets;
shahan602cc852018-06-06 20:09:57 -070032import com.google.devtools.build.lib.actions.FileStateValue;
brandjon38dafdd2020-08-06 09:49:13 -070033import com.google.devtools.build.lib.analysis.ConfiguredRuleClassProvider;
Florian Weikertcca703a2015-12-07 09:56:38 +000034import com.google.devtools.build.lib.analysis.util.BuildViewTestCase;
philwo3bcb9f62017-09-06 12:52:21 +020035import com.google.devtools.build.lib.clock.BlazeClock;
Kristina Chodorow335f0672015-11-16 23:19:13 +000036import com.google.devtools.build.lib.cmdline.Label;
37import com.google.devtools.build.lib.cmdline.PackageIdentifier;
michajloa96b2d22020-02-26 14:54:50 -080038import com.google.devtools.build.lib.events.Event;
39import com.google.devtools.build.lib.events.ExtendedEventHandler;
adonovan87b46082020-07-08 15:58:04 -070040import com.google.devtools.build.lib.packages.BazelModuleContext;
nharmata58613102018-10-11 15:10:04 -070041import com.google.devtools.build.lib.packages.BuildFileNotFoundException;
Kristina Chodorow335f0672015-11-16 23:19:13 +000042import com.google.devtools.build.lib.packages.ConstantRuleVisibility;
nharmataff688bf2017-06-07 17:03:52 -040043import com.google.devtools.build.lib.packages.NoSuchPackageException;
Nathan Harmata2022ad82016-02-22 23:04:14 +000044import com.google.devtools.build.lib.packages.NoSuchTargetException;
nharmatac9fbe952019-06-13 09:39:23 -070045import com.google.devtools.build.lib.packages.Package;
michajlof05777b2021-03-18 05:51:36 -070046import com.google.devtools.build.lib.packages.PackageOverheadEstimator;
michajlod05a9232020-02-25 16:39:00 -080047import com.google.devtools.build.lib.packages.PackageValidator;
48import com.google.devtools.build.lib.packages.PackageValidator.InvalidPackageException;
adonovan240bdea2020-09-03 15:24:12 -070049import com.google.devtools.build.lib.packages.semantics.BuildLanguageOptions;
ajurkowskid74b0ec2020-04-13 10:58:21 -070050import com.google.devtools.build.lib.pkgcache.PackageOptions;
Kristina Chodorow335f0672015-11-16 23:19:13 +000051import com.google.devtools.build.lib.pkgcache.PathPackageLocator;
Klaus Aehlig8eb47482018-09-17 09:14:58 -070052import com.google.devtools.build.lib.rules.repository.RepositoryDelegatorFunction;
Googler74178a52020-06-29 17:42:47 -070053import com.google.devtools.build.lib.server.FailureDetails.PackageLoading;
Kristina Chodorow335f0672015-11-16 23:19:13 +000054import com.google.devtools.build.lib.skyframe.util.SkyframeExecutorTestUtils;
55import com.google.devtools.build.lib.testutil.ManualClock;
brandjon38dafdd2020-08-06 09:49:13 -070056import com.google.devtools.build.lib.testutil.TestRuleClassProvider;
Googler74178a52020-06-29 17:42:47 -070057import com.google.devtools.build.lib.util.DetailedExitCode;
58import com.google.devtools.build.lib.util.ExitCode;
Benjamin Peterson723eca62019-07-22 17:24:10 -070059import com.google.devtools.build.lib.util.Pair;
Ulf Adamsc73051c62016-03-23 09:18:13 +000060import com.google.devtools.build.lib.util.io.TimestampGranularityMonitor;
janakr97c0bd12020-09-08 13:19:03 -070061import com.google.devtools.build.lib.vfs.DigestHashFunction;
Kristina Chodorow335f0672015-11-16 23:19:13 +000062import com.google.devtools.build.lib.vfs.Dirent;
63import com.google.devtools.build.lib.vfs.FileStatus;
64import com.google.devtools.build.lib.vfs.FileSystem;
65import com.google.devtools.build.lib.vfs.FileSystemUtils;
66import com.google.devtools.build.lib.vfs.ModifiedFileSet;
67import com.google.devtools.build.lib.vfs.Path;
68import com.google.devtools.build.lib.vfs.PathFragment;
tomluee6a6862018-01-17 14:36:26 -080069import com.google.devtools.build.lib.vfs.Root;
Kristina Chodorow335f0672015-11-16 23:19:13 +000070import com.google.devtools.build.lib.vfs.RootedPath;
nharmatace0335a2019-11-13 15:48:05 -080071import com.google.devtools.build.lib.vfs.Symlinks;
janakrd1b2f962022-03-04 13:33:44 -080072import com.google.devtools.build.lib.vfs.SyscallCache;
Kristina Chodorow335f0672015-11-16 23:19:13 +000073import com.google.devtools.build.lib.vfs.inmemoryfs.InMemoryFileSystem;
Kristina Chodorow335f0672015-11-16 23:19:13 +000074import com.google.devtools.build.skyframe.EvaluationResult;
75import com.google.devtools.build.skyframe.RecordingDifferencer;
76import com.google.devtools.build.skyframe.SkyKey;
77import com.google.devtools.build.skyframe.SkyValue;
Janak Ramakrishnan326c6982016-09-27 14:58:26 +000078import com.google.devtools.common.options.Options;
Kristina Chodorow335f0672015-11-16 23:19:13 +000079import java.io.IOException;
nharmataff688bf2017-06-07 17:03:52 -040080import java.io.InputStream;
Janak Ramakrishnanffbc28a2017-02-13 22:51:45 +000081import java.util.ArrayList;
tomluee6a6862018-01-17 14:36:26 -080082import java.util.Arrays;
Kristina Chodorow335f0672015-11-16 23:19:13 +000083import java.util.Collection;
Janak Ramakrishnanffbc28a2017-02-13 22:51:45 +000084import java.util.List;
Kristina Chodorow335f0672015-11-16 23:19:13 +000085import java.util.Map;
jhorvitzdd1d8412020-08-01 05:59:14 -070086import java.util.Optional;
michajlof05777b2021-03-18 05:51:36 -070087import java.util.OptionalLong;
Googlerc804c662016-12-01 16:53:28 +000088import java.util.Set;
Kristina Chodorow335f0672015-11-16 23:19:13 +000089import java.util.UUID;
michajlo90810222020-03-02 09:36:23 -080090import java.util.concurrent.atomic.AtomicInteger;
Kristina Chodorow335f0672015-11-16 23:19:13 +000091import javax.annotation.Nullable;
adonovan450c7ad2020-09-14 13:00:21 -070092import net.starlark.java.eval.Module;
adonovan3ed7ed52020-09-30 12:03:28 -070093import net.starlark.java.eval.StarlarkInt;
michajlod05a9232020-02-25 16:39:00 -080094import org.junit.Rule;
John Cater94695912016-08-03 12:09:39 +000095import org.junit.Test;
96import org.junit.runner.RunWith;
97import org.junit.runners.JUnit4;
michajlod05a9232020-02-25 16:39:00 -080098import org.mockito.Mock;
99import org.mockito.junit.MockitoJUnit;
100import org.mockito.junit.MockitoRule;
Kristina Chodorow335f0672015-11-16 23:19:13 +0000101
102/**
laurentlbc0bd2102018-10-17 07:05:25 -0700103 * Unit tests of specific functionality of PackageFunction. Note that it's already tested indirectly
104 * in several other places.
Kristina Chodorow335f0672015-11-16 23:19:13 +0000105 */
Florian Weikert92b22362015-12-03 10:17:18 +0000106@RunWith(JUnit4.class)
Florian Weikertcca703a2015-12-07 09:56:38 +0000107public class PackageFunctionTest extends BuildViewTestCase {
Kristina Chodorow335f0672015-11-16 23:19:13 +0000108
michajlod05a9232020-02-25 16:39:00 -0800109 @Rule public final MockitoRule mockito = MockitoJUnit.rule();
110
111 @Mock private PackageValidator mockPackageValidator;
112
michajlof05777b2021-03-18 05:51:36 -0700113 @Mock private PackageOverheadEstimator mockPackageOverheadEstimator;
114
Kristina Chodorow335f0672015-11-16 23:19:13 +0000115 private CustomInMemoryFs fs = new CustomInMemoryFs(new ManualClock());
116
Ulf Adamsc73051c62016-03-23 09:18:13 +0000117 private void preparePackageLoading(Path... roots) {
nharmatac9fbe952019-06-13 09:39:23 -0700118 preparePackageLoadingWithCustomStarklarkSemanticsOptions(
adonovan240bdea2020-09-03 15:24:12 -0700119 Options.getDefaults(BuildLanguageOptions.class), roots);
nharmatac9fbe952019-06-13 09:39:23 -0700120 }
121
122 private void preparePackageLoadingWithCustomStarklarkSemanticsOptions(
brandjon6dbfa3e2020-10-06 19:31:08 -0700123 BuildLanguageOptions buildLanguageOptions, Path... roots) {
ajurkowskid74b0ec2020-04-13 10:58:21 -0700124 PackageOptions packageOptions = Options.getDefaults(PackageOptions.class);
125 packageOptions.defaultVisibility = ConstantRuleVisibility.PUBLIC;
126 packageOptions.showLoadingProgress = true;
127 packageOptions.globbingThreads = 7;
Klaus Aehlig6f33a1c2016-09-13 16:46:10 +0000128 getSkyframeExecutor()
129 .preparePackageLoading(
John Catere0d1d0e2017-11-28 20:47:41 -0800130 new PathPackageLocator(
131 outputBase,
tomluee6a6862018-01-17 14:36:26 -0800132 Arrays.stream(roots).map(Root::fromPath).collect(ImmutableList.toImmutableList()),
John Catere0d1d0e2017-11-28 20:47:41 -0800133 BazelSkyframeExecutorConstants.BUILD_FILES_BY_PRIORITY),
ajurkowskid74b0ec2020-04-13 10:58:21 -0700134 packageOptions,
brandjon6dbfa3e2020-10-06 19:31:08 -0700135 buildLanguageOptions,
Klaus Aehlig6f33a1c2016-09-13 16:46:10 +0000136 UUID.randomUUID(),
137 ImmutableMap.<String, String>of(),
138 new TimestampGranularityMonitor(BlazeClock.instance()));
juliexxia651797f2018-08-01 11:45:13 -0700139 skyframeExecutor.setActionEnv(ImmutableMap.<String, String>of());
Ulf Adamsc73051c62016-03-23 09:18:13 +0000140 }
141
Kristina Chodorow335f0672015-11-16 23:19:13 +0000142 @Override
Kristina Chodorow335f0672015-11-16 23:19:13 +0000143 protected FileSystem createFileSystem() {
144 return fs;
145 }
146
michajlod05a9232020-02-25 16:39:00 -0800147 @Override
148 protected PackageValidator getPackageValidator() {
149 return mockPackageValidator;
150 }
151
michajlof05777b2021-03-18 05:51:36 -0700152 @Override
153 protected PackageOverheadEstimator getPackageOverheadEstimator() {
154 return mockPackageOverheadEstimator;
155 }
156
nharmatac9fbe952019-06-13 09:39:23 -0700157 private Package validPackageWithoutErrors(SkyKey skyKey) throws InterruptedException {
158 return validPackageInternal(skyKey, /*checkPackageError=*/ true);
159 }
160
161 private Package validPackage(SkyKey skyKey) throws InterruptedException {
162 return validPackageInternal(skyKey, /*checkPackageError=*/ false);
163 }
164
165 private Package validPackageInternal(SkyKey skyKey, boolean checkPackageError)
166 throws InterruptedException {
Klaus Aehlig8eb47482018-09-17 09:14:58 -0700167 SkyframeExecutor skyframeExecutor = getSkyframeExecutor();
168 skyframeExecutor.injectExtraPrecomputedValues(
169 ImmutableList.of(
170 PrecomputedValue.injected(
pcloudyc25376a2021-07-16 08:25:42 -0700171 RepositoryDelegatorFunction.RESOLVED_FILE_INSTEAD_OF_WORKSPACE, Optional.empty()),
172 PrecomputedValue.injected(RepositoryDelegatorFunction.ENABLE_BZLMOD, false)));
Klaus Aehlig8eb47482018-09-17 09:14:58 -0700173 EvaluationResult<PackageValue> result =
174 SkyframeExecutorTestUtils.evaluate(
175 skyframeExecutor, skyKey, /*keepGoing=*/ false, reporter);
Kristina Chodorow335f0672015-11-16 23:19:13 +0000176 if (result.hasError()) {
177 fail(result.getError(skyKey).getException().getMessage());
178 }
179 PackageValue value = result.get(skyKey);
nharmatac9fbe952019-06-13 09:39:23 -0700180 if (checkPackageError) {
181 assertThat(value.getPackage().containsErrors()).isFalse();
182 }
183 return value.getPackage();
Kristina Chodorow335f0672015-11-16 23:19:13 +0000184 }
185
nharmatac68908f2021-12-10 11:14:41 -0800186 private Exception evaluatePackageToException(String pkg) throws Exception {
187 return evaluatePackageToException(pkg, /*keepGoing=*/ false);
188 }
189
brandjon54a7fcf2021-02-01 09:16:42 -0800190 /**
191 * Helper that evaluates the given package and returns the expected exception.
192 *
193 * <p>Disables the failFastHandler as a side-effect.
194 */
nharmatac68908f2021-12-10 11:14:41 -0800195 private Exception evaluatePackageToException(String pkg, boolean keepGoing) throws Exception {
brandjon54a7fcf2021-02-01 09:16:42 -0800196 reporter.removeHandler(failFastHandler);
197
198 SkyKey skyKey = PackageValue.key(PackageIdentifier.parse(pkg));
199 EvaluationResult<PackageValue> result =
nharmatac68908f2021-12-10 11:14:41 -0800200 SkyframeExecutorTestUtils.evaluate(getSkyframeExecutor(), skyKey, keepGoing, reporter);
brandjon54a7fcf2021-02-01 09:16:42 -0800201 assertThat(result.hasError()).isTrue();
202 return result.getError(skyKey).getException();
203 }
204
Florian Weikert92b22362015-12-03 10:17:18 +0000205 @Test
John Cater94695912016-08-03 12:09:39 +0000206 public void testValidPackage() throws Exception {
207 scratch.file("pkg/BUILD");
nharmatac9fbe952019-06-13 09:39:23 -0700208 validPackageWithoutErrors(PackageValue.key(PackageIdentifier.parse("@//pkg")));
John Cater94695912016-08-03 12:09:39 +0000209 }
210
211 @Test
michajlod05a9232020-02-25 16:39:00 -0800212 public void testInvalidPackage() throws Exception {
213 scratch.file("pkg/BUILD", "sh_library(name='foo', srcs=['foo.sh'])");
214 scratch.file("pkg/foo.sh");
215
216 doAnswer(
217 inv -> {
218 Package pkg = inv.getArgument(0, Package.class);
219 if (pkg.getName().equals("pkg")) {
michajlof05777b2021-03-18 05:51:36 -0700220 inv.getArgument(2, ExtendedEventHandler.class).handle(Event.warn("warning event"));
michajlod05a9232020-02-25 16:39:00 -0800221 throw new InvalidPackageException(pkg.getPackageIdentifier(), "no good");
222 }
223 return null;
224 })
225 .when(mockPackageValidator)
michajlof05777b2021-03-18 05:51:36 -0700226 .validate(any(Package.class), any(OptionalLong.class), any(ExtendedEventHandler.class));
michajlod05a9232020-02-25 16:39:00 -0800227
228 invalidatePackages();
229
brandjon54a7fcf2021-02-01 09:16:42 -0800230 Exception ex = evaluatePackageToException("@//pkg");
231 assertThat(ex).isInstanceOf(InvalidPackageException.class);
232 assertThat(ex).hasMessageThat().contains("no such package 'pkg': no good");
michajloa96b2d22020-02-26 14:54:50 -0800233 assertContainsEvent("warning event");
michajlod05a9232020-02-25 16:39:00 -0800234 }
235
236 @Test
michajlof05777b2021-03-18 05:51:36 -0700237 public void testPackageOverheadPassedToValidationLogic() throws Exception {
238 scratch.file("pkg/BUILD", "# Contents doesn't matter, it's all fake");
239
240 when(mockPackageOverheadEstimator.estimatePackageOverhead(any(Package.class)))
241 .thenReturn(OptionalLong.of(42));
242
243 invalidatePackages();
244
245 SkyframeExecutorTestUtils.evaluate(
246 getSkyframeExecutor(),
247 PackageValue.key(PackageIdentifier.parse("@//pkg")),
248 /*keepGoing=*/ false,
249 reporter);
250
251 verify(mockPackageValidator)
252 .validate(any(Package.class), eq(OptionalLong.of(42)), any(ExtendedEventHandler.class));
253 }
254
255 @Test
michajlo90810222020-03-02 09:36:23 -0800256 public void testSkyframeExecutorClearedPackagesResultsInReload() throws Exception {
257 scratch.file("pkg/BUILD", "sh_library(name='foo', srcs=['foo.sh'])");
258 scratch.file("pkg/foo.sh");
259
260 invalidatePackages();
261
262 // Use number of times the package was validated as a proxy for number of times it was loaded.
263 AtomicInteger validationCount = new AtomicInteger();
264 doAnswer(
265 inv -> {
266 if (inv.getArgument(0, Package.class).getName().equals("pkg")) {
267 validationCount.incrementAndGet();
268 }
269 return null;
270 })
271 .when(mockPackageValidator)
michajlof05777b2021-03-18 05:51:36 -0700272 .validate(any(Package.class), any(OptionalLong.class), any(ExtendedEventHandler.class));
michajlo90810222020-03-02 09:36:23 -0800273
274 SkyKey skyKey = PackageValue.key(PackageIdentifier.parse("@//pkg"));
275 EvaluationResult<PackageValue> result1 =
276 SkyframeExecutorTestUtils.evaluate(
277 getSkyframeExecutor(), skyKey, /*keepGoing=*/ false, reporter);
278 assertThatEvaluationResult(result1).hasNoError();
279
280 skyframeExecutor.clearLoadedPackages();
281
282 EvaluationResult<PackageValue> result2 =
283 SkyframeExecutorTestUtils.evaluate(
284 getSkyframeExecutor(), skyKey, /*keepGoing=*/ false, reporter);
285 assertThatEvaluationResult(result2).hasNoError();
286
287 assertThat(validationCount.get()).isEqualTo(2);
288 }
289
290 @Test
Kristina Chodorow335f0672015-11-16 23:19:13 +0000291 public void testPropagatesFilesystemInconsistencies() throws Exception {
Kristina Chodorow335f0672015-11-16 23:19:13 +0000292 RecordingDifferencer differencer = getSkyframeExecutor().getDifferencerForTesting();
tomluee6a6862018-01-17 14:36:26 -0800293 Root pkgRoot = getSkyframeExecutor().getPathEntries().get(0);
Kristina Chodorow335f0672015-11-16 23:19:13 +0000294 Path fooBuildFile = scratch.file("foo/BUILD");
295 Path fooDir = fooBuildFile.getParentDirectory();
296
neerajen89188eb2018-07-19 13:03:43 -0700297 // Our custom filesystem says that fooDir is neither a file nor directory nor symlink
laurentlbc0bd2102018-10-17 07:05:25 -0700298 FileStatus inconsistentFileStatus =
299 new FileStatus() {
300 @Override
301 public boolean isFile() {
302 return false;
303 }
Kristina Chodorow335f0672015-11-16 23:19:13 +0000304
laurentlbc0bd2102018-10-17 07:05:25 -0700305 @Override
306 public boolean isDirectory() {
307 return false;
308 }
Kristina Chodorow335f0672015-11-16 23:19:13 +0000309
laurentlbc0bd2102018-10-17 07:05:25 -0700310 @Override
311 public boolean isSymbolicLink() {
312 return false;
313 }
Kristina Chodorow335f0672015-11-16 23:19:13 +0000314
laurentlbc0bd2102018-10-17 07:05:25 -0700315 @Override
316 public boolean isSpecialFile() {
317 return false;
318 }
Kristina Chodorow335f0672015-11-16 23:19:13 +0000319
laurentlbc0bd2102018-10-17 07:05:25 -0700320 @Override
321 public long getSize() throws IOException {
322 return 0;
323 }
Kristina Chodorow335f0672015-11-16 23:19:13 +0000324
laurentlbc0bd2102018-10-17 07:05:25 -0700325 @Override
326 public long getLastModifiedTime() throws IOException {
327 return 0;
328 }
Kristina Chodorow335f0672015-11-16 23:19:13 +0000329
laurentlbc0bd2102018-10-17 07:05:25 -0700330 @Override
331 public long getLastChangeTime() throws IOException {
332 return 0;
333 }
Kristina Chodorow335f0672015-11-16 23:19:13 +0000334
laurentlbc0bd2102018-10-17 07:05:25 -0700335 @Override
336 public long getNodeId() throws IOException {
337 return 0;
338 }
339 };
neerajen89188eb2018-07-19 13:03:43 -0700340
341 fs.stubStat(fooBuildFile, inconsistentFileStatus);
Kristina Chodorow335f0672015-11-16 23:19:13 +0000342 RootedPath pkgRootedPath = RootedPath.toRootedPath(pkgRoot, fooDir);
janakrd1b2f962022-03-04 13:33:44 -0800343 SkyValue fooDirValue = FileStateValue.create(pkgRootedPath, SyscallCache.NO_CACHE, tsgm);
Kristina Chodorow335f0672015-11-16 23:19:13 +0000344 differencer.inject(ImmutableMap.of(FileStateValue.key(pkgRootedPath), fooDirValue));
brandjon54a7fcf2021-02-01 09:16:42 -0800345
346 Exception ex = evaluatePackageToException("@//foo");
347 String msg = ex.getMessage();
348 assertThat(msg).contains("Inconsistent filesystem operations");
349 assertThat(msg)
350 .contains(
351 "according to stat, existing path /workspace/foo/BUILD is neither"
352 + " a file nor directory nor symlink.");
janakra4a564a2021-03-18 12:40:11 -0700353 assertDetailedExitCode(
354 ex,
355 PackageLoading.Code.PERSISTENT_INCONSISTENT_FILESYSTEM_ERROR,
356 ExitCode.LOCAL_ENVIRONMENTAL_ERROR);
Kristina Chodorow335f0672015-11-16 23:19:13 +0000357 }
358
Florian Weikert92b22362015-12-03 10:17:18 +0000359 @Test
jmmvbfbd95f2020-08-31 13:12:38 -0700360 public void testPropagatesFilesystemInconsistencies_globbing() throws Exception {
Kristina Chodorow335f0672015-11-16 23:19:13 +0000361 RecordingDifferencer differencer = getSkyframeExecutor().getDifferencerForTesting();
tomluee6a6862018-01-17 14:36:26 -0800362 Root pkgRoot = getSkyframeExecutor().getPathEntries().get(0);
laurentlbc0bd2102018-10-17 07:05:25 -0700363 scratch.file(
364 "foo/BUILD",
Googlerbb0d36a2019-09-26 13:19:28 -0700365 "sh_library(name = 'foo', srcs = glob(['bar/**/baz.sh']))",
366 "x = 1//0" // causes 'foo' to be marked in error
367 );
Kristina Chodorow335f0672015-11-16 23:19:13 +0000368 Path bazFile = scratch.file("foo/bar/baz/baz.sh");
369 Path bazDir = bazFile.getParentDirectory();
370 Path barDir = bazDir.getParentDirectory();
371
Kristina Chodorow335f0672015-11-16 23:19:13 +0000372 // Our custom filesystem says "foo/bar/baz" does not exist but it also says that "foo/bar"
373 // has a child directory "baz".
aehligc801c392017-12-19 07:12:25 -0800374 fs.stubStat(bazDir, null);
Kristina Chodorow335f0672015-11-16 23:19:13 +0000375 RootedPath barDirRootedPath = RootedPath.toRootedPath(pkgRoot, barDir);
janakre54491e2018-07-11 16:29:13 -0700376 differencer.inject(
377 ImmutableMap.of(
378 DirectoryListingStateValue.key(barDirRootedPath),
379 DirectoryListingStateValue.create(
380 ImmutableList.of(new Dirent("baz", Dirent.Type.DIRECTORY)))));
brandjon54a7fcf2021-02-01 09:16:42 -0800381
382 Exception ex = evaluatePackageToException("@//foo");
383 String msg = ex.getMessage();
384 assertThat(msg).contains("Inconsistent filesystem operations");
385 assertThat(msg).contains("/workspace/foo/bar/baz is no longer an existing directory");
janakra4a564a2021-03-18 12:40:11 -0700386 assertDetailedExitCode(
387 ex,
388 PackageLoading.Code.PERSISTENT_INCONSISTENT_FILESYSTEM_ERROR,
389 ExitCode.LOCAL_ENVIRONMENTAL_ERROR);
Kristina Chodorow335f0672015-11-16 23:19:13 +0000390 }
391
392 /** Regression test for unexpected exception type from PackageValue. */
Florian Weikert92b22362015-12-03 10:17:18 +0000393 @Test
nharmata11d8ecf2021-06-09 12:27:08 -0700394 public void testDiscrepancyBetweenGlobbingErrors() throws Exception {
laurentlbc0bd2102018-10-17 07:05:25 -0700395 Path fooBuildFile =
396 scratch.file("foo/BUILD", "sh_library(name = 'foo', srcs = glob(['bar/*.sh']))");
Kristina Chodorow335f0672015-11-16 23:19:13 +0000397 Path fooDir = fooBuildFile.getParentDirectory();
398 Path barDir = fooDir.getRelative("bar");
399 scratch.file("foo/bar/baz.sh");
aehligc801c392017-12-19 07:12:25 -0800400 fs.scheduleMakeUnreadableAfterReaddir(barDir);
Kristina Chodorow335f0672015-11-16 23:19:13 +0000401
nharmatac68908f2021-12-10 11:14:41 -0800402 Exception ex =
403 evaluatePackageToException(
404 "@//foo",
405 // Use --keep_going, not --nokeep_going, semantics so as to exercise the situation we
406 // want to exercise.
407 //
408 // In --nokeep_going semantics, the GlobValue node's error would halt normal evaluation
409 // and trigger error bubbling. Then, during error bubbling we would freshly compute the
410 // PackageValue node again, meaning we would do non-Skyframe globbing except this time
411 // non-Skyframe globbing would encounter the io error, meaning there actually wouldn't
412 // be a discrepancy.
413 /*keepGoing=*/ true);
brandjon54a7fcf2021-02-01 09:16:42 -0800414 String msg = ex.getMessage();
415 assertThat(msg).contains("Inconsistent filesystem operations");
jhorvitz4e5e52b2021-02-26 14:02:45 -0800416 assertThat(msg).contains("Encountered error '/workspace/foo/bar (Permission denied)'");
janakra4a564a2021-03-18 12:40:11 -0700417 assertDetailedExitCode(
418 ex,
419 PackageLoading.Code.TRANSIENT_INCONSISTENT_FILESYSTEM_ERROR,
420 ExitCode.LOCAL_ENVIRONMENTAL_ERROR);
Kristina Chodorow335f0672015-11-16 23:19:13 +0000421 }
422
Janak Ramakrishnan063b4882016-07-18 20:33:28 +0000423 @SuppressWarnings("unchecked") // Cast of srcs attribute to Iterable<Label>.
424 @Test
425 public void testGlobOrderStable() throws Exception {
426 scratch.file("foo/BUILD", "sh_library(name = 'foo', srcs = glob(['**/*.txt']))");
427 scratch.file("foo/b.txt");
428 scratch.file("foo/c/c.txt");
429 preparePackageLoading(rootDirectory);
430 SkyKey skyKey = PackageValue.key(PackageIdentifier.parse("@//foo"));
nharmatac9fbe952019-06-13 09:39:23 -0700431 Package pkg = validPackageWithoutErrors(skyKey);
Googlera40c64a2020-08-11 16:39:39 -0700432 assertThat((Iterable<Label>) pkg.getTarget("foo").getAssociatedRule().getAttr("srcs"))
Janak Ramakrishnan063b4882016-07-18 20:33:28 +0000433 .containsExactly(
434 Label.parseAbsoluteUnchecked("//foo:b.txt"),
435 Label.parseAbsoluteUnchecked("//foo:c/c.txt"))
436 .inOrder();
437 scratch.file("foo/d.txt");
438 getSkyframeExecutor()
439 .invalidateFilesUnderPathForTesting(
440 reporter,
nharmatab4060b62017-04-04 17:11:39 +0000441 ModifiedFileSet.builder().modify(PathFragment.create("foo/d.txt")).build(),
tomluee6a6862018-01-17 14:36:26 -0800442 Root.fromPath(rootDirectory));
nharmatac9fbe952019-06-13 09:39:23 -0700443 pkg = validPackageWithoutErrors(skyKey);
Googlera40c64a2020-08-11 16:39:39 -0700444 assertThat((Iterable<Label>) pkg.getTarget("foo").getAssociatedRule().getAttr("srcs"))
Janak Ramakrishnan063b4882016-07-18 20:33:28 +0000445 .containsExactly(
446 Label.parseAbsoluteUnchecked("//foo:b.txt"),
447 Label.parseAbsoluteUnchecked("//foo:c/c.txt"),
448 Label.parseAbsoluteUnchecked("//foo:d.txt"))
449 .inOrder();
450 }
451
Janak Ramakrishnan063b4882016-07-18 20:33:28 +0000452 @Test
nharmata11d8ecf2021-06-09 12:27:08 -0700453 public void testGlobOrderStableWithNonSkyframeAndSkyframeComponents() throws Exception {
Janak Ramakrishnan063b4882016-07-18 20:33:28 +0000454 scratch.file("foo/BUILD", "sh_library(name = 'foo', srcs = glob(['*.txt']))");
455 scratch.file("foo/b.txt");
456 scratch.file("foo/a.config");
457 preparePackageLoading(rootDirectory);
458 SkyKey skyKey = PackageValue.key(PackageIdentifier.parse("@//foo"));
nharmatac9fbe952019-06-13 09:39:23 -0700459 assertSrcs(validPackageWithoutErrors(skyKey), "foo", "//foo:b.txt");
Janak Ramakrishnan063b4882016-07-18 20:33:28 +0000460 scratch.overwriteFile(
461 "foo/BUILD", "sh_library(name = 'foo', srcs = glob(['*.txt', '*.config']))");
462 getSkyframeExecutor()
463 .invalidateFilesUnderPathForTesting(
464 reporter,
nharmatab4060b62017-04-04 17:11:39 +0000465 ModifiedFileSet.builder().modify(PathFragment.create("foo/BUILD")).build(),
tomluee6a6862018-01-17 14:36:26 -0800466 Root.fromPath(rootDirectory));
nharmatac9fbe952019-06-13 09:39:23 -0700467 assertSrcs(validPackageWithoutErrors(skyKey), "foo", "//foo:a.config", "//foo:b.txt");
Janak Ramakrishnan063b4882016-07-18 20:33:28 +0000468 scratch.overwriteFile(
469 "foo/BUILD", "sh_library(name = 'foo', srcs = glob(['*.txt', '*.config'])) # comment");
470 getSkyframeExecutor()
471 .invalidateFilesUnderPathForTesting(
472 reporter,
nharmatab4060b62017-04-04 17:11:39 +0000473 ModifiedFileSet.builder().modify(PathFragment.create("foo/BUILD")).build(),
tomluee6a6862018-01-17 14:36:26 -0800474 Root.fromPath(rootDirectory));
nharmatac9fbe952019-06-13 09:39:23 -0700475 assertSrcs(validPackageWithoutErrors(skyKey), "foo", "//foo:a.config", "//foo:b.txt");
Nathan Harmata44e1e3a2016-08-23 21:22:17 +0000476 getSkyframeExecutor().resetEvaluator();
ajurkowskid74b0ec2020-04-13 10:58:21 -0700477 PackageOptions packageOptions = Options.getDefaults(PackageOptions.class);
478 packageOptions.defaultVisibility = ConstantRuleVisibility.PUBLIC;
479 packageOptions.showLoadingProgress = true;
480 packageOptions.globbingThreads = 7;
Klaus Aehlig6f33a1c2016-09-13 16:46:10 +0000481 getSkyframeExecutor()
482 .preparePackageLoading(
John Catere0d1d0e2017-11-28 20:47:41 -0800483 new PathPackageLocator(
484 outputBase,
tomluee6a6862018-01-17 14:36:26 -0800485 ImmutableList.of(Root.fromPath(rootDirectory)),
John Catere0d1d0e2017-11-28 20:47:41 -0800486 BazelSkyframeExecutorConstants.BUILD_FILES_BY_PRIORITY),
ajurkowskid74b0ec2020-04-13 10:58:21 -0700487 packageOptions,
adonovan240bdea2020-09-03 15:24:12 -0700488 Options.getDefaults(BuildLanguageOptions.class),
Klaus Aehlig6f33a1c2016-09-13 16:46:10 +0000489 UUID.randomUUID(),
Klaus Aehlig03b9cfd2016-09-14 13:14:39 +0000490 ImmutableMap.<String, String>of(),
Klaus Aehlig6f33a1c2016-09-13 16:46:10 +0000491 tsgm);
juliexxia651797f2018-08-01 11:45:13 -0700492 getSkyframeExecutor().setActionEnv(ImmutableMap.<String, String>of());
nharmatac9fbe952019-06-13 09:39:23 -0700493 assertSrcs(validPackageWithoutErrors(skyKey), "foo", "//foo:a.config", "//foo:b.txt");
Janak Ramakrishnan063b4882016-07-18 20:33:28 +0000494 }
495
Benjamin Peterson3b39e1e2020-01-31 10:21:38 -0800496 @Test
497 public void globEscapesAt() throws Exception {
498 scratch.file("foo/BUILD", "filegroup(name = 'foo', srcs = glob(['*.txt']))");
499 scratch.file("foo/@f.txt");
500 preparePackageLoading(rootDirectory);
501 SkyKey skyKey = PackageValue.key(PackageIdentifier.parse("@//foo"));
502 assertSrcs(validPackageWithoutErrors(skyKey), "foo", "//foo:@f.txt");
503
504 scratch.overwriteFile("foo/BUILD", "filegroup(name = 'foo', srcs = glob(['*.txt'])) # comment");
505 getSkyframeExecutor()
506 .invalidateFilesUnderPathForTesting(
507 reporter,
508 ModifiedFileSet.builder().modify(PathFragment.create("foo/BUILD")).build(),
509 Root.fromPath(rootDirectory));
510 assertSrcs(validPackageWithoutErrors(skyKey), "foo", "//foo:@f.txt");
511 }
512
Janak Ramakrishnanffbc28a2017-02-13 22:51:45 +0000513 /**
514 * Tests that a symlink to a file outside of the package root is handled consistently. If the
515 * default behavior of Bazel was changed from {@code
516 * ExternalFileAction#DEPEND_ON_EXTERNAL_PKG_FOR_EXTERNAL_REPO_PATHS} to {@code
517 * ExternalFileAction#ASSUME_NON_EXISTENT_AND_IMMUTABLE_FOR_EXTERNAL_PATHS} then foo/link.sh
518 * should no longer appear in the srcs of //foo:foo. However, either way the srcs should be the
519 * same independent of the evaluation being incremental or clean.
520 */
521 @Test
522 public void testGlobWithExternalSymlink() throws Exception {
523 scratch.file(
524 "foo/BUILD",
525 "sh_library(name = 'foo', srcs = glob(['*.sh']))",
526 "sh_library(name = 'bar', srcs = glob(['link.sh']))",
527 "sh_library(name = 'baz', srcs = glob(['subdir_link/*.txt']))");
528 scratch.file("foo/ordinary.sh");
529 Path externalTarget = scratch.file("../ops/target.txt");
530 FileSystemUtils.ensureSymbolicLink(scratch.resolve("foo/link.sh"), externalTarget);
531 FileSystemUtils.ensureSymbolicLink(
532 scratch.resolve("foo/subdir_link"), externalTarget.getParentDirectory());
533 preparePackageLoading(rootDirectory);
534 SkyKey fooKey = PackageValue.key(PackageIdentifier.parse("@//foo"));
nharmatac9fbe952019-06-13 09:39:23 -0700535 Package fooPkg = validPackageWithoutErrors(fooKey);
536 assertSrcs(fooPkg, "foo", "//foo:link.sh", "//foo:ordinary.sh");
537 assertSrcs(fooPkg, "bar", "//foo:link.sh");
538 assertSrcs(fooPkg, "baz", "//foo:subdir_link/target.txt");
Janak Ramakrishnanffbc28a2017-02-13 22:51:45 +0000539 scratch.overwriteFile(
540 "foo/BUILD",
541 "sh_library(name = 'foo', srcs = glob(['*.sh'])) #comment",
542 "sh_library(name = 'bar', srcs = glob(['link.sh']))",
543 "sh_library(name = 'baz', srcs = glob(['subdir_link/*.txt']))");
544 getSkyframeExecutor()
545 .invalidateFilesUnderPathForTesting(
546 reporter,
nharmatab4060b62017-04-04 17:11:39 +0000547 ModifiedFileSet.builder().modify(PathFragment.create("foo/BUILD")).build(),
tomluee6a6862018-01-17 14:36:26 -0800548 Root.fromPath(rootDirectory));
nharmatac9fbe952019-06-13 09:39:23 -0700549 Package fooPkg2 = validPackageWithoutErrors(fooKey);
550 assertThat(fooPkg2).isNotEqualTo(fooPkg);
551 assertSrcs(fooPkg2, "foo", "//foo:link.sh", "//foo:ordinary.sh");
552 assertSrcs(fooPkg2, "bar", "//foo:link.sh");
553 assertSrcs(fooPkg2, "baz", "//foo:subdir_link/target.txt");
Janak Ramakrishnanffbc28a2017-02-13 22:51:45 +0000554 }
555
nharmatac9fbe952019-06-13 09:39:23 -0700556 private static void assertSrcs(Package pkg, String targetName, String... expected)
Janak Ramakrishnanffbc28a2017-02-13 22:51:45 +0000557 throws NoSuchTargetException {
558 List<Label> expectedLabels = new ArrayList<>();
559 for (String item : expected) {
560 expectedLabels.add(Label.parseAbsoluteUnchecked(item));
561 }
nharmatac9fbe952019-06-13 09:39:23 -0700562 assertThat(getSrcs(pkg, targetName)).containsExactlyElementsIn(expectedLabels).inOrder();
Janak Ramakrishnanffbc28a2017-02-13 22:51:45 +0000563 }
564
565 @SuppressWarnings("unchecked")
nharmatac9fbe952019-06-13 09:39:23 -0700566 private static Iterable<Label> getSrcs(Package pkg, String targetName)
Janak Ramakrishnanffbc28a2017-02-13 22:51:45 +0000567 throws NoSuchTargetException {
Googlera40c64a2020-08-11 16:39:39 -0700568 return (Iterable<Label>) pkg.getTarget(targetName).getAssociatedRule().getAttr("srcs");
Janak Ramakrishnanffbc28a2017-02-13 22:51:45 +0000569 }
laurentlbc0bd2102018-10-17 07:05:25 -0700570
Florian Weikert92b22362015-12-03 10:17:18 +0000571 @Test
Janak Ramakrishnan4e2e4082017-01-06 06:28:45 +0000572 public void testOneNewElementInMultipleGlob() throws Exception {
573 scratch.file(
574 "foo/BUILD",
575 "sh_library(name = 'foo', srcs = glob(['*.sh']))",
576 "sh_library(name = 'bar', srcs = glob(['*.sh', '*.txt']))");
577 preparePackageLoading(rootDirectory);
578 SkyKey skyKey = PackageValue.key(PackageIdentifier.parse("@//foo"));
nharmatac9fbe952019-06-13 09:39:23 -0700579 Package pkg = validPackageWithoutErrors(skyKey);
580 scratch.file("foo/irrelevant");
Janak Ramakrishnan4e2e4082017-01-06 06:28:45 +0000581 getSkyframeExecutor()
582 .invalidateFilesUnderPathForTesting(
583 reporter,
nharmatab4060b62017-04-04 17:11:39 +0000584 ModifiedFileSet.builder().modify(PathFragment.create("foo/irrelevant")).build(),
tomluee6a6862018-01-17 14:36:26 -0800585 Root.fromPath(rootDirectory));
nharmatac9fbe952019-06-13 09:39:23 -0700586 assertThat(validPackageWithoutErrors(skyKey)).isSameInstanceAs(pkg);
Janak Ramakrishnan4e2e4082017-01-06 06:28:45 +0000587 }
588
589 @Test
590 public void testNoNewElementInMultipleGlob() throws Exception {
591 scratch.file(
592 "foo/BUILD",
593 "sh_library(name = 'foo', srcs = glob(['*.sh', '*.txt']))",
594 "sh_library(name = 'bar', srcs = glob(['*.sh', '*.txt']))");
595 preparePackageLoading(rootDirectory);
596 SkyKey skyKey = PackageValue.key(PackageIdentifier.parse("@//foo"));
nharmatac9fbe952019-06-13 09:39:23 -0700597 Package pkg = validPackageWithoutErrors(skyKey);
598 scratch.file("foo/irrelevant");
Janak Ramakrishnan4e2e4082017-01-06 06:28:45 +0000599 getSkyframeExecutor()
600 .invalidateFilesUnderPathForTesting(
601 reporter,
nharmatab4060b62017-04-04 17:11:39 +0000602 ModifiedFileSet.builder().modify(PathFragment.create("foo/irrelevant")).build(),
tomluee6a6862018-01-17 14:36:26 -0800603 Root.fromPath(rootDirectory));
nharmatac9fbe952019-06-13 09:39:23 -0700604 assertThat(validPackageWithoutErrors(skyKey)).isSameInstanceAs(pkg);
Janak Ramakrishnan4e2e4082017-01-06 06:28:45 +0000605 }
606
607 @Test
gregceb100b1d2020-05-20 10:22:17 -0700608 public void testTransitiveStarlarkDepsStoredInPackage() throws Exception {
laurentlbc0bd2102018-10-17 07:05:25 -0700609 scratch.file("foo/BUILD", "load('//bar:ext.bzl', 'a')");
Kristina Chodorow335f0672015-11-16 23:19:13 +0000610 scratch.file("bar/BUILD");
laurentlbc0bd2102018-10-17 07:05:25 -0700611 scratch.file("bar/ext.bzl", "load('//baz:ext.bzl', 'b')", "a = b");
Kristina Chodorow335f0672015-11-16 23:19:13 +0000612 scratch.file("baz/BUILD");
laurentlbc0bd2102018-10-17 07:05:25 -0700613 scratch.file("baz/ext.bzl", "b = 1");
Kristina Chodorow335f0672015-11-16 23:19:13 +0000614 scratch.file("qux/BUILD");
laurentlbc0bd2102018-10-17 07:05:25 -0700615 scratch.file("qux/ext.bzl", "c = 1");
Kristina Chodorow335f0672015-11-16 23:19:13 +0000616
Ulf Adamsc73051c62016-03-23 09:18:13 +0000617 preparePackageLoading(rootDirectory);
Kristina Chodorow335f0672015-11-16 23:19:13 +0000618
Brian Silvermand7d6d622016-03-17 09:53:39 +0000619 SkyKey skyKey = PackageValue.key(PackageIdentifier.parse("@//foo"));
nharmatac9fbe952019-06-13 09:39:23 -0700620 Package pkg = validPackageWithoutErrors(skyKey);
gregce7ecc2d62020-04-17 15:32:47 -0700621 assertThat(pkg.getStarlarkFileDependencies())
dannark90e2b4b2018-06-27 13:35:04 -0700622 .containsExactly(
623 Label.parseAbsolute("//bar:ext.bzl", ImmutableMap.of()),
624 Label.parseAbsolute("//baz:ext.bzl", ImmutableMap.of()));
Kristina Chodorow335f0672015-11-16 23:19:13 +0000625
laurentlbc0bd2102018-10-17 07:05:25 -0700626 scratch.overwriteFile("bar/ext.bzl", "load('//qux:ext.bzl', 'c')", "a = c");
tomluee6a6862018-01-17 14:36:26 -0800627 getSkyframeExecutor()
628 .invalidateFilesUnderPathForTesting(
629 reporter,
630 ModifiedFileSet.builder().modify(PathFragment.create("bar/ext.bzl")).build(),
631 Root.fromPath(rootDirectory));
Kristina Chodorow335f0672015-11-16 23:19:13 +0000632
nharmatac9fbe952019-06-13 09:39:23 -0700633 pkg = validPackageWithoutErrors(skyKey);
gregce7ecc2d62020-04-17 15:32:47 -0700634 assertThat(pkg.getStarlarkFileDependencies())
dannark90e2b4b2018-06-27 13:35:04 -0700635 .containsExactly(
636 Label.parseAbsolute("//bar:ext.bzl", ImmutableMap.of()),
637 Label.parseAbsolute("//qux:ext.bzl", ImmutableMap.of()));
Kristina Chodorow335f0672015-11-16 23:19:13 +0000638 }
639
Florian Weikert92b22362015-12-03 10:17:18 +0000640 @Test
gregceb100b1d2020-05-20 10:22:17 -0700641 public void testNonExistingStarlarkExtension() throws Exception {
adonovanc0e86902020-11-19 15:50:29 -0800642 scratch.file("test/starlark/BUILD", "load('//test/starlark:bad_extension.bzl', 'some_symbol')");
Kristina Chodorow335f0672015-11-16 23:19:13 +0000643 invalidatePackages();
644
brandjon54a7fcf2021-02-01 09:16:42 -0800645 Exception ex = evaluatePackageToException("@//test/starlark");
646 assertThat(ex)
647 .hasMessageThat()
648 .isEqualTo(
649 "error loading package 'test/starlark': "
650 + "cannot load '//test/starlark:bad_extension.bzl': no such file");
janakra4a564a2021-03-18 12:40:11 -0700651 assertDetailedExitCode(
652 ex, PackageLoading.Code.IMPORT_STARLARK_FILE_ERROR, ExitCode.BUILD_FAILURE);
Kristina Chodorow335f0672015-11-16 23:19:13 +0000653 }
654
Florian Weikert92b22362015-12-03 10:17:18 +0000655 @Test
gregceb100b1d2020-05-20 10:22:17 -0700656 public void testNonExistingStarlarkExtensionFromExtension() throws Exception {
laurentlbc0bd2102018-10-17 07:05:25 -0700657 scratch.file(
gregce0503fee2020-06-11 09:22:27 -0700658 "test/starlark/extension.bzl",
659 "load('//test/starlark:bad_extension.bzl', 'some_symbol')",
Kristina Chodorow335f0672015-11-16 23:19:13 +0000660 "a = 'a'");
adonovanc0e86902020-11-19 15:50:29 -0800661 scratch.file("test/starlark/BUILD", "load('//test/starlark:extension.bzl', 'a')");
Kristina Chodorow335f0672015-11-16 23:19:13 +0000662 invalidatePackages();
663
brandjon54a7fcf2021-02-01 09:16:42 -0800664 Exception ex = evaluatePackageToException("@//test/starlark");
665 assertThat(ex)
diamondmd6df9802019-03-19 06:53:43 -0700666 .hasMessageThat()
667 .isEqualTo(
gregce0503fee2020-06-11 09:22:27 -0700668 "error loading package 'test/starlark': "
adonovandcc23002020-12-10 12:09:16 -0800669 + "at /workspace/test/starlark/extension.bzl:1:6: "
gregce0503fee2020-06-11 09:22:27 -0700670 + "cannot load '//test/starlark:bad_extension.bzl': no such file");
janakra4a564a2021-03-18 12:40:11 -0700671 assertDetailedExitCode(
672 ex, PackageLoading.Code.IMPORT_STARLARK_FILE_ERROR, ExitCode.BUILD_FAILURE);
Kristina Chodorow335f0672015-11-16 23:19:13 +0000673 }
674
Florian Weikert92b22362015-12-03 10:17:18 +0000675 @Test
brandjon6c63b8f2021-02-01 10:17:42 -0800676 public void testBuiltinsInjectionFailure() throws Exception {
677 setBuildLanguageOptions("--experimental_builtins_bzl_path=tools/builtins_staging");
678 scratch.file(
679 "tools/builtins_staging/exports.bzl",
680 "1 // 0 # <-- dynamic error",
681 "exported_toplevels = {}",
682 "exported_rules = {}",
683 "exported_to_java = {}");
684 scratch.file("pkg/BUILD");
685
686 Exception ex = evaluatePackageToException("@//pkg");
687 assertThat(ex)
688 .hasMessageThat()
689 .isEqualTo(
690 "error loading package 'pkg': Internal error while loading Starlark builtins: Failed"
691 + " to load builtins sources: initialization of module 'exports.bzl' (internal)"
692 + " failed");
janakra4a564a2021-03-18 12:40:11 -0700693 assertDetailedExitCode(
694 ex, PackageLoading.Code.BUILTINS_INJECTION_FAILURE, ExitCode.BUILD_FAILURE);
brandjon6c63b8f2021-02-01 10:17:42 -0800695 }
696
697 @Test
gregceb100b1d2020-05-20 10:22:17 -0700698 public void testSymlinkCycleWithStarlarkExtension() throws Exception {
gregce0503fee2020-06-11 09:22:27 -0700699 Path extensionFilePath = scratch.resolve("/workspace/test/starlark/extension.bzl");
nharmatab4060b62017-04-04 17:11:39 +0000700 FileSystemUtils.ensureSymbolicLink(extensionFilePath, PathFragment.create("extension.bzl"));
adonovanc0e86902020-11-19 15:50:29 -0800701 scratch.file("test/starlark/BUILD", "load('//test/starlark:extension.bzl', 'a')");
Kristina Chodorow335f0672015-11-16 23:19:13 +0000702 invalidatePackages();
703
brandjon54a7fcf2021-02-01 09:16:42 -0800704 Exception ex = evaluatePackageToException("@//test/starlark");
705 assertThat(ex)
diamondmd6df9802019-03-19 06:53:43 -0700706 .hasMessageThat()
707 .isEqualTo(
gregce0503fee2020-06-11 09:22:27 -0700708 "error loading package 'test/starlark': Encountered error while reading extension "
709 + "file 'test/starlark/extension.bzl': Symlink cycle");
janakra4a564a2021-03-18 12:40:11 -0700710 assertDetailedExitCode(
711 ex, PackageLoading.Code.IMPORT_STARLARK_FILE_ERROR, ExitCode.BUILD_FAILURE);
Kristina Chodorow335f0672015-11-16 23:19:13 +0000712 }
713
Florian Weikert92b22362015-12-03 10:17:18 +0000714 @Test
Kristina Chodorow335f0672015-11-16 23:19:13 +0000715 public void testIOErrorLookingForSubpackageForLabelIsHandled() throws Exception {
brandjon54a7fcf2021-02-01 09:16:42 -0800716 scratch.file(
717 "foo/BUILD", //
718 "sh_library(name = 'foo', srcs = ['bar/baz.sh'])");
Kristina Chodorow335f0672015-11-16 23:19:13 +0000719 Path barBuildFile = scratch.file("foo/bar/BUILD");
aehligc801c392017-12-19 07:12:25 -0800720 fs.stubStatError(barBuildFile, new IOException("nope"));
brandjon54a7fcf2021-02-01 09:16:42 -0800721
722 evaluatePackageToException("@//foo");
Kristina Chodorow335f0672015-11-16 23:19:13 +0000723 assertContainsEvent("nope");
724 }
725
Florian Weikert92b22362015-12-03 10:17:18 +0000726 @Test
Googler83439e62019-09-24 12:11:30 -0700727 public void testLoadOK() throws Exception {
728 scratch.file("p/a.bzl", "a = 1; b = 1; d = 1");
729 scratch.file("p/subdir/a.bzl", "c = 1; e = 1");
730 scratch.file(
731 "p/BUILD",
732 //
733 "load(':a.bzl', 'a')",
734 "load('a.bzl', 'b')",
735 "load('subdir/a.bzl', 'c')",
736 "load('//p:a.bzl', 'd')",
737 "load('//p:subdir/a.bzl', 'e')");
738 validPackageWithoutErrors(PackageValue.key(PackageIdentifier.parse("@//p")));
739 }
740
adonovane0bd9d32020-09-18 12:51:15 -0700741 // See WorkspaceFileFunctionTest for tests that exercise load('@repo...').
Googler83439e62019-09-24 12:11:30 -0700742
743 @Test
744 public void testLoadBadLabel() throws Exception {
745 scratch.file("p/BUILD", "load('this\tis not a label', 'a')");
746 reporter.removeHandler(failFastHandler);
747 SkyKey key = PackageValue.key(PackageIdentifier.parse("@//p"));
748 SkyframeExecutorTestUtils.evaluate(skyframeExecutor, key, /*keepGoing=*/ false, reporter);
749 assertContainsEvent(
750 "in load statement: invalid target name 'this<?>is not a label': target names may not"
751 + " contain non-printable characters");
Kristina Chodorow335f0672015-11-16 23:19:13 +0000752 }
753
Florian Weikert92b22362015-12-03 10:17:18 +0000754 @Test
Googler83439e62019-09-24 12:11:30 -0700755 public void testLoadFromExternalPackage() throws Exception {
756 scratch.file("p/BUILD", "load('//external:file.bzl', 'a')");
757 reporter.removeHandler(failFastHandler);
758 SkyKey key = PackageValue.key(PackageIdentifier.parse("@//p"));
759 SkyframeExecutorTestUtils.evaluate(skyframeExecutor, key, /*keepGoing=*/ false, reporter);
760 assertContainsEvent("Starlark files may not be loaded from the //external package");
761 }
762
763 @Test
764 public void testLoadWithoutBzlSuffix() throws Exception {
765 scratch.file("p/BUILD", "load('//p:file.starlark', 'a')");
766 reporter.removeHandler(failFastHandler);
767 SkyKey key = PackageValue.key(PackageIdentifier.parse("@//p"));
768 SkyframeExecutorTestUtils.evaluate(skyframeExecutor, key, /*keepGoing=*/ false, reporter);
769 assertContainsEvent("The label must reference a file with extension '.bzl'");
Kristina Chodorow335f0672015-11-16 23:19:13 +0000770 }
771
Florian Weikert92b22362015-12-03 10:17:18 +0000772 @Test
Kristina Chodorow335f0672015-11-16 23:19:13 +0000773 public void testBadWorkspaceFile() throws Exception {
774 Path workspacePath = scratch.overwriteFile("WORKSPACE", "junk");
Brian Silvermand7d6d622016-03-17 09:53:39 +0000775 SkyKey skyKey = PackageValue.key(PackageIdentifier.createInMainRepo("external"));
Kristina Chodorow335f0672015-11-16 23:19:13 +0000776 getSkyframeExecutor()
777 .invalidate(
778 Predicates.equalTo(
shahan602cc852018-06-06 20:09:57 -0700779 FileStateValue.key(
Kristina Chodorow335f0672015-11-16 23:19:13 +0000780 RootedPath.toRootedPath(
tomluee6a6862018-01-17 14:36:26 -0800781 Root.fromPath(workspacePath.getParentDirectory()),
nharmatab4060b62017-04-04 17:11:39 +0000782 PathFragment.create(workspacePath.getBaseName())))));
Kristina Chodorow335f0672015-11-16 23:19:13 +0000783
784 reporter.removeHandler(failFastHandler);
785 EvaluationResult<PackageValue> result =
786 SkyframeExecutorTestUtils.evaluate(
787 getSkyframeExecutor(), skyKey, /*keepGoing=*/ false, reporter);
lberkiaea56b32017-05-30 12:35:33 +0200788 assertThat(result.hasError()).isFalse();
789 assertThat(result.get(skyKey).getPackage().containsErrors()).isTrue();
Kristina Chodorow335f0672015-11-16 23:19:13 +0000790 }
791
Nathan Harmata2022ad82016-02-22 23:04:14 +0000792 // Regression test for the two ugly consequences of a bug where GlobFunction incorrectly matched
793 // dangling symlinks.
794 @Test
795 public void testIncrementalSkyframeHybridGlobbingOnDanglingSymlink() throws Exception {
laurentlbc0bd2102018-10-17 07:05:25 -0700796 Path packageDirPath =
797 scratch.file("foo/BUILD", "exports_files(glob(['*.txt']))").getParentDirectory();
Nathan Harmata2022ad82016-02-22 23:04:14 +0000798 scratch.file("foo/existing.txt");
799 FileSystemUtils.ensureSymbolicLink(packageDirPath.getChild("dangling.txt"), "nope");
800
Ulf Adamsc73051c62016-03-23 09:18:13 +0000801 preparePackageLoading(rootDirectory);
Nathan Harmata2022ad82016-02-22 23:04:14 +0000802
Brian Silvermand7d6d622016-03-17 09:53:39 +0000803 SkyKey skyKey = PackageValue.key(PackageIdentifier.parse("@//foo"));
nharmatac9fbe952019-06-13 09:39:23 -0700804 Package pkg = validPackageWithoutErrors(skyKey);
805 assertThat(pkg.containsErrors()).isFalse();
806 assertThat(pkg.getTarget("existing.txt").getName()).isEqualTo("existing.txt");
807 assertThrows(NoSuchTargetException.class, () -> pkg.getTarget("dangling.txt"));
Nathan Harmata2022ad82016-02-22 23:04:14 +0000808
laurentlbc0bd2102018-10-17 07:05:25 -0700809 scratch.overwriteFile(
810 "foo/BUILD", "exports_files(glob(['*.txt']))", "#some-irrelevant-comment");
Nathan Harmata2022ad82016-02-22 23:04:14 +0000811
tomluee6a6862018-01-17 14:36:26 -0800812 getSkyframeExecutor()
813 .invalidateFilesUnderPathForTesting(
814 reporter,
815 ModifiedFileSet.builder().modify(PathFragment.create("foo/BUILD")).build(),
816 Root.fromPath(rootDirectory));
Nathan Harmata2022ad82016-02-22 23:04:14 +0000817
nharmatac9fbe952019-06-13 09:39:23 -0700818 Package pkg2 = validPackageWithoutErrors(skyKey);
819 assertThat(pkg2.containsErrors()).isFalse();
820 assertThat(pkg2.getTarget("existing.txt").getName()).isEqualTo("existing.txt");
821 assertThrows(NoSuchTargetException.class, () -> pkg2.getTarget("dangling.txt"));
jcater83130f42019-04-30 14:29:28 -0700822 // One consequence of the bug was that dangling symlinks were matched by globs evaluated by
823 // Skyframe globbing, meaning there would incorrectly be corresponding targets in packages
824 // that had skyframe cache hits during skyframe hybrid globbing.
Nathan Harmata2022ad82016-02-22 23:04:14 +0000825
826 scratch.file("foo/nope");
tomluee6a6862018-01-17 14:36:26 -0800827 getSkyframeExecutor()
828 .invalidateFilesUnderPathForTesting(
829 reporter,
830 ModifiedFileSet.builder().modify(PathFragment.create("foo/nope")).build(),
831 Root.fromPath(rootDirectory));
Nathan Harmata2022ad82016-02-22 23:04:14 +0000832
nharmatac9fbe952019-06-13 09:39:23 -0700833 Package newPkg = validPackageWithoutErrors(skyKey);
834 assertThat(newPkg.containsErrors()).isFalse();
835 assertThat(newPkg.getTarget("existing.txt").getName()).isEqualTo("existing.txt");
Nathan Harmata2022ad82016-02-22 23:04:14 +0000836 // Another consequence of the bug is that change pruning would incorrectly cut off changes that
837 // caused a dangling symlink potentially matched by a glob to come into existence.
nharmatac9fbe952019-06-13 09:39:23 -0700838 assertThat(newPkg.getTarget("dangling.txt").getName()).isEqualTo("dangling.txt");
839 assertThat(newPkg).isNotSameInstanceAs(pkg);
Nathan Harmata2022ad82016-02-22 23:04:14 +0000840 }
841
Nathan Harmata86c319e2016-02-25 01:12:22 +0000842 // Regression test for Skyframe globbing incorrectly matching the package's directory path on
843 // 'glob(['**'], exclude_directories = 0)'. We test for this directly by triggering
nharmata11d8ecf2021-06-09 12:27:08 -0700844 // hybrid globbing (gives coverage for both non-skyframe globbing and skyframe globbing).
Nathan Harmata86c319e2016-02-25 01:12:22 +0000845 @Test
846 public void testRecursiveGlobNeverMatchesPackageDirectory() throws Exception {
laurentlbc0bd2102018-10-17 07:05:25 -0700847 scratch.file(
848 "foo/BUILD",
Nathan Harmata86c319e2016-02-25 01:12:22 +0000849 "[sh_library(name = x + '-matched') for x in glob(['**'], exclude_directories = 0)]");
850 scratch.file("foo/bar");
851
Ulf Adamsc73051c62016-03-23 09:18:13 +0000852 preparePackageLoading(rootDirectory);
Nathan Harmata86c319e2016-02-25 01:12:22 +0000853
Brian Silvermand7d6d622016-03-17 09:53:39 +0000854 SkyKey skyKey = PackageValue.key(PackageIdentifier.parse("@//foo"));
nharmatac9fbe952019-06-13 09:39:23 -0700855 Package pkg = validPackageWithoutErrors(skyKey);
856 assertThat(pkg.containsErrors()).isFalse();
857 assertThat(pkg.getTarget("bar-matched").getName()).isEqualTo("bar-matched");
858 assertThrows(NoSuchTargetException.class, () -> pkg.getTarget("-matched"));
Nathan Harmata86c319e2016-02-25 01:12:22 +0000859
laurentlbc0bd2102018-10-17 07:05:25 -0700860 scratch.overwriteFile(
861 "foo/BUILD",
Nathan Harmata86c319e2016-02-25 01:12:22 +0000862 "[sh_library(name = x + '-matched') for x in glob(['**'], exclude_directories = 0)]",
863 "#some-irrelevant-comment");
tomluee6a6862018-01-17 14:36:26 -0800864 getSkyframeExecutor()
865 .invalidateFilesUnderPathForTesting(
866 reporter,
867 ModifiedFileSet.builder().modify(PathFragment.create("foo/BUILD")).build(),
868 Root.fromPath(rootDirectory));
Nathan Harmata86c319e2016-02-25 01:12:22 +0000869
nharmatac9fbe952019-06-13 09:39:23 -0700870 Package pkg2 = validPackageWithoutErrors(skyKey);
871 assertThat(pkg2.containsErrors()).isFalse();
872 assertThat(pkg2.getTarget("bar-matched").getName()).isEqualTo("bar-matched");
873 assertThrows(NoSuchTargetException.class, () -> pkg2.getTarget("-matched"));
Nathan Harmata86c319e2016-02-25 01:12:22 +0000874 }
875
nharmataff688bf2017-06-07 17:03:52 -0400876 @Test
877 public void testPackageLoadingErrorOnIOExceptionReadingBuildFile() throws Exception {
878 Path fooBuildFilePath = scratch.file("foo/BUILD");
879 IOException exn = new IOException("nope");
aehligc801c392017-12-19 07:12:25 -0800880 fs.throwExceptionOnGetInputStream(fooBuildFilePath, exn);
nharmataff688bf2017-06-07 17:03:52 -0400881
brandjon54a7fcf2021-02-01 09:16:42 -0800882 Exception ex = evaluatePackageToException("@//foo");
883 assertThat(ex).hasMessageThat().contains("nope");
884 assertThat(ex).isInstanceOf(NoSuchPackageException.class);
885 assertThat(ex).hasCauseThat().isInstanceOf(IOException.class);
janakra4a564a2021-03-18 12:40:11 -0700886 assertDetailedExitCode(ex, PackageLoading.Code.BUILD_FILE_MISSING, ExitCode.BUILD_FAILURE);
nharmataff688bf2017-06-07 17:03:52 -0400887 }
888
nharmatabea67e92017-06-16 00:26:27 +0200889 @Test
890 public void testPackageLoadingErrorOnIOExceptionReadingBzlFile() throws Exception {
891 scratch.file("foo/BUILD", "load('//foo:bzl.bzl', 'x')");
892 Path fooBzlFilePath = scratch.file("foo/bzl.bzl");
893 IOException exn = new IOException("nope");
aehligc801c392017-12-19 07:12:25 -0800894 fs.throwExceptionOnGetInputStream(fooBzlFilePath, exn);
nharmatabea67e92017-06-16 00:26:27 +0200895
brandjon54a7fcf2021-02-01 09:16:42 -0800896 Exception ex = evaluatePackageToException("@//foo");
897 assertThat(ex).hasMessageThat().contains("nope");
898 assertThat(ex).isInstanceOf(NoSuchPackageException.class);
899 assertThat(ex).hasCauseThat().isInstanceOf(IOException.class);
janakra4a564a2021-03-18 12:40:11 -0700900 assertDetailedExitCode(
901 ex, PackageLoading.Code.IMPORT_STARLARK_FILE_ERROR, ExitCode.BUILD_FAILURE);
nharmatabea67e92017-06-16 00:26:27 +0200902 }
903
nharmata58613102018-10-11 15:10:04 -0700904 @Test
905 public void testLabelsCrossesSubpackageBoundaries() throws Exception {
906 reporter.removeHandler(failFastHandler);
907
908 scratch.file("pkg/BUILD", "exports_files(['sub/blah'])");
909 scratch.file("pkg/sub/BUILD");
910 invalidatePackages();
911
912 SkyKey skyKey = PackageValue.key(PackageIdentifier.parse("@//pkg"));
laurentlbc0bd2102018-10-17 07:05:25 -0700913 EvaluationResult<PackageValue> result =
914 SkyframeExecutorTestUtils.evaluate(
915 getSkyframeExecutor(), skyKey, /*keepGoing=*/ false, reporter);
nharmata58613102018-10-11 15:10:04 -0700916 assertThatEvaluationResult(result).hasNoError();
917 assertThat(result.get(skyKey).getPackage().containsErrors()).isTrue();
laurentlbaf489f22019-07-29 08:19:03 -0700918 assertContainsEvent("Label '//pkg:sub/blah' is invalid because 'pkg/sub' is a subpackage");
nharmata58613102018-10-11 15:10:04 -0700919 }
920
921 @Test
922 public void testSymlinkCycleEncounteredWhileHandlingLabelCrossingSubpackageBoundaries()
923 throws Exception {
nharmata58613102018-10-11 15:10:04 -0700924 scratch.file("pkg/BUILD", "exports_files(['sub/blah'])");
925 Path subBuildFilePath = scratch.dir("pkg/sub").getChild("BUILD");
926 FileSystemUtils.ensureSymbolicLink(subBuildFilePath, subBuildFilePath);
927 invalidatePackages();
928
brandjon54a7fcf2021-02-01 09:16:42 -0800929 Exception ex = evaluatePackageToException("@//pkg");
930 assertThat(ex).isInstanceOf(BuildFileNotFoundException.class);
931 assertThat(ex)
nharmata58613102018-10-11 15:10:04 -0700932 .hasMessageThat()
933 .contains(
934 "no such package 'pkg/sub': Symlink cycle detected while trying to find BUILD file");
935 assertContainsEvent("circular symlinks detected");
936 }
937
arostovtsevde789892021-11-17 11:29:23 -0800938 // Regression test for b/206459361.
939 @Test
940 public void nonSkyframeGlobbingIOException_andLabelCrossingSubpackageBoundaries_withKeepGoing()
941 throws Exception {
942 reporter.removeHandler(failFastHandler);
943
944 // When a package's BUILD file and the relevant filesystem state is such that non-Skyframe
945 // globbing will encounter an IOException due to a directory symlink cycle *and* the BUILD file
946 // defines a target with a label that crosses subpackage boundaries,
947 Path pkgBUILDPath =
948 scratch.file(
949 "pkg/BUILD",
950 "exports_files(['sub/blah']) # label crossing subpackage boundaries", //
951 "glob(['globcycle/**/foo.txt']) # triggers non-Skyframe globbing error");
952 scratch.file("pkg/sub/BUILD");
953 Path pkgGlobcyclePath = pkgBUILDPath.getParentDirectory().getChild("globcycle");
954 FileSystemUtils.ensureSymbolicLink(pkgGlobcyclePath, pkgGlobcyclePath);
955 assertThrows(IOException.class, () -> pkgGlobcyclePath.statIfFound(Symlinks.FOLLOW));
956
957 invalidatePackages();
958
959 // ... and we evaluate the package with keepGoing == true, we expect the evaluation to fail with
960 // the non-Skyframe globbing error, but for the label crossing event to *not* get added (because
961 // the globbing IOException would put Package.Builder in a state on which we cannot run
962 // handleLabelsCrossingSubpackagesAndPropagateInconsistentFilesystemExceptions).
963 SkyKey pkgKey = PackageValue.key(PackageIdentifier.parse("@//pkg"));
964 EvaluationResult<PackageValue> result =
965 SkyframeExecutorTestUtils.evaluate(
966 getSkyframeExecutor(), pkgKey, /*keepGoing=*/ true, reporter);
967 assertThatEvaluationResult(result)
968 .hasErrorEntryForKeyThat(pkgKey)
969 .hasExceptionThat()
970 .isInstanceOf(NoSuchPackageException.class);
971 assertThatEvaluationResult(result)
972 .hasErrorEntryForKeyThat(pkgKey)
973 .hasExceptionThat()
974 .hasMessageThat()
975 .contains("Symlink cycle: /workspace/pkg/globcycle");
976 assertDoesNotContainEvent(
977 "Label '//pkg:sub/blah' is invalid because 'pkg/sub' is a subpackage");
978 }
979
nharmatac9fbe952019-06-13 09:39:23 -0700980 @Test
jmmvbfbd95f2020-08-31 13:12:38 -0700981 public void testGlobAllowEmpty_paramValueMustBeBoolean() throws Exception {
nharmatac9fbe952019-06-13 09:39:23 -0700982 reporter.removeHandler(failFastHandler);
983
984 scratch.file("pkg/BUILD", "x = " + "glob(['*.foo'], allow_empty = 5)");
985 invalidatePackages();
986
987 SkyKey skyKey = PackageValue.key(PackageIdentifier.parse("@//pkg"));
michajloc17b80e2020-07-20 10:00:28 -0700988 validPackage(skyKey);
nharmatac9fbe952019-06-13 09:39:23 -0700989
michajloc17b80e2020-07-20 10:00:28 -0700990 assertContainsEvent("expected boolean for argument `allow_empty`, got `5`");
nharmatac9fbe952019-06-13 09:39:23 -0700991 }
992
993 @Test
jmmvbfbd95f2020-08-31 13:12:38 -0700994 public void testGlobAllowEmpty_functionParam() throws Exception {
nharmatac9fbe952019-06-13 09:39:23 -0700995 scratch.file("pkg/BUILD", "x = " + "glob(['*.foo'], allow_empty=True)");
996 invalidatePackages();
997
998 SkyKey skyKey = PackageValue.key(PackageIdentifier.parse("@//pkg"));
999 Package pkg = validPackage(skyKey);
1000 assertThat(pkg.containsErrors()).isFalse();
michajloc17b80e2020-07-20 10:00:28 -07001001 assertNoEvents();
nharmatac9fbe952019-06-13 09:39:23 -07001002 }
1003
1004 @Test
jmmvbfbd95f2020-08-31 13:12:38 -07001005 public void testGlobAllowEmpty_starlarkOption() throws Exception {
nharmatac9fbe952019-06-13 09:39:23 -07001006 preparePackageLoadingWithCustomStarklarkSemanticsOptions(
adonovan240bdea2020-09-03 15:24:12 -07001007 Options.parse(BuildLanguageOptions.class, "--incompatible_disallow_empty_glob=false")
nharmatac9fbe952019-06-13 09:39:23 -07001008 .getOptions(),
1009 rootDirectory);
1010
1011 scratch.file("pkg/BUILD", "x = " + "glob(['*.foo'])");
1012 invalidatePackages();
1013
1014 SkyKey skyKey = PackageValue.key(PackageIdentifier.parse("@//pkg"));
1015 Package pkg = validPackage(skyKey);
1016 assertThat(pkg.containsErrors()).isFalse();
michajloc17b80e2020-07-20 10:00:28 -07001017 assertNoEvents();
nharmatac9fbe952019-06-13 09:39:23 -07001018 }
1019
1020 @Test
jmmvbfbd95f2020-08-31 13:12:38 -07001021 public void testGlobDisallowEmpty_functionParam_wasNonEmptyAndBecomesEmpty() throws Exception {
nharmatac9fbe952019-06-13 09:39:23 -07001022 scratch.file("pkg/BUILD", "x = " + "glob(['*.foo'], allow_empty=False)");
1023 scratch.file("pkg/blah.foo");
1024 invalidatePackages();
1025
1026 SkyKey skyKey = PackageValue.key(PackageIdentifier.parse("@//pkg"));
1027
1028 Package pkg = validPackage(skyKey);
1029 assertThat(pkg.containsErrors()).isFalse();
michajloc17b80e2020-07-20 10:00:28 -07001030 assertNoEvents();
nharmatac9fbe952019-06-13 09:39:23 -07001031
1032 scratch.deleteFile("pkg/blah.foo");
1033 getSkyframeExecutor()
1034 .invalidateFilesUnderPathForTesting(
1035 reporter,
1036 ModifiedFileSet.builder().modify(PathFragment.create("pkg/blah.foo")).build(),
1037 Root.fromPath(rootDirectory));
1038
1039 reporter.removeHandler(failFastHandler);
1040 pkg = validPackage(skyKey);
1041 assertThat(pkg.containsErrors()).isTrue();
michajloc17b80e2020-07-20 10:00:28 -07001042 assertContainsEvent(
laurentlb7679c302020-07-17 03:16:11 -07001043 "glob pattern '*.foo' didn't match anything, but allow_empty is set to False (the "
michajloc17b80e2020-07-20 10:00:28 -07001044 + "default value of allow_empty can be set with --incompatible_disallow_empty_glob).");
nharmatac9fbe952019-06-13 09:39:23 -07001045 }
1046
1047 @Test
jmmvbfbd95f2020-08-31 13:12:38 -07001048 public void testGlobDisallowEmpty_starlarkOption_wasNonEmptyAndBecomesEmpty() throws Exception {
nharmatac9fbe952019-06-13 09:39:23 -07001049 preparePackageLoadingWithCustomStarklarkSemanticsOptions(
adonovan240bdea2020-09-03 15:24:12 -07001050 Options.parse(BuildLanguageOptions.class, "--incompatible_disallow_empty_glob=true")
nharmatac9fbe952019-06-13 09:39:23 -07001051 .getOptions(),
1052 rootDirectory);
1053
1054 scratch.file("pkg/BUILD", "x = " + "glob(['*.foo'])");
1055 scratch.file("pkg/blah.foo");
1056 invalidatePackages();
1057
1058 SkyKey skyKey = PackageValue.key(PackageIdentifier.parse("@//pkg"));
1059
1060 Package pkg = validPackage(skyKey);
1061 assertThat(pkg.containsErrors()).isFalse();
michajloc17b80e2020-07-20 10:00:28 -07001062 assertNoEvents();
nharmatac9fbe952019-06-13 09:39:23 -07001063
1064 scratch.deleteFile("pkg/blah.foo");
1065 getSkyframeExecutor()
1066 .invalidateFilesUnderPathForTesting(
1067 reporter,
1068 ModifiedFileSet.builder().modify(PathFragment.create("pkg/blah.foo")).build(),
1069 Root.fromPath(rootDirectory));
1070
1071 reporter.removeHandler(failFastHandler);
1072 pkg = validPackage(skyKey);
1073 assertThat(pkg.containsErrors()).isTrue();
michajloc17b80e2020-07-20 10:00:28 -07001074 assertContainsEvent(
laurentlb7679c302020-07-17 03:16:11 -07001075 "glob pattern '*.foo' didn't match anything, but allow_empty is set to False (the "
michajloc17b80e2020-07-20 10:00:28 -07001076 + "default value of allow_empty can be set with --incompatible_disallow_empty_glob).");
nharmatac9fbe952019-06-13 09:39:23 -07001077 }
1078
1079 @Test
jmmvbfbd95f2020-08-31 13:12:38 -07001080 public void testGlobDisallowEmpty_functionParam_wasEmptyAndStaysEmpty() throws Exception {
nharmatac9fbe952019-06-13 09:39:23 -07001081 scratch.file("pkg/BUILD", "x = " + "glob(['*.foo'], allow_empty=False)");
1082 invalidatePackages();
1083
1084 SkyKey skyKey = PackageValue.key(PackageIdentifier.parse("@//pkg"));
1085 reporter.removeHandler(failFastHandler);
1086
1087 Package pkg = validPackage(skyKey);
1088 assertThat(pkg.containsErrors()).isTrue();
1089 String expectedEventString =
laurentlb7679c302020-07-17 03:16:11 -07001090 "glob pattern '*.foo' didn't match anything, but allow_empty is set to False (the "
1091 + "default value of allow_empty can be set with --incompatible_disallow_empty_glob).";
nharmatac9fbe952019-06-13 09:39:23 -07001092 assertContainsEvent(expectedEventString);
1093
1094 scratch.overwriteFile("pkg/BUILD", "x = " + "glob(['*.foo'], allow_empty=False) #comment");
1095 getSkyframeExecutor()
1096 .invalidateFilesUnderPathForTesting(
1097 reporter,
1098 ModifiedFileSet.builder().modify(PathFragment.create("pkg/BUILD")).build(),
1099 Root.fromPath(rootDirectory));
1100
1101 pkg = validPackage(skyKey);
1102 assertThat(pkg.containsErrors()).isTrue();
nharmatac9fbe952019-06-13 09:39:23 -07001103 assertContainsEvent(expectedEventString);
1104 }
1105
1106 @Test
jmmvbfbd95f2020-08-31 13:12:38 -07001107 public void testGlobDisallowEmpty_starlarkOption_wasEmptyAndStaysEmpty() throws Exception {
nharmatac9fbe952019-06-13 09:39:23 -07001108 preparePackageLoadingWithCustomStarklarkSemanticsOptions(
adonovan240bdea2020-09-03 15:24:12 -07001109 Options.parse(BuildLanguageOptions.class, "--incompatible_disallow_empty_glob=true")
nharmatac9fbe952019-06-13 09:39:23 -07001110 .getOptions(),
1111 rootDirectory);
1112
1113 scratch.file("pkg/BUILD", "x = " + "glob(['*.foo'])");
1114 invalidatePackages();
1115
1116 SkyKey skyKey = PackageValue.key(PackageIdentifier.parse("@//pkg"));
1117 reporter.removeHandler(failFastHandler);
1118
1119 Package pkg = validPackage(skyKey);
1120 assertThat(pkg.containsErrors()).isTrue();
1121 String expectedEventString =
laurentlb7679c302020-07-17 03:16:11 -07001122 "glob pattern '*.foo' didn't match anything, but allow_empty is set to False (the "
1123 + "default value of allow_empty can be set with --incompatible_disallow_empty_glob).";
nharmatac9fbe952019-06-13 09:39:23 -07001124 assertContainsEvent(expectedEventString);
1125
1126 scratch.overwriteFile("pkg/BUILD", "x = " + "glob(['*.foo']) #comment");
1127 getSkyframeExecutor()
1128 .invalidateFilesUnderPathForTesting(
1129 reporter,
1130 ModifiedFileSet.builder().modify(PathFragment.create("pkg/BUILD")).build(),
1131 Root.fromPath(rootDirectory));
1132
1133 pkg = validPackage(skyKey);
1134 assertThat(pkg.containsErrors()).isTrue();
nharmatac9fbe952019-06-13 09:39:23 -07001135 assertContainsEvent(expectedEventString);
1136 }
1137
1138 @Test
jmmvbfbd95f2020-08-31 13:12:38 -07001139 public void testGlobDisallowEmpty_functionParam_wasEmptyDueToExcludeAndStaysEmpty()
nharmatac9fbe952019-06-13 09:39:23 -07001140 throws Exception {
1141 scratch.file("pkg/BUILD", "x = glob(include=['*.foo'], exclude=['blah.*'], allow_empty=False)");
1142 scratch.file("pkg/blah.foo");
1143 invalidatePackages();
1144
1145 SkyKey skyKey = PackageValue.key(PackageIdentifier.parse("@//pkg"));
1146 reporter.removeHandler(failFastHandler);
1147
1148 Package pkg = validPackage(skyKey);
1149 assertThat(pkg.containsErrors()).isTrue();
1150 String expectedEventString =
laurentlb7679c302020-07-17 03:16:11 -07001151 "all files in the glob have been excluded, but allow_empty is set to False (the "
1152 + "default value of allow_empty can be set with --incompatible_disallow_empty_glob).";
nharmatac9fbe952019-06-13 09:39:23 -07001153 assertContainsEvent(expectedEventString);
1154
1155 scratch.overwriteFile(
1156 "pkg/BUILD",
1157 "x = glob(include=['*.foo'], exclude=['blah.*'], allow_empty=False) # comment");
1158 getSkyframeExecutor()
1159 .invalidateFilesUnderPathForTesting(
1160 reporter,
1161 ModifiedFileSet.builder().modify(PathFragment.create("pkg/BUILD")).build(),
1162 Root.fromPath(rootDirectory));
1163
1164 pkg = validPackage(skyKey);
1165 assertThat(pkg.containsErrors()).isTrue();
nharmatac9fbe952019-06-13 09:39:23 -07001166 assertContainsEvent(expectedEventString);
1167 }
1168
1169 @Test
jmmvbfbd95f2020-08-31 13:12:38 -07001170 public void testGlobDisallowEmpty_starlarkOption_wasEmptyDueToExcludeAndStaysEmpty()
nharmatac9fbe952019-06-13 09:39:23 -07001171 throws Exception {
1172 preparePackageLoadingWithCustomStarklarkSemanticsOptions(
adonovan240bdea2020-09-03 15:24:12 -07001173 Options.parse(BuildLanguageOptions.class, "--incompatible_disallow_empty_glob=true")
nharmatac9fbe952019-06-13 09:39:23 -07001174 .getOptions(),
1175 rootDirectory);
1176
1177 scratch.file("pkg/BUILD", "x = glob(include=['*.foo'], exclude=['blah.*'])");
1178 scratch.file("pkg/blah.foo");
1179 invalidatePackages();
1180
1181 SkyKey skyKey = PackageValue.key(PackageIdentifier.parse("@//pkg"));
1182 reporter.removeHandler(failFastHandler);
1183
1184 Package pkg = validPackage(skyKey);
1185 assertThat(pkg.containsErrors()).isTrue();
1186 String expectedEventString =
laurentlb7679c302020-07-17 03:16:11 -07001187 "all files in the glob have been excluded, but allow_empty is set to False (the "
1188 + "default value of allow_empty can be set with --incompatible_disallow_empty_glob).";
nharmatac9fbe952019-06-13 09:39:23 -07001189 assertContainsEvent(expectedEventString);
1190
1191 scratch.overwriteFile("pkg/BUILD", "x = glob(include=['*.foo'], exclude=['blah.*']) # comment");
1192 getSkyframeExecutor()
1193 .invalidateFilesUnderPathForTesting(
1194 reporter,
1195 ModifiedFileSet.builder().modify(PathFragment.create("pkg/BUILD")).build(),
1196 Root.fromPath(rootDirectory));
1197
1198 pkg = validPackage(skyKey);
1199 assertThat(pkg.containsErrors()).isTrue();
nharmatac9fbe952019-06-13 09:39:23 -07001200 assertContainsEvent(expectedEventString);
1201 }
1202
1203 @Test
jmmvbfbd95f2020-08-31 13:12:38 -07001204 public void testGlobDisallowEmpty_functionParam_wasEmptyAndBecomesNonEmpty() throws Exception {
nharmatac9fbe952019-06-13 09:39:23 -07001205 scratch.file("pkg/BUILD", "x = " + "glob(['*.foo'], allow_empty=False)");
1206 invalidatePackages();
1207
1208 SkyKey skyKey = PackageValue.key(PackageIdentifier.parse("@//pkg"));
1209
1210 reporter.removeHandler(failFastHandler);
1211 Package pkg = validPackage(skyKey);
1212 assertThat(pkg.containsErrors()).isTrue();
michajloc17b80e2020-07-20 10:00:28 -07001213 assertContainsEvent(
laurentlb7679c302020-07-17 03:16:11 -07001214 "glob pattern '*.foo' didn't match anything, but allow_empty is set to False (the "
michajloc17b80e2020-07-20 10:00:28 -07001215 + "default value of allow_empty can be set with --incompatible_disallow_empty_glob).");
nharmatac9fbe952019-06-13 09:39:23 -07001216
1217 scratch.file("pkg/blah.foo");
1218 getSkyframeExecutor()
1219 .invalidateFilesUnderPathForTesting(
1220 reporter,
1221 ModifiedFileSet.builder().modify(PathFragment.create("pkg/blah.foo")).build(),
1222 Root.fromPath(rootDirectory));
1223
1224 reporter.addHandler(failFastHandler);
michajloc17b80e2020-07-20 10:00:28 -07001225 eventCollector.clear();
nharmatac9fbe952019-06-13 09:39:23 -07001226 pkg = validPackage(skyKey);
1227 assertThat(pkg.containsErrors()).isFalse();
michajloc17b80e2020-07-20 10:00:28 -07001228 assertNoEvents();
nharmatac9fbe952019-06-13 09:39:23 -07001229 }
1230
1231 @Test
jmmvbfbd95f2020-08-31 13:12:38 -07001232 public void testGlobDisallowEmpty_starlarkOption_wasEmptyAndBecomesNonEmpty() throws Exception {
nharmatac9fbe952019-06-13 09:39:23 -07001233 preparePackageLoadingWithCustomStarklarkSemanticsOptions(
adonovan240bdea2020-09-03 15:24:12 -07001234 Options.parse(BuildLanguageOptions.class, "--incompatible_disallow_empty_glob=true")
nharmatac9fbe952019-06-13 09:39:23 -07001235 .getOptions(),
1236 rootDirectory);
1237
1238 scratch.file("pkg/BUILD", "x = " + "glob(['*.foo'])");
1239 invalidatePackages();
1240
1241 SkyKey skyKey = PackageValue.key(PackageIdentifier.parse("@//pkg"));
1242
1243 reporter.removeHandler(failFastHandler);
1244 Package pkg = validPackage(skyKey);
1245 assertThat(pkg.containsErrors()).isTrue();
michajloc17b80e2020-07-20 10:00:28 -07001246
1247 assertContainsEvent(
laurentlb7679c302020-07-17 03:16:11 -07001248 "glob pattern '*.foo' didn't match anything, but allow_empty is set to False (the "
michajloc17b80e2020-07-20 10:00:28 -07001249 + "default value of allow_empty can be set with --incompatible_disallow_empty_glob).");
nharmatac9fbe952019-06-13 09:39:23 -07001250
1251 scratch.file("pkg/blah.foo");
1252 getSkyframeExecutor()
1253 .invalidateFilesUnderPathForTesting(
1254 reporter,
1255 ModifiedFileSet.builder().modify(PathFragment.create("pkg/blah.foo")).build(),
1256 Root.fromPath(rootDirectory));
1257
1258 reporter.addHandler(failFastHandler);
michajloc17b80e2020-07-20 10:00:28 -07001259 eventCollector.clear();
nharmatac9fbe952019-06-13 09:39:23 -07001260 pkg = validPackage(skyKey);
1261 assertThat(pkg.containsErrors()).isFalse();
michajloc17b80e2020-07-20 10:00:28 -07001262 assertNoEvents();
nharmatac9fbe952019-06-13 09:39:23 -07001263 }
1264
Benjamin Peterson723eca62019-07-22 17:24:10 -07001265 @Test
adonovan87b46082020-07-08 15:58:04 -07001266 public void testPackageRecordsLoadedModules() throws Exception {
1267 scratch.file("p/BUILD", "load('a.bzl', 'a'); load(':b.bzl', 'b')");
1268 scratch.file("p/a.bzl", "load('c.bzl', 'c'); a = c");
1269 scratch.file("p/b.bzl", "load(':c.bzl', 'c'); b = c");
1270 scratch.file("p/c.bzl", "c = 0");
1271
1272 // load p
1273 preparePackageLoading(rootDirectory);
1274 SkyKey skyKey = PackageValue.key(PackageIdentifier.parse("@//p"));
1275 Package p = validPackageWithoutErrors(skyKey);
1276
1277 // Keys are load strings as they appear in the source (notice ":" in one of them).
1278 Map<String, Module> pLoads = p.getLoads();
1279 assertThat(pLoads.keySet().toString()).isEqualTo("[a.bzl, :b.bzl]");
1280
1281 // subgraph a
1282 Module a = pLoads.get("a.bzl");
1283 assertThat(a.toString()).isEqualTo("<module //p:a.bzl>");
1284 Map<String, Module> aLoads = BazelModuleContext.of(a).loads();
1285 assertThat(aLoads.keySet().toString()).isEqualTo("[c.bzl]");
1286 Module cViaA = aLoads.get("c.bzl");
1287 assertThat(cViaA.toString()).isEqualTo("<module //p:c.bzl>");
1288
1289 // subgraph b
1290 Module b = pLoads.get(":b.bzl");
1291 assertThat(b.toString()).isEqualTo("<module //p:b.bzl>");
1292 Map<String, Module> bLoads = BazelModuleContext.of(b).loads();
1293 assertThat(bLoads.keySet().toString()).isEqualTo("[:c.bzl]");
1294 Module cViaB = bLoads.get(":c.bzl");
1295 assertThat(cViaB).isSameInstanceAs(cViaA);
1296
adonovan3ed7ed52020-09-30 12:03:28 -07001297 assertThat(cViaA.getGlobal("c")).isEqualTo(StarlarkInt.of(0));
adonovan87b46082020-07-08 15:58:04 -07001298 }
1299
1300 @Test
Benjamin Peterson723eca62019-07-22 17:24:10 -07001301 public void veryBrokenPackagePostsDoneToProgressReceiver() throws Exception {
1302 reporter.removeHandler(failFastHandler);
1303
adonovanc0e86902020-11-19 15:50:29 -08001304 // Note: syntax error (recovered), non-existent .bzl file.
Benjamin Peterson723eca62019-07-22 17:24:10 -07001305 scratch.file("pkg/BUILD", "load('//does_not:exist.bzl', 'broken'");
1306 SkyKey key = PackageValue.key(PackageIdentifier.parse("@//pkg"));
1307 EvaluationResult<PackageValue> result =
1308 SkyframeExecutorTestUtils.evaluate(getSkyframeExecutor(), key, false, reporter);
adonovanc0e86902020-11-19 15:50:29 -08001309 assertThatEvaluationResult(result).hasErrorEntryForKeyThat(key);
1310 assertContainsEvent("syntax error at 'newline': expected ,");
Benjamin Peterson723eca62019-07-22 17:24:10 -07001311 assertThat(getSkyframeExecutor().getPackageProgressReceiver().progressState())
1312 .isEqualTo(new Pair<String, String>("1 packages loaded", ""));
1313 }
1314
nharmatace0335a2019-11-13 15:48:05 -08001315 @Test
nharmata11d8ecf2021-06-09 12:27:08 -07001316 public void testNonSkyframeGlobbingEncountersSymlinkCycleAndThrowsIOException() throws Exception {
nharmatace0335a2019-11-13 15:48:05 -08001317 reporter.removeHandler(failFastHandler);
nharmatace0335a2019-11-13 15:48:05 -08001318
nharmata11d8ecf2021-06-09 12:27:08 -07001319 // When a package's BUILD file and the relevant filesystem state is such that non-Skyframe
1320 // globbing will encounter an IOException due to a directory symlink cycle,
nharmatace0335a2019-11-13 15:48:05 -08001321 Path fooBUILDPath = scratch.file("foo/BUILD", "glob(['cycle/**/foo.txt'])");
1322 Path fooCyclePath = fooBUILDPath.getParentDirectory().getChild("cycle");
1323 FileSystemUtils.ensureSymbolicLink(fooCyclePath, fooCyclePath);
1324 IOException ioExnFromFS =
1325 assertThrows(IOException.class, () -> fooCyclePath.statIfFound(Symlinks.FOLLOW));
1326 // And it is indeed the case that the FileSystem throws an IOException when the cycle's Path is
nharmata11d8ecf2021-06-09 12:27:08 -07001327 // stat'd (following symlinks, as non-Skyframe globbing does).
nharmatace0335a2019-11-13 15:48:05 -08001328 assertThat(ioExnFromFS).hasMessageThat().contains("Too many levels of symbolic links");
1329
1330 // Then, when we evaluate the PackageValue node for the Package in keepGoing mode,
1331 SkyKey pkgKey = PackageValue.key(PackageIdentifier.parse("@//foo"));
1332 EvaluationResult<PackageValue> result =
1333 SkyframeExecutorTestUtils.evaluate(
1334 getSkyframeExecutor(), pkgKey, /*keepGoing=*/ true, reporter);
1335 // The result is a *non-transient* Skyframe error.
1336 assertThatEvaluationResult(result).hasErrorEntryForKeyThat(pkgKey).isNotTransient();
1337 // And that error is a NoSuchPackageException
1338 assertThatEvaluationResult(result)
1339 .hasErrorEntryForKeyThat(pkgKey)
1340 .hasExceptionThat()
1341 .isInstanceOf(NoSuchPackageException.class);
1342 // With a useful error message,
1343 assertThatEvaluationResult(result)
1344 .hasErrorEntryForKeyThat(pkgKey)
1345 .hasExceptionThat()
1346 .hasMessageThat()
1347 .contains("Symlink cycle: /workspace/foo/cycle");
1348 // And appropriate Skyframe root cause (N.B. since we want PackageFunction to rethrow in
1349 // situations like this, we want the PackageValue node to be its own root cause).
janakr4aeb7b32020-11-16 13:11:38 -08001350 assertThatEvaluationResult(result).hasErrorEntryForKeyThat(pkgKey);
nharmatace0335a2019-11-13 15:48:05 -08001351
1352 // Then, when we modify the BUILD file so as to force package loading,
1353 scratch.overwriteFile(
1354 "foo/BUILD", "glob(['cycle/**/foo.txt']) # dummy comment to force package loading");
1355 // But we don't make any filesystem changes that would invalidate the GlobValues, meaning that
1356 // PackageFunction will observe cache hits from Skyframe globbing,
1357 //
1358 // And we also have our filesystem blow up if the directory symlink cycle is encountered (thus,
nharmata11d8ecf2021-06-09 12:27:08 -07001359 // the absence of a crash indicates the lack of non-Skyframe globbing),
nharmatace0335a2019-11-13 15:48:05 -08001360 fs.stubStatError(
1361 fooCyclePath,
1362 new IOException() {
1363 @Override
1364 public String getMessage() {
1365 throw new IllegalStateException("should't get here!");
1366 }
1367 });
1368 // And we evaluate the PackageValue node for the Package in keepGoing mode,
1369 getSkyframeExecutor()
1370 .invalidateFilesUnderPathForTesting(
1371 reporter,
1372 ModifiedFileSet.builder().modify(PathFragment.create("foo/BUILD")).build(),
1373 Root.fromPath(rootDirectory));
1374 // The results are exactly the same as before,
1375 result =
1376 SkyframeExecutorTestUtils.evaluate(
1377 getSkyframeExecutor(), pkgKey, /*keepGoing=*/ true, reporter);
1378 assertThatEvaluationResult(result).hasErrorEntryForKeyThat(pkgKey).isNotTransient();
1379 assertThatEvaluationResult(result)
1380 .hasErrorEntryForKeyThat(pkgKey)
1381 .hasExceptionThat()
1382 .isInstanceOf(NoSuchPackageException.class);
1383 assertThatEvaluationResult(result)
1384 .hasErrorEntryForKeyThat(pkgKey)
1385 .hasExceptionThat()
1386 .hasMessageThat()
1387 .contains("Symlink cycle: /workspace/foo/cycle");
nharmatace0335a2019-11-13 15:48:05 -08001388 // Thus showing that clean and incremental package loading have the same semantics in the
1389 // presence of a symlink cycle encountered during glob evaluation.
1390 }
1391
Googler74178a52020-06-29 17:42:47 -07001392 private static void assertDetailedExitCode(
janakra4a564a2021-03-18 12:40:11 -07001393 Exception exception, PackageLoading.Code expectedPackageLoadingCode, ExitCode exitCode) {
Googler74178a52020-06-29 17:42:47 -07001394 assertThat(exception).isInstanceOf(DetailedException.class);
1395 DetailedExitCode detailedExitCode = ((DetailedException) exception).getDetailedExitCode();
janakra4a564a2021-03-18 12:40:11 -07001396 assertThat(detailedExitCode.getExitCode()).isEqualTo(exitCode);
Googler74178a52020-06-29 17:42:47 -07001397 assertThat(detailedExitCode.getFailureDetail().getPackageLoading().getCode())
1398 .isEqualTo(expectedPackageLoadingCode);
janakra4a564a2021-03-18 12:40:11 -07001399 assertThat(DetailedExitCode.getExitCode(detailedExitCode.getFailureDetail()))
1400 .isEqualTo(exitCode);
Googler74178a52020-06-29 17:42:47 -07001401 }
1402
brandjon38dafdd2020-08-06 09:49:13 -07001403 /**
1404 * Tests of the prelude file functionality.
1405 *
1406 * <p>This is in a separate BuildViewTestCase because we override the prelude label for the test.
1407 * (The prelude label is configured differently between Bazel and Blaze.)
1408 */
1409 @RunWith(JUnit4.class)
1410 public static class PreludeTest extends BuildViewTestCase {
1411
1412 private final CustomInMemoryFs fs = new CustomInMemoryFs(new ManualClock());
1413
1414 @Override
1415 protected FileSystem createFileSystem() {
1416 return fs;
1417 }
1418
1419 @Override
1420 protected ConfiguredRuleClassProvider createRuleClassProvider() {
1421 ConfiguredRuleClassProvider.Builder builder = new ConfiguredRuleClassProvider.Builder();
1422 // addStandardRules() may call setPrelude(), so do it first.
1423 TestRuleClassProvider.addStandardRules(builder);
1424 builder.setPrelude("//tools/build_rules:test_prelude");
1425 return builder.build();
1426 }
1427
1428 @Test
1429 public void testPreludeDefinedSymbolIsUsable() throws Exception {
1430 scratch.file("tools/build_rules/BUILD");
1431 scratch.file(
1432 "tools/build_rules/test_prelude", //
1433 "foo = 'FOO'");
1434 scratch.file(
1435 "pkg/BUILD", //
1436 "print(foo)");
1437
1438 getConfiguredTarget("//pkg:BUILD");
1439 assertContainsEvent("FOO");
1440 }
1441
1442 @Test
1443 public void testPreludeAutomaticallyReexportsLoadedSymbols() throws Exception {
1444 scratch.file("tools/build_rules/BUILD");
1445 scratch.file(
1446 "tools/build_rules/test_prelude", //
1447 "load('//util:common.bzl', 'foo')");
1448 scratch.file("util/BUILD");
1449 scratch.file(
1450 "util/common.bzl", //
1451 "foo = 'FOO'");
1452 scratch.file(
1453 "pkg/BUILD", //
1454 "print(foo)");
1455
1456 getConfiguredTarget("//pkg:BUILD");
1457 assertContainsEvent("FOO");
1458 }
1459
1460 // TODO(brandjon): Invert this test once the prelude is a module instead of a syntactic
1461 // mutation on BUILD files.
1462 @Test
1463 public void testPreludeCanExportUnderscoreSymbols() throws Exception {
1464 scratch.file("tools/build_rules/BUILD");
1465 scratch.file(
1466 "tools/build_rules/test_prelude", //
1467 "_foo = 'FOO'");
1468 scratch.file(
1469 "pkg/BUILD", //
1470 "print(_foo)");
1471
1472 getConfiguredTarget("//pkg:BUILD");
1473 assertContainsEvent("FOO");
1474 }
1475
1476 @Test
brandjon6c63b8f2021-02-01 10:17:42 -08001477 public void testPreludeCanShadowUniversal() throws Exception {
1478 scratch.file("tools/build_rules/BUILD");
1479 scratch.file(
1480 "tools/build_rules/test_prelude", //
1481 "len = 'FOO'");
1482 scratch.file(
1483 "pkg/BUILD", //
1484 "print(len)");
1485
1486 getConfiguredTarget("//pkg:BUILD");
1487 assertContainsEvent("FOO");
1488 }
1489
1490 @Test
brandjon38dafdd2020-08-06 09:49:13 -07001491 public void testPreludeCanShadowPredeclareds() throws Exception {
1492 scratch.file("tools/build_rules/BUILD");
1493 scratch.file(
1494 "tools/build_rules/test_prelude", //
1495 "cc_library = 'FOO'");
1496 scratch.file(
1497 "pkg/BUILD", //
1498 "print(cc_library)");
1499
1500 getConfiguredTarget("//pkg:BUILD");
1501 assertContainsEvent("FOO");
1502 }
1503
brandjon38dafdd2020-08-06 09:49:13 -07001504 @Test
brandjon6c63b8f2021-02-01 10:17:42 -08001505 public void testPreludeCanShadowInjectedPredeclareds() throws Exception {
1506 setBuildLanguageOptions("--experimental_builtins_bzl_path=tools/builtins_staging");
1507 scratch.file(
1508 "tools/builtins_staging/exports.bzl",
1509 "exported_toplevels = {}",
1510 "exported_rules = {'cc_library': 'BAR'}",
1511 "exported_to_java = {}");
1512 scratch.file("tools/build_rules/BUILD");
1513 scratch.file(
1514 "tools/build_rules/test_prelude", //
1515 "cc_library = 'FOO'");
1516 scratch.file(
1517 "pkg/BUILD", //
1518 "print(cc_library)");
1519
1520 getConfiguredTarget("//pkg:BUILD");
1521 assertContainsEvent("FOO");
1522 }
1523
1524 @Test
brandjon2a73a732020-08-08 06:41:14 -07001525 public void testPreludeSymbolCannotBeMutated() throws Exception {
brandjon38dafdd2020-08-06 09:49:13 -07001526 scratch.file("tools/build_rules/BUILD");
1527 scratch.file(
1528 "tools/build_rules/test_prelude", //
1529 "foo = ['FOO']");
1530 scratch.file(
1531 "pkg/BUILD", //
brandjon2a73a732020-08-08 06:41:14 -07001532 "foo.append('BAR')");
brandjon38dafdd2020-08-06 09:49:13 -07001533
1534 reporter.removeHandler(failFastHandler);
1535 getConfiguredTarget("//pkg:BUILD");
brandjon2a73a732020-08-08 06:41:14 -07001536 assertContainsEvent("trying to mutate a frozen list value");
1537 }
1538
1539 @Test
1540 public void testPreludeCanAccessBzlDialectFeatures() throws Exception {
1541 scratch.file("tools/build_rules/BUILD");
1542 // Test both bzl symbols and syntax (e.g. function defs).
1543 scratch.file(
1544 "tools/build_rules/test_prelude", //
1545 "def foo():",
1546 " return native.glob");
1547 scratch.file(
1548 "pkg/BUILD", //
1549 "print(foo())");
1550
1551 getConfiguredTarget("//pkg:BUILD");
1552 // Prelude can access native.glob (though only a BUILD thread can call it).
adonovan800117e2020-09-18 10:30:06 -07001553 assertContainsEvent("<built-in method glob of native value>");
brandjon38dafdd2020-08-06 09:49:13 -07001554 }
1555
1556 @Test
1557 public void testPreludeNeedNotBePresent() throws Exception {
1558 scratch.file(
1559 "pkg/BUILD", //
1560 "print('FOO')");
1561
1562 getConfiguredTarget("//pkg:BUILD");
1563 assertContainsEvent("FOO");
1564 }
1565
1566 @Test
brandjon2a73a732020-08-08 06:41:14 -07001567 public void testPreludeNeedNotBePresent_evenWhenPackageIs() throws Exception {
1568 scratch.file("tools/build_rules/BUILD");
1569 scratch.file(
1570 "pkg/BUILD", //
1571 "print('FOO')");
1572
1573 getConfiguredTarget("//pkg:BUILD");
1574 assertContainsEvent("FOO");
1575 }
1576
1577 @Test
brandjon38dafdd2020-08-06 09:49:13 -07001578 public void testPreludeFileNotRecognizedWithoutPackage() throws Exception {
1579 scratch.file(
1580 "tools/build_rules/test_prelude", //
1581 "foo = 'FOO'");
1582 scratch.file(
1583 "pkg/BUILD", //
1584 "print(foo)");
1585
1586 // The prelude file is not found without a corresponding package to contain it. BUILD files
1587 // get processed as if no prelude file is present.
1588 reporter.removeHandler(failFastHandler);
1589 getConfiguredTarget("//pkg:BUILD");
1590 assertContainsEvent("name 'foo' is not defined");
1591 }
1592
1593 @Test
1594 public void testPreludeFailsWhenErrorInPreludeFile() throws Exception {
1595 scratch.file("tools/build_rules/BUILD");
1596 scratch.file(
1597 "tools/build_rules/test_prelude", //
1598 "1//0", // <-- dynamic error
1599 "foo = 'FOO'");
1600 scratch.file(
1601 "pkg/BUILD", //
1602 "print(foo)");
1603
1604 reporter.removeHandler(failFastHandler);
1605 getConfiguredTarget("//pkg:BUILD");
1606 assertContainsEvent(
1607 "File \"/workspace/tools/build_rules/test_prelude\", line 1, column 2, in <toplevel>");
1608 assertContainsEvent("Error: integer division by zero");
1609 }
1610
1611 @Test
1612 public void testPreludeWorksEvenWhenPreludePackageInError() throws Exception {
1613 scratch.file(
1614 "tools/build_rules/BUILD", //
1615 "1//0"); // <-- dynamic error
1616 scratch.file(
1617 "tools/build_rules/test_prelude", //
1618 "foo = 'FOO'");
1619 scratch.file(
1620 "pkg/BUILD", //
1621 "print(foo)");
1622
1623 // Succeeds because prelude loading is only dependent on the prelude package's existence, not
1624 // its evaluation.
1625 getConfiguredTarget("//pkg:BUILD");
1626 assertContainsEvent("FOO");
1627 }
1628
1629 // Another hypothetical test case we could try: Confirm that it's possible to explicitly load
1630 // the prelude file as a regular .bzl. We don't bother testing this use case because, aside from
1631 // being arguably pathological, it is currently impossible in practice: The prelude label
1632 // doesn't end with ".bzl" and isn't configurable by the user. We also want to eliminate the
1633 // prelude, so there's no intention of adding such a feature.
1634
1635 // Another possible test case: Verify how prelude applies to WORKSPACE files.
1636 }
1637
Kristina Chodorow335f0672015-11-16 23:19:13 +00001638 private static class CustomInMemoryFs extends InMemoryFileSystem {
Googlerc804c662016-12-01 16:53:28 +00001639 private abstract static class FileStatusOrException {
1640 abstract FileStatus get() throws IOException;
1641
1642 private static class ExceptionImpl extends FileStatusOrException {
1643 private final IOException exn;
1644
1645 private ExceptionImpl(IOException exn) {
1646 this.exn = exn;
1647 }
1648
1649 @Override
1650 FileStatus get() throws IOException {
1651 throw exn;
1652 }
1653 }
1654
1655 private static class FileStatusImpl extends FileStatusOrException {
1656
laurentlbc0bd2102018-10-17 07:05:25 -07001657 @Nullable private final FileStatus fileStatus;
Googlerc804c662016-12-01 16:53:28 +00001658
laurentlbc0bd2102018-10-17 07:05:25 -07001659 private FileStatusImpl(@Nullable FileStatus fileStatus) {
Googlerc804c662016-12-01 16:53:28 +00001660 this.fileStatus = fileStatus;
1661 }
1662
1663 @Override
1664 @Nullable
1665 FileStatus get() {
1666 return fileStatus;
1667 }
1668 }
1669 }
1670
ajurkowski8883c612021-03-08 08:12:37 -08001671 private final Map<PathFragment, FileStatusOrException> stubbedStats = Maps.newHashMap();
1672 private final Set<PathFragment> makeUnreadableAfterReaddir = Sets.newHashSet();
1673 private final Map<PathFragment, IOException> pathsToErrorOnGetInputStream = Maps.newHashMap();
Kristina Chodorow335f0672015-11-16 23:19:13 +00001674
1675 public CustomInMemoryFs(ManualClock manualClock) {
janakr97c0bd12020-09-08 13:19:03 -07001676 super(manualClock, DigestHashFunction.SHA256);
Kristina Chodorow335f0672015-11-16 23:19:13 +00001677 }
1678
aehligc801c392017-12-19 07:12:25 -08001679 public void stubStat(Path path, @Nullable FileStatus stubbedResult) {
ajurkowski8883c612021-03-08 08:12:37 -08001680 stubbedStats.put(path.asFragment(), new FileStatusOrException.FileStatusImpl(stubbedResult));
Kristina Chodorow335f0672015-11-16 23:19:13 +00001681 }
1682
aehligc801c392017-12-19 07:12:25 -08001683 public void stubStatError(Path path, IOException stubbedResult) {
ajurkowski8883c612021-03-08 08:12:37 -08001684 stubbedStats.put(path.asFragment(), new FileStatusOrException.ExceptionImpl(stubbedResult));
Kristina Chodorow335f0672015-11-16 23:19:13 +00001685 }
1686
1687 @Override
ajurkowski8883c612021-03-08 08:12:37 -08001688 public FileStatus statIfFound(PathFragment path, boolean followSymlinks) throws IOException {
Googlerc804c662016-12-01 16:53:28 +00001689 if (stubbedStats.containsKey(path)) {
1690 return stubbedStats.get(path).get();
Kristina Chodorow335f0672015-11-16 23:19:13 +00001691 }
fellya205ed82018-09-10 11:52:34 -07001692 return super.statIfFound(path, followSymlinks);
Kristina Chodorow335f0672015-11-16 23:19:13 +00001693 }
1694
aehligc801c392017-12-19 07:12:25 -08001695 public void scheduleMakeUnreadableAfterReaddir(Path path) {
ajurkowski8883c612021-03-08 08:12:37 -08001696 makeUnreadableAfterReaddir.add(path.asFragment());
Kristina Chodorow335f0672015-11-16 23:19:13 +00001697 }
1698
1699 @Override
ajurkowski8883c612021-03-08 08:12:37 -08001700 public Collection<Dirent> readdir(PathFragment path, boolean followSymlinks)
1701 throws IOException {
Kristina Chodorow335f0672015-11-16 23:19:13 +00001702 Collection<Dirent> result = super.readdir(path, followSymlinks);
Googlerc804c662016-12-01 16:53:28 +00001703 if (makeUnreadableAfterReaddir.contains(path)) {
ajurkowski8883c612021-03-08 08:12:37 -08001704 setReadable(path, false);
Kristina Chodorow335f0672015-11-16 23:19:13 +00001705 }
1706 return result;
1707 }
nharmataff688bf2017-06-07 17:03:52 -04001708
aehligc801c392017-12-19 07:12:25 -08001709 public void throwExceptionOnGetInputStream(Path path, IOException exn) {
ajurkowski8883c612021-03-08 08:12:37 -08001710 pathsToErrorOnGetInputStream.put(path.asFragment(), exn);
nharmataff688bf2017-06-07 17:03:52 -04001711 }
1712
1713 @Override
janakra4a564a2021-03-18 12:40:11 -07001714 protected synchronized InputStream getInputStream(PathFragment path) throws IOException {
nharmataff688bf2017-06-07 17:03:52 -04001715 IOException exnToThrow = pathsToErrorOnGetInputStream.get(path);
1716 if (exnToThrow != null) {
1717 throw exnToThrow;
1718 }
1719 return super.getInputStream(path);
1720 }
Kristina Chodorow335f0672015-11-16 23:19:13 +00001721 }
Kristina Chodorow335f0672015-11-16 23:19:13 +00001722}