blob: 053c5ebde69a4f831faf580dd2bf7c09cd3d2d77 [file] [log] [blame]
Ulf Adams5d058c42015-12-09 16:22:01 +00001// Copyright 2015 The Bazel Authors. All rights reserved.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14package com.google.devtools.build.lib.pkgcache;
15
16import static com.google.common.truth.Truth.assertThat;
17import static org.junit.Assert.assertNotNull;
18import static org.junit.Assert.assertNotSame;
19import static org.junit.Assert.assertSame;
20import static org.junit.Assert.fail;
21
22import com.google.common.base.Joiner;
Ulf Adams5d058c42015-12-09 16:22:01 +000023import com.google.common.base.Predicates;
24import com.google.common.collect.ImmutableList;
25import com.google.common.collect.ImmutableMap;
26import com.google.devtools.build.lib.analysis.BlazeDirectories;
27import com.google.devtools.build.lib.cmdline.Label;
28import com.google.devtools.build.lib.events.Reporter;
29import com.google.devtools.build.lib.packages.ConstantRuleVisibility;
30import com.google.devtools.build.lib.packages.NoSuchPackageException;
31import com.google.devtools.build.lib.packages.NoSuchTargetException;
32import com.google.devtools.build.lib.packages.NoSuchThingException;
33import com.google.devtools.build.lib.packages.Package;
34import com.google.devtools.build.lib.packages.PackageFactory;
35import com.google.devtools.build.lib.packages.Preprocessor;
36import com.google.devtools.build.lib.packages.Rule;
37import com.google.devtools.build.lib.packages.Target;
38import com.google.devtools.build.lib.skyframe.DiffAwareness;
39import com.google.devtools.build.lib.skyframe.PrecomputedValue;
40import com.google.devtools.build.lib.skyframe.SequencedSkyframeExecutor;
41import com.google.devtools.build.lib.skyframe.SkyValueDirtinessChecker;
42import com.google.devtools.build.lib.skyframe.SkyframeExecutor;
43import com.google.devtools.build.lib.syntax.GlobList;
44import com.google.devtools.build.lib.testutil.ManualClock;
45import com.google.devtools.build.lib.testutil.TestRuleClassProvider;
46import com.google.devtools.build.lib.util.BlazeClock;
Mark Schaller6df81792015-12-10 18:47:47 +000047import com.google.devtools.build.lib.util.Preconditions;
Ulf Adams5d058c42015-12-09 16:22:01 +000048import com.google.devtools.build.lib.util.io.TimestampGranularityMonitor;
49import com.google.devtools.build.lib.vfs.Dirent;
50import com.google.devtools.build.lib.vfs.FileStatus;
51import com.google.devtools.build.lib.vfs.FileSystem;
52import com.google.devtools.build.lib.vfs.FileSystemUtils;
53import com.google.devtools.build.lib.vfs.ModifiedFileSet;
54import com.google.devtools.build.lib.vfs.Path;
55import com.google.devtools.build.lib.vfs.PathFragment;
56import com.google.devtools.build.lib.vfs.inmemoryfs.InMemoryFileSystem;
57import com.google.devtools.build.skyframe.SkyFunction;
58import com.google.devtools.build.skyframe.SkyFunctionName;
59
60import org.junit.Before;
61import org.junit.Test;
62import org.junit.runner.RunWith;
63import org.junit.runners.JUnit4;
64
65import java.io.FileNotFoundException;
66import java.io.IOException;
67import java.util.ArrayList;
68import java.util.Collection;
69import java.util.List;
70import java.util.UUID;
71
72import javax.annotation.Nullable;
73
74/**
75 * Tests for incremental loading; these cover both normal operation and diff awareness, for which a
76 * list of modified / added / removed files is available.
77 */
78@RunWith(JUnit4.class)
79public class IncrementalLoadingTest {
80 protected PackageCacheTester tester;
81
82 private Path throwOnReaddir = null;
83 private Path throwOnStat = null;
84
85 @Before
86 public final void createTester() throws Exception {
87 ManualClock clock = new ManualClock();
88 FileSystem fs =
89 new InMemoryFileSystem(clock) {
90 @Override
91 public Collection<Dirent> readdir(Path path, boolean followSymlinks) throws IOException {
92 if (path.equals(throwOnReaddir)) {
93 throw new FileNotFoundException(path.getPathString());
94 }
95 return super.readdir(path, followSymlinks);
96 }
97
98 @Nullable
99 @Override
100 public FileStatus stat(Path path, boolean followSymlinks) throws IOException {
101 if (path.equals(throwOnStat)) {
102 throw new IOException("bork " + path.getPathString());
103 }
104 return super.stat(path, followSymlinks);
105 }
106 };
107 tester = createTester(fs, clock);
108 }
109
110 protected PackageCacheTester createTester(FileSystem fs, ManualClock clock) throws Exception {
111 return new PackageCacheTester(fs, clock, Preprocessor.Factory.Supplier.NullSupplier.INSTANCE);
112 }
113
114 @Test
115 public void testNoChange() throws Exception {
116 tester.addFile("base/BUILD",
117 "filegroup(name = 'hello', srcs = ['foo.txt'])");
118 tester.sync();
119 Target oldTarget = tester.getTarget("//base:hello");
120 assertNotNull(oldTarget);
121
122 tester.sync();
123 Target newTarget = tester.getTarget("//base:hello");
124 assertSame(oldTarget, newTarget);
125 }
126
127 @Test
128 public void testModifyBuildFile() throws Exception {
129 tester.addFile("base/BUILD", "filegroup(name = 'hello', srcs = ['foo.txt'])");
130 tester.sync();
131 Target oldTarget = tester.getTarget("//base:hello");
132
133 tester.modifyFile("base/BUILD", "filegroup(name = 'hello', srcs = ['bar.txt'])");
134 tester.sync();
135 Target newTarget = tester.getTarget("//base:hello");
136 assertNotSame(oldTarget, newTarget);
137 }
138
139 @Test
140 public void testModifyNonBuildFile() throws Exception {
141 tester.addFile("base/BUILD", "filegroup(name = 'hello', srcs = ['foo.txt'])");
142 tester.addFile("base/foo.txt", "nothing");
143 tester.sync();
144 Target oldTarget = tester.getTarget("//base:hello");
145
146 tester.modifyFile("base/foo.txt", "other");
147 tester.sync();
148 Target newTarget = tester.getTarget("//base:hello");
149 assertSame(oldTarget, newTarget);
150 }
151
152 @Test
153 public void testRemoveNonBuildFile() throws Exception {
154 tester.addFile("base/BUILD", "filegroup(name = 'hello', srcs = ['foo.txt'])");
155 tester.addFile("base/foo.txt", "nothing");
156 tester.sync();
157 Target oldTarget = tester.getTarget("//base:hello");
158
159 tester.removeFile("base/foo.txt");
160 tester.sync();
161 Target newTarget = tester.getTarget("//base:hello");
162 assertSame(oldTarget, newTarget);
163 }
164
165 @Test
166 public void testModifySymlinkedFileSamePackage() throws Exception {
167 tester.addSymlink("base/BUILD", "mybuild");
168 tester.addFile("base/mybuild", "filegroup(name = 'hello', srcs = ['foo.txt'])");
169 tester.sync();
170 Target oldTarget = tester.getTarget("//base:hello");
171 tester.modifyFile("base/mybuild", "filegroup(name = 'hello', srcs = ['bar.txt'])");
172 tester.sync();
173 Target newTarget = tester.getTarget("//base:hello");
174 assertNotSame(oldTarget, newTarget);
175 }
176
177 @Test
178 public void testModifySymlinkedFileDifferentPackage() throws Exception {
179 tester.addSymlink("base/BUILD", "../other/BUILD");
180 tester.addFile("other/BUILD", "filegroup(name = 'hello', srcs = ['foo.txt'])");
181 tester.sync();
182 Target oldTarget = tester.getTarget("//base:hello");
183
184 tester.modifyFile("other/BUILD", "filegroup(name = 'hello', srcs = ['bar.txt'])");
185 tester.sync();
186 Target newTarget = tester.getTarget("//base:hello");
187 assertNotSame(oldTarget, newTarget);
188 }
189
190 @Test
191 public void testBUILDSymlinkModifiedThenChanges() throws Exception {
192 // We need to ensure that the timestamps of "one" and "two" are different, because Blaze
193 // currently does not recognize changes to symlinks if the timestamps of the old and the new
194 // file pointed to by the symlink are the same.
195 tester.addFile("one", "filegroup(name='a', srcs=['1'])");
196 tester.sync();
197
198 tester.addFile("two", "filegroup(name='a', srcs=['2'])");
199 tester.addSymlink("oldlink", "one");
200 tester.addSymlink("newlink", "one");
201 tester.addSymlink("a/BUILD", "../oldlink");
202 tester.sync();
203 Target a1 = tester.getTarget("//a:a");
204
205 tester.modifySymlink("a/BUILD", "../newlink");
206 tester.sync();
207
208 tester.getTarget("//a:a");
209
210 tester.modifySymlink("newlink", "two");
211 tester.sync();
212
213 Target a3 = tester.getTarget("//a:a");
214 assertNotSame(a1, a3);
215 }
216
217 @Test
218 public void testBUILDFileIsExternalSymlinkAndChanges() throws Exception {
219 tester.addFile("/nonroot/file", "filegroup(name='a', srcs=['file'])");
220 tester.addSymlink("a/BUILD", "/nonroot/file");
221 tester.sync();
222
223 Target a1 = tester.getTarget("//a:a");
224 tester.modifyFile("/nonroot/file", "filegroup(name='a', srcs=['file2'])");
225 tester.sync();
226
227 Target a2 = tester.getTarget("//a:a");
228 tester.sync();
229
230 assertNotSame(a1, a2);
231 }
232
233 @Test
234 public void testLabelWithTwoSegmentsAndTotalInvalidation() throws Exception {
235 tester.addFile("a/BUILD", "filegroup(name='fg', srcs=['b/c'])");
236 tester.addFile("a/b/BUILD");
237 tester.sync();
238
239 Target fg1 = tester.getTarget("//a:fg");
240 tester.everythingModified();
241 tester.sync();
242
243 Target fg2 = tester.getTarget("//a:fg");
244 assertSame(fg1, fg2);
245 }
246
247 @Test
248 public void testAddGlobFile() throws Exception {
249 tester.addFile("base/BUILD", "filegroup(name = 'hello', srcs = glob(['*.txt']))");
250 tester.addFile("base/foo.txt", "nothing");
251 tester.sync();
252 Target oldTarget = tester.getTarget("//base:hello");
253
254 tester.addFile("base/bar.txt", "also nothing");
255 tester.sync();
256 Target newTarget = tester.getTarget("//base:hello");
257 assertNotSame(oldTarget, newTarget);
258 }
259
260 @Test
261 public void testRemoveGlobFile() throws Exception {
262 tester.addFile("base/BUILD", "filegroup(name = 'hello', srcs = glob(['*.txt']))");
263 tester.addFile("base/foo.txt", "nothing");
264 tester.addFile("base/bar.txt", "also nothing");
265 tester.sync();
266 Target oldTarget = tester.getTarget("//base:hello");
267
268 tester.removeFile("base/bar.txt");
269 tester.sync();
270 Target newTarget = tester.getTarget("//base:hello");
271 assertNotSame(oldTarget, newTarget);
272 }
273
274 @Test
275 public void testPackageNotInLastBuildReplaced() throws Exception {
276 tester.addFile("a/BUILD", "filegroup(name='a', srcs=['bad.sh'])");
277 tester.sync();
278 Target a1 = tester.getTarget("//a:a");
279
280 tester.addFile("b/BUILD", "filegroup(name='b', srcs=['b.sh'])");
281 tester.modifyFile("a/BUILD", "filegroup(name='a', srcs=['good.sh'])");
282 tester.sync();
283 tester.getTarget("//b:b");
284
285 tester.sync();
286 Target a2 = tester.getTarget("//a:a");
287 assertNotSame(a1, a2);
288 }
289
290 @Test
291 public void testBrokenSymlinkAddedThenFixed() throws Exception {
292 tester.addFile("a/BUILD", "filegroup(name='a', srcs=glob(['**']))");
293 tester.sync();
294 Target a1 = tester.getTarget("//a:a");
295
296 tester.addSymlink("a/b", "../c");
297 tester.sync();
298 tester.getTarget("//a:a");
299
300 tester.addFile("c");
301 tester.sync();
302 Target a3 = tester.getTarget("//a:a");
303 assertNotSame(a1, a3);
304 }
305
306 @Test
307 public void testBuildFileWithSyntaxError() throws Exception {
308 tester.addFile("a/BUILD", "sh_library(xyz='a')");
309 tester.sync();
310 try {
311 tester.getTarget("//a:a");
312 fail();
313 } catch (NoSuchThingException e) {
314 // Expected
315 }
316
317 tester.modifyFile("a/BUILD", "sh_library(name='a')");
318 tester.sync();
319 tester.getTarget("//a:a");
320 }
321
322 @Test
323 public void testSymlinkedBuildFileWithSyntaxError() throws Exception {
324 tester.addFile("a/BUILD.real", "sh_library(xyz='a')");
325 tester.addSymlink("a/BUILD", "BUILD.real");
326 tester.sync();
327 try {
328 tester.getTarget("//a:a");
329 fail();
330 } catch (NoSuchThingException e) {
331 // Expected
332 }
333 tester.modifyFile("a/BUILD.real", "sh_library(name='a')");
334 tester.sync();
335 tester.getTarget("//a:a");
336 }
337
338 @Test
339 public void testTransientErrorsInGlobbing() throws Exception {
340 Path buildFile = tester.addFile("e/BUILD", "sh_library(name = 'e', data = glob(['*.txt']))");
341 Path parentDir = buildFile.getParentDirectory();
342 tester.addFile("e/data.txt");
343 throwOnReaddir = parentDir;
344 tester.sync();
345 Target target = tester.getTarget("//e:e");
346 assertThat(((Rule) target).containsErrors()).isTrue();
347 GlobList<?> globList = (GlobList<?>) ((Rule) target).getAttributeContainer().getAttr("data");
348 assertThat(globList).isEmpty();
349 throwOnReaddir = null;
350 tester.sync();
351 target = tester.getTarget("//e:e");
352 assertThat(((Rule) target).containsErrors()).isFalse();
353 globList = (GlobList<?>) ((Rule) target).getAttributeContainer().getAttr("data");
354 assertThat(globList).containsExactly(Label.parseAbsolute("//e:data.txt"));
355 }
356
357 @Test
358 public void testIrrelevantFileInSubdirDoesntReloadPackage() throws Exception {
359 tester.addFile("pkg/BUILD", "sh_library(name = 'pkg', srcs = glob(['**/*.sh']))");
360 tester.addFile("pkg/pkg.sh", "#!/bin/bash");
361 tester.addFile("pkg/bar/bar.sh", "#!/bin/bash");
362 Package pkg = tester.getTarget("//pkg:pkg").getPackage();
363
364 // Write file in directory to force reload of top-level glob.
365 tester.addFile("pkg/irrelevant_file");
366 tester.addFile("pkg/bar/irrelevant_file"); // Subglob is also reloaded.
367 assertSame(pkg, tester.getTarget("//pkg:pkg").getPackage());
368 }
369
370 @Test
371 public void testMissingPackages() throws Exception {
372 tester.sync();
373
374 try {
375 tester.getTarget("//a:a");
376 fail();
377 } catch (NoSuchThingException e) {
378 // expected
379 }
380
381 tester.addFile("a/BUILD", "sh_library(name='a')");
382 tester.sync();
383 tester.getTarget("//a:a");
384 }
385
Lukacs Berkide2183d2015-12-16 11:25:36 +0000386 @Test
387 public void testChangedExternalFile() throws Exception {
388 tester.addFile("a/BUILD",
389 "load('/a/b', 'b')",
390 "b()");
391
392 tester.addFile("/b.bzl",
393 "def b():",
394 " pass");
395 tester.addSymlink("a/b.bzl", "/b.bzl");
396 tester.sync();
397 tester.getTarget("//a:BUILD");
398 tester.modifyFile("/b.bzl", "ERROR ERROR");
399 tester.sync();
400
401 try {
402 tester.getTarget("//a:BUILD");
403 fail();
404 } catch (NoSuchThingException e) {
405 // expected
406 }
407 }
408
409
Ulf Adams5d058c42015-12-09 16:22:01 +0000410 static class PackageCacheTester {
Lukacs Berkide2183d2015-12-16 11:25:36 +0000411 private class ManualDiffAwareness implements DiffAwareness {
412 private View lastView;
413 private View currentView;
414
415 @Override
416 public View getCurrentView() {
417 lastView = currentView;
418 currentView = new View() {};
419 return currentView;
420 }
421
422 @Override
423 public ModifiedFileSet getDiff(View oldView, View newView) {
424 if (oldView == lastView && newView == currentView) {
425 return Preconditions.checkNotNull(modifiedFileSet);
426 } else {
427 return ModifiedFileSet.EVERYTHING_MODIFIED;
428 }
429 }
430
431 @Override
432 public String name() {
433 return "PackageCacheTester.DiffAwareness";
434 }
435
436 @Override
437 public void close() {
438 }
439 }
440
441 private class ManualDiffAwarenessFactory implements DiffAwareness.Factory {
442 @Nullable
443 @Override
444 public DiffAwareness maybeCreate(Path pathEntry) {
445 return pathEntry == workspace ? new ManualDiffAwareness() : null;
446 }
447 }
448
Ulf Adams5d058c42015-12-09 16:22:01 +0000449 private final ManualClock clock;
450 private final Path workspace;
451 private final Path outputBase;
452 private final Reporter reporter = new Reporter();
453 private final SkyframeExecutor skyframeExecutor;
454 private final List<Path> changes = new ArrayList<>();
455 private boolean everythingModified = false;
Lukacs Berkide2183d2015-12-16 11:25:36 +0000456 private ModifiedFileSet modifiedFileSet;
Ulf Adams5d058c42015-12-09 16:22:01 +0000457
458 public PackageCacheTester(
459 FileSystem fs, ManualClock clock, Preprocessor.Factory.Supplier supplier)
460 throws IOException {
461 this.clock = clock;
462 workspace = fs.getPath("/workspace");
463 workspace.createDirectory();
464 outputBase = fs.getPath("/output_base");
465 outputBase.createDirectory();
466 addFile("WORKSPACE");
467
468 skyframeExecutor =
469 SequencedSkyframeExecutor.create(
470 new PackageFactory(TestRuleClassProvider.getRuleClassProvider()),
Ulf Adams5d058c42015-12-09 16:22:01 +0000471 new BlazeDirectories(fs.getPath("/install"), fs.getPath("/output"), workspace),
472 null, /* BinTools */
473 null, /* workspaceStatusActionFactory */
474 TestRuleClassProvider.getRuleClassProvider().getBuildInfoFactories(),
Lukacs Berkide2183d2015-12-16 11:25:36 +0000475 ImmutableList.of(new ManualDiffAwarenessFactory()),
Ulf Adams5d058c42015-12-09 16:22:01 +0000476 Predicates.<PathFragment>alwaysFalse(),
477 supplier,
478 ImmutableMap.<SkyFunctionName, SkyFunction>of(),
479 ImmutableList.<PrecomputedValue.Injected>of(),
480 ImmutableList.<SkyValueDirtinessChecker>of());
481 skyframeExecutor.preparePackageLoading(
482 new PathPackageLocator(outputBase, ImmutableList.of(workspace)),
483 ConstantRuleVisibility.PUBLIC, true, 7, "",
Ulf Adamsc73051c62016-03-23 09:18:13 +0000484 UUID.randomUUID(), new TimestampGranularityMonitor(BlazeClock.instance()));
Ulf Adams5d058c42015-12-09 16:22:01 +0000485 }
486
487 Path addFile(String fileName, String... content) throws IOException {
488 Path buildFile = workspace.getRelative(fileName);
489 Preconditions.checkState(!buildFile.exists());
490 Path currentPath = buildFile;
491
492 // Add the new file and all the directories that will be created by
493 // createDirectoryAndParents()
494 while (!currentPath.exists()) {
495 changes.add(currentPath);
496 currentPath = currentPath.getParentDirectory();
497 }
498
499 FileSystemUtils.createDirectoryAndParents(buildFile.getParentDirectory());
500 FileSystemUtils.writeContentAsLatin1(buildFile, Joiner.on('\n').join(content));
501 return buildFile;
502 }
503
504 void addSymlink(String fileName, String target) throws IOException {
505 Path path = workspace.getRelative(fileName);
506 Preconditions.checkState(!path.exists());
507 FileSystemUtils.createDirectoryAndParents(path.getParentDirectory());
508 path.createSymbolicLink(new PathFragment(target));
509 changes.add(path);
510 }
511
512 void removeFile(String fileName) throws IOException {
513 Path path = workspace.getRelative(fileName);
514 Preconditions.checkState(path.delete());
515 changes.add(path);
516 }
517
518 void modifyFile(String fileName, String... content) throws IOException {
519 Path path = workspace.getRelative(fileName);
520 Preconditions.checkState(path.exists());
521 Preconditions.checkState(path.delete());
522 FileSystemUtils.writeContentAsLatin1(path, Joiner.on('\n').join(content));
523 changes.add(path);
524 }
525
526 void modifySymlink(String fileName, String newTarget) throws IOException {
527 Path symlink = workspace.getRelative(fileName);
528 Preconditions.checkState(symlink.exists());
529 symlink.delete();
530 symlink.createSymbolicLink(new PathFragment(newTarget));
531 changes.add(symlink);
532 }
533
534 void everythingModified() {
535 everythingModified = true;
536 }
537
538 private ModifiedFileSet getModifiedFileSet() {
539 if (everythingModified) {
540 everythingModified = false;
541 return ModifiedFileSet.EVERYTHING_MODIFIED;
542 }
543
544 ModifiedFileSet.Builder builder = ModifiedFileSet.builder();
545 for (Path path : changes) {
546 if (!path.startsWith(workspace)) {
547 continue;
548 }
549
550 PathFragment workspacePath = path.relativeTo(workspace);
551 builder.modify(workspacePath);
552 }
553 return builder.build();
554 }
555
556 void sync() throws InterruptedException {
557 clock.advanceMillis(1);
558
Lukacs Berkide2183d2015-12-16 11:25:36 +0000559 modifiedFileSet = getModifiedFileSet();
Ulf Adams5d058c42015-12-09 16:22:01 +0000560 skyframeExecutor.preparePackageLoading(
561 new PathPackageLocator(outputBase, ImmutableList.of(workspace)),
562 ConstantRuleVisibility.PUBLIC, true, 7, "",
Ulf Adamsc73051c62016-03-23 09:18:13 +0000563 UUID.randomUUID(), new TimestampGranularityMonitor(BlazeClock.instance()));
Ulf Adams5d058c42015-12-09 16:22:01 +0000564 skyframeExecutor.invalidateFilesUnderPathForTesting(
Lukacs Berkide2183d2015-12-16 11:25:36 +0000565 new Reporter(), modifiedFileSet, workspace);
Ulf Adams5d058c42015-12-09 16:22:01 +0000566 ((SequencedSkyframeExecutor) skyframeExecutor).handleDiffs(new Reporter());
567
568 changes.clear();
569 }
570
571 Target getTarget(String targetName)
572 throws NoSuchPackageException, NoSuchTargetException, InterruptedException {
573 Label label = Label.parseAbsoluteUnchecked(targetName);
574 return skyframeExecutor.getPackageManager().getTarget(reporter, label);
575 }
576 }
577}