blob: 925cc451603e70f74013b588f17f873df2b526b4 [file] [log] [blame]
Chi Wangc785f022021-08-01 22:33:22 -07001// Copyright 2021 The Bazel Authors. All rights reserved.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8// Unless required by applicable law or agreed to in writing, software
9// distributed under the License is distributed on an "AS IS" BASIS,
10// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11// See the License for the specific language governing permissions and
12// limitations under the License.
13package com.google.devtools.build.lib.remote;
14
15import static com.google.common.base.Preconditions.checkNotNull;
16import static com.google.common.truth.Truth.assertThat;
chiwang581c81a2021-09-06 19:00:55 -070017import static com.google.common.util.concurrent.MoreExecutors.directExecutor;
Chi Wangc785f022021-08-01 22:33:22 -070018import static com.google.devtools.build.lib.actions.ExecutionRequirements.REMOTE_EXECUTION_INLINE_OUTPUTS;
19import static com.google.devtools.build.lib.remote.util.DigestUtil.toBinaryDigest;
chiwang4a12a2c2021-09-02 21:15:54 -070020import static com.google.devtools.build.lib.remote.util.Utils.getFromFuture;
Chi Wangc785f022021-08-01 22:33:22 -070021import static com.google.devtools.build.lib.vfs.FileSystemUtils.readContent;
22import static java.nio.charset.StandardCharsets.UTF_8;
23import static org.junit.Assert.assertThrows;
24import static org.mockito.ArgumentMatchers.any;
25import static org.mockito.ArgumentMatchers.argThat;
26import static org.mockito.ArgumentMatchers.eq;
chiwang7f08b782021-09-05 21:04:35 -070027import static org.mockito.Mockito.doReturn;
Chi Wangc785f022021-08-01 22:33:22 -070028import static org.mockito.Mockito.mock;
29import static org.mockito.Mockito.never;
30import static org.mockito.Mockito.spy;
31import static org.mockito.Mockito.times;
32import static org.mockito.Mockito.verify;
33import static org.mockito.Mockito.when;
34
35import build.bazel.remote.execution.v2.ActionResult;
36import build.bazel.remote.execution.v2.Digest;
37import build.bazel.remote.execution.v2.Directory;
38import build.bazel.remote.execution.v2.DirectoryNode;
39import build.bazel.remote.execution.v2.FileNode;
40import build.bazel.remote.execution.v2.OutputDirectory;
41import build.bazel.remote.execution.v2.OutputFile;
42import build.bazel.remote.execution.v2.OutputSymlink;
Chi Wang8c2c78c2021-11-16 19:09:58 -080043import build.bazel.remote.execution.v2.Platform;
Chi Wangc785f022021-08-01 22:33:22 -070044import build.bazel.remote.execution.v2.RequestMetadata;
45import build.bazel.remote.execution.v2.SymlinkNode;
46import build.bazel.remote.execution.v2.Tree;
47import com.google.common.base.Throwables;
48import com.google.common.collect.ImmutableClassToInstanceMap;
49import com.google.common.collect.ImmutableList;
50import com.google.common.collect.ImmutableMap;
51import com.google.common.collect.ImmutableSet;
chiwang7f08b782021-09-05 21:04:35 -070052import com.google.common.eventbus.EventBus;
53import com.google.common.util.concurrent.Futures;
Chi Wang702df842022-03-21 05:37:13 -070054import com.google.common.util.concurrent.SettableFuture;
Chi Wangc785f022021-08-01 22:33:22 -070055import com.google.devtools.build.lib.actions.ActionInput;
chiwang4a12a2c2021-09-02 21:15:54 -070056import com.google.devtools.build.lib.actions.ActionInputHelper;
Chi Wang003e2d02021-09-21 22:52:08 -070057import com.google.devtools.build.lib.actions.ActionUploadFinishedEvent;
58import com.google.devtools.build.lib.actions.ActionUploadStartedEvent;
Chi Wangc785f022021-08-01 22:33:22 -070059import com.google.devtools.build.lib.actions.Artifact;
60import com.google.devtools.build.lib.actions.Artifact.SpecialArtifact;
61import com.google.devtools.build.lib.actions.Artifact.TreeFileArtifact;
62import com.google.devtools.build.lib.actions.ArtifactRoot;
63import com.google.devtools.build.lib.actions.ArtifactRoot.RootType;
Chi Wang8c2c78c2021-11-16 19:09:58 -080064import com.google.devtools.build.lib.actions.ExecutionRequirements;
Chi Wangc785f022021-08-01 22:33:22 -070065import com.google.devtools.build.lib.actions.FileArtifactValue.RemoteFileArtifactValue;
66import com.google.devtools.build.lib.actions.ResourceSet;
67import com.google.devtools.build.lib.actions.SimpleSpawn;
68import com.google.devtools.build.lib.actions.Spawn;
chiwang4a12a2c2021-09-02 21:15:54 -070069import com.google.devtools.build.lib.actions.SpawnResult;
chiwang7f08b782021-09-05 21:04:35 -070070import com.google.devtools.build.lib.actions.SpawnResult.Status;
Chi Wangc785f022021-08-01 22:33:22 -070071import com.google.devtools.build.lib.actions.cache.MetadataInjector;
72import com.google.devtools.build.lib.actions.util.ActionsTestUtil;
73import com.google.devtools.build.lib.clock.JavaClock;
chiwangdb15e472021-09-05 20:06:30 -070074import com.google.devtools.build.lib.collect.nestedset.NestedSet;
Chi Wangc785f022021-08-01 22:33:22 -070075import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
76import com.google.devtools.build.lib.collect.nestedset.Order;
chiwang7f08b782021-09-05 21:04:35 -070077import com.google.devtools.build.lib.events.Event;
78import com.google.devtools.build.lib.events.EventKind;
79import com.google.devtools.build.lib.events.Reporter;
80import com.google.devtools.build.lib.events.StoredEventHandler;
Chi Wangc785f022021-08-01 22:33:22 -070081import com.google.devtools.build.lib.exec.util.FakeOwner;
Chi Wang8c2c78c2021-11-16 19:09:58 -080082import com.google.devtools.build.lib.exec.util.SpawnBuilder;
Chi Wangc785f022021-08-01 22:33:22 -070083import com.google.devtools.build.lib.remote.RemoteExecutionService.RemoteActionResult;
chiwang862fd5e2021-09-05 20:39:35 -070084import com.google.devtools.build.lib.remote.common.BulkTransferException;
Chi Wangc785f022021-08-01 22:33:22 -070085import com.google.devtools.build.lib.remote.common.RemoteActionExecutionContext;
ron-stripecf57d032021-08-26 06:07:30 -070086import com.google.devtools.build.lib.remote.common.RemoteCacheClient.CachedActionResult;
chiwangdb15e472021-09-05 20:06:30 -070087import com.google.devtools.build.lib.remote.common.RemoteExecutionClient;
Chi Wangc785f022021-08-01 22:33:22 -070088import com.google.devtools.build.lib.remote.common.RemotePathResolver;
89import com.google.devtools.build.lib.remote.common.RemotePathResolver.DefaultRemotePathResolver;
90import com.google.devtools.build.lib.remote.common.RemotePathResolver.SiblingRepositoryLayoutResolver;
91import com.google.devtools.build.lib.remote.options.RemoteOptions;
92import com.google.devtools.build.lib.remote.options.RemoteOutputsMode;
93import com.google.devtools.build.lib.remote.util.DigestUtil;
94import com.google.devtools.build.lib.remote.util.FakeSpawnExecutionContext;
chiwangdb15e472021-09-05 20:06:30 -070095import com.google.devtools.build.lib.remote.util.RxNoGlobalErrorsRule;
Chi Wangc785f022021-08-01 22:33:22 -070096import com.google.devtools.build.lib.remote.util.TracingMetadataUtils;
97import com.google.devtools.build.lib.remote.util.Utils.InMemoryOutput;
98import com.google.devtools.build.lib.skyframe.TreeArtifactValue;
99import com.google.devtools.build.lib.util.io.FileOutErr;
100import com.google.devtools.build.lib.vfs.DigestHashFunction;
101import com.google.devtools.build.lib.vfs.FileSystem;
102import com.google.devtools.build.lib.vfs.Path;
103import com.google.devtools.build.lib.vfs.PathFragment;
104import com.google.devtools.build.lib.vfs.Symlinks;
janakr491e4412022-01-28 15:35:34 -0800105import com.google.devtools.build.lib.vfs.SyscallCache;
Chi Wangc785f022021-08-01 22:33:22 -0700106import com.google.devtools.build.lib.vfs.inmemoryfs.InMemoryFileSystem;
107import com.google.devtools.common.options.Options;
108import com.google.protobuf.ByteString;
109import java.io.IOException;
110import java.nio.charset.StandardCharsets;
111import java.util.Arrays;
112import java.util.Collection;
Chi Wang702df842022-03-21 05:37:13 -0700113import java.util.Random;
chiwangdb15e472021-09-05 20:06:30 -0700114import java.util.concurrent.ExecutorService;
115import java.util.concurrent.Executors;
116import java.util.concurrent.Semaphore;
117import java.util.concurrent.atomic.AtomicReference;
Chi Wangc785f022021-08-01 22:33:22 -0700118import org.junit.Before;
chiwangdb15e472021-09-05 20:06:30 -0700119import org.junit.Rule;
Chi Wangc785f022021-08-01 22:33:22 -0700120import org.junit.Test;
121import org.junit.runner.RunWith;
122import org.junit.runners.JUnit4;
123
124/** Tests for {@link RemoteExecutionService}. */
125@RunWith(JUnit4.class)
126public class RemoteExecutionServiceTest {
chiwangdb15e472021-09-05 20:06:30 -0700127 @Rule public final RxNoGlobalErrorsRule rxNoGlobalErrorsRule = new RxNoGlobalErrorsRule();
128
janakr491e4412022-01-28 15:35:34 -0800129 private final DigestUtil digestUtil =
130 new DigestUtil(SyscallCache.NO_CACHE, DigestHashFunction.SHA256);
chiwang7f08b782021-09-05 21:04:35 -0700131 private final Reporter reporter = new Reporter(new EventBus());
132 private final StoredEventHandler eventHandler = new StoredEventHandler();
Chi Wangc785f022021-08-01 22:33:22 -0700133
134 RemoteOptions remoteOptions;
135 private Path execRoot;
136 private ArtifactRoot artifactRoot;
137 private FakeActionInputFileCache fakeFileCache;
138 private RemotePathResolver remotePathResolver;
139 private FileOutErr outErr;
140 private InMemoryRemoteCache cache;
chiwangdb15e472021-09-05 20:06:30 -0700141 private RemoteExecutionClient executor;
Chi Wangc785f022021-08-01 22:33:22 -0700142 private RemoteActionExecutionContext remoteActionExecutionContext;
143
144 @Before
145 public final void setUp() throws Exception {
chiwang7f08b782021-09-05 21:04:35 -0700146 reporter.addHandler(eventHandler);
147
Chi Wangc785f022021-08-01 22:33:22 -0700148 remoteOptions = Options.getDefaults(RemoteOptions.class);
149
150 FileSystem fs = new InMemoryFileSystem(new JavaClock(), DigestHashFunction.SHA256);
151 execRoot = fs.getPath("/execroot");
152 execRoot.createDirectoryAndParents();
153 artifactRoot = ArtifactRoot.asDerivedRoot(execRoot, RootType.Output, "outputs");
154 checkNotNull(artifactRoot.getRoot().asPath()).createDirectoryAndParents();
155 fakeFileCache = new FakeActionInputFileCache(execRoot);
156
157 remotePathResolver = new DefaultRemotePathResolver(execRoot);
158
159 Path stdout = fs.getPath("/tmp/stdout");
160 Path stderr = fs.getPath("/tmp/stderr");
161 checkNotNull(stdout.getParentDirectory()).createDirectoryAndParents();
162 checkNotNull(stderr.getParentDirectory()).createDirectoryAndParents();
163 outErr = new FileOutErr(stdout, stderr);
164
Chi Wang003e2d02021-09-21 22:52:08 -0700165 cache = spy(new InMemoryRemoteCache(remoteOptions, digestUtil));
chiwangdb15e472021-09-05 20:06:30 -0700166 executor = mock(RemoteExecutionClient.class);
Chi Wangc785f022021-08-01 22:33:22 -0700167
168 RequestMetadata metadata =
169 TracingMetadataUtils.buildMetadata("none", "none", "action-id", null);
170 remoteActionExecutionContext = RemoteActionExecutionContext.create(metadata);
171 }
172
173 @Test
Chi Wang8c2c78c2021-11-16 19:09:58 -0800174 public void buildRemoteAction_differentiateWorkspace_generateActionSalt() throws Exception {
175 Spawn spawn =
176 new SpawnBuilder("dummy")
177 .withExecutionInfo(ExecutionRequirements.DIFFERENTIATE_WORKSPACE_CACHE, "aa")
178 .build();
179 FakeSpawnExecutionContext context = newSpawnExecutionContext(spawn);
180 RemoteExecutionService service = newRemoteExecutionService();
181
182 RemoteAction remoteAction = service.buildRemoteAction(spawn, context);
183
184 Platform expected =
185 Platform.newBuilder()
186 .addProperties(Platform.Property.newBuilder().setName("workspace").setValue("aa"))
187 .build();
188 assertThat(remoteAction.getAction().getSalt()).isEqualTo(expected.toByteString());
189 }
190
191 @Test
Chi Wang11066c72021-09-17 00:30:45 -0700192 public void downloadOutputs_outputFiles_executableBitIgnored() throws Exception {
193 // Test that executable bit of downloaded output files are ignored since it will be chmod 555
194 // after action
195 // execution.
Chi Wangc785f022021-08-01 22:33:22 -0700196
197 // arrange
198 Digest fooDigest = cache.addContents(remoteActionExecutionContext, "foo-contents");
199 Digest barDigest = cache.addContents(remoteActionExecutionContext, "bar-contents");
200 ActionResult.Builder builder = ActionResult.newBuilder();
201 builder.addOutputFilesBuilder().setPath("outputs/foo").setDigest(fooDigest);
202 builder
203 .addOutputFilesBuilder()
204 .setPath("outputs/bar")
205 .setDigest(barDigest)
206 .setIsExecutable(true);
ron-stripecf57d032021-08-26 06:07:30 -0700207 RemoteActionResult result =
208 RemoteActionResult.createFromCache(CachedActionResult.remote(builder.build()));
Chi Wangc785f022021-08-01 22:33:22 -0700209 Spawn spawn = newSpawnFromResult(result);
210 FakeSpawnExecutionContext context = newSpawnExecutionContext(spawn);
211 RemoteExecutionService service = newRemoteExecutionService();
212 RemoteAction action = service.buildRemoteAction(spawn, context);
213
214 // act
215 service.downloadOutputs(action, result);
216
217 // assert
218 assertThat(digestUtil.compute(execRoot.getRelative("outputs/foo"))).isEqualTo(fooDigest);
219 assertThat(digestUtil.compute(execRoot.getRelative("outputs/bar"))).isEqualTo(barDigest);
220 assertThat(execRoot.getRelative("outputs/foo").isExecutable()).isFalse();
Chi Wang11066c72021-09-17 00:30:45 -0700221 assertThat(execRoot.getRelative("outputs/bar").isExecutable()).isFalse();
Chi Wangc785f022021-08-01 22:33:22 -0700222 assertThat(context.isLockOutputFilesCalled()).isTrue();
223 }
224
225 @Test
226 public void downloadOutputs_siblingLayoutAndRelativeToInputRoot_works() throws Exception {
227 // arrange
228 remotePathResolver = new SiblingRepositoryLayoutResolver(execRoot, true);
229
230 Digest fooDigest = cache.addContents(remoteActionExecutionContext, "foo-contents");
231 Digest barDigest = cache.addContents(remoteActionExecutionContext, "bar-contents");
232 ActionResult.Builder builder = ActionResult.newBuilder();
233 builder.addOutputFilesBuilder().setPath("execroot/outputs/foo").setDigest(fooDigest);
234 builder.addOutputFilesBuilder().setPath("execroot/outputs/bar").setDigest(barDigest);
ron-stripecf57d032021-08-26 06:07:30 -0700235 RemoteActionResult result =
236 RemoteActionResult.createFromCache(CachedActionResult.remote(builder.build()));
Chi Wangc785f022021-08-01 22:33:22 -0700237 Spawn spawn = newSpawnFromResult(result);
238 FakeSpawnExecutionContext context = newSpawnExecutionContext(spawn);
239 RemoteExecutionService service = newRemoteExecutionService();
240 RemoteAction action = service.buildRemoteAction(spawn, context);
241
242 // act
243 service.downloadOutputs(action, result);
244
245 // assert
246 assertThat(digestUtil.compute(execRoot.getRelative("outputs/foo"))).isEqualTo(fooDigest);
247 assertThat(digestUtil.compute(execRoot.getRelative("outputs/bar"))).isEqualTo(barDigest);
248 assertThat(context.isLockOutputFilesCalled()).isTrue();
249 }
250
251 @Test
252 public void downloadOutputs_outputDirectories_works() throws Exception {
253 // Test that downloading an output directory works.
254
255 // arrange
256 Digest fooDigest = cache.addContents(remoteActionExecutionContext, "foo-contents");
257 Digest quxDigest = cache.addContents(remoteActionExecutionContext, "qux-contents");
258 Tree barTreeMessage =
259 Tree.newBuilder()
260 .setRoot(
261 Directory.newBuilder()
262 .addFiles(
263 FileNode.newBuilder()
264 .setName("qux")
265 .setDigest(quxDigest)
266 .setIsExecutable(true)))
267 .build();
268 Digest barTreeDigest =
269 cache.addContents(remoteActionExecutionContext, barTreeMessage.toByteArray());
270 ActionResult.Builder builder = ActionResult.newBuilder();
271 builder.addOutputFilesBuilder().setPath("outputs/a/foo").setDigest(fooDigest);
272 builder.addOutputDirectoriesBuilder().setPath("outputs/a/bar").setTreeDigest(barTreeDigest);
ron-stripecf57d032021-08-26 06:07:30 -0700273 RemoteActionResult result =
274 RemoteActionResult.createFromCache(CachedActionResult.remote(builder.build()));
Chi Wangc785f022021-08-01 22:33:22 -0700275 Spawn spawn = newSpawnFromResult(result);
276 FakeSpawnExecutionContext context = newSpawnExecutionContext(spawn);
277 RemoteExecutionService service = newRemoteExecutionService();
278 RemoteAction action = service.buildRemoteAction(spawn, context);
279
280 // act
281 service.downloadOutputs(action, result);
282
283 // assert
284 assertThat(digestUtil.compute(execRoot.getRelative("outputs/a/foo"))).isEqualTo(fooDigest);
285 assertThat(digestUtil.compute(execRoot.getRelative("outputs/a/bar/qux"))).isEqualTo(quxDigest);
Chi Wang11066c72021-09-17 00:30:45 -0700286 assertThat(execRoot.getRelative("outputs/a/bar/qux").isExecutable()).isFalse();
Chi Wangc785f022021-08-01 22:33:22 -0700287 assertThat(context.isLockOutputFilesCalled()).isTrue();
288 }
289
290 @Test
291 public void downloadOutputs_emptyOutputDirectories_works() throws Exception {
292 // Test that downloading an empty output directory works.
293
294 // arrange
Googlerf33d1672021-09-06 03:28:56 -0700295 Tree barTreeMessage = Tree.newBuilder().setRoot(Directory.getDefaultInstance()).build();
Chi Wangc785f022021-08-01 22:33:22 -0700296 Digest barTreeDigest =
297 cache.addContents(remoteActionExecutionContext, barTreeMessage.toByteArray());
298 ActionResult.Builder builder = ActionResult.newBuilder();
299 builder.addOutputDirectoriesBuilder().setPath("outputs/a/bar").setTreeDigest(barTreeDigest);
ron-stripecf57d032021-08-26 06:07:30 -0700300 RemoteActionResult result =
301 RemoteActionResult.createFromCache(CachedActionResult.remote(builder.build()));
Chi Wangc785f022021-08-01 22:33:22 -0700302 Spawn spawn = newSpawnFromResult(result);
303 FakeSpawnExecutionContext context = newSpawnExecutionContext(spawn);
304 RemoteExecutionService service = newRemoteExecutionService();
305 RemoteAction action = service.buildRemoteAction(spawn, context);
306
307 // act
308 service.downloadOutputs(action, result);
309
310 // assert
311 assertThat(execRoot.getRelative("outputs/a/bar").isDirectory()).isTrue();
312 assertThat(context.isLockOutputFilesCalled()).isTrue();
313 }
314
315 @Test
316 public void downloadOutputs_nestedOutputDirectories_works() throws Exception {
317 // Test that downloading a nested output directory works.
318
319 // arrange
320 Digest fooDigest = cache.addContents(remoteActionExecutionContext, "foo-contents");
321 Digest quxDigest = cache.addContents(remoteActionExecutionContext, "qux-contents");
322 Directory wobbleDirMessage =
323 Directory.newBuilder()
324 .addFiles(FileNode.newBuilder().setName("qux").setDigest(quxDigest))
325 .build();
326 Digest wobbleDirDigest =
327 cache.addContents(remoteActionExecutionContext, wobbleDirMessage.toByteArray());
328 Tree barTreeMessage =
329 Tree.newBuilder()
330 .setRoot(
331 Directory.newBuilder()
332 .addFiles(
333 FileNode.newBuilder()
334 .setName("qux")
335 .setDigest(quxDigest)
336 .setIsExecutable(true))
337 .addDirectories(
338 DirectoryNode.newBuilder().setName("wobble").setDigest(wobbleDirDigest)))
339 .addChildren(wobbleDirMessage)
340 .build();
341 Digest barTreeDigest =
342 cache.addContents(remoteActionExecutionContext, barTreeMessage.toByteArray());
343 ActionResult.Builder builder = ActionResult.newBuilder();
344 builder.addOutputFilesBuilder().setPath("outputs/a/foo").setDigest(fooDigest);
345 builder.addOutputDirectoriesBuilder().setPath("outputs/a/bar").setTreeDigest(barTreeDigest);
ron-stripecf57d032021-08-26 06:07:30 -0700346 RemoteActionResult result =
347 RemoteActionResult.createFromCache(CachedActionResult.remote(builder.build()));
Chi Wangc785f022021-08-01 22:33:22 -0700348 Spawn spawn = newSpawnFromResult(result);
349 FakeSpawnExecutionContext context = newSpawnExecutionContext(spawn);
350 RemoteExecutionService service = newRemoteExecutionService();
351 RemoteAction action = service.buildRemoteAction(spawn, context);
352
353 // act
354 service.downloadOutputs(action, result);
355
356 // assert
357 assertThat(digestUtil.compute(execRoot.getRelative("outputs/a/foo"))).isEqualTo(fooDigest);
358 assertThat(digestUtil.compute(execRoot.getRelative("outputs/a/bar/wobble/qux")))
359 .isEqualTo(quxDigest);
360 assertThat(execRoot.getRelative("outputs/a/bar/wobble/qux").isExecutable()).isFalse();
361 assertThat(context.isLockOutputFilesCalled()).isTrue();
362 }
363
364 @Test
John Millikinff6ac532022-04-26 06:32:12 -0700365 public void downloadOutputs_outputDirectoriesWithNestedFile_works() throws Exception {
366 // Test that downloading an output directory containing a named output file works.
367
368 // arrange
369 Digest fooDigest = cache.addContents(remoteActionExecutionContext, "foo-contents");
370 Digest barDigest = cache.addContents(remoteActionExecutionContext, "bar-ontents");
371 Tree subdirTreeMessage =
372 Tree.newBuilder()
373 .setRoot(
374 Directory.newBuilder()
375 .addFiles(FileNode.newBuilder().setName("foo").setDigest(fooDigest))
376 .addFiles(FileNode.newBuilder().setName("bar").setDigest(barDigest)))
377 .build();
378 Digest subdirTreeDigest =
379 cache.addContents(remoteActionExecutionContext, subdirTreeMessage.toByteArray());
380 ActionResult.Builder builder = ActionResult.newBuilder();
381 builder.addOutputFilesBuilder().setPath("outputs/subdir/foo").setDigest(fooDigest);
382 builder.addOutputDirectoriesBuilder().setPath("outputs/subdir").setTreeDigest(subdirTreeDigest);
383 RemoteActionResult result =
384 RemoteActionResult.createFromCache(CachedActionResult.remote(builder.build()));
385 Spawn spawn = newSpawnFromResult(result);
386 FakeSpawnExecutionContext context = newSpawnExecutionContext(spawn);
387 RemoteExecutionService service = newRemoteExecutionService();
388 RemoteAction action = service.buildRemoteAction(spawn, context);
389
390 // act
391 service.downloadOutputs(action, result);
392
393 // assert
394 assertThat(digestUtil.compute(execRoot.getRelative("outputs/subdir/foo"))).isEqualTo(fooDigest);
395 assertThat(digestUtil.compute(execRoot.getRelative("outputs/subdir/bar"))).isEqualTo(barDigest);
396 assertThat(context.isLockOutputFilesCalled()).isTrue();
397 }
398
399 @Test
Chi Wangc785f022021-08-01 22:33:22 -0700400 public void downloadOutputs_outputDirectoriesWithSameHash_works() throws Exception {
401 // Test that downloading an output directory works when two Directory
402 // protos have the same hash i.e. because they have the same name and contents or are empty.
403
404 /*
405 * /bar/foo/file
406 * /foo/file
407 */
408
409 // arrange
410 Digest fileDigest = cache.addContents(remoteActionExecutionContext, "file");
411 FileNode file = FileNode.newBuilder().setName("file").setDigest(fileDigest).build();
412 Directory fooDir = Directory.newBuilder().addFiles(file).build();
413 Digest fooDigest = cache.addContents(remoteActionExecutionContext, fooDir.toByteArray());
414 DirectoryNode fooDirNode =
415 DirectoryNode.newBuilder().setName("foo").setDigest(fooDigest).build();
416 Directory barDir = Directory.newBuilder().addDirectories(fooDirNode).build();
417 Digest barDigest = cache.addContents(remoteActionExecutionContext, barDir.toByteArray());
418 DirectoryNode barDirNode =
419 DirectoryNode.newBuilder().setName("bar").setDigest(barDigest).build();
420 Directory rootDir =
421 Directory.newBuilder().addDirectories(fooDirNode).addDirectories(barDirNode).build();
422 Tree tree =
423 Tree.newBuilder()
424 .setRoot(rootDir)
425 .addChildren(barDir)
426 .addChildren(fooDir)
427 .addChildren(fooDir)
428 .build();
429 Digest treeDigest = cache.addContents(remoteActionExecutionContext, tree.toByteArray());
430 ActionResult.Builder builder = ActionResult.newBuilder();
431 builder.addOutputDirectoriesBuilder().setPath("outputs/a/").setTreeDigest(treeDigest);
ron-stripecf57d032021-08-26 06:07:30 -0700432 RemoteActionResult result =
433 RemoteActionResult.createFromCache(CachedActionResult.remote(builder.build()));
Chi Wangc785f022021-08-01 22:33:22 -0700434 Spawn spawn = newSpawnFromResult(result);
435 FakeSpawnExecutionContext context = newSpawnExecutionContext(spawn);
436 RemoteExecutionService service = newRemoteExecutionService();
437 RemoteAction action = service.buildRemoteAction(spawn, context);
438
439 // act
440 service.downloadOutputs(action, result);
441
442 // assert
443 assertThat(digestUtil.compute(execRoot.getRelative("outputs/a/bar/foo/file")))
444 .isEqualTo(fileDigest);
445 assertThat(digestUtil.compute(execRoot.getRelative("outputs/a/foo/file")))
446 .isEqualTo(fileDigest);
447 assertThat(context.isLockOutputFilesCalled()).isTrue();
448 }
449
450 @Test
451 public void downloadOutputs_relativeFileSymlink_success() throws Exception {
452 ActionResult.Builder builder = ActionResult.newBuilder();
453 builder.addOutputFileSymlinksBuilder().setPath("outputs/a/b/link").setTarget("../../foo");
ron-stripecf57d032021-08-26 06:07:30 -0700454 RemoteActionResult result =
455 RemoteActionResult.createFromCache(CachedActionResult.remote(builder.build()));
Chi Wangc785f022021-08-01 22:33:22 -0700456 Spawn spawn = newSpawnFromResult(result);
457 FakeSpawnExecutionContext context = newSpawnExecutionContext(spawn);
458 RemoteExecutionService service = newRemoteExecutionService();
459 RemoteAction action = service.buildRemoteAction(spawn, context);
460
461 // Doesn't check for dangling links, hence download succeeds.
462 service.downloadOutputs(action, result);
463
464 Path path = execRoot.getRelative("outputs/a/b/link");
465 assertThat(path.isSymbolicLink()).isTrue();
466 assertThat(path.readSymbolicLink()).isEqualTo(PathFragment.create("../../foo"));
467 assertThat(context.isLockOutputFilesCalled()).isTrue();
468 }
469
470 @Test
471 public void downloadOutputs_relativeDirectorySymlink_success() throws Exception {
472 ActionResult.Builder builder = ActionResult.newBuilder();
473 builder.addOutputDirectorySymlinksBuilder().setPath("outputs/a/b/link").setTarget("foo");
ron-stripecf57d032021-08-26 06:07:30 -0700474 RemoteActionResult result =
475 RemoteActionResult.createFromCache(CachedActionResult.remote(builder.build()));
Chi Wangc785f022021-08-01 22:33:22 -0700476 Spawn spawn = newSpawnFromResult(result);
477 FakeSpawnExecutionContext context = newSpawnExecutionContext(spawn);
478 RemoteExecutionService service = newRemoteExecutionService();
479 RemoteAction action = service.buildRemoteAction(spawn, context);
480
481 // Doesn't check for dangling links, hence download succeeds.
482 service.downloadOutputs(action, result);
483
484 Path path = execRoot.getRelative("outputs/a/b/link");
485 assertThat(path.isSymbolicLink()).isTrue();
486 assertThat(path.readSymbolicLink()).isEqualTo(PathFragment.create("foo"));
487 assertThat(context.isLockOutputFilesCalled()).isTrue();
488 }
489
490 @Test
491 public void downloadOutputs_relativeSymlinkInDirectory_success() throws Exception {
492 Tree tree =
493 Tree.newBuilder()
494 .setRoot(
495 Directory.newBuilder()
496 .addSymlinks(SymlinkNode.newBuilder().setName("link").setTarget("../foo")))
497 .build();
498 Digest treeDigest = cache.addContents(remoteActionExecutionContext, tree.toByteArray());
499 ActionResult.Builder builder = ActionResult.newBuilder();
500 builder.addOutputDirectoriesBuilder().setPath("outputs/dir").setTreeDigest(treeDigest);
ron-stripecf57d032021-08-26 06:07:30 -0700501 RemoteActionResult result =
502 RemoteActionResult.createFromCache(CachedActionResult.remote(builder.build()));
Chi Wangc785f022021-08-01 22:33:22 -0700503 Spawn spawn = newSpawnFromResult(result);
504 FakeSpawnExecutionContext context = newSpawnExecutionContext(spawn);
505 RemoteExecutionService service = newRemoteExecutionService();
506 RemoteAction action = service.buildRemoteAction(spawn, context);
507
508 // Doesn't check for dangling links, hence download succeeds.
509 service.downloadOutputs(action, result);
510
511 Path path = execRoot.getRelative("outputs/dir/link");
512 assertThat(path.isSymbolicLink()).isTrue();
513 assertThat(path.readSymbolicLink()).isEqualTo(PathFragment.create("../foo"));
514 assertThat(context.isLockOutputFilesCalled()).isTrue();
515 }
516
517 @Test
518 public void downloadOutputs_absoluteFileSymlink_error() throws Exception {
519 ActionResult.Builder builder = ActionResult.newBuilder();
520 builder.addOutputFileSymlinksBuilder().setPath("outputs/foo").setTarget("/abs/link");
ron-stripecf57d032021-08-26 06:07:30 -0700521 RemoteActionResult result =
522 RemoteActionResult.createFromCache(CachedActionResult.remote(builder.build()));
Chi Wangc785f022021-08-01 22:33:22 -0700523 Spawn spawn = newSpawnFromResult(result);
524 FakeSpawnExecutionContext context = newSpawnExecutionContext(spawn);
525 RemoteExecutionService service = newRemoteExecutionService();
526 RemoteAction action = service.buildRemoteAction(spawn, context);
527
528 IOException expected =
529 assertThrows(IOException.class, () -> service.downloadOutputs(action, result));
530
531 assertThat(expected).hasMessageThat().contains("/abs/link");
532 assertThat(expected).hasMessageThat().contains("absolute path");
533 assertThat(context.isLockOutputFilesCalled()).isTrue();
534 }
535
536 @Test
537 public void downloadOutputs_absoluteDirectorySymlink_error() throws Exception {
538 ActionResult.Builder builder = ActionResult.newBuilder();
539 builder.addOutputDirectorySymlinksBuilder().setPath("outputs/foo").setTarget("/abs/link");
ron-stripecf57d032021-08-26 06:07:30 -0700540 RemoteActionResult result =
541 RemoteActionResult.createFromCache(CachedActionResult.remote(builder.build()));
Chi Wangc785f022021-08-01 22:33:22 -0700542 Spawn spawn = newSpawnFromResult(result);
543 FakeSpawnExecutionContext context = newSpawnExecutionContext(spawn);
544 RemoteExecutionService service = newRemoteExecutionService();
545 RemoteAction action = service.buildRemoteAction(spawn, context);
546
547 IOException expected =
548 assertThrows(IOException.class, () -> service.downloadOutputs(action, result));
549
550 assertThat(expected).hasMessageThat().contains("/abs/link");
551 assertThat(expected).hasMessageThat().contains("absolute path");
552 assertThat(context.isLockOutputFilesCalled()).isTrue();
553 }
554
555 @Test
556 public void downloadOutputs_absoluteSymlinkInDirectory_error() throws Exception {
557 Tree tree =
558 Tree.newBuilder()
559 .setRoot(
560 Directory.newBuilder()
561 .addSymlinks(SymlinkNode.newBuilder().setName("link").setTarget("/foo")))
562 .build();
563 Digest treeDigest = cache.addContents(remoteActionExecutionContext, tree.toByteArray());
564 ActionResult.Builder builder = ActionResult.newBuilder();
565 builder.addOutputDirectoriesBuilder().setPath("outputs/dir").setTreeDigest(treeDigest);
ron-stripecf57d032021-08-26 06:07:30 -0700566 RemoteActionResult result =
567 RemoteActionResult.createFromCache(CachedActionResult.remote(builder.build()));
Chi Wangc785f022021-08-01 22:33:22 -0700568 Spawn spawn = newSpawnFromResult(result);
569 FakeSpawnExecutionContext context = newSpawnExecutionContext(spawn);
570 RemoteExecutionService service = newRemoteExecutionService();
571 RemoteAction action = service.buildRemoteAction(spawn, context);
572
573 IOException expected =
574 assertThrows(IOException.class, () -> service.downloadOutputs(action, result));
575
576 assertThat(expected.getSuppressed()).isEmpty();
577 assertThat(expected).hasMessageThat().contains("outputs/dir/link");
578 assertThat(expected).hasMessageThat().contains("/foo");
579 assertThat(expected).hasMessageThat().contains("absolute path");
580 assertThat(context.isLockOutputFilesCalled()).isTrue();
581 }
582
583 @Test
584 public void downloadOutputs_onFailure_maintainDirectories() throws Exception {
585 // Test that output directories are not deleted on download failure. See
586 // https://github.com/bazelbuild/bazel/issues/6260.
Googlerf33d1672021-09-06 03:28:56 -0700587 Tree tree = Tree.newBuilder().setRoot(Directory.getDefaultInstance()).build();
Chi Wangc785f022021-08-01 22:33:22 -0700588 Digest treeDigest = cache.addContents(remoteActionExecutionContext, tree.toByteArray());
589 Digest outputFileDigest =
590 cache.addException("outputdir/outputfile", new IOException("download failed"));
591 Digest otherFileDigest = cache.addContents(remoteActionExecutionContext, "otherfile");
592 ActionResult.Builder builder = ActionResult.newBuilder();
593 builder.addOutputDirectoriesBuilder().setPath("outputs/outputdir").setTreeDigest(treeDigest);
594 builder.addOutputFiles(
595 OutputFile.newBuilder()
596 .setPath("outputs/outputdir/outputfile")
597 .setDigest(outputFileDigest));
598 builder.addOutputFiles(
599 OutputFile.newBuilder().setPath("outputs/otherfile").setDigest(otherFileDigest));
ron-stripecf57d032021-08-26 06:07:30 -0700600 RemoteActionResult result =
601 RemoteActionResult.createFromCache(CachedActionResult.remote(builder.build()));
Chi Wangc785f022021-08-01 22:33:22 -0700602 Spawn spawn = newSpawnFromResult(result);
603 FakeSpawnExecutionContext context = newSpawnExecutionContext(spawn);
604 RemoteExecutionService service = newRemoteExecutionService();
605 RemoteAction action = service.buildRemoteAction(spawn, context);
606
607 assertThrows(BulkTransferException.class, () -> service.downloadOutputs(action, result));
608
609 assertThat(cache.getNumFailedDownloads()).isEqualTo(1);
610 assertThat(execRoot.getRelative("outputs/outputdir").exists()).isTrue();
611 assertThat(execRoot.getRelative("outputs/outputdir/outputfile").exists()).isFalse();
612 assertThat(execRoot.getRelative("outputs/otherfile").exists()).isFalse();
613 assertThat(context.isLockOutputFilesCalled()).isFalse();
614 }
615
616 @Test
617 public void downloadOutputs_onError_waitForRemainingDownloadsToComplete() throws Exception {
618 // If one or more downloads of output files / directories fail then the code should
619 // wait for all downloads to have been completed before it tries to clean up partially
620 // downloaded files.
621 Digest digest1 = cache.addContents(remoteActionExecutionContext, "file1");
622 Digest digest2 = cache.addException("file2", new IOException("download failed"));
623 Digest digest3 = cache.addContents(remoteActionExecutionContext, "file3");
624 ActionResult actionResult =
625 ActionResult.newBuilder()
626 .setExitCode(0)
627 .addOutputFiles(OutputFile.newBuilder().setPath("outputs/file1").setDigest(digest1))
628 .addOutputFiles(OutputFile.newBuilder().setPath("outputs/file2").setDigest(digest2))
629 .addOutputFiles(OutputFile.newBuilder().setPath("outputs/file3").setDigest(digest3))
630 .build();
ron-stripecf57d032021-08-26 06:07:30 -0700631 RemoteActionResult result =
632 RemoteActionResult.createFromCache(CachedActionResult.remote(actionResult));
Chi Wangc785f022021-08-01 22:33:22 -0700633 Spawn spawn = newSpawnFromResult(result);
634 FakeSpawnExecutionContext context = newSpawnExecutionContext(spawn);
635 RemoteExecutionService service = newRemoteExecutionService();
636 RemoteAction action = service.buildRemoteAction(spawn, context);
637
638 BulkTransferException downloadException =
639 assertThrows(BulkTransferException.class, () -> service.downloadOutputs(action, result));
640
641 assertThat(downloadException.getSuppressed()).hasLength(1);
642 assertThat(cache.getNumSuccessfulDownloads()).isEqualTo(2);
643 assertThat(cache.getNumFailedDownloads()).isEqualTo(1);
644 assertThat(downloadException.getSuppressed()[0]).isInstanceOf(IOException.class);
645 IOException e = (IOException) downloadException.getSuppressed()[0];
646 assertThat(Throwables.getRootCause(e)).hasMessageThat().isEqualTo("download failed");
647 assertThat(context.isLockOutputFilesCalled()).isFalse();
648 }
649
650 @Test
651 public void downloadOutputs_withMultipleErrors_addsThemAsSuppressed() throws Exception {
652 Digest digest1 = cache.addContents(remoteActionExecutionContext, "file1");
653 Digest digest2 = cache.addException("file2", new IOException("file2 failed"));
654 Digest digest3 = cache.addException("file3", new IOException("file3 failed"));
655 ActionResult actionResult =
656 ActionResult.newBuilder()
657 .setExitCode(0)
658 .addOutputFiles(OutputFile.newBuilder().setPath("outputs/file1").setDigest(digest1))
659 .addOutputFiles(OutputFile.newBuilder().setPath("outputs/file2").setDigest(digest2))
660 .addOutputFiles(OutputFile.newBuilder().setPath("outputs/file3").setDigest(digest3))
661 .build();
ron-stripecf57d032021-08-26 06:07:30 -0700662 RemoteActionResult result =
663 RemoteActionResult.createFromCache(CachedActionResult.remote(actionResult));
Chi Wangc785f022021-08-01 22:33:22 -0700664 Spawn spawn = newSpawnFromResult(result);
665 FakeSpawnExecutionContext context = newSpawnExecutionContext(spawn);
666 RemoteExecutionService service = newRemoteExecutionService();
667 RemoteAction action = service.buildRemoteAction(spawn, context);
668
669 BulkTransferException e =
670 assertThrows(BulkTransferException.class, () -> service.downloadOutputs(action, result));
671
672 assertThat(e.getSuppressed()).hasLength(2);
673 assertThat(e.getSuppressed()[0]).isInstanceOf(IOException.class);
674 assertThat(e.getSuppressed()[0]).hasMessageThat().isAnyOf("file2 failed", "file3 failed");
675 assertThat(e.getSuppressed()[1]).isInstanceOf(IOException.class);
676 assertThat(e.getSuppressed()[1]).hasMessageThat().isAnyOf("file2 failed", "file3 failed");
677 }
678
679 @Test
680 public void downloadOutputs_withDuplicateIOErrors_doesNotSuppress() throws Exception {
681 Digest digest1 = cache.addContents(remoteActionExecutionContext, "file1");
682 IOException reusedException = new IOException("reused io exception");
683 Digest digest2 = cache.addException("file2", reusedException);
684 Digest digest3 = cache.addException("file3", reusedException);
685 ActionResult actionResult =
686 ActionResult.newBuilder()
687 .setExitCode(0)
688 .addOutputFiles(OutputFile.newBuilder().setPath("outputs/file1").setDigest(digest1))
689 .addOutputFiles(OutputFile.newBuilder().setPath("outputs/file2").setDigest(digest2))
690 .addOutputFiles(OutputFile.newBuilder().setPath("outputs/file3").setDigest(digest3))
691 .build();
ron-stripecf57d032021-08-26 06:07:30 -0700692 RemoteActionResult result =
693 RemoteActionResult.createFromCache(CachedActionResult.remote(actionResult));
Chi Wangc785f022021-08-01 22:33:22 -0700694 Spawn spawn = newSpawnFromResult(result);
695 FakeSpawnExecutionContext context = newSpawnExecutionContext(spawn);
696 RemoteExecutionService service = newRemoteExecutionService();
697 RemoteAction action = service.buildRemoteAction(spawn, context);
698
699 BulkTransferException downloadException =
700 assertThrows(BulkTransferException.class, () -> service.downloadOutputs(action, result));
701
702 for (Throwable t : downloadException.getSuppressed()) {
703 assertThat(t).isInstanceOf(IOException.class);
704 IOException e = (IOException) t;
705 assertThat(Throwables.getRootCause(e)).hasMessageThat().isEqualTo("reused io exception");
706 }
707 }
708
709 @Test
710 public void downloadOutputs_withDuplicateInterruptions_doesNotSuppress() throws Exception {
711 Digest digest1 = cache.addContents(remoteActionExecutionContext, "file1");
712 InterruptedException reusedInterruption = new InterruptedException("reused interruption");
713 Digest digest2 = cache.addException("file2", reusedInterruption);
714 Digest digest3 = cache.addException("file3", reusedInterruption);
715 ActionResult actionResult =
716 ActionResult.newBuilder()
717 .setExitCode(0)
718 .addOutputFiles(OutputFile.newBuilder().setPath("outputs/file1").setDigest(digest1))
719 .addOutputFiles(OutputFile.newBuilder().setPath("outputs/file2").setDigest(digest2))
720 .addOutputFiles(OutputFile.newBuilder().setPath("outputs/file3").setDigest(digest3))
721 .build();
ron-stripecf57d032021-08-26 06:07:30 -0700722 RemoteActionResult result =
723 RemoteActionResult.createFromCache(CachedActionResult.remote(actionResult));
Chi Wangc785f022021-08-01 22:33:22 -0700724 Spawn spawn = newSpawnFromResult(result);
725 FakeSpawnExecutionContext context = newSpawnExecutionContext(spawn);
726 RemoteExecutionService service = newRemoteExecutionService();
727 RemoteAction action = service.buildRemoteAction(spawn, context);
728
729 InterruptedException e =
730 assertThrows(InterruptedException.class, () -> service.downloadOutputs(action, result));
731
732 assertThat(e.getSuppressed()).isEmpty();
733 assertThat(Throwables.getRootCause(e)).hasMessageThat().isEqualTo("reused interruption");
734 }
735
736 @Test
737 public void downloadOutputs_withStdoutStderrOnSuccess_writable() throws Exception {
738 // Tests that fetching stdout/stderr as a digest works and that OutErr is still
739 // writable afterwards.
740 FileOutErr childOutErr = outErr.childOutErr();
741 FileOutErr spyOutErr = spy(outErr);
742 FileOutErr spyChildOutErr = spy(childOutErr);
743 when(spyOutErr.childOutErr()).thenReturn(spyChildOutErr);
744 Digest digestStdout = cache.addContents(remoteActionExecutionContext, "stdout");
745 Digest digestStderr = cache.addContents(remoteActionExecutionContext, "stderr");
746 ActionResult actionResult =
747 ActionResult.newBuilder()
748 .setExitCode(0)
749 .setStdoutDigest(digestStdout)
750 .setStderrDigest(digestStderr)
751 .build();
ron-stripecf57d032021-08-26 06:07:30 -0700752 RemoteActionResult result =
753 RemoteActionResult.createFromCache(CachedActionResult.remote(actionResult));
Chi Wangc785f022021-08-01 22:33:22 -0700754 Spawn spawn = newSpawnFromResult(result);
755 FakeSpawnExecutionContext context = newSpawnExecutionContext(spawn, spyOutErr);
756 RemoteExecutionService service = newRemoteExecutionService();
757 RemoteAction action = service.buildRemoteAction(spawn, context);
758
759 service.downloadOutputs(action, result);
760
761 verify(spyOutErr, times(2)).childOutErr();
762 verify(spyChildOutErr).clearOut();
763 verify(spyChildOutErr).clearErr();
764 assertThat(outErr.getOutputPath().exists()).isTrue();
765 assertThat(outErr.getErrorPath().exists()).isTrue();
766 try {
767 outErr.getOutputStream().write(0);
768 outErr.getErrorStream().write(0);
769 } catch (IOException err) {
770 throw new AssertionError("outErr should still be writable after download finished.", err);
771 }
772 assertThat(context.isLockOutputFilesCalled()).isTrue();
773 }
774
775 @Test
776 public void downloadOutputs_withStdoutStderrOnFailure_writableAndEmpty() throws Exception {
777 // Test that when downloading stdout/stderr fails the OutErr is still writable
778 // and empty.
779 FileOutErr childOutErr = outErr.childOutErr();
780 FileOutErr spyOutErr = spy(outErr);
781 FileOutErr spyChildOutErr = spy(childOutErr);
782 when(spyOutErr.childOutErr()).thenReturn(spyChildOutErr);
783 // Don't add stdout/stderr as a known blob to the remote cache so that downloading it will fail
784 Digest digestStdout = digestUtil.computeAsUtf8("stdout");
785 Digest digestStderr = digestUtil.computeAsUtf8("stderr");
786 ActionResult actionResult =
787 ActionResult.newBuilder()
788 .setExitCode(0)
789 .setStdoutDigest(digestStdout)
790 .setStderrDigest(digestStderr)
791 .build();
ron-stripecf57d032021-08-26 06:07:30 -0700792 RemoteActionResult result =
793 RemoteActionResult.createFromCache(CachedActionResult.remote(actionResult));
Chi Wangc785f022021-08-01 22:33:22 -0700794 Spawn spawn = newSpawnFromResult(result);
795 FakeSpawnExecutionContext context = newSpawnExecutionContext(spawn, spyOutErr);
796 RemoteExecutionService service = newRemoteExecutionService();
797 RemoteAction action = service.buildRemoteAction(spawn, context);
798
799 assertThrows(BulkTransferException.class, () -> service.downloadOutputs(action, result));
800
801 verify(spyOutErr, times(2)).childOutErr();
802 verify(spyChildOutErr).clearOut();
803 verify(spyChildOutErr).clearErr();
804 assertThat(outErr.getOutputPath().exists()).isFalse();
805 assertThat(outErr.getErrorPath().exists()).isFalse();
806 try {
807 outErr.getOutputStream().write(0);
808 outErr.getErrorStream().write(0);
809 } catch (IOException err) {
810 throw new AssertionError("outErr should still be writable after download failed.", err);
811 }
812 assertThat(context.isLockOutputFilesCalled()).isFalse();
813 }
814
815 @Test
816 public void downloadOutputs_outputNameClashesWithTempName_success() throws Exception {
817 Digest d1 = cache.addContents(remoteActionExecutionContext, "content1");
818 Digest d2 = cache.addContents(remoteActionExecutionContext, "content2");
819 ActionResult r =
820 ActionResult.newBuilder()
821 .setExitCode(0)
822 .addOutputFiles(OutputFile.newBuilder().setPath("outputs/foo.tmp").setDigest(d1))
823 .addOutputFiles(OutputFile.newBuilder().setPath("outputs/foo").setDigest(d2))
824 .build();
ron-stripecf57d032021-08-26 06:07:30 -0700825 RemoteActionResult result = RemoteActionResult.createFromCache(CachedActionResult.remote(r));
Chi Wangc785f022021-08-01 22:33:22 -0700826 Spawn spawn = newSpawnFromResult(result);
827 FakeSpawnExecutionContext context = newSpawnExecutionContext(spawn);
828 RemoteExecutionService service = newRemoteExecutionService();
829 RemoteAction action = service.buildRemoteAction(spawn, context);
830
831 service.downloadOutputs(action, result);
832
833 assertThat(readContent(execRoot.getRelative("outputs/foo.tmp"), StandardCharsets.UTF_8))
834 .isEqualTo("content1");
835 assertThat(readContent(execRoot.getRelative("outputs/foo"), StandardCharsets.UTF_8))
836 .isEqualTo("content2");
837 assertThat(context.isLockOutputFilesCalled()).isTrue();
838 }
839
840 @Test
841 public void downloadOutputs_outputFilesWithTopLevel_download() throws Exception {
842 // arrange
843 Digest d1 = cache.addContents(remoteActionExecutionContext, "content1");
844 ActionResult r =
845 ActionResult.newBuilder()
846 .setExitCode(0)
847 .addOutputFiles(OutputFile.newBuilder().setPath("outputs/file1").setDigest(d1))
848 .build();
849
ron-stripecf57d032021-08-26 06:07:30 -0700850 RemoteActionResult result = RemoteActionResult.createFromCache(CachedActionResult.remote(r));
Chi Wangc785f022021-08-01 22:33:22 -0700851 Spawn spawn = newSpawnFromResult(result);
852 MetadataInjector injector = mock(MetadataInjector.class);
853 FakeSpawnExecutionContext context = newSpawnExecutionContext(spawn, injector);
854 RemoteOptions remoteOptions = Options.getDefaults(RemoteOptions.class);
855 remoteOptions.remoteOutputsMode = RemoteOutputsMode.TOPLEVEL;
856 RemoteExecutionService service =
857 newRemoteExecutionService(remoteOptions, spawn.getOutputFiles());
858 RemoteAction action = service.buildRemoteAction(spawn, context);
859
860 // act
861 service.downloadOutputs(action, result);
862
863 // assert
864 verify(injector, never()).injectFile(any(), any());
865 assertThat(digestUtil.compute(execRoot.getRelative("outputs/file1"))).isEqualTo(d1);
866 assertThat(context.isLockOutputFilesCalled()).isTrue();
867 }
868
869 @Test
870 public void downloadOutputs_outputFilesWithoutTopLevel_inject() throws Exception {
871 // arrange
872 Digest d1 = cache.addContents(remoteActionExecutionContext, "content1");
873 ActionResult r =
874 ActionResult.newBuilder()
875 .setExitCode(0)
876 .addOutputFiles(OutputFile.newBuilder().setPath("outputs/file1").setDigest(d1))
877 .build();
878
ron-stripecf57d032021-08-26 06:07:30 -0700879 RemoteActionResult result = RemoteActionResult.createFromCache(CachedActionResult.remote(r));
Chi Wangc785f022021-08-01 22:33:22 -0700880 Spawn spawn = newSpawnFromResult(result);
881 MetadataInjector injector = mock(MetadataInjector.class);
882 FakeSpawnExecutionContext context = newSpawnExecutionContext(spawn, injector);
883 RemoteOptions remoteOptions = Options.getDefaults(RemoteOptions.class);
884 remoteOptions.remoteOutputsMode = RemoteOutputsMode.TOPLEVEL;
885 RemoteExecutionService service = newRemoteExecutionService(remoteOptions);
886 RemoteAction action = service.buildRemoteAction(spawn, context);
887
888 // act
889 InMemoryOutput inMemoryOutput = service.downloadOutputs(action, result);
890
891 // assert
892 assertThat(inMemoryOutput).isNull();
893 Artifact a1 = ActionsTestUtil.createArtifact(artifactRoot, "file1");
894 verify(injector).injectFile(eq(a1), remoteFileMatchingDigest(d1));
895 Path outputBase = checkNotNull(artifactRoot.getRoot().asPath());
896 assertThat(outputBase.readdir(Symlinks.NOFOLLOW)).isEmpty();
897 assertThat(context.isLockOutputFilesCalled()).isTrue();
898 }
899
900 @Test
901 public void downloadOutputs_outputFilesWithMinimal_injectingMetadata() throws Exception {
902 // Test that injecting the metadata for a remote output file works
903
904 // arrange
905 Digest d1 = cache.addContents(remoteActionExecutionContext, "content1");
906 Digest d2 = cache.addContents(remoteActionExecutionContext, "content2");
907 ActionResult r =
908 ActionResult.newBuilder()
909 .setExitCode(0)
910 .addOutputFiles(OutputFile.newBuilder().setPath("outputs/file1").setDigest(d1))
911 .addOutputFiles(OutputFile.newBuilder().setPath("outputs/file2").setDigest(d2))
912 .build();
913
ron-stripecf57d032021-08-26 06:07:30 -0700914 RemoteActionResult result = RemoteActionResult.createFromCache(CachedActionResult.remote(r));
Chi Wangc785f022021-08-01 22:33:22 -0700915 Spawn spawn = newSpawnFromResult(result);
916 MetadataInjector injector = mock(MetadataInjector.class);
917 FakeSpawnExecutionContext context = newSpawnExecutionContext(spawn, injector);
918 RemoteOptions remoteOptions = Options.getDefaults(RemoteOptions.class);
919 remoteOptions.remoteOutputsMode = RemoteOutputsMode.MINIMAL;
920 RemoteExecutionService service = newRemoteExecutionService(remoteOptions);
921 RemoteAction action = service.buildRemoteAction(spawn, context);
922
923 // act
924 InMemoryOutput inMemoryOutput = service.downloadOutputs(action, result);
925
926 // assert
927 assertThat(inMemoryOutput).isNull();
928 Artifact a1 = ActionsTestUtil.createArtifact(artifactRoot, "file1");
929 Artifact a2 = ActionsTestUtil.createArtifact(artifactRoot, "file2");
930 verify(injector).injectFile(eq(a1), remoteFileMatchingDigest(d1));
931 verify(injector).injectFile(eq(a2), remoteFileMatchingDigest(d2));
932 Path outputBase = checkNotNull(artifactRoot.getRoot().asPath());
933 assertThat(outputBase.readdir(Symlinks.NOFOLLOW)).isEmpty();
934 assertThat(context.isLockOutputFilesCalled()).isTrue();
935 }
936
937 @Test
938 public void downloadOutputs_outputDirectoriesWithMinimal_injectingMetadata() throws Exception {
939 // Test that injecting the metadata for a tree artifact / remote output directory works
940
941 // arrange
942
943 // Output Directory:
944 // dir/file1
945 // dir/a/file2
946 Digest d1 = cache.addContents(remoteActionExecutionContext, "content1");
947 Digest d2 = cache.addContents(remoteActionExecutionContext, "content2");
948 FileNode file1 = FileNode.newBuilder().setName("file1").setDigest(d1).build();
949 FileNode file2 = FileNode.newBuilder().setName("file2").setDigest(d2).build();
950 Directory a = Directory.newBuilder().addFiles(file2).build();
951 Digest da = cache.addContents(remoteActionExecutionContext, a);
952 Directory root =
953 Directory.newBuilder()
954 .addFiles(file1)
955 .addDirectories(DirectoryNode.newBuilder().setName("a").setDigest(da))
956 .build();
957 Tree t = Tree.newBuilder().setRoot(root).addChildren(a).build();
958 Digest dt = cache.addContents(remoteActionExecutionContext, t);
959 ActionResult r =
960 ActionResult.newBuilder()
961 .setExitCode(0)
962 .addOutputDirectories(
963 OutputDirectory.newBuilder().setPath("outputs/dir").setTreeDigest(dt))
964 .build();
965
ron-stripecf57d032021-08-26 06:07:30 -0700966 RemoteActionResult result = RemoteActionResult.createFromCache(CachedActionResult.remote(r));
Chi Wangc785f022021-08-01 22:33:22 -0700967 Spawn spawn = newSpawnFromResult(result);
968 MetadataInjector injector = mock(MetadataInjector.class);
969 FakeSpawnExecutionContext context = newSpawnExecutionContext(spawn, injector);
970 RemoteOptions remoteOptions = Options.getDefaults(RemoteOptions.class);
971 remoteOptions.remoteOutputsMode = RemoteOutputsMode.MINIMAL;
972 RemoteExecutionService service = newRemoteExecutionService(remoteOptions);
973 RemoteAction action = service.buildRemoteAction(spawn, context);
974
975 // act
976 InMemoryOutput inMemoryOutput = service.downloadOutputs(action, result);
977
978 // assert
979 assertThat(inMemoryOutput).isNull();
980 SpecialArtifact dir =
981 ActionsTestUtil.createTreeArtifactWithGeneratingAction(
982 artifactRoot, PathFragment.create("outputs/dir"));
983 TreeArtifactValue tree =
984 TreeArtifactValue.newBuilder(dir)
985 .putChild(
986 TreeFileArtifact.createTreeOutput(dir, "file1"),
987 new RemoteFileArtifactValue(
988 toBinaryDigest(d1), d1.getSizeBytes(), 1, action.getActionId()))
989 .putChild(
990 TreeFileArtifact.createTreeOutput(dir, "a/file2"),
991 new RemoteFileArtifactValue(
992 toBinaryDigest(d2), d2.getSizeBytes(), 1, action.getActionId()))
993 .build();
994 verify(injector).injectTree(dir, tree);
995 Path outputBase = checkNotNull(artifactRoot.getRoot().asPath());
996 assertThat(outputBase.readdir(Symlinks.NOFOLLOW)).isEmpty();
997 assertThat(context.isLockOutputFilesCalled()).isTrue();
998 }
999
1000 @Test
1001 public void downloadOutputs_outputDirectoriesWithMinimalOnFailure_failProperly()
1002 throws Exception {
1003 // Test that we properly fail when downloading the metadata of an output
1004 // directory fails
1005
1006 // arrange
1007
1008 // Output Directory:
1009 // dir/file1
1010 // dir/a/file2
1011 Digest d1 = cache.addContents(remoteActionExecutionContext, "content1");
1012 Digest d2 = cache.addContents(remoteActionExecutionContext, "content2");
1013 FileNode file1 = FileNode.newBuilder().setName("file1").setDigest(d1).build();
1014 FileNode file2 = FileNode.newBuilder().setName("file2").setDigest(d2).build();
1015 Directory a = Directory.newBuilder().addFiles(file2).build();
1016 Digest da = cache.addContents(remoteActionExecutionContext, a);
1017 Directory root =
1018 Directory.newBuilder()
1019 .addFiles(file1)
1020 .addDirectories(DirectoryNode.newBuilder().setName("a").setDigest(da))
1021 .build();
1022 Tree t = Tree.newBuilder().setRoot(root).addChildren(a).build();
1023 // Downloading the tree will fail
1024 IOException downloadTreeException = new IOException("entry not found");
1025 Digest dt = cache.addException(t, downloadTreeException);
1026 ActionResult r =
1027 ActionResult.newBuilder()
1028 .setExitCode(0)
1029 .addOutputDirectories(
1030 OutputDirectory.newBuilder().setPath("outputs/dir").setTreeDigest(dt))
1031 .build();
1032
ron-stripecf57d032021-08-26 06:07:30 -07001033 RemoteActionResult result = RemoteActionResult.createFromCache(CachedActionResult.remote(r));
Chi Wangc785f022021-08-01 22:33:22 -07001034 Spawn spawn = newSpawnFromResult(result);
1035 MetadataInjector injector = mock(MetadataInjector.class);
1036 FakeSpawnExecutionContext context = newSpawnExecutionContext(spawn, injector);
1037 RemoteOptions remoteOptions = Options.getDefaults(RemoteOptions.class);
1038 remoteOptions.remoteOutputsMode = RemoteOutputsMode.MINIMAL;
1039 RemoteExecutionService service = newRemoteExecutionService(remoteOptions);
1040 RemoteAction action = service.buildRemoteAction(spawn, context);
1041
1042 // act
1043 BulkTransferException e =
1044 assertThrows(BulkTransferException.class, () -> service.downloadOutputs(action, result));
1045
1046 // assert
1047 assertThat(e.getSuppressed()).hasLength(1);
1048 assertThat(e.getSuppressed()[0]).isEqualTo(downloadTreeException);
1049 assertThat(context.isLockOutputFilesCalled()).isFalse();
1050 }
1051
1052 @Test
1053 public void downloadOutputs_stdoutAndStdErrWithMinimal_works() throws Exception {
1054 // Test that downloading of non-embedded stdout and stderr works
1055
1056 // arrange
1057 Digest dOut = cache.addContents(remoteActionExecutionContext, "stdout");
1058 Digest dErr = cache.addContents(remoteActionExecutionContext, "stderr");
1059 ActionResult r =
1060 ActionResult.newBuilder()
1061 .setExitCode(0)
1062 .setStdoutDigest(dOut)
1063 .setStderrDigest(dErr)
1064 .build();
1065
ron-stripecf57d032021-08-26 06:07:30 -07001066 RemoteActionResult result = RemoteActionResult.createFromCache(CachedActionResult.remote(r));
Chi Wangc785f022021-08-01 22:33:22 -07001067 Spawn spawn = newSpawnFromResult(result);
1068 MetadataInjector injector = mock(MetadataInjector.class);
1069 FakeSpawnExecutionContext context = newSpawnExecutionContext(spawn, injector);
1070 RemoteOptions remoteOptions = Options.getDefaults(RemoteOptions.class);
1071 remoteOptions.remoteOutputsMode = RemoteOutputsMode.MINIMAL;
1072 RemoteExecutionService service = newRemoteExecutionService(remoteOptions);
1073 RemoteAction action = service.buildRemoteAction(spawn, context);
1074
1075 // act
1076 InMemoryOutput inMemoryOutput = service.downloadOutputs(action, result);
1077
1078 // assert
1079 assertThat(inMemoryOutput).isNull();
1080 assertThat(outErr.outAsLatin1()).isEqualTo("stdout");
1081 assertThat(outErr.errAsLatin1()).isEqualTo("stderr");
1082 Path outputBase = checkNotNull(artifactRoot.getRoot().asPath());
1083 assertThat(outputBase.readdir(Symlinks.NOFOLLOW)).isEmpty();
1084 assertThat(context.isLockOutputFilesCalled()).isTrue();
1085 }
1086
1087 @Test
1088 public void downloadOutputs_inMemoryOutputWithMinimal_downloadIt() throws Exception {
1089 // Test that downloading an in memory output works
1090
1091 // arrange
1092 Digest d1 = cache.addContents(remoteActionExecutionContext, "content1");
1093 Digest d2 = cache.addContents(remoteActionExecutionContext, "content2");
1094 ActionResult r =
1095 ActionResult.newBuilder()
1096 .setExitCode(0)
1097 .addOutputFiles(OutputFile.newBuilder().setPath("outputs/file1").setDigest(d1))
1098 .addOutputFiles(OutputFile.newBuilder().setPath("outputs/file2").setDigest(d2))
1099 .build();
1100
ron-stripecf57d032021-08-26 06:07:30 -07001101 RemoteActionResult result = RemoteActionResult.createFromCache(CachedActionResult.remote(r));
Chi Wangc785f022021-08-01 22:33:22 -07001102 // a1 should be provided as an InMemoryOutput
1103 PathFragment inMemoryOutputPathFragment = PathFragment.create("outputs/file1");
1104 Spawn spawn = newSpawnFromResultWithInMemoryOutput(result, inMemoryOutputPathFragment);
1105 MetadataInjector injector = mock(MetadataInjector.class);
1106 FakeSpawnExecutionContext context = newSpawnExecutionContext(spawn, injector);
1107 RemoteOptions remoteOptions = Options.getDefaults(RemoteOptions.class);
1108 remoteOptions.remoteOutputsMode = RemoteOutputsMode.MINIMAL;
1109 RemoteExecutionService service = newRemoteExecutionService(remoteOptions);
1110 RemoteAction action = service.buildRemoteAction(spawn, context);
1111
1112 // act
1113 InMemoryOutput inMemoryOutput = service.downloadOutputs(action, result);
1114
1115 // assert
1116 assertThat(inMemoryOutput).isNotNull();
1117 ByteString expectedContents = ByteString.copyFrom("content1", UTF_8);
1118 assertThat(inMemoryOutput.getContents()).isEqualTo(expectedContents);
1119 Artifact a1 = ActionsTestUtil.createArtifact(artifactRoot, "file1");
1120 Artifact a2 = ActionsTestUtil.createArtifact(artifactRoot, "file2");
1121 assertThat(inMemoryOutput.getOutput()).isEqualTo(a1);
1122 // The in memory file also needs to be injected as an output
1123 verify(injector).injectFile(eq(a1), remoteFileMatchingDigest(d1));
1124 verify(injector).injectFile(eq(a2), remoteFileMatchingDigest(d2));
1125 Path outputBase = checkNotNull(artifactRoot.getRoot().asPath());
1126 assertThat(outputBase.readdir(Symlinks.NOFOLLOW)).isEmpty();
1127 assertThat(context.isLockOutputFilesCalled()).isTrue();
1128 }
1129
1130 @Test
1131 public void downloadOutputs_missingInMemoryOutputWithMinimal_returnsNull() throws Exception {
1132 // Test that downloadOutputs returns null if a declared in-memory output is missing from action
1133 // result.
1134
1135 // arrange
1136 Digest d1 = cache.addContents(remoteActionExecutionContext, "in-memory output");
1137 ActionResult r = ActionResult.newBuilder().setExitCode(0).build();
ron-stripecf57d032021-08-26 06:07:30 -07001138 RemoteActionResult result = RemoteActionResult.createFromCache(CachedActionResult.remote(r));
Chi Wangc785f022021-08-01 22:33:22 -07001139 Artifact a1 = ActionsTestUtil.createArtifact(artifactRoot, "file1");
Chi Wanga1511162022-03-17 04:14:41 -07001140 // set file1 as declared output but not mandatory output
Chi Wangc785f022021-08-01 22:33:22 -07001141 Spawn spawn =
Chi Wanga1511162022-03-17 04:14:41 -07001142 new SimpleSpawn(
1143 new FakeOwner("foo", "bar", "//dummy:label"),
1144 /*arguments=*/ ImmutableList.of(),
1145 /*environment=*/ ImmutableMap.of(),
1146 /*executionInfo=*/ ImmutableMap.of(REMOTE_EXECUTION_INLINE_OUTPUTS, "outputs/file1"),
1147 /*runfilesSupplier=*/ null,
1148 /*filesetMappings=*/ ImmutableMap.of(),
1149 /*inputs=*/ NestedSetBuilder.emptySet(Order.STABLE_ORDER),
1150 /*tools=*/ NestedSetBuilder.emptySet(Order.STABLE_ORDER),
1151 /*outputs=*/ ImmutableSet.of(a1),
1152 /*mandatoryOutputs=*/ ImmutableSet.of(),
1153 ResourceSet.ZERO);
1154
Chi Wangc785f022021-08-01 22:33:22 -07001155 MetadataInjector injector = mock(MetadataInjector.class);
1156 FakeSpawnExecutionContext context = newSpawnExecutionContext(spawn, injector);
1157 RemoteOptions remoteOptions = Options.getDefaults(RemoteOptions.class);
1158 remoteOptions.remoteOutputsMode = RemoteOutputsMode.MINIMAL;
1159 RemoteExecutionService service = newRemoteExecutionService(remoteOptions);
1160 RemoteAction action = service.buildRemoteAction(spawn, context);
1161
1162 // act
1163 InMemoryOutput inMemoryOutput = service.downloadOutputs(action, result);
1164
1165 // assert
1166 assertThat(inMemoryOutput).isNull();
1167 // The in memory file metadata also should not have been injected.
1168 verify(injector, never()).injectFile(eq(a1), remoteFileMatchingDigest(d1));
1169 }
1170
chiwang4a12a2c2021-09-02 21:15:54 -07001171 @Test
Chi Wanga1511162022-03-17 04:14:41 -07001172 public void downloadOutputs_missingMandatoryOutputs_reportError() throws Exception {
1173 // Test that an AC which misses mandatory outputs is correctly ignored.
1174 Digest fooDigest = cache.addContents(remoteActionExecutionContext, "foo-contents");
1175 ActionResult.Builder builder = ActionResult.newBuilder();
1176 builder.addOutputFilesBuilder().setPath("outputs/foo").setDigest(fooDigest);
1177 RemoteActionResult result =
1178 RemoteActionResult.createFromCache(CachedActionResult.remote(builder.build()));
1179 ImmutableSet.Builder<Artifact> outputs = ImmutableSet.builder();
1180 ImmutableList<String> expectedOutputFiles = ImmutableList.of("outputs/foo", "outputs/bar");
1181 for (String outputFile : expectedOutputFiles) {
1182 Path path = remotePathResolver.outputPathToLocalPath(outputFile);
1183 Artifact output = ActionsTestUtil.createArtifact(artifactRoot, path);
1184 outputs.add(output);
1185 }
1186 Spawn spawn = newSpawn(ImmutableMap.of(), outputs.build());
1187 FakeSpawnExecutionContext context = newSpawnExecutionContext(spawn);
1188 RemoteExecutionService service = newRemoteExecutionService();
1189 RemoteAction action = service.buildRemoteAction(spawn, context);
1190
1191 IOException error =
1192 assertThrows(IOException.class, () -> service.downloadOutputs(action, result));
1193
1194 assertThat(error).hasMessageThat().containsMatch("expected output .+ does not exist.");
1195 }
1196
1197 @Test
chiwang4a12a2c2021-09-02 21:15:54 -07001198 public void uploadOutputs_uploadDirectory_works() throws Exception {
1199 // Test that uploading a directory works.
1200
1201 // arrange
1202 Digest fooDigest =
1203 fakeFileCache.createScratchInput(ActionInputHelper.fromPath("outputs/a/foo"), "xyz");
1204 Digest quxDigest =
1205 fakeFileCache.createScratchInput(ActionInputHelper.fromPath("outputs/bar/qux"), "abc");
1206 Digest barDigest =
1207 fakeFileCache.createScratchInputDirectory(
1208 ActionInputHelper.fromPath("outputs/bar"),
1209 Tree.newBuilder()
1210 .setRoot(
1211 Directory.newBuilder()
1212 .addFiles(
1213 FileNode.newBuilder()
1214 .setIsExecutable(true)
1215 .setName("qux")
1216 .setDigest(quxDigest)
1217 .build())
1218 .build())
1219 .build());
1220 Path fooFile = execRoot.getRelative("outputs/a/foo");
1221 Path quxFile = execRoot.getRelative("outputs/bar/qux");
1222 quxFile.setExecutable(true);
1223 Path barDir = execRoot.getRelative("outputs/bar");
1224 Artifact outputFile = ActionsTestUtil.createArtifact(artifactRoot, fooFile);
1225 Artifact outputDirectory =
1226 ActionsTestUtil.createTreeArtifactWithGeneratingAction(
1227 artifactRoot, barDir.relativeTo(execRoot));
1228 RemoteExecutionService service = newRemoteExecutionService();
1229 Spawn spawn = newSpawn(ImmutableMap.of(), ImmutableSet.of(outputFile, outputDirectory));
1230 FakeSpawnExecutionContext context = newSpawnExecutionContext(spawn);
1231 RemoteAction action = service.buildRemoteAction(spawn, context);
1232 SpawnResult spawnResult =
1233 new SpawnResult.Builder()
1234 .setExitCode(0)
1235 .setStatus(SpawnResult.Status.SUCCESS)
1236 .setRunnerName("test")
1237 .build();
1238
1239 // act
1240 UploadManifest manifest = service.buildUploadManifest(action, spawnResult);
1241 service.uploadOutputs(action, spawnResult);
1242
1243 // assert
1244 ActionResult.Builder expectedResult = ActionResult.newBuilder();
Chi Wang11066c72021-09-17 00:30:45 -07001245 expectedResult
1246 .addOutputFilesBuilder()
1247 .setPath("outputs/a/foo")
1248 .setDigest(fooDigest)
1249 .setIsExecutable(true);
chiwang4a12a2c2021-09-02 21:15:54 -07001250 expectedResult.addOutputDirectoriesBuilder().setPath("outputs/bar").setTreeDigest(barDigest);
1251 assertThat(manifest.getActionResult()).isEqualTo(expectedResult.build());
1252
1253 ImmutableList<Digest> toQuery = ImmutableList.of(fooDigest, quxDigest, barDigest);
1254 assertThat(getFromFuture(cache.findMissingDigests(remoteActionExecutionContext, toQuery)))
1255 .isEmpty();
1256 }
1257
1258 @Test
1259 public void uploadOutputs_uploadEmptyDirectory_works() throws Exception {
1260 // Test that uploading an empty directory works.
1261
1262 // arrange
1263 Digest barDigest =
1264 fakeFileCache.createScratchInputDirectory(
1265 ActionInputHelper.fromPath("outputs/bar"),
1266 Tree.newBuilder().setRoot(Directory.getDefaultInstance()).build());
1267 Path barDir = execRoot.getRelative("outputs/bar");
1268 Artifact outputDirectory =
1269 ActionsTestUtil.createTreeArtifactWithGeneratingAction(
1270 artifactRoot, barDir.relativeTo(execRoot));
1271 RemoteExecutionService service = newRemoteExecutionService();
1272 Spawn spawn = newSpawn(ImmutableMap.of(), ImmutableSet.of(outputDirectory));
1273 FakeSpawnExecutionContext context = newSpawnExecutionContext(spawn);
1274 RemoteAction action = service.buildRemoteAction(spawn, context);
1275 SpawnResult spawnResult =
1276 new SpawnResult.Builder()
1277 .setExitCode(0)
1278 .setStatus(SpawnResult.Status.SUCCESS)
1279 .setRunnerName("test")
1280 .build();
1281
1282 // act
1283 UploadManifest manifest = service.buildUploadManifest(action, spawnResult);
1284 service.uploadOutputs(action, spawnResult);
1285
1286 // assert
1287 ActionResult.Builder expectedResult = ActionResult.newBuilder();
1288 expectedResult.addOutputDirectoriesBuilder().setPath("outputs/bar").setTreeDigest(barDigest);
1289 assertThat(manifest.getActionResult()).isEqualTo(expectedResult.build());
1290 assertThat(
1291 getFromFuture(
1292 cache.findMissingDigests(
1293 remoteActionExecutionContext, ImmutableList.of(barDigest))))
1294 .isEmpty();
1295 }
1296
1297 @Test
1298 public void uploadOutputs_uploadNestedDirectory_works() throws Exception {
1299 // Test that uploading a nested directory works.
1300
1301 // arrange
1302 final Digest wobbleDigest =
1303 fakeFileCache.createScratchInput(
1304 ActionInputHelper.fromPath("outputs/bar/test/wobble"), "xyz");
1305 final Digest quxDigest =
1306 fakeFileCache.createScratchInput(ActionInputHelper.fromPath("outputs/bar/qux"), "abc");
1307 final Directory testDirMessage =
1308 Directory.newBuilder()
1309 .addFiles(FileNode.newBuilder().setName("wobble").setDigest(wobbleDigest).build())
1310 .build();
1311 final Digest testDigest = digestUtil.compute(testDirMessage);
1312 final Tree barTree =
1313 Tree.newBuilder()
1314 .setRoot(
1315 Directory.newBuilder()
1316 .addFiles(
1317 FileNode.newBuilder()
1318 .setIsExecutable(true)
1319 .setName("qux")
1320 .setDigest(quxDigest))
1321 .addDirectories(
1322 DirectoryNode.newBuilder().setName("test").setDigest(testDigest)))
1323 .addChildren(testDirMessage)
1324 .build();
1325 final Digest barDigest =
1326 fakeFileCache.createScratchInputDirectory(
1327 ActionInputHelper.fromPath("outputs/bar"), barTree);
1328
1329 final Path quxFile = execRoot.getRelative("outputs/bar/qux");
1330 quxFile.setExecutable(true);
1331 final Path barDir = execRoot.getRelative("outputs/bar");
1332
1333 Artifact outputDirectory =
1334 ActionsTestUtil.createTreeArtifactWithGeneratingAction(
1335 artifactRoot, barDir.relativeTo(execRoot));
1336 RemoteExecutionService service = newRemoteExecutionService();
1337 Spawn spawn = newSpawn(ImmutableMap.of(), ImmutableSet.of(outputDirectory));
1338 FakeSpawnExecutionContext context = newSpawnExecutionContext(spawn);
1339 RemoteAction action = service.buildRemoteAction(spawn, context);
1340 SpawnResult spawnResult =
1341 new SpawnResult.Builder()
1342 .setExitCode(0)
1343 .setStatus(SpawnResult.Status.SUCCESS)
1344 .setRunnerName("test")
1345 .build();
1346
1347 // act
1348 UploadManifest manifest = service.buildUploadManifest(action, spawnResult);
1349 service.uploadOutputs(action, spawnResult);
1350
1351 // assert
1352 ActionResult.Builder expectedResult = ActionResult.newBuilder();
1353 expectedResult.addOutputDirectoriesBuilder().setPath("outputs/bar").setTreeDigest(barDigest);
1354 assertThat(manifest.getActionResult()).isEqualTo(expectedResult.build());
1355
1356 ImmutableList<Digest> toQuery = ImmutableList.of(wobbleDigest, quxDigest, barDigest);
1357 assertThat(getFromFuture(cache.findMissingDigests(remoteActionExecutionContext, toQuery)))
1358 .isEmpty();
1359 }
1360
1361 @Test
1362 public void uploadOutputs_emptyOutputs_doNotPerformUpload() throws Exception {
1363 // Test that uploading an empty output does not try to perform an upload.
1364
1365 // arrange
1366 Digest emptyDigest =
1367 fakeFileCache.createScratchInput(ActionInputHelper.fromPath("outputs/bar/test/wobble"), "");
1368 Path file = execRoot.getRelative("outputs/bar/test/wobble");
1369 Artifact outputFile = ActionsTestUtil.createArtifact(artifactRoot, file);
1370 RemoteExecutionService service = newRemoteExecutionService();
1371 Spawn spawn = newSpawn(ImmutableMap.of(), ImmutableSet.of(outputFile));
1372 FakeSpawnExecutionContext context = newSpawnExecutionContext(spawn);
1373 RemoteAction action = service.buildRemoteAction(spawn, context);
1374 SpawnResult spawnResult =
1375 new SpawnResult.Builder()
1376 .setExitCode(0)
1377 .setStatus(SpawnResult.Status.SUCCESS)
1378 .setRunnerName("test")
1379 .build();
1380
1381 // act
1382 service.uploadOutputs(action, spawnResult);
1383
1384 // assert
1385 assertThat(
1386 getFromFuture(
1387 cache.findMissingDigests(
1388 remoteActionExecutionContext, ImmutableSet.of(emptyDigest))))
1389 .containsExactly(emptyDigest);
1390 }
1391
chiwangdb15e472021-09-05 20:06:30 -07001392 @Test
chiwang7f08b782021-09-05 21:04:35 -07001393 public void uploadOutputs_uploadFails_printWarning() throws Exception {
1394 RemoteExecutionService service = newRemoteExecutionService();
1395 Spawn spawn = newSpawn(ImmutableMap.of(), ImmutableSet.of());
1396 FakeSpawnExecutionContext context = newSpawnExecutionContext(spawn);
1397 RemoteAction action = service.buildRemoteAction(spawn, context);
1398 SpawnResult spawnResult =
1399 new SpawnResult.Builder()
1400 .setExitCode(0)
1401 .setStatus(Status.SUCCESS)
1402 .setRunnerName("test")
1403 .build();
1404 doReturn(Futures.immediateFailedFuture(new IOException("cache down")))
1405 .when(cache)
1406 .uploadActionResult(any(), any(), any());
1407
1408 service.uploadOutputs(action, spawnResult);
1409
1410 assertThat(eventHandler.getEvents()).hasSize(1);
1411 Event evt = eventHandler.getEvents().get(0);
1412 assertThat(evt.getKind()).isEqualTo(EventKind.WARNING);
1413 assertThat(evt.getMessage()).contains("cache down");
1414 }
1415
1416 @Test
Chi Wang003e2d02021-09-21 22:52:08 -07001417 public void uploadOutputs_firesUploadEvents() throws Exception {
1418 Digest digest =
1419 fakeFileCache.createScratchInput(ActionInputHelper.fromPath("outputs/file"), "content");
1420 Path file = execRoot.getRelative("outputs/file");
1421 Artifact outputFile = ActionsTestUtil.createArtifact(artifactRoot, file);
1422 RemoteExecutionService service = newRemoteExecutionService();
1423 Spawn spawn = newSpawn(ImmutableMap.of(), ImmutableSet.of(outputFile));
1424 FakeSpawnExecutionContext context = newSpawnExecutionContext(spawn);
1425 RemoteAction action = service.buildRemoteAction(spawn, context);
1426 SpawnResult spawnResult =
1427 new SpawnResult.Builder()
1428 .setExitCode(0)
1429 .setStatus(SpawnResult.Status.SUCCESS)
1430 .setRunnerName("test")
1431 .build();
1432
1433 service.uploadOutputs(action, spawnResult);
1434
1435 assertThat(eventHandler.getPosts())
1436 .containsAtLeast(
1437 ActionUploadStartedEvent.create(spawn.getResourceOwner(), "cas/" + digest.getHash()),
1438 ActionUploadFinishedEvent.create(spawn.getResourceOwner(), "cas/" + digest.getHash()),
1439 ActionUploadStartedEvent.create(spawn.getResourceOwner(), "ac/" + action.getActionId()),
1440 ActionUploadFinishedEvent.create(
1441 spawn.getResourceOwner(), "ac/" + action.getActionId()));
1442 }
1443
1444 @Test
Chi Wanga1511162022-03-17 04:14:41 -07001445 public void uploadOutputs_missingMandatoryOutputs_dontUpload() throws Exception {
Chi Wang5b545882022-03-14 06:34:32 -07001446 Path file = execRoot.getRelative("outputs/file");
1447 Artifact outputFile = ActionsTestUtil.createArtifact(artifactRoot, file);
1448 RemoteExecutionService service = newRemoteExecutionService();
1449 Spawn spawn = newSpawn(ImmutableMap.of(), ImmutableSet.of(outputFile));
1450 FakeSpawnExecutionContext context = newSpawnExecutionContext(spawn);
1451 RemoteAction action = service.buildRemoteAction(spawn, context);
1452 SpawnResult spawnResult =
1453 new SpawnResult.Builder()
1454 .setExitCode(0)
1455 .setStatus(SpawnResult.Status.SUCCESS)
1456 .setRunnerName("test")
1457 .build();
1458
1459 service.uploadOutputs(action, spawnResult);
1460
1461 // assert
1462 assertThat(cache.getNumFindMissingDigests()).isEmpty();
1463 }
1464
1465 @Test
chiwangdb15e472021-09-05 20:06:30 -07001466 public void uploadInputsIfNotPresent_deduplicateFindMissingBlobCalls() throws Exception {
1467 int taskCount = 100;
1468 ExecutorService executorService = Executors.newFixedThreadPool(taskCount);
1469 AtomicReference<Throwable> error = new AtomicReference<>(null);
1470 Semaphore semaphore = new Semaphore(0);
1471 ActionInput input = ActionInputHelper.fromPath("inputs/foo");
1472 Digest inputDigest = fakeFileCache.createScratchInput(input, "input-foo");
1473 RemoteExecutionService service = newRemoteExecutionService();
Chi Wang702df842022-03-21 05:37:13 -07001474 Spawn spawn =
1475 newSpawn(
1476 ImmutableMap.of(),
1477 ImmutableSet.of(),
1478 NestedSetBuilder.create(Order.STABLE_ORDER, input));
1479 FakeSpawnExecutionContext context = newSpawnExecutionContext(spawn);
1480 RemoteAction action = service.buildRemoteAction(spawn, context);
chiwangdb15e472021-09-05 20:06:30 -07001481
1482 for (int i = 0; i < taskCount; ++i) {
1483 executorService.execute(
1484 () -> {
1485 try {
chiwangdb15e472021-09-05 20:06:30 -07001486 service.uploadInputsIfNotPresent(action, /*force=*/ false);
1487 } catch (Throwable e) {
1488 if (e instanceof InterruptedException) {
1489 Thread.currentThread().interrupt();
1490 }
1491 error.set(e);
1492 } finally {
1493 semaphore.release();
1494 }
1495 });
1496 }
1497 semaphore.acquire(taskCount);
1498
1499 assertThat(error.get()).isNull();
1500 assertThat(cache.getNumFindMissingDigests()).containsEntry(inputDigest, 1);
1501 for (Integer num : cache.getNumFindMissingDigests().values()) {
1502 assertThat(num).isEqualTo(1);
1503 }
1504 }
1505
Fredrik Medleybecd1492021-10-26 19:44:10 -07001506 @Test
Chi Wang702df842022-03-21 05:37:13 -07001507 public void uploadInputsIfNotPresent_sameInputs_interruptOne_keepOthers() throws Exception {
1508 int taskCount = 100;
1509 ExecutorService executorService = Executors.newFixedThreadPool(taskCount);
1510 AtomicReference<Throwable> error = new AtomicReference<>(null);
1511 Semaphore semaphore = new Semaphore(0);
1512 ActionInput input = ActionInputHelper.fromPath("inputs/foo");
1513 fakeFileCache.createScratchInput(input, "input-foo");
1514 RemoteExecutionService service = newRemoteExecutionService();
1515 Spawn spawn =
1516 newSpawn(
1517 ImmutableMap.of(),
1518 ImmutableSet.of(),
1519 NestedSetBuilder.create(Order.STABLE_ORDER, input));
1520 FakeSpawnExecutionContext context = newSpawnExecutionContext(spawn);
1521 RemoteAction action = service.buildRemoteAction(spawn, context);
1522 Random random = new Random();
1523
1524 for (int i = 0; i < taskCount; ++i) {
1525 boolean shouldInterrupt = random.nextBoolean();
1526 executorService.execute(
1527 () -> {
1528 try {
1529 if (shouldInterrupt) {
1530 Thread.currentThread().interrupt();
1531 }
1532 service.uploadInputsIfNotPresent(action, /*force=*/ false);
1533 } catch (Throwable e) {
1534 if (!(shouldInterrupt && e instanceof InterruptedException)) {
1535 error.set(e);
1536 }
1537 } finally {
1538 semaphore.release();
1539 }
1540 });
1541 }
1542 semaphore.acquire(taskCount);
1543
1544 assertThat(error.get()).isNull();
1545 }
1546
1547 @Test
1548 public void uploadInputsIfNotPresent_interrupted_requestCancelled() throws Exception {
1549 SettableFuture<ImmutableSet<Digest>> future = SettableFuture.create();
1550 doReturn(future).when(cache).findMissingDigests(any(), any());
1551 ActionInput input = ActionInputHelper.fromPath("inputs/foo");
1552 fakeFileCache.createScratchInput(input, "input-foo");
1553 RemoteExecutionService service = newRemoteExecutionService();
1554 Spawn spawn =
1555 newSpawn(
1556 ImmutableMap.of(),
1557 ImmutableSet.of(),
1558 NestedSetBuilder.create(Order.STABLE_ORDER, input));
1559 FakeSpawnExecutionContext context = newSpawnExecutionContext(spawn);
1560 RemoteAction action = service.buildRemoteAction(spawn, context);
1561
1562 try {
1563 Thread.currentThread().interrupt();
1564 service.uploadInputsIfNotPresent(action, /*force=*/ false);
1565 } catch (InterruptedException ignored) {
1566 // Intentionally left empty
1567 }
1568
1569 assertThat(future.isCancelled()).isTrue();
1570 }
1571
1572 @Test
Fredrik Medleybecd1492021-10-26 19:44:10 -07001573 public void buildMerkleTree_withMemoization_works() throws Exception {
1574 // Test that Merkle tree building can be memoized.
1575
1576 // TODO: Would like to check that NestedSet.getNonLeaves() is only called once per node, but
1577 // cannot Mockito.spy on NestedSet as it is final.
1578
1579 // arrange
1580 /*
1581 * First:
1582 * /bar/file
1583 * /foo1/file
1584 * Second:
1585 * /bar/file
1586 * /foo2/file
1587 */
1588
1589 // arrange
1590 // Single node NestedSets are folded, so always add a dummy file everywhere.
1591 ActionInput dummyFile = ActionInputHelper.fromPath("dummy");
1592 fakeFileCache.createScratchInput(dummyFile, "dummy");
1593
1594 ActionInput barFile = ActionInputHelper.fromPath("bar/file");
1595 NestedSet<ActionInput> nodeBar =
1596 NestedSetBuilder.create(Order.STABLE_ORDER, dummyFile, barFile);
1597 fakeFileCache.createScratchInput(barFile, "bar");
1598
1599 ActionInput foo1File = ActionInputHelper.fromPath("foo1/file");
1600 NestedSet<ActionInput> nodeFoo1 =
1601 NestedSetBuilder.create(Order.STABLE_ORDER, dummyFile, foo1File);
1602 fakeFileCache.createScratchInput(foo1File, "foo1");
1603
1604 ActionInput foo2File = ActionInputHelper.fromPath("foo2/file");
1605 NestedSet<ActionInput> nodeFoo2 =
1606 NestedSetBuilder.create(Order.STABLE_ORDER, dummyFile, foo2File);
1607 fakeFileCache.createScratchInput(foo2File, "foo2");
1608
1609 NestedSet<ActionInput> nodeRoot1 =
1610 new NestedSetBuilder<ActionInput>(Order.STABLE_ORDER)
1611 .add(dummyFile)
1612 .addTransitive(nodeBar)
1613 .addTransitive(nodeFoo1)
1614 .build();
1615 NestedSet<ActionInput> nodeRoot2 =
1616 new NestedSetBuilder<ActionInput>(Order.STABLE_ORDER)
1617 .add(dummyFile)
1618 .addTransitive(nodeBar)
1619 .addTransitive(nodeFoo2)
1620 .build();
1621
1622 Spawn spawn1 =
1623 new SimpleSpawn(
1624 new FakeOwner("foo", "bar", "//dummy:label"),
1625 /*arguments=*/ ImmutableList.of(),
1626 /*environment=*/ ImmutableMap.of(),
1627 /*executionInfo=*/ ImmutableMap.of(),
1628 /*inputs=*/ nodeRoot1,
1629 /*outputs=*/ ImmutableSet.of(),
1630 ResourceSet.ZERO);
1631 Spawn spawn2 =
1632 new SimpleSpawn(
1633 new FakeOwner("foo", "bar", "//dummy:label"),
1634 /*arguments=*/ ImmutableList.of(),
1635 /*environment=*/ ImmutableMap.of(),
1636 /*executionInfo=*/ ImmutableMap.of(),
1637 /*inputs=*/ nodeRoot2,
1638 /*outputs=*/ ImmutableSet.of(),
1639 ResourceSet.ZERO);
1640
1641 FakeSpawnExecutionContext context1 = newSpawnExecutionContext(spawn1);
1642 FakeSpawnExecutionContext context2 = newSpawnExecutionContext(spawn2);
1643 RemoteOptions remoteOptions = Options.getDefaults(RemoteOptions.class);
1644 remoteOptions.remoteMerkleTreeCache = true;
1645 remoteOptions.remoteMerkleTreeCacheSize = 0;
1646 RemoteExecutionService service = spy(newRemoteExecutionService(remoteOptions));
1647
1648 // act first time
1649 service.buildRemoteAction(spawn1, context1);
1650
1651 // assert first time
1652 // Called for: manifests, runfiles, nodeRoot1, nodeFoo1 and nodeBar.
1653 verify(service, times(5)).uncachedBuildMerkleTreeVisitor(any(), any());
1654
1655 // act second time
1656 service.buildRemoteAction(spawn2, context2);
1657
1658 // assert second time
1659 // Called again for: manifests, runfiles, nodeRoot2 and nodeFoo2 but not nodeBar (cached).
1660 verify(service, times(5 + 4)).uncachedBuildMerkleTreeVisitor(any(), any());
1661 }
1662
Chi Wangc785f022021-08-01 22:33:22 -07001663 private Spawn newSpawnFromResult(RemoteActionResult result) {
1664 return newSpawnFromResult(ImmutableMap.of(), result);
1665 }
1666
1667 private Spawn newSpawnFromResult(
1668 ImmutableMap<String, String> executionInfo, RemoteActionResult result) {
1669 ImmutableSet.Builder<Artifact> outputs = ImmutableSet.builder();
1670 for (OutputFile file : result.getOutputFiles()) {
1671 Path path = remotePathResolver.outputPathToLocalPath(file.getPath());
1672 Artifact output = ActionsTestUtil.createArtifact(artifactRoot, path);
1673 outputs.add(output);
1674 }
1675
1676 for (OutputDirectory directory : result.getOutputDirectories()) {
1677 Path path = remotePathResolver.outputPathToLocalPath(directory.getPath());
1678 Artifact output =
1679 ActionsTestUtil.createTreeArtifactWithGeneratingAction(
1680 artifactRoot, path.relativeTo(execRoot));
1681 outputs.add(output);
1682 }
1683
1684 for (OutputSymlink fileSymlink : result.getOutputFileSymlinks()) {
1685 Path path = remotePathResolver.outputPathToLocalPath(fileSymlink.getPath());
1686 Artifact output = ActionsTestUtil.createArtifact(artifactRoot, path);
1687 outputs.add(output);
1688 }
1689
1690 for (OutputSymlink directorySymlink : result.getOutputDirectorySymlinks()) {
1691 Path path = remotePathResolver.outputPathToLocalPath(directorySymlink.getPath());
1692 Artifact output =
1693 ActionsTestUtil.createTreeArtifactWithGeneratingAction(
1694 artifactRoot, path.relativeTo(execRoot));
1695 outputs.add(output);
1696 }
1697
1698 return newSpawn(executionInfo, outputs.build());
1699 }
1700
1701 private Spawn newSpawnFromResultWithInMemoryOutput(
1702 RemoteActionResult result, PathFragment inMemoryOutput) {
1703 return newSpawnFromResult(
1704 ImmutableMap.of(REMOTE_EXECUTION_INLINE_OUTPUTS, inMemoryOutput.getPathString()), result);
1705 }
1706
1707 private Spawn newSpawn(
1708 ImmutableMap<String, String> executionInfo, ImmutableSet<Artifact> outputs) {
chiwangdb15e472021-09-05 20:06:30 -07001709 return newSpawn(executionInfo, outputs, NestedSetBuilder.emptySet(Order.STABLE_ORDER));
1710 }
1711
1712 private Spawn newSpawn(
1713 ImmutableMap<String, String> executionInfo,
1714 ImmutableSet<Artifact> outputs,
1715 NestedSet<? extends ActionInput> inputs) {
Chi Wangc785f022021-08-01 22:33:22 -07001716 return new SimpleSpawn(
1717 new FakeOwner("foo", "bar", "//dummy:label"),
1718 /*arguments=*/ ImmutableList.of(),
1719 /*environment=*/ ImmutableMap.of(),
1720 /*executionInfo=*/ executionInfo,
chiwangdb15e472021-09-05 20:06:30 -07001721 /*inputs=*/ inputs,
Chi Wangc785f022021-08-01 22:33:22 -07001722 /*outputs=*/ outputs,
1723 ResourceSet.ZERO);
1724 }
1725
1726 private FakeSpawnExecutionContext newSpawnExecutionContext(Spawn spawn) {
1727 return new FakeSpawnExecutionContext(spawn, fakeFileCache, execRoot, outErr);
1728 }
1729
1730 private FakeSpawnExecutionContext newSpawnExecutionContext(Spawn spawn, FileOutErr outErr) {
1731 return new FakeSpawnExecutionContext(spawn, fakeFileCache, execRoot, outErr);
1732 }
1733
1734 private FakeSpawnExecutionContext newSpawnExecutionContext(
1735 Spawn spawn, MetadataInjector metadataInjector) {
1736 return new FakeSpawnExecutionContext(
1737 spawn, fakeFileCache, execRoot, outErr, ImmutableClassToInstanceMap.of(), metadataInjector);
1738 }
1739
1740 private RemoteExecutionService newRemoteExecutionService() {
1741 return newRemoteExecutionService(remoteOptions);
1742 }
1743
1744 private RemoteExecutionService newRemoteExecutionService(RemoteOptions remoteOptions) {
1745 return newRemoteExecutionService(remoteOptions, ImmutableList.of());
1746 }
1747
1748 private RemoteExecutionService newRemoteExecutionService(
1749 RemoteOptions remoteOptions, Collection<? extends ActionInput> topLevelOutputs) {
1750 return new RemoteExecutionService(
chiwang581c81a2021-09-06 19:00:55 -07001751 directExecutor(),
chiwang7f08b782021-09-05 21:04:35 -07001752 reporter,
1753 /*verboseFailures=*/ true,
Chi Wangc785f022021-08-01 22:33:22 -07001754 execRoot,
1755 remotePathResolver,
1756 "none",
1757 "none",
1758 digestUtil,
1759 remoteOptions,
1760 cache,
chiwangdb15e472021-09-05 20:06:30 -07001761 executor,
Chi Wangc785f022021-08-01 22:33:22 -07001762 ImmutableSet.copyOf(topLevelOutputs),
1763 null);
1764 }
1765
1766 private static RemoteFileArtifactValue remoteFileMatchingDigest(Digest expectedDigest) {
1767 return argThat(
1768 metadata ->
1769 Arrays.equals(metadata.getDigest(), toBinaryDigest(expectedDigest))
1770 && metadata.getSize() == expectedDigest.getSizeBytes());
1771 }
1772}