Adding stable sorting for execlog
RELNOTES: Introducing --execution_log_binary_file and --execution_log_json_file that output a stable sorted execution log. They will offer a stable replacement to --experimental_execution_log_file.
PiperOrigin-RevId: 243808737
diff --git a/src/main/java/com/google/devtools/build/lib/BUILD b/src/main/java/com/google/devtools/build/lib/BUILD
index 34a6a8d..1c1bc6d 100644
--- a/src/main/java/com/google/devtools/build/lib/BUILD
+++ b/src/main/java/com/google/devtools/build/lib/BUILD
@@ -140,6 +140,7 @@
"//src/main/java/com/google/devtools/build/lib/vfs",
"//third_party:guava",
"//third_party/protobuf:protobuf_java",
+ "//third_party/protobuf:protobuf_java_util",
],
)
@@ -867,6 +868,7 @@
":runtime",
":util",
"//src/main/java/com/google/devtools/build/lib/actions",
+ "//src/main/java/com/google/devtools/build/lib/bazel/execlog:stable_sort",
"//src/main/java/com/google/devtools/build/lib/shell",
"//src/main/java/com/google/devtools/build/lib/skyframe/serialization/autocodec",
"//src/main/java/com/google/devtools/build/lib/vfs",
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/SpawnLogModule.java b/src/main/java/com/google/devtools/build/lib/bazel/SpawnLogModule.java
index ab03ba8..55779ab 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/SpawnLogModule.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/SpawnLogModule.java
@@ -13,6 +13,7 @@
// limitations under the License.
package com.google.devtools.build.lib.bazel;
+import com.google.devtools.build.lib.bazel.execlog.StableSort;
import com.google.devtools.build.lib.buildtool.BuildRequest;
import com.google.devtools.build.lib.events.Event;
import com.google.devtools.build.lib.exec.ExecutionOptions;
@@ -23,46 +24,136 @@
import com.google.devtools.build.lib.util.AbruptExitException;
import com.google.devtools.build.lib.util.ExitCode;
import com.google.devtools.build.lib.util.io.AsynchronousFileOutputStream;
+import com.google.devtools.build.lib.util.io.MessageOutputStreamWrapper.BinaryOutputStreamWrapper;
+import com.google.devtools.build.lib.util.io.MessageOutputStreamWrapper.JsonOutputStreamWrapper;
+import com.google.devtools.build.lib.util.io.MessageOutputStreamWrapper.MessageOutputStreamCollection;
+import com.google.devtools.build.lib.vfs.FileSystem;
+import com.google.devtools.build.lib.vfs.Path;
+import java.io.File;
import java.io.IOException;
+import java.io.InputStream;
/**
* Module providing on-demand spawn logging.
*/
public final class SpawnLogModule extends BlazeModule {
+ /**
+ * SpawnLogContext will log to a temporary file as the execution is being performed. rawOutput is
+ * the path to that temporary file.
+ */
private SpawnLogContext spawnLogContext;
+ private Path rawOutput;
+
+ /**
+ * After the execution is done, the temporary file contents will be sorted and logged as the user
+ * requested, to binary and/or json files. We will open the streams at the beginning of the
+ * command so that any errors (e.g., unwritable location) will be surfaced before the execution
+ * begins.
+ */
+ private MessageOutputStreamCollection outputStreams;
+
+ private CommandEnvironment env;
+
+ private void clear() {
+ spawnLogContext = null;
+ outputStreams = new MessageOutputStreamCollection();
+ rawOutput = null;
+ env = null;
+ }
+
+ private void initOutputs(CommandEnvironment env) throws IOException {
+ clear();
+ this.env = env;
+
+ ExecutionOptions executionOptions = env.getOptions().getOptions(ExecutionOptions.class);
+ if (executionOptions == null) {
+ return;
+ }
+ FileSystem fileSystem = env.getRuntime().getFileSystem();
+ Path workingDirectory = env.getWorkingDirectory();
+
+ if (executionOptions.executionLogBinaryFile != null) {
+ outputStreams.addStream(
+ new BinaryOutputStreamWrapper(
+ workingDirectory
+ .getRelative(executionOptions.executionLogBinaryFile)
+ .getOutputStream()));
+ }
+
+ if (executionOptions.executionLogJsonFile != null) {
+ outputStreams.addStream(
+ new JsonOutputStreamWrapper(
+ workingDirectory
+ .getRelative(executionOptions.executionLogJsonFile)
+ .getOutputStream()));
+ }
+
+ AsynchronousFileOutputStream outStream = null;
+ if (executionOptions.executionLogFile != null) {
+ rawOutput = env.getRuntime().getFileSystem().getPath(executionOptions.executionLogFile);
+ outStream =
+ new AsynchronousFileOutputStream(
+ workingDirectory.getRelative(executionOptions.executionLogFile));
+ } else if (!outputStreams.isEmpty()) {
+ // Execution log requested but raw log file not specified
+ File file = File.createTempFile("exec", ".log");
+ rawOutput = fileSystem.getPath(file.getAbsolutePath());
+ outStream = new AsynchronousFileOutputStream(rawOutput);
+ }
+
+ if (outStream == null) {
+ // No logging needed
+ clear();
+ return;
+ }
+
+ spawnLogContext = new SpawnLogContext(env.getExecRoot(), outStream);
+ }
+
@Override
public void executorInit(CommandEnvironment env, BuildRequest request, ExecutorBuilder builder) {
env.getEventBus().register(this);
- ExecutionOptions executionOptions = env.getOptions().getOptions(ExecutionOptions.class);
- if (executionOptions != null
- && executionOptions.executionLogFile != null
- && !executionOptions.executionLogFile.isEmpty()) {
- try {
- spawnLogContext = new SpawnLogContext(
- env.getExecRoot(),
- new AsynchronousFileOutputStream(executionOptions.executionLogFile));
- } catch (IOException e) {
- env.getReporter().handle(Event.error(e.getMessage()));
- env.getBlazeModuleEnvironment().exit(new AbruptExitException(
- "Error found creating SpawnLogContext", ExitCode.COMMAND_LINE_ERROR));
- }
+
+ try {
+ initOutputs(env);
+ } catch (IOException e) {
+ env.getReporter().handle(Event.error(e.getMessage()));
+ env.getBlazeModuleEnvironment()
+ .exit(
+ new AbruptExitException(
+ "Error initializing execution log", ExitCode.COMMAND_LINE_ERROR, e));
+ }
+
+ if (spawnLogContext != null) {
builder.addActionContext(spawnLogContext);
builder.addStrategyByContext(SpawnLogContext.class, "");
- } else {
- spawnLogContext = null;
}
}
@Override
- public void afterCommand() {
+ public void afterCommand() throws AbruptExitException {
+ boolean done = false;
if (spawnLogContext != null) {
try {
spawnLogContext.close();
+ if (!outputStreams.isEmpty()) {
+ InputStream in = rawOutput.getInputStream();
+ StableSort.stableSort(in, outputStreams);
+ outputStreams.close();
+ }
+ done = true;
} catch (IOException e) {
- throw new RuntimeException(e);
+ throw new AbruptExitException(ExitCode.LOCAL_ENVIRONMENTAL_ERROR, e);
} finally {
- spawnLogContext = null;
+ if (!done && !outputStreams.isEmpty()) {
+ env.getReporter()
+ .handle(
+ Event.warn(
+ "Execution log might not have been populated. Raw execution log is at "
+ + rawOutput));
+ }
+ clear();
}
}
}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/debug/BUILD b/src/main/java/com/google/devtools/build/lib/bazel/debug/BUILD
index 59f1202..8cb3158 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/debug/BUILD
+++ b/src/main/java/com/google/devtools/build/lib/bazel/debug/BUILD
@@ -57,6 +57,7 @@
"//src/main/java/com/google/devtools/build/lib:io",
"//src/main/java/com/google/devtools/build/lib:runtime",
"//src/main/java/com/google/devtools/build/lib:util",
+ "//src/main/java/com/google/devtools/build/lib/vfs:pathfragment",
"//src/main/java/com/google/devtools/common/options",
"//third_party:guava",
],
@@ -66,6 +67,8 @@
name = "debugging-options",
srcs = ["DebuggingOptions.java"],
deps = [
+ "//src/main/java/com/google/devtools/build/lib:util",
+ "//src/main/java/com/google/devtools/build/lib/vfs:pathfragment",
"//src/main/java/com/google/devtools/common/options",
],
)
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/debug/DebuggingOptions.java b/src/main/java/com/google/devtools/build/lib/bazel/debug/DebuggingOptions.java
index 4bed8b0..db2fef8 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/debug/DebuggingOptions.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/debug/DebuggingOptions.java
@@ -14,6 +14,8 @@
package com.google.devtools.build.lib.bazel.debug;
+import com.google.devtools.build.lib.util.OptionsUtils;
+import com.google.devtools.build.lib.vfs.PathFragment;
import com.google.devtools.common.options.Option;
import com.google.devtools.common.options.OptionDocumentationCategory;
import com.google.devtools.common.options.OptionEffectTag;
@@ -27,7 +29,8 @@
category = "verbosity",
documentationCategory = OptionDocumentationCategory.LOGGING,
effectTags = {OptionEffectTag.UNKNOWN},
+ converter = OptionsUtils.PathFragmentConverter.class,
help =
"Log certain Workspace Rules events into this file as delimited WorkspaceEvent protos.")
- public String workspaceRulesLogFile;
+ public PathFragment workspaceRulesLogFile;
}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/debug/WorkspaceRuleModule.java b/src/main/java/com/google/devtools/build/lib/bazel/debug/WorkspaceRuleModule.java
index 9efbcf6..90f6da2 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/debug/WorkspaceRuleModule.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/debug/WorkspaceRuleModule.java
@@ -23,6 +23,7 @@
import com.google.devtools.build.lib.util.AbruptExitException;
import com.google.devtools.build.lib.util.ExitCode;
import com.google.devtools.build.lib.util.io.AsynchronousFileOutputStream;
+import com.google.devtools.build.lib.vfs.PathFragment;
import com.google.devtools.common.options.OptionsBase;
import java.io.IOException;
@@ -42,10 +43,12 @@
return;
}
- String logFile = env.getOptions().getOptions(DebuggingOptions.class).workspaceRulesLogFile;
- if (logFile != null && !logFile.isEmpty()) {
+ PathFragment logFile =
+ env.getOptions().getOptions(DebuggingOptions.class).workspaceRulesLogFile;
+ if (logFile != null) {
try {
- outFileStream = new AsynchronousFileOutputStream(logFile);
+ outFileStream =
+ new AsynchronousFileOutputStream(env.getWorkingDirectory().getRelative(logFile));
} catch (IOException e) {
env.getReporter().handle(Event.error(e.getMessage()));
env.getBlazeModuleEnvironment()
diff --git a/src/main/java/com/google/devtools/build/lib/exec/ExecutionOptions.java b/src/main/java/com/google/devtools/build/lib/exec/ExecutionOptions.java
index 3247b924..843178d 100644
--- a/src/main/java/com/google/devtools/build/lib/exec/ExecutionOptions.java
+++ b/src/main/java/com/google/devtools/build/lib/exec/ExecutionOptions.java
@@ -436,14 +436,36 @@
public boolean statsSummary;
@Option(
- name = "experimental_execution_log_file",
- defaultValue = "",
- category = "verbosity",
- documentationCategory = OptionDocumentationCategory.UNCATEGORIZED,
- effectTags = {OptionEffectTag.UNKNOWN},
- help = "Log the executed spawns into this file as delimited Spawn protos."
- )
- public String executionLogFile;
+ name = "experimental_execution_log_file",
+ defaultValue = "null",
+ category = "verbosity",
+ documentationCategory = OptionDocumentationCategory.UNCATEGORIZED,
+ effectTags = {OptionEffectTag.UNKNOWN},
+ converter = OptionsUtils.PathFragmentConverter.class,
+ help = "Log the executed spawns into this file as delimited Spawn protos.")
+ public PathFragment executionLogFile;
+
+ @Option(
+ name = "execution_log_binary_file",
+ defaultValue = "null",
+ category = "verbosity",
+ documentationCategory = OptionDocumentationCategory.UNCATEGORIZED,
+ effectTags = {OptionEffectTag.UNKNOWN},
+ converter = OptionsUtils.PathFragmentConverter.class,
+ help = "Log the executed spawns into this file as delimited Spawn protos.")
+ public PathFragment executionLogBinaryFile;
+
+ @Option(
+ name = "execution_log_json_file",
+ defaultValue = "null",
+ category = "verbosity",
+ documentationCategory = OptionDocumentationCategory.UNCATEGORIZED,
+ effectTags = {OptionEffectTag.UNKNOWN},
+ converter = OptionsUtils.PathFragmentConverter.class,
+ help =
+ "Log the executed spawns into this file as json representation of the delimited Spawn"
+ + " protos.")
+ public PathFragment executionLogJsonFile;
@Option(
name = "experimental_split_xml_generation",
diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteModule.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteModule.java
index 9420d8c..894b767 100644
--- a/src/main/java/com/google/devtools/build/lib/remote/RemoteModule.java
+++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteModule.java
@@ -174,8 +174,10 @@
try {
List<ClientInterceptor> interceptors = new ArrayList<>();
- if (!remoteOptions.experimentalRemoteGrpcLog.isEmpty()) {
- rpcLogFile = new AsynchronousFileOutputStream(remoteOptions.experimentalRemoteGrpcLog);
+ if (remoteOptions.experimentalRemoteGrpcLog != null) {
+ rpcLogFile =
+ new AsynchronousFileOutputStream(
+ env.getWorkingDirectory().getRelative(remoteOptions.experimentalRemoteGrpcLog));
interceptors.add(new LoggingInterceptor(rpcLogFile, env.getRuntime().getClock()));
}
diff --git a/src/main/java/com/google/devtools/build/lib/remote/options/RemoteOptions.java b/src/main/java/com/google/devtools/build/lib/remote/options/RemoteOptions.java
index 961b108..755b6e5 100644
--- a/src/main/java/com/google/devtools/build/lib/remote/options/RemoteOptions.java
+++ b/src/main/java/com/google/devtools/build/lib/remote/options/RemoteOptions.java
@@ -205,9 +205,10 @@
@Option(
name = "experimental_remote_grpc_log",
- defaultValue = "",
+ defaultValue = "null",
category = "remote",
documentationCategory = OptionDocumentationCategory.REMOTE,
+ converter = OptionsUtils.PathFragmentConverter.class,
effectTags = {OptionEffectTag.UNKNOWN},
help =
"If specified, a path to a file to log gRPC call related details. This log consists of a"
@@ -216,7 +217,7 @@
+ "protobufs with each message prefixed by a varint denoting the size of the"
+ " following serialized protobuf message, as performed by the method "
+ "LogEntry.writeDelimitedTo(OutputStream).")
- public String experimentalRemoteGrpcLog;
+ public PathFragment experimentalRemoteGrpcLog;
@Option(
name = "incompatible_remote_symlinks",
diff --git a/src/main/java/com/google/devtools/build/lib/util/io/AsynchronousFileOutputStream.java b/src/main/java/com/google/devtools/build/lib/util/io/AsynchronousFileOutputStream.java
index 172750b..1482e54 100644
--- a/src/main/java/com/google/devtools/build/lib/util/io/AsynchronousFileOutputStream.java
+++ b/src/main/java/com/google/devtools/build/lib/util/io/AsynchronousFileOutputStream.java
@@ -22,6 +22,7 @@
import com.google.common.util.concurrent.SettableFuture;
import com.google.common.util.concurrent.Uninterruptibles;
import com.google.devtools.build.lib.concurrent.ThreadSafety;
+import com.google.devtools.build.lib.vfs.Path;
import com.google.protobuf.CodedOutputStream;
import com.google.protobuf.Message;
import com.google.protobuf.MessageLite;
@@ -29,8 +30,6 @@
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.OutputStream;
-import java.nio.file.Files;
-import java.nio.file.Paths;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.LinkedBlockingDeque;
@@ -50,11 +49,11 @@
// To store any exception raised from the writes.
private final AtomicReference<Throwable> exception = new AtomicReference<>();
- public AsynchronousFileOutputStream(String filename) throws IOException {
+ public AsynchronousFileOutputStream(Path path) throws IOException {
this(
- filename,
+ path.toString(),
new BufferedOutputStream( // Use a buffer of 100 kByte, scientifically chosen at random.
- Files.newOutputStream(Paths.get(filename)), 100000));
+ path.getOutputStream(), 100000));
}
@VisibleForTesting
diff --git a/src/main/java/com/google/devtools/build/lib/util/io/MessageOutputStreamWrapper.java b/src/main/java/com/google/devtools/build/lib/util/io/MessageOutputStreamWrapper.java
new file mode 100644
index 0000000..58aea9c
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/util/io/MessageOutputStreamWrapper.java
@@ -0,0 +1,107 @@
+// Copyright 2019 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.util.io;
+
+import com.google.common.base.Preconditions;
+import com.google.protobuf.Message;
+import com.google.protobuf.util.JsonFormat;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+
+/** Creating a MessageOutputStream from an OutputStream */
+public class MessageOutputStreamWrapper {
+ /** Outputs the messages in binary format */
+ public static class BinaryOutputStreamWrapper implements MessageOutputStream {
+ private final OutputStream stream;
+
+ public BinaryOutputStreamWrapper(OutputStream stream) {
+ this.stream = Preconditions.checkNotNull(stream);
+ }
+
+ @Override
+ public void write(Message m) throws IOException {
+ Preconditions.checkNotNull(m);
+ m.writeDelimitedTo(stream);
+ }
+
+ @Override
+ public void close() throws IOException {
+ stream.close();
+ }
+ }
+
+ /** Outputs the messages in JSON text format */
+ public static class JsonOutputStreamWrapper implements MessageOutputStream {
+ private final OutputStream stream;
+ private final JsonFormat.Printer printer = JsonFormat.printer().includingDefaultValueFields();
+
+ public JsonOutputStreamWrapper(OutputStream stream) {
+ Preconditions.checkNotNull(stream);
+ this.stream = stream;
+ }
+
+ @Override
+ public void write(Message m) throws IOException {
+ Preconditions.checkNotNull(m);
+ stream.write(printer.print(m).getBytes(StandardCharsets.UTF_8));
+ }
+
+ @Override
+ public void close() throws IOException {
+ stream.close();
+ }
+ }
+
+ /** Outputs the messages in JSON text format */
+ public static class MessageOutputStreamCollection implements MessageOutputStream {
+ private final ArrayList<MessageOutputStream> streams = new ArrayList<>();
+
+ public boolean isEmpty() {
+ return streams.isEmpty();
+ }
+
+ public void addStream(MessageOutputStream m) {
+ streams.add(m);
+ }
+
+ @Override
+ public void write(Message m) throws IOException {
+ for (MessageOutputStream stream : streams) {
+ stream.write(m);
+ }
+ }
+
+ @Override
+ public void close() throws IOException {
+ IOException firstException = null;
+ for (MessageOutputStream stream : streams) {
+ try {
+ stream.close();
+ } catch (IOException e) {
+ if (firstException == null) {
+ firstException = e;
+ } else {
+ firstException.addSuppressed(e);
+ }
+ }
+ }
+ if (firstException != null) {
+ throw firstException;
+ }
+ }
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/BUILD b/src/test/java/com/google/devtools/build/lib/BUILD
index 6c10c22..bdcc604 100644
--- a/src/test/java/com/google/devtools/build/lib/BUILD
+++ b/src/test/java/com/google/devtools/build/lib/BUILD
@@ -386,6 +386,8 @@
"//src/main/java/com/google/devtools/build/lib:io",
"//src/main/java/com/google/devtools/build/lib:out-err",
"//src/main/java/com/google/devtools/build/lib:util",
+ "//src/main/java/com/google/devtools/build/lib/vfs",
+ "//src/main/java/com/google/devtools/build/lib/vfs/inmemoryfs",
"//src/main/protobuf:bazel_flags_java_proto",
"//third_party:mockito",
],
diff --git a/src/test/java/com/google/devtools/build/lib/util/io/AsynchronousFileOutputStreamTest.java b/src/test/java/com/google/devtools/build/lib/util/io/AsynchronousFileOutputStreamTest.java
index b9e6077..2d5cc77 100644
--- a/src/test/java/com/google/devtools/build/lib/util/io/AsynchronousFileOutputStreamTest.java
+++ b/src/test/java/com/google/devtools/build/lib/util/io/AsynchronousFileOutputStreamTest.java
@@ -18,22 +18,21 @@
import com.google.common.io.ByteStreams;
import com.google.devtools.build.lib.runtime.commands.proto.BazelFlagsProto.FlagInfo;
+import com.google.devtools.build.lib.vfs.FileSystem;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.inmemoryfs.InMemoryFileSystem;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.charset.StandardCharsets;
-import java.nio.file.Files;
-import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Random;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ThreadLocalRandom;
import org.junit.After;
import org.junit.Before;
-import org.junit.Rule;
import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
import org.mockito.Mockito;
@@ -42,7 +41,6 @@
/** Tests {@link AsynchronousFileOutputStream}. */
@RunWith(JUnit4.class)
public class AsynchronousFileOutputStreamTest {
- @Rule public TemporaryFolder tmp = new TemporaryFolder();
private final Random random = ThreadLocalRandom.current();
private static final char[] RAND_CHARS = "abcdefghijklmnopqrstuvwxzy0123456789-".toCharArray();
private static final int RAND_STRING_LENGTH = 10;
@@ -80,8 +78,9 @@
@Test
public void testConcurrentWrites() throws Exception {
- Path logPath = tmp.newFile().toPath();
- AsynchronousFileOutputStream out = new AsynchronousFileOutputStream(logPath.toString());
+ FileSystem fileSystem = new InMemoryFileSystem();
+ Path logPath = fileSystem.getPath("/logFile");
+ AsynchronousFileOutputStream out = new AsynchronousFileOutputStream(logPath);
Thread[] writers = new Thread[10];
final CountDownLatch start = new CountDownLatch(writers.length);
for (int i = 0; i < writers.length; ++i) {
@@ -108,7 +107,7 @@
}
out.close();
String contents =
- new String(ByteStreams.toByteArray(Files.newInputStream(logPath)), StandardCharsets.UTF_8);
+ new String(ByteStreams.toByteArray(logPath.getInputStream()), StandardCharsets.UTF_8);
for (int i = 0; i < writers.length; ++i) {
for (int j = 0; j < 10; ++j) {
assertThat(contents).contains("Thread # " + i + " time # " + j + "\n");
@@ -118,8 +117,10 @@
@Test
public void testConcurrentProtoWrites() throws Exception {
- Path logPath = tmp.newFile().toPath();
- AsynchronousFileOutputStream out = new AsynchronousFileOutputStream(logPath.toString());
+ final String filename = "/logFile";
+ FileSystem fileSystem = new InMemoryFileSystem();
+ Path logPath = fileSystem.getPath(filename);
+ AsynchronousFileOutputStream out = new AsynchronousFileOutputStream(logPath);
ArrayList<FlagInfo> messages = new ArrayList<>();
for (int i = 0; i < 100; ++i) {
messages.add(generateRandomMessage());
@@ -150,7 +151,7 @@
}
out.close();
ArrayList<FlagInfo> readMessages = new ArrayList<>();
- try (InputStream in = Files.newInputStream(logPath)) {
+ try (InputStream in = fileSystem.getPath(filename).getInputStream()) {
for (int i = 0; i < messages.size(); ++i) {
readMessages.add(FlagInfo.parseDelimitedFrom(in));
}
diff --git a/src/test/shell/bazel/BUILD b/src/test/shell/bazel/BUILD
index e3500ad..b8536ca 100644
--- a/src/test/shell/bazel/BUILD
+++ b/src/test/shell/bazel/BUILD
@@ -855,6 +855,15 @@
)
sh_test(
+ name = "bazel_execlog_test",
+ srcs = ["bazel_execlog_test.sh"],
+ data = [
+ ":test-deps",
+ ],
+ tags = ["no_windows"],
+)
+
+sh_test(
name = "new_local_repo_test",
srcs = ["new_local_repo_test.sh"],
data = [":test-deps"],
diff --git a/src/test/shell/bazel/bazel_execlog_test.sh b/src/test/shell/bazel/bazel_execlog_test.sh
new file mode 100755
index 0000000..389e6da
--- /dev/null
+++ b/src/test/shell/bazel/bazel_execlog_test.sh
@@ -0,0 +1,110 @@
+#!/bin/bash
+#
+# Copyright 2019 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Tests verbosity behavior on workspace initialization
+
+CURRENT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+source "${CURRENT_DIR}/../integration_test_setup.sh" \
+ || { echo "integration_test_setup.sh not found!" >&2; exit 1; }
+
+function test_dir_depends() {
+ create_new_workspace
+ cat > skylark.bzl <<'EOF'
+def _dir_impl(ctx):
+ output_dir = ctx.actions.declare_directory(ctx.attr.outdir)
+ ctx.actions.run_shell(
+ outputs = [output_dir],
+ inputs = [],
+ command = """
+ mkdir -p $1/sub; \
+ echo "first file!" > $1/foo; \
+ echo "second file." > $1/sub/bar; \
+ """,
+ arguments = [output_dir.path],
+ )
+ return [ DefaultInfo(files = depset(direct = [output_dir])) ]
+
+gen_dir = rule(
+ implementation = _dir_impl,
+ attrs = {
+ "outdir": attr.string(mandatory = True),
+ }
+)
+EOF
+ cat > BUILD <<'EOF'
+load(":skylark.bzl", "gen_dir")
+
+gen_dir(
+ name = "dir",
+ outdir = "dir_name1",
+)
+
+genrule(
+ name = "rule1",
+ srcs = ["dir"],
+ outs = ["combined1.txt"],
+ cmd = "cat $(location dir)/foo $(location dir)/sub/bar > $(location combined1.txt)"
+)
+
+gen_dir(
+ name = "dir2",
+ outdir = "dir_name2",
+)
+
+genrule(
+ name = "rule2",
+ srcs = ["dir2"],
+ outs = ["combined2.txt"],
+ cmd = "cat $(location dir2)/foo $(location dir2)/sub/bar > $(location combined2.txt)"
+)
+EOF
+
+ bazel build //:all --execution_log_json_file output.json 2>&1 >> $TEST_log || fail "could not build"
+
+ # dir and dir2 are skylark functions that create a directory output
+ # rule1 and rule2 are functions that consume the directory output
+ #
+ # the output files are named such that the rule's would be placed first in the
+ # stable sorting order, followed by the dir's.
+ # Since rule1 depends on dir1 and rule2 depends on dir2, we expect the
+ # following order:
+ # dir1, rule1, dir2, rule2
+ #
+ # If dependencies were not properly accounted for, the order would have been:
+ # rule1, rule2, dir1, dir2
+
+ dir1Num=`grep "SkylarkAction dir_name1" -n output.json | grep -Eo '^[^:]+'`
+ dir2Num=`grep "SkylarkAction dir_name2" -n output.json | grep -Eo '^[^:]+'`
+ rule1Num=`grep "Executing genrule //:rule1" -n output.json | grep -Eo '^[^:]+'`
+ rule2Num=`grep "Executing genrule //:rule2" -n output.json | grep -Eo '^[^:]+'`
+
+ if [ "$rule1Num" -lt "$dir1Num" ]
+ then
+ fail "rule1 dependency on dir1 is not recornized"
+ fi
+
+ if [ "$dir2Num" -lt "$rule1Num" ]
+ then
+ fail "rule1 comes after dir2"
+ fi
+
+ if [ "$rule2Num" -lt "$dir2Num" ]
+ then
+ fail "rule2 dependency on dir2 is not recornized"
+ fi
+}
+
+run_suite "execlog_tests"
