Stream blaze JSON profile output via remote uploader with --experimental_stream_log_file_uploads. When enabled, we do not write the profile to disk at all.
RELNOTES: None
PiperOrigin-RevId: 281272627
diff --git a/src/main/java/com/google/devtools/build/lib/buildtool/buildevent/ProfilerStartedEvent.java b/src/main/java/com/google/devtools/build/lib/buildtool/buildevent/ProfilerStartedEvent.java
index c3cf5e9..2d1a2a6 100644
--- a/src/main/java/com/google/devtools/build/lib/buildtool/buildevent/ProfilerStartedEvent.java
+++ b/src/main/java/com/google/devtools/build/lib/buildtool/buildevent/ProfilerStartedEvent.java
@@ -13,20 +13,34 @@
// limitations under the License.
package com.google.devtools.build.lib.buildtool.buildevent;
+import com.google.devtools.build.lib.buildeventstream.BuildEventArtifactUploader.UploadContext;
import com.google.devtools.build.lib.events.ExtendedEventHandler;
import com.google.devtools.build.lib.vfs.Path;
+import javax.annotation.Nullable;
/**
* This event is fired when the profiler is started.
*/
public class ProfilerStartedEvent implements ExtendedEventHandler.Postable {
- private final Path profilePath;
+ @Nullable private final Path profilePath;
+ @Nullable private final UploadContext streamingContext;
+ @Nullable private final String name;
- public ProfilerStartedEvent(Path profilePath) {
+ public ProfilerStartedEvent(String name, Path profilePath, UploadContext streamingContext) {
this.profilePath = profilePath;
+ this.streamingContext = streamingContext;
+ this.name = name;
}
public Path getProfilePath() {
return profilePath;
}
+
+ public UploadContext getStreamingContext() {
+ return streamingContext;
+ }
+
+ public String getName() {
+ return name;
+ }
}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/BlazeCommandDispatcher.java b/src/main/java/com/google/devtools/build/lib/runtime/BlazeCommandDispatcher.java
index 7e0b9bc..4e41757 100644
--- a/src/main/java/com/google/devtools/build/lib/runtime/BlazeCommandDispatcher.java
+++ b/src/main/java/com/google/devtools/build/lib/runtime/BlazeCommandDispatcher.java
@@ -27,6 +27,7 @@
import com.google.devtools.build.lib.analysis.NoBuildEvent;
import com.google.devtools.build.lib.bugreport.BugReport;
import com.google.devtools.build.lib.bugreport.BugReporter;
+import com.google.devtools.build.lib.buildeventstream.BuildEventProtocolOptions;
import com.google.devtools.build.lib.buildtool.buildevent.ProfilerStartedEvent;
import com.google.devtools.build.lib.clock.BlazeClock;
import com.google.devtools.build.lib.events.Event;
@@ -312,16 +313,17 @@
// TODO(ulfjack): Move the profiler initialization as early in the startup sequence as possible.
// Profiler setup and shutdown must always happen in pairs. Shutdown is currently performed in
// the afterCommand call in the finally block below.
- Path profilePath =
+ ProfilerStartedEvent profilerStartedEvent =
runtime.initProfiler(
storedEventHandler,
workspace,
commonOptions,
- env.getCommandId(),
+ options.getOptions(BuildEventProtocolOptions.class),
+ env,
execStartTimeNanos,
waitTimeInMs);
if (commonOptions.postProfileStartedEvent) {
- storedEventHandler.post(new ProfilerStartedEvent(profilePath));
+ storedEventHandler.post(profilerStartedEvent);
}
BlazeCommandResult result = BlazeCommandResult.exitCode(ExitCode.BLAZE_INTERNAL_ERROR);
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/BlazeRuntime.java b/src/main/java/com/google/devtools/build/lib/runtime/BlazeRuntime.java
index 2bbfd58..2a115ff 100644
--- a/src/main/java/com/google/devtools/build/lib/runtime/BlazeRuntime.java
+++ b/src/main/java/com/google/devtools/build/lib/runtime/BlazeRuntime.java
@@ -35,10 +35,14 @@
import com.google.devtools.build.lib.analysis.test.CoverageReportActionFactory;
import com.google.devtools.build.lib.bugreport.BugReport;
import com.google.devtools.build.lib.bugreport.BugReporter;
+import com.google.devtools.build.lib.buildeventstream.BuildEvent.LocalFile.LocalFileType;
+import com.google.devtools.build.lib.buildeventstream.BuildEventArtifactUploader;
+import com.google.devtools.build.lib.buildeventstream.BuildEventArtifactUploader.UploadContext;
+import com.google.devtools.build.lib.buildeventstream.BuildEventProtocolOptions;
+import com.google.devtools.build.lib.buildtool.buildevent.ProfilerStartedEvent;
import com.google.devtools.build.lib.clock.BlazeClock;
import com.google.devtools.build.lib.clock.Clock;
import com.google.devtools.build.lib.events.Event;
-import com.google.devtools.build.lib.events.EventHandler;
import com.google.devtools.build.lib.events.ExtendedEventHandler;
import com.google.devtools.build.lib.events.OutputFilter;
import com.google.devtools.build.lib.exec.BinTools;
@@ -278,12 +282,18 @@
return moduleInvocationPolicy;
}
+ private BuildEventArtifactUploader newUploader(
+ CommandEnvironment env, String buildEventUploadStrategy) {
+ return getBuildEventArtifactUploaderFactoryMap().select(buildEventUploadStrategy).create(env);
+ }
+
/** Configure profiling based on the provided options. */
- Path initProfiler(
- EventHandler eventHandler,
+ ProfilerStartedEvent initProfiler(
+ ExtendedEventHandler eventHandler,
BlazeWorkspace workspace,
CommonCommandOptions options,
- UUID buildID,
+ BuildEventProtocolOptions bepOptions,
+ CommandEnvironment env,
long execStartTimeNanos,
long waitTimeInMs) {
OutputStream out = null;
@@ -291,6 +301,8 @@
ImmutableSet.Builder<ProfilerTask> profiledTasksBuilder = ImmutableSet.builder();
Profiler.Format format = Profiler.Format.BINARY_BAZEL_FORMAT;
Path profilePath = null;
+ String profileName = null;
+ UploadContext streamingContext = null;
try {
if (options.enableTracer || (options.removeBinaryProfile && options.profilePath != null)) {
if (options.enableTracerCompression == TriState.YES
@@ -303,15 +315,23 @@
}
if (options.profilePath != null) {
profilePath = workspace.getWorkspace().getRelative(options.profilePath);
+ out = profilePath.getOutputStream();
} else {
- String profileName = "command.profile";
+ profileName = "command.profile";
if (format == Format.JSON_TRACE_FILE_COMPRESSED_FORMAT) {
profileName = "command.profile.gz";
}
- profilePath = workspace.getOutputBase().getRelative(profileName);
+ if (bepOptions.streamingLogFileUploads) {
+ BuildEventArtifactUploader buildEventArtifactUploader =
+ newUploader(env, bepOptions.buildEventUploadStrategy);
+ streamingContext = buildEventArtifactUploader.startUpload(LocalFileType.LOG);
+ out = streamingContext.getOutputStream();
+ } else {
+ profilePath = workspace.getOutputBase().getRelative(profileName);
+ out = profilePath.getOutputStream();
+ }
}
- out = profilePath.getOutputStream();
- if (options.announceProfilePath) {
+ if (profilePath != null && options.announceProfilePath) {
eventHandler.handle(Event.info("Writing tracer profile to '" + profilePath + "'"));
}
for (ProfilerTask profilerTask : ProfilerTask.values()) {
@@ -354,7 +374,7 @@
format,
getProductName(),
workspace.getOutputBase().toString(),
- buildID,
+ env.getCommandId(),
recordFullProfilerData,
clock,
execStartTimeNanos,
@@ -399,7 +419,7 @@
} catch (IOException e) {
eventHandler.handle(Event.error("Error while creating profile file: " + e.getMessage()));
}
- return profilePath;
+ return new ProfilerStartedEvent(profileName, profilePath, streamingContext);
}
public FileSystem getFileSystem() {
