src/main/protobuf/spawn.proto

// Copyright 2018 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

syntax = "proto3";

package tools.protos;

import "google/protobuf/duration.proto";
import "google/protobuf/timestamp.proto";

option java_package = "com.google.devtools.build.lib.exec";
option java_outer_classname = "Protos";

// This file defines the output formats emitted by the --execution_log_* flags.
// The purpose of the execution log is to enable comparisons of multiple builds
// to diagnose output differences or more subtle problems such as remote cache
// misses.

// Digest of a file or action cache entry.
message Digest {
  // The content hash as a lowercase hex string including any leading zeroes.
  string hash = 1;

  // The original content size in bytes.
  int64 size_bytes = 2;

  // The name of the digest function used to compute the hash.
  string hash_function_name = 3;
}

message File {
  // Path to the file relative to the execution root.
  string path = 1;

  // Symlink target path.
  // Only set for unresolved symlinks.
  string symlink_target_path = 4;

  // File digest.
  // Always omitted for unresolved symlinks. May be omitted for empty files.
  Digest digest = 2;

  // Whether the file is a tool.
  // Only set for inputs, never for outputs.
  bool is_tool = 3;
}

// Contents of command environment.
message EnvironmentVariable {
  string name = 1;
  string value = 2;
}

// Command execution platform. This message needs to be kept in sync
// with [Platform][google.devtools.remoteexecution.v1test.Platform].
message Platform {
  message Property {
    string name = 1;
    string value = 2;
  }
  repeated Property properties = 1;
}

// Timing, size, and memory statistics for a SpawnExec.
message SpawnMetrics {
  // Total wall time spent running a spawn, measured locally.
  google.protobuf.Duration total_time = 1;
  // Time taken to convert the spawn into a network request.
  google.protobuf.Duration parse_time = 2;
  // Time spent communicating over the network.
  google.protobuf.Duration network_time = 3;
  // Time spent fetching remote outputs.
  google.protobuf.Duration fetch_time = 4;
  // Time spent waiting in queues.
  google.protobuf.Duration queue_time = 5;
  // Time spent setting up the environment in which the spawn is run.
  google.protobuf.Duration setup_time = 6;
  // Time spent uploading outputs to a remote store.
  google.protobuf.Duration upload_time = 7;
  // Time spent running the subprocess.
  google.protobuf.Duration execution_wall_time = 8;
  // Time spent by the execution framework processing outputs.
  google.protobuf.Duration process_outputs_time = 9;
  // Time spent in previous failed attempts, not including queue time.
  google.protobuf.Duration retry_time = 10;
  // Total size in bytes of inputs or 0 if unavailable.
  int64 input_bytes = 11;
  // Total number of input files or 0 if unavailable.
  int64 input_files = 12;
  // Estimated memory usage or 0 if unavailable.
  int64 memory_estimate_bytes = 13;
  // Limit of total size of inputs or 0 if unavailable.
  int64 input_bytes_limit = 14;
  // Limit of total number of input files or 0 if unavailable.
  int64 input_files_limit = 15;
  // Limit of total size of outputs or 0 if unavailable.
  int64 output_bytes_limit = 16;
  // Limit of total number of output files or 0 if unavailable.
  int64 output_files_limit = 17;
  // Memory limit or 0 if unavailable.
  int64 memory_bytes_limit = 18;
  // Time limit or 0 if unavailable.
  google.protobuf.Duration time_limit = 19;
  // Instant when the spawn started to execute.
  google.protobuf.Timestamp start_time = 20;
}

// Details of an executed spawn.
// This is the format generated by --execution_log_{binary,json}_file.

// Each message contains an executed command, its full inputs and outputs, and
// other information. This format is relatively costly to produce and results
// in very large files, due to the amount of repeated information. The
// --experimental_execution_log_compact_file format provides a better
// alternative.
message SpawnExec {
  // The command that was run.
  repeated string command_args = 1;

  // The command environment.
  repeated EnvironmentVariable environment_variables = 2;

  // The command execution platform.
  Platform platform = 3;

  // The inputs at the time of the execution.
  repeated File inputs = 4;

  // All the listed outputs paths. The paths are relative to the execution root.
  // Actual outputs are a subset of the listed outputs. These paths are sorted.
  repeated string listed_outputs = 5;

  // Whether the spawn was allowed to run remotely.
  bool remotable = 6;

  // Whether the spawn was allowed to be cached.
  bool cacheable = 7;

  // The spawn timeout.
  int64 timeout_millis = 8;

  // The mnemonic of the action this spawn belongs to.
  string mnemonic = 10;

  // The outputs generated by the execution.
  // In order for one of the listed_outputs to appear here, it must have been
  // produced and have the expected type (file, directory or symlink).
  repeated File actual_outputs = 11;

  // If the spawn did not hit a disk or remote cache, this will be the name of
  // the runner, e.g. "remote", "linux-sandbox" or "worker".
  //
  // If the spawn hit a disk or remote cache, this will be "disk cache hit" or
  // "remote cache hit", respectively. This includes the case where a remote
  // cache was hit while executing the spawn remotely.
  //
  // Note that spawns whose owning action hits the persistent action cache
  // are never reported at all.
  //
  // This won't always match the spawn strategy. For the dynamic strategy, it
  // will be the runner for the first branch to complete. For the remote
  // strategy, it might be a local runner in case of a fallback.
  string runner = 12;

  // Whether the spawn hit a disk or remote cache.
  bool cache_hit = 13;

  // A text status describing an execution error. Empty in case of success.
  string status = 14;

  // This field contains the contents of SpawnResult.exitCode.
  // Its semantics varies greatly depending on the status field.
  // Dependable: if status is empty, exit_code is guaranteed to be zero.
  int32 exit_code = 15;

  // Whether the spawn was allowed to be cached remotely.
  bool remote_cacheable = 16;

  // The canonical label of the target this spawn belongs to.
  string target_label = 18;

  // The action cache digest.
  // Only available when remote execution, remote cache or disk cache was
  // enabled for this spawn.
  Digest digest = 19;

  // Timing, size and memory statistics.
  SpawnMetrics metrics = 20;

  reserved 9, 17;
}

// An entry in the compact log format.
// This is the format generated by --experimental_execution_log_compact_file.
//
// Each entry describes either an executed spawn or non-spawn action or a piece
// of data referenced by other entries. This considerably reduces the runtime
// overhead and the size of the log when compared to the
// --execution_log_{binary,json}_file formats.
//
// To ensure that the log can be parsed in a single pass, every entry must be
// serialized after all other entries it references by ID. However, entries
// aren't guaranteed to be serialized in increasing ID order.
//
// Entries other than spawns and non-spawn actions may not be assumed to be
// canonical. For performance reasons, the same file, directory or input set may
// be serialized multiple times with a different ID.
message ExecLogEntry {
  // Information pertaining to the entire invocation.
  // May appear at most once in the initial position.
  message Invocation {
    // The hash function used to compute digests.
    string hash_function_name = 1;

    // The name of the subdirectory of the runfiles tree corresponding to the
    // main repository (also known as the "workspace name").
    //
    // With --enable_bzlmod, this is always "_main", but can vary when using
    // WORKSPACE.
    string workspace_runfiles_directory = 2;

    // Whether --experimental_sibling_repository_layout is enabled.
    bool sibling_repository_layout = 3;
  }

  // An input or output file.
  message File {
    // The file path.
    string path = 1;
    // A digest of the file contents.
    // The hash function name is omitted. It can be obtained from Invocation.
    // May be omitted for empty files.
    Digest digest = 2;
  }

  // An input or output directory.
  // May be a source directory, a fileset tree, or a tree artifact.
  message Directory {
    // The directory path.
    string path = 1;
    // The contained files, whose paths are relative to the directory.
    repeated File files = 2;
  }

  // An unresolved symlink.
  message UnresolvedSymlink {
    // The symlink path.
    string path = 1;
    // The path the symlink points to.
    string target_path = 2;
  }

  // A set of spawn inputs.
  // The contents of the set are the directly contained entries in addition to
  // the contents of all transitively referenced sets. When order matters,
  // transitive sets come before direct entries and within a set, entries are
  // considered in left-to-right order ("postorder").
  // Sets are not canonical: two sets with different structure may yield the
  // same contents.
  message InputSet {
    // Entry IDs of files, directories, unresolved symlinks or runfiles trees
    // belonging to this set.
    repeated uint32 input_ids = 5;
    // Entry IDs of other input sets contained in this set.
    repeated uint32 transitive_set_ids = 4;

    reserved 1, 2, 3;
  }

  // A collection of runfiles symlinked at custom locations.
  // The contents of the set are the directly contained entries in addition to
  // the contents of all transitively referenced sets. When order matters,
  // transitive sets come before direct entries and within a set, entries are
  // considered in left-to-right order ("postorder").
  // Sets are not canonical: two sets with different structure may yield the
  // same contents.
  message SymlinkEntrySet {
    // A map from relative paths of runfiles symlinks to the entry IDs of the
    // symlink target, which may be a file, directory, or unresolved symlink.
    map<string, uint32> direct_entries = 1;
    // Entry IDs of other symlink entry sets transitively contained in this set.
    repeated uint32 transitive_set_ids = 2;
  }

  // A structured representation of the .runfiles directory of an executable.
  //
  // Instead of storing the directory directly, the tree is represented
  // similarly to its in-memory representation in Bazel and needs to be
  // reassembled from the following parts (in case of path collisions, later
  // entries overwrite earlier ones):
  //
  // 1. symlinks (symlinks_id)
  // 2. artifacts at canonical locations (input_set_id)
  // 3. empty files (empty_files)
  // 4. root symlinks (root_symlinks_id)
  // 5. the _repo_mapping file with the repo mapping manifest
  // (repo_mapping_manifest)
  // 6. the <workspace runfiles directory>/.runfile file (if the workspace
  // runfiles directory
  //    wouldn't exist otherwise)
  //
  // See SpawnLogReconstructor#reconstructRunfilesDir for details.
  message RunfilesTree {
    // The runfiles tree path.
    string path = 1;
    // The entry ID of the set of artifacts in the runfiles tree that are
    // symlinked at their canonical locations relative to the tree path.
    // See SpawnLogReconstructor#getRunfilesPaths for how to recover the
    // tree-relative paths of the artifacts from their exec paths.
    //
    // In case of path collisions, later artifacts overwrite earlier ones and
    // artifacts override custom symlinks.
    //
    // The referenced set must not transitively contain any runfile trees.
    uint32 input_set_id = 2;
    // The entry ID of the set of symlink entries with paths relative to the
    // subdirectory of the runfiles tree root corresponding to the main
    // repository.
    uint32 symlinks_id = 3;
    // The entry ID of the set of symlink entries with paths relative to the
    // root of the runfiles tree.
    uint32 root_symlinks_id = 4;
    // The paths of empty files relative to the subdirectory of the runfiles
    // tree root corresponding to the main repository.
    repeated string empty_files = 5;
    // The "_repo_mapping" file at the root of the runfiles tree, if it exists.
    // Only the digest is stored as the relative path is fixed.
    File repo_mapping_manifest = 6;
    // Whether the runfiles tree contains external runfiles at their legacy
    // locations (e.g. _main/external/bazel_tools/tools/bash/runfiles.bash)
    // in addition to the default locations (e.g.
    // bazel_tools/tools/bash/runfiles.bash).
    bool legacy_external_runfiles = 7;
  }

  // A spawn output.
  message Output {
    oneof type {
      // The ID of a file (ctx.actions.declare_file), directory
      // (ctx.actions.declare_directory) or unresolved symlink
      // (ctx.actions.declare_symlink) that is an output of the spawn.
      uint32 output_id = 5;
      // A declared output that is either missing or has the wrong type
      // (e.g., a file where a directory was expected).
      string invalid_output_path = 4;
    }

    reserved 1, 2, 3;
  }

  // An executed spawn.
  message Spawn {
    // The command line arguments.
    repeated string args = 1;

    // The environment variables.
    repeated EnvironmentVariable env_vars = 2;

    // The execution platform.
    Platform platform = 3;

    // Entry ID of the set of inputs. Unset means empty.
    uint32 input_set_id = 4;

    // Entry ID of the subset of inputs that are tools. Unset means empty.
    uint32 tool_set_id = 5;

    // The set of outputs.
    repeated Output outputs = 6;

    // See SpawnExec.label.
    string target_label = 7;

    // See SpawnExec.mnemonic.
    string mnemonic = 8;

    // See SpawnExec.exit_code.
    int32 exit_code = 9;

    // See SpawnExec.status.
    string status = 10;

    // See SpawnExec.runner.
    string runner = 11;

    // See SpawnExec.cache_hit.
    bool cache_hit = 12;

    // See SpawnExec.remotable.
    bool remotable = 13;

    // See SpawnExec.cacheable.
    bool cacheable = 14;

    // See SpawnExec.remote_cacheable.
    bool remote_cacheable = 15;

    // See SpawnExec.digest.
    // The hash function name is omitted. It can be obtained from Invocation.
    // Unset if the file is empty.
    Digest digest = 16;

    // See SpawnExec.timeout_millis.
    int64 timeout_millis = 17;

    // See SpawnExec.metrics.
    SpawnMetrics metrics = 18;
  }

  // A symlink action, which is not backed by a spawn.
  message SymlinkAction {
    // The path of the input file of the action (i.e., the target of the
    // symlink).
    string input_path = 1;

    // The path of the output file of the action (i.e., the symlink itself).
    string output_path = 2;

    // The canonical label of the target this action belongs to.
    string target_label = 3;

    // The mnemonic of the action.
    string mnemonic = 4;
  }

  // If nonzero, then this entry may be referenced by later entries by this ID.
  // Nonzero IDs are unique within an execution log, but may not be contiguous.
  uint32 id = 1;

  // The entry payload.
  oneof type {
    Invocation invocation = 2;
    File file = 3;
    Directory directory = 4;
    UnresolvedSymlink unresolved_symlink = 5;
    InputSet input_set = 6;
    Spawn spawn = 7;
    SymlinkAction symlink_action = 8;
    SymlinkEntrySet symlink_entry_set = 9;
    RunfilesTree runfiles_tree = 10;
  }
}