Fifth cl for verbose workspaces (ability to log certain potentially non-hermetic events that happen as part of repository rules). * Allow to specify log file rather than dumping to INFO. * Include a parser that is able to convert that binary file to text, optionally filtering out specific rules. In the future: - Log levels, full or alerts only RELNOTES: None PiperOrigin-RevId: 210620591
diff --git a/src/BUILD b/src/BUILD index 50ec65e..c717547 100644 --- a/src/BUILD +++ b/src/BUILD
@@ -350,6 +350,7 @@ "//src/test/shell:srcs", "//src/tools/android/java/com/google/devtools/build/android:srcs", "//src/tools/execlog:srcs", + "//src/tools/workspacelog:srcs", "//src/tools/launcher:srcs", "//src/tools/package_printer/java/com/google/devtools/build/packageprinter:srcs", "//src/tools/skylark/java/com/google/devtools/skylark/skylint:srcs",
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/debug/BUILD b/src/main/java/com/google/devtools/build/lib/bazel/debug/BUILD index ad9343c..59f1202 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/debug/BUILD +++ b/src/main/java/com/google/devtools/build/lib/bazel/debug/BUILD
@@ -53,7 +53,10 @@ ":debugging-options", ":workspace-rule-event", "//src/main/java/com/google/devtools/build/lib:events", + "//src/main/java/com/google/devtools/build/lib:exitcode-external", + "//src/main/java/com/google/devtools/build/lib:io", "//src/main/java/com/google/devtools/build/lib:runtime", + "//src/main/java/com/google/devtools/build/lib:util", "//src/main/java/com/google/devtools/common/options", "//third_party:guava", ],
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/debug/DebuggingOptions.java b/src/main/java/com/google/devtools/build/lib/bazel/debug/DebuggingOptions.java index 2761b61..4bed8b0 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/debug/DebuggingOptions.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/debug/DebuggingOptions.java
@@ -22,10 +22,12 @@ /** Options for debugging and verbosity tools. */ public final class DebuggingOptions extends OptionsBase { @Option( - name = "experimental_workspace_rules_logging", + name = "experimental_workspace_rules_log_file", defaultValue = "null", + category = "verbosity", documentationCategory = OptionDocumentationCategory.LOGGING, effectTags = {OptionEffectTag.UNKNOWN}, - help = "Log certain Workspace Rules events") - public String workspaceRulesLogging; + help = + "Log certain Workspace Rules events into this file as delimited WorkspaceEvent protos.") + public String workspaceRulesLogFile; }
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/debug/WorkspaceRuleModule.java b/src/main/java/com/google/devtools/build/lib/bazel/debug/WorkspaceRuleModule.java index bd5406d..9efbcf6 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/debug/WorkspaceRuleModule.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/debug/WorkspaceRuleModule.java
@@ -20,16 +20,20 @@ import com.google.devtools.build.lib.events.Reporter; import com.google.devtools.build.lib.runtime.BlazeModule; import com.google.devtools.build.lib.runtime.CommandEnvironment; +import com.google.devtools.build.lib.util.AbruptExitException; +import com.google.devtools.build.lib.util.ExitCode; +import com.google.devtools.build.lib.util.io.AsynchronousFileOutputStream; import com.google.devtools.common.options.OptionsBase; +import java.io.IOException; /** A module for logging workspace rule events */ public final class WorkspaceRuleModule extends BlazeModule { private Reporter reporter; private EventBus eventBus; + private AsynchronousFileOutputStream outFileStream; @Override public void beforeCommand(CommandEnvironment env) { - reporter = env.getReporter(); eventBus = env.getEventBus(); @@ -38,18 +42,43 @@ return; } - if (env.getOptions().getOptions(DebuggingOptions.class).workspaceRulesLogging != null) { + String logFile = env.getOptions().getOptions(DebuggingOptions.class).workspaceRulesLogFile; + if (logFile != null && !logFile.isEmpty()) { + try { + outFileStream = new AsynchronousFileOutputStream(logFile); + } catch (IOException e) { + env.getReporter().handle(Event.error(e.getMessage())); + env.getBlazeModuleEnvironment() + .exit( + new AbruptExitException( + "Error initializing workspace rule log file.", ExitCode.COMMAND_LINE_ERROR)); + } eventBus.register(this); } } @Override + public void afterCommand() { + if (outFileStream != null) { + try { + outFileStream.close(); + } catch (IOException e) { + throw new RuntimeException(e); + } finally { + outFileStream = null; + } + } + } + + @Override public Iterable<Class<? extends OptionsBase>> getCommonCommandOptions() { return ImmutableList.<Class<? extends OptionsBase>>of(DebuggingOptions.class); } @Subscribe public void workspaceRuleEventReceived(WorkspaceRuleEvent event) { - reporter.handle(Event.info(event.logMessage())); + if (outFileStream != null) { + outFileStream.write(event.getLogEvent()); + } } }
diff --git a/src/test/shell/bazel/BUILD b/src/test/shell/bazel/BUILD index 671a9c2..17fef97 100644 --- a/src/test/shell/bazel/BUILD +++ b/src/test/shell/bazel/BUILD
@@ -679,6 +679,9 @@ sh_test( name = "bazel_workspaces_test", srcs = ["bazel_workspaces_test.sh"], - data = [":test-deps"], + data = [ + ":test-deps", + "//src/tools/workspacelog:parser", + ], tags = ["no_windows"], )
diff --git a/src/test/shell/bazel/bazel_workspaces_test.sh b/src/test/shell/bazel/bazel_workspaces_test.sh index 8bb03be..efc1c6d 100755 --- a/src/test/shell/bazel/bazel_workspaces_test.sh +++ b/src/test/shell/bazel/bazel_workspaces_test.sh
@@ -23,7 +23,9 @@ source "${CURRENT_DIR}/remote_helpers.sh" \ || { echo "remote_helpers.sh not found!" >&2; exit 1; } -function test_execute() { +# Sets up a workspace with the given commands inserted into the repository rule +# that will be executed when doing bazel build //:test +function set_workspace_command() { create_new_workspace cat > BUILD <<'EOF' genrule( @@ -35,7 +37,7 @@ EOF cat >> repos.bzl <<EOF def _executeMe(repository_ctx): - repository_ctx.execute(["echo", "testing!"]) + $1 build_contents = "package(default_visibility = ['//visibility:public'])\n\n" build_contents += "exports_files([\"t.txt\"])\n" repository_ctx.file("BUILD", build_contents, False) @@ -50,24 +52,42 @@ load("//:repos.bzl", "ex_repo") ex_repo(name = "repo") EOF +} - bazel build //:test --experimental_workspace_rules_logging=yes &> $TEST_log || fail "could not build //:test" - executes=`grep "location: .*repos.bzl:2:3" $TEST_log | wc -l` - if [ "$executes" -ne "1" ] - then - fail "Expected exactly 1 occurrence of the given command, got $executes" - fi +function build_and_process_log() { + bazel build //:test --experimental_workspace_rules_log_file=output 2>&1 >> $TEST_log || fail "could not build //:test" + ${BAZEL_RUNFILES}/src/tools/workspacelog/parser --log_path=output > output.log.txt "$@" || fail "error parsing output" +} - # Cached executions are not replayed - bazel build //:test --experimental_workspace_rules_logging=yes &> output || fail "could not build //:test" - cat output >> $TEST_log - executes=`grep "location: .*repos.bzl:2:3" output | wc -l` - if [ "$executes" -ne "0" ] +function ensure_contains_exactly() { + num=`grep "${1}" output.log.txt | wc -l` + if [ "$num" -ne $2 ] then - fail "Expected exactly 0 occurrence of the given command, got $executes" + fail "Expected exactly $2 occurences of $1, got $num: " `cat output.log.txt` fi } +function ensure_contains_atleast() { + num=`grep "${1}" output.log.txt | wc -l` + if [ "$num" -lt $2 ] + then + fail "Expected at least $2 occurences of $1, got $num: " `cat output.log.txt` + fi +} + +function test_execute() { + set_workspace_command 'repository_ctx.execute(["echo", "testing!"])' + build_and_process_log + + ensure_contains_exactly "location: .*repos.bzl:2:3" 1 + + # Cached executions are not replayed + build_and_process_log + ensure_contains_exactly "location: .*repos.bzl:2:3" 0 +} + +# The workspace is set up so that the function is interrupted and re-executed. +# The log should contain both instances. function test_reexecute() { create_new_workspace cat > BUILD <<'EOF' @@ -108,60 +128,46 @@ a_repo(name = "another") EOF - bazel build //:test --experimental_workspace_rules_logging=yes &> $TEST_log || fail "could not build //:test" - executes=`grep "location: .*repos.bzl:2:3" $TEST_log | wc -l` - if [ "$executes" -le "2" ] - then - fail "Expected at least 2 occurrences of the given command, got $executes" - fi + build_and_process_log + + ensure_contains_atleast "location: .*repos.bzl:2:3" 2 } -# Sets up a workspace with the given commands inserted into the repository rule -# that will be executed when doing bazel build //:test -function set_workspace_command() { - create_new_workspace - cat > BUILD <<'EOF' -genrule( - name="test", - srcs=["@repo//:t.txt"], - outs=["out.txt"], - cmd="echo Result > $(location out.txt)" -) -EOF - cat >> repos.bzl <<EOF -def _executeMe(repository_ctx): - $1 - build_contents = "package(default_visibility = ['//visibility:public'])\n\n" - build_contents += "exports_files([\"t.txt\"])\n" - repository_ctx.file("BUILD", build_contents, False) - repository_ctx.file("t.txt", "HELLO!\n", False) - -ex_repo = repository_rule( - implementation = _executeMe, - local = True, -) -EOF - cat >> WORKSPACE <<EOF -load("//:repos.bzl", "ex_repo") -ex_repo(name = "repo") -EOF -} # Ensure details of the specific functions are present function test_execute2() { set_workspace_command 'repository_ctx.execute(["echo", "test_contents"], 21, {"Arg1": "Val1"}, True)' - bazel build //:test --experimental_workspace_rules_logging=yes &> ${TEST_log} || fail "could not build //:test\n" - expect_log "location: .*repos.bzl:2:3" - expect_log "arguments: \"echo\"" - expect_log "arguments: \"test_contents\"" - expect_log "timeout_seconds: 21" - expect_log "quiet: true" - expect_log "key: \"Arg1\"" - expect_log "value: \"Val1\"" - expect_log "rule: \"//external:repo\"" + build_and_process_log --exclude_rule "//external:local_config_cc" + + ensure_contains_exactly 'location: .*repos.bzl:2:3' 1 + ensure_contains_exactly 'arguments: "echo"' 1 + ensure_contains_exactly 'arguments: "test_contents"' 1 + ensure_contains_exactly 'timeout_seconds: 21' 1 + ensure_contains_exactly 'quiet: true' 1 + ensure_contains_exactly 'key: "Arg1"' 1 + ensure_contains_exactly 'value: "Val1"' 1 + # Workspace contains 2 file commands + ensure_contains_atleast 'rule: "//external:repo"' 3 } +function test_execute_quiet2() { + set_workspace_command 'repository_ctx.execute(["echo", "test2"], 32, {"A1": "V1"}, False)' + + build_and_process_log --exclude_rule "//external:local_config_cc" + + ensure_contains_exactly 'location: .*repos.bzl:2:3' 1 + ensure_contains_exactly 'arguments: "echo"' 1 + ensure_contains_exactly 'arguments: "test2"' 1 + ensure_contains_exactly 'timeout_seconds: 32' 1 + # quiet: false does not show up when printing protos + # since it's the default value + ensure_contains_exactly 'quiet: ' 0 + ensure_contains_exactly 'key: "A1"' 1 + ensure_contains_exactly 'value: "V1"' 1 + # Workspace contains 2 file commands + ensure_contains_atleast 'rule: "//external:repo"' 3 +} function test_download() { # Prepare HTTP server with Python @@ -176,13 +182,14 @@ set_workspace_command "repository_ctx.download(\"http://localhost:${fileserver_port}/file.txt\", \"file.txt\", \"${file_sha256}\")" - bazel build //:test --experimental_workspace_rules_logging=yes &> ${TEST_log} && shutdown_server || fail "could not build //:test\n" - expect_log "location: .*repos.bzl:2:3" - expect_log "rule: \"//external:repo\"" - expect_log "download_event" - expect_log "url: \"http://localhost:${fileserver_port}/file.txt\"" - expect_log "output: \"file.txt\"" - expect_log "sha256: \"${file_sha256}\"" + build_and_process_log --exclude_rule "//external:local_config_cc" + + ensure_contains_exactly 'location: .*repos.bzl:2:3' 1 + ensure_contains_atleast 'rule: "//external:repo"' 1 + ensure_contains_exactly 'download_event' 1 + ensure_contains_exactly "url: \"http://localhost:${fileserver_port}/file.txt\"" 1 + ensure_contains_exactly 'output: "file.txt"' 1 + ensure_contains_exactly "sha256: \"${file_sha256}\"" 1 } function test_download_multiple() { @@ -197,13 +204,14 @@ set_workspace_command "repository_ctx.download([\"http://localhost:${fileserver_port}/file1.txt\",\"http://localhost:${fileserver_port}/file2.txt\"], \"out_for_list.txt\")" - bazel build //:test --experimental_workspace_rules_logging=yes &> $TEST_log && shutdown_server || fail "could not build //:test\n" - expect_log "location: .*repos.bzl:2:3" - expect_log "rule: \"//external:repo\"" - expect_log "download_event" - expect_log "url: \"http://localhost:${fileserver_port}/file1.txt\"" - expect_log "url: \"http://localhost:${fileserver_port}/file2.txt\"" - expect_log "output: \"out_for_list.txt\"" + build_and_process_log --exclude_rule "//external:local_config_cc" + + ensure_contains_exactly 'location: .*repos.bzl:2:3' 1 + ensure_contains_atleast 'rule: "//external:repo"' 1 + ensure_contains_exactly 'download_event' 1 + ensure_contains_exactly "url: \"http://localhost:${fileserver_port}/file1.txt\"" 1 + ensure_contains_exactly "url: \"http://localhost:${fileserver_port}/file2.txt\"" 1 + ensure_contains_exactly 'output: "out_for_list.txt"' 1 } function test_download_and_extract() { @@ -223,74 +231,81 @@ set_workspace_command "repository_ctx.download_and_extract(\"http://localhost:${fileserver_port}/download_and_extract.zip\", \"out_dir\", \"${file_sha256}\", \"zip\", \"server_dir/\")" - bazel build //:test --experimental_workspace_rules_logging=yes &> ${TEST_log} && shutdown_server || fail "could not build //:test\n" + build_and_process_log --exclude_rule "//external:local_config_cc" - expect_log "location: .*repos.bzl:2:3" - expect_log "rule: \"//external:repo\"" - expect_log "download_and_extract_event" - expect_log "url: \"http://localhost:${fileserver_port}/download_and_extract.zip\"" - expect_log "output: \"out_dir\"" - expect_log "sha256: \"${file_sha256}\"" - expect_log "type: \"zip\"" - expect_log "strip_prefix: \"server_dir/\"" + ensure_contains_exactly 'location: .*repos.bzl:2:3' 1 + ensure_contains_atleast 'rule: "//external:repo"' 1 + ensure_contains_exactly 'download_and_extract_event' 1 + ensure_contains_exactly "url: \"http://localhost:${fileserver_port}/download_and_extract.zip\"" 1 + ensure_contains_exactly 'output: "out_dir"' 1 + ensure_contains_exactly "sha256: \"${file_sha256}\"" 1 + ensure_contains_exactly 'type: "zip"' 1 + ensure_contains_exactly 'strip_prefix: "server_dir/"' 1 } function test_file() { set_workspace_command 'repository_ctx.file("filefile.sh", "echo filefile", True)' - bazel build //:test --experimental_workspace_rules_logging=yes &> $TEST_log || fail "could not build //:test\n" - expect_log 'location: .*repos.bzl:2:3' - expect_log 'rule: "//external:repo"' - expect_log 'file_event' - expect_log 'path: ".*filefile.sh"' - expect_log 'content: "echo filefile"' - expect_log 'executable: true' + build_and_process_log --exclude_rule "//external:local_config_cc" + + ensure_contains_exactly 'location: .*repos.bzl:2:3' 1 + ensure_contains_atleast 'rule: "//external:repo"' 1 + + # There are 3 file_event in external:repo as it is currently set up + ensure_contains_exactly 'file_event' 3 + ensure_contains_exactly 'path: ".*filefile.sh"' 1 + ensure_contains_exactly 'content: "echo filefile"' 1 + ensure_contains_exactly 'executable: true' 1 } function test_os() { set_workspace_command 'print(repository_ctx.os.name)' - bazel build //:test --experimental_workspace_rules_logging=yes &> $TEST_log || fail "could not build //:test\n" - expect_log 'location: .*repos.bzl:2:9' - expect_log 'rule: "//external:repo"' - expect_log 'os_event' + build_and_process_log --exclude_rule "//external:local_config_cc" + + ensure_contains_exactly 'location: .*repos.bzl:2:9' 1 + ensure_contains_atleast 'rule: "//external:repo"' 1 + ensure_contains_exactly 'os_event' 1 } function test_symlink() { set_workspace_command 'repository_ctx.file("symlink.txt", "something") repository_ctx.symlink("symlink.txt", "symlink_out.txt")' - bazel build //:test --experimental_workspace_rules_logging=yes &> $TEST_log || fail "could not build //:test\n" - expect_log 'location: .*repos.bzl:3:3' - expect_log 'rule: "//external:repo"' - expect_log 'symlink_event' - expect_log 'from: ".*symlink.txt"' - expect_log 'to: ".*symlink_out.txt"' + build_and_process_log --exclude_rule "//external:local_config_cc" + + ensure_contains_exactly 'location: .*repos.bzl:3:3' 1 + ensure_contains_atleast 'rule: "//external:repo"' 1 + ensure_contains_exactly 'symlink_event' 1 + ensure_contains_exactly 'from: ".*symlink.txt"' 1 + ensure_contains_exactly 'to: ".*symlink_out.txt"' 1 } function test_template() { - set_workspace_command 'repository_ctx.file("template_in.txt", "%{subKey}") + set_workspace_command 'repository_ctx.file("template_in.txt", "%{subKey}", False) repository_ctx.template("template_out.txt", "template_in.txt", {"subKey": "subVal"}, True)' - bazel build //:test --experimental_workspace_rules_logging=yes &> $TEST_log || fail "could not build //:test\n" - expect_log 'location: .*repos.bzl:3:3' - expect_log 'rule: "//external:repo"' - expect_log 'template_event' - expect_log 'path: ".*template_out.txt"' - expect_log 'template: ".*template_in.txt"' - expect_log 'key: "subKey"' - expect_log 'value: "subVal"' - expect_log 'executable: true' + build_and_process_log --exclude_rule "//external:local_config_cc" + + ensure_contains_exactly 'location: .*repos.bzl:3:3' 1 + ensure_contains_atleast 'rule: "//external:repo"' 1 + ensure_contains_exactly 'template_event' 1 + ensure_contains_exactly 'path: ".*template_out.txt"' 1 + ensure_contains_exactly 'template: ".*template_in.txt"' 1 + ensure_contains_exactly 'key: "subKey"' 1 + ensure_contains_exactly 'value: "subVal"' 1 + ensure_contains_exactly 'executable: true' 1 } function test_which() { set_workspace_command 'print(repository_ctx.which("which_prog"))' - bazel build //:test --experimental_workspace_rules_logging=yes &> $TEST_log || fail "could not build //:test\n" - expect_log 'location: .*repos.bzl:2:9' - expect_log 'rule: "//external:repo"' - expect_log 'which_event' - expect_log 'program: "which_prog"' + build_and_process_log --exclude_rule "//external:local_config_cc" + + ensure_contains_exactly 'location: .*repos.bzl:2:9' 1 + ensure_contains_atleast 'rule: "//external:repo"' 1 + ensure_contains_exactly 'which_event' 1 + ensure_contains_exactly 'program: "which_prog"' 1 } function tear_down() { @@ -302,3 +317,4 @@ } run_suite "workspaces_tests" +
diff --git a/src/tools/workspacelog/BUILD b/src/tools/workspacelog/BUILD new file mode 100644 index 0000000..6e741b0 --- /dev/null +++ b/src/tools/workspacelog/BUILD
@@ -0,0 +1,15 @@ +filegroup( + name = "srcs", + srcs = glob(["**"]) + [ + "//src/tools/workspacelog/src/main/java/com/google/devtools/build/workspacelog:srcs", + "//src/tools/workspacelog/test/main/java/com/google/devtools/build/workspacelog:srcs", + ], + visibility = ["//src:__pkg__"], +) + +java_binary( + name = "parser", + main_class = "com.google.devtools.build.workspacelog.WorkspaceLogParser", + visibility = ["//visibility:public"], + runtime_deps = ["//src/tools/workspacelog/src/main/java/com/google/devtools/build/workspacelog:parser"], +)
diff --git a/src/tools/workspacelog/README.md b/src/tools/workspacelog/README.md new file mode 100644 index 0000000..2c6b9bd --- /dev/null +++ b/src/tools/workspacelog/README.md
@@ -0,0 +1,38 @@ +# Workspace Log Parser + +This tool is used to inspect and parse the Bazel workspace logs. +To generate the workspace log, run e.g.,: + + bazel build \ + --experimental_workspace_rules_log_file=/tmp/workspace.log :hello_world + +Then build the parser and run it. + + bazel build src/tools/workspacelog:all + bazel-bin/src/tools/workspacelog/parser --log_path=/tmp/workspace.log + +This will simply print the log contents to stdout in text form. + + +To output results to a file, use `--output_path`: + + bazel-bin/src/tools/workspacelog/parser --log_path=/tmp/workspace.log \ + --output_path=/tmp/workspace.log.txt + + +To exclude all events produced by a certain rule, use `--exclude_rule`: + + bazel build src/tools/workspacelog:all + bazel-bin/src/tools/workspacelog/parser --log_path=/tmp/workspace.log \ + --exclude_rule "//external:local_config_cc" + +Note that `--exclude_rule` may be specified multiple times. + + bazel build src/tools/workspacelog:all + bazel-bin/src/tools/workspacelog/parser --log_path=/tmp/workspace.log \ + --exclude_rule "//external:local_config_cc" \ + --exclude_rule "//external:dep" + +For example, the above will filter out any events produced by rules +`//external:local_config_cc` or `//external:dep` +
diff --git a/src/tools/workspacelog/src/main/java/com/google/devtools/build/workspacelog/BUILD b/src/tools/workspacelog/src/main/java/com/google/devtools/build/workspacelog/BUILD new file mode 100644 index 0000000..b3df80d --- /dev/null +++ b/src/tools/workspacelog/src/main/java/com/google/devtools/build/workspacelog/BUILD
@@ -0,0 +1,16 @@ +filegroup( + name = "srcs", + srcs = glob(["**"]), + visibility = ["//src/tools/workspacelog:__pkg__"], +) + +java_library( + name = "parser", + srcs = glob(["*.java"]), + visibility = ["//src/tools/workspacelog:__subpackages__"], + deps = [ + "//src/main/java/com/google/devtools/build/lib/bazel/debug:workspace_log_java_proto", + "//src/main/java/com/google/devtools/common/options", + "//third_party:guava", + ], +)
diff --git a/src/tools/workspacelog/src/main/java/com/google/devtools/build/workspacelog/WorkspaceLogParser.java b/src/tools/workspacelog/src/main/java/com/google/devtools/build/workspacelog/WorkspaceLogParser.java new file mode 100644 index 0000000..79f25a9 --- /dev/null +++ b/src/tools/workspacelog/src/main/java/com/google/devtools/build/workspacelog/WorkspaceLogParser.java
@@ -0,0 +1,111 @@ +// Copyright 2018 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.workspacelog; + +import static java.nio.charset.StandardCharsets.UTF_8; + +import com.google.common.annotations.VisibleForTesting; +import com.google.devtools.build.lib.bazel.debug.proto.WorkspaceLogProtos.WorkspaceEvent; +import com.google.devtools.common.options.OptionsParser; +import java.io.BufferedWriter; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.io.PrintWriter; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +/** A tool to inspect and parse the Bazel workspace rules log. */ +final class WorkspaceLogParser { + + static final String DELIMITER = "\n---------------------------------------------------------\n"; + + @VisibleForTesting + static class ExcludingLogParser { + final InputStream in; + final Set<String> excludedRules; + + ExcludingLogParser(InputStream in, Set<String> excludedRules) { + this.in = in; + if (excludedRules == null) { + this.excludedRules = Collections.emptySet(); + } else { + this.excludedRules = excludedRules; + } + } + + public WorkspaceEvent getNext() throws IOException { + WorkspaceEvent w; + // Find the next record whose runner matches + do { + if (in.available() <= 0) { + // End of file + return null; + } + w = WorkspaceEvent.parseDelimitedFrom(in); + } while (excludedRules.contains(w.getRule())); + return w; + } + } + + public static void output(ExcludingLogParser p, OutputStream outStream) throws IOException { + PrintWriter out = + new PrintWriter(new BufferedWriter(new OutputStreamWriter(outStream, UTF_8)), true); + WorkspaceEvent w; + while ((w = p.getNext()) != null) { + out.println(w); + out.println(DELIMITER); + } + } + + public static void main(String[] args) throws Exception { + OptionsParser op = OptionsParser.newOptionsParser(WorkspaceLogParserOptions.class); + op.parseAndExitUponError(args); + + WorkspaceLogParserOptions options = op.getOptions(WorkspaceLogParserOptions.class); + List<String> remainingArgs = op.getResidue(); + + if (!remainingArgs.isEmpty()) { + System.err.println("Unexpected options: " + String.join(" ", remainingArgs)); + System.exit(1); + } + + if (options.logPath == null || options.logPath.isEmpty()) { + System.err.println("--log_path needs to be specified."); + System.exit(1); + } + + try (InputStream input = new FileInputStream(options.logPath)) { + ExcludingLogParser parser; + if (options.excludeRule == null) { + parser = new ExcludingLogParser(input, null); + } else { + parser = new ExcludingLogParser(input, new HashSet<String>(options.excludeRule)); + } + + if (options.outputPath == null) { + output(parser, System.out); + } else { + try (OutputStream output = new FileOutputStream(options.outputPath)) { + output(parser, output); + } + } + } + } +}
diff --git a/src/tools/workspacelog/src/main/java/com/google/devtools/build/workspacelog/WorkspaceLogParserOptions.java b/src/tools/workspacelog/src/main/java/com/google/devtools/build/workspacelog/WorkspaceLogParserOptions.java new file mode 100644 index 0000000..2598b26 --- /dev/null +++ b/src/tools/workspacelog/src/main/java/com/google/devtools/build/workspacelog/WorkspaceLogParserOptions.java
@@ -0,0 +1,52 @@ +// Copyright 2018 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.workspacelog; + +import com.google.devtools.common.options.Option; +import com.google.devtools.common.options.OptionDocumentationCategory; +import com.google.devtools.common.options.OptionEffectTag; +import com.google.devtools.common.options.OptionsBase; +import java.util.List; + +/** Options for workspace log parser. */ +public class WorkspaceLogParserOptions extends OptionsBase { + @Option( + name = "log_path", + defaultValue = "null", + category = "logging", + documentationCategory = OptionDocumentationCategory.LOGGING, + effectTags = {OptionEffectTag.UNKNOWN}, + help = "Location of the workspace rules log file to parse.") + public String logPath; + + @Option( + name = "output_path", + defaultValue = "null", + category = "logging", + documentationCategory = OptionDocumentationCategory.LOGGING, + effectTags = {OptionEffectTag.UNKNOWN}, + help = "Location where to put the output. If left empty, the log will be output to stdout.") + public String outputPath; + + @Option( + name = "exclude_rule", + defaultValue = "null", + category = "logging", + documentationCategory = OptionDocumentationCategory.LOGGING, + effectTags = {OptionEffectTag.UNKNOWN}, + allowMultiple = true, + help = "Rule(s) to filter out while parsing.") + public List<String> excludeRule; +}
diff --git a/src/tools/workspacelog/test/main/java/com/google/devtools/build/workspacelog/BUILD b/src/tools/workspacelog/test/main/java/com/google/devtools/build/workspacelog/BUILD new file mode 100644 index 0000000..759eb3e --- /dev/null +++ b/src/tools/workspacelog/test/main/java/com/google/devtools/build/workspacelog/BUILD
@@ -0,0 +1,30 @@ +package( + default_testonly = 1, + default_visibility = ["//src:__subpackages__"], +) + +filegroup( + name = "srcs", + testonly = 0, + srcs = glob( + ["**"], + exclude = [ + "*~", + ], + ), + visibility = ["//src/tools/workspacelog:__pkg__"], +) + +java_test( + name = "WorkspaceLogParserTest", + size = "small", + srcs = ["WorkspaceLogParserTest.java"], + test_class = "com.google.devtools.build.workspacelog.WorkspaceLogParserTest", + deps = [ + "//src/main/java/com/google/devtools/build/lib/bazel/debug:workspace_log_java_proto", + "//src/main/java/com/google/devtools/common/options", + "//src/tools/execlog/src/main/java/com/google/devtools/build/workspacelog:parser", + "//third_party:junit4", + "//third_party:truth", + ], +)
diff --git a/src/tools/workspacelog/test/main/java/com/google/devtools/build/workspacelog/WorkspaceLogParserTest.java b/src/tools/workspacelog/test/main/java/com/google/devtools/build/workspacelog/WorkspaceLogParserTest.java new file mode 100644 index 0000000..1d53fdf --- /dev/null +++ b/src/tools/workspacelog/test/main/java/com/google/devtools/build/workspacelog/WorkspaceLogParserTest.java
@@ -0,0 +1,125 @@ +// Copyright 2018 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.workspacelog; + +import static com.google.common.truth.Truth.assertThat; + +import com.google.devtools.build.lib.bazel.debug.proto.WorkspaceLogProtos.WorkspaceEvent; +import com.google.devtools.build.workspacelog.WorkspaceLogParser.ExcludingLogParser; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Testing WorkspaceLogParser */ +@RunWith(JUnit4.class) +public final class WorkspaceLogParserTest { + + private InputStream toInputStream(List<WorkspaceEvent> list) throws Exception { + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + for (WorkspaceEvent event : list) { + event.writeDelimitedTo(bos); + } + + return new ByteArrayInputStream(bos.toByteArray()); + } + + @Test + public void getNextEmpty() throws Exception { + ExcludingLogParser p = + new ExcludingLogParser(toInputStream(new ArrayList<WorkspaceEvent>()), null); + assertThat(p.getNext()).isNull(); + } + + @Test + public void getNextEmptyWithExclusions() throws Exception { + ExcludingLogParser p = + new ExcludingLogParser( + toInputStream(new ArrayList<WorkspaceEvent>()), new HashSet<>(Arrays.asList("a", "b"))); + assertThat(p.getNext()).isNull(); + } + + @Test + public void getNextSingleExcluded1() throws Exception { + WorkspaceEvent a = WorkspaceEvent.newBuilder().setRule("a").setLocation("SomeLocation").build(); + + // Excluded by first exclusion + ExcludingLogParser p = + new ExcludingLogParser( + toInputStream(Arrays.asList(a)), new HashSet<>(Arrays.asList("a", "b"))); + assertThat(p.getNext()).isNull(); + } + + @Test + public void getNextSingleExcluded2() throws Exception { + WorkspaceEvent a = WorkspaceEvent.newBuilder().setRule("a").setLocation("SomeLocation").build(); + + // Excluded by second exclusion + ExcludingLogParser p = + new ExcludingLogParser( + toInputStream(Arrays.asList(a)), new HashSet<>(Arrays.asList("b", "a"))); + assertThat(p.getNext()).isNull(); + } + + @Test + public void getNextSingleIncluded() throws Exception { + WorkspaceEvent a = + WorkspaceEvent.newBuilder().setRule("onOnList").setLocation("SomeLocation").build(); + + ExcludingLogParser p = + new ExcludingLogParser( + toInputStream(Arrays.asList(a)), new HashSet<>(Arrays.asList("b", "a"))); + assertThat(p.getNext()).isEqualTo(a); + assertThat(p.getNext()).isNull(); + } + + @Test + public void getNextSingleLongerList1() throws Exception { + WorkspaceEvent a = WorkspaceEvent.newBuilder().setRule("a").setLocation("a1").build(); + WorkspaceEvent b = WorkspaceEvent.newBuilder().setRule("b").setLocation("b1").build(); + WorkspaceEvent c = WorkspaceEvent.newBuilder().setRule("a").setLocation("a2").build(); + WorkspaceEvent d = WorkspaceEvent.newBuilder().setRule("b").setLocation("b2").build(); + WorkspaceEvent e = WorkspaceEvent.newBuilder().setRule("d").build(); + + ExcludingLogParser p = + new ExcludingLogParser( + toInputStream(Arrays.asList(a, b, c, d, e)), new HashSet<>(Arrays.asList("b", "a"))); + assertThat(p.getNext()).isEqualTo(e); + assertThat(p.getNext()).isNull(); + } + + @Test + public void getNextSingleLongerList2() throws Exception { + WorkspaceEvent a = WorkspaceEvent.newBuilder().setRule("a").setLocation("a1").build(); + WorkspaceEvent b = WorkspaceEvent.newBuilder().setRule("b").setLocation("b1").build(); + WorkspaceEvent c = WorkspaceEvent.newBuilder().setRule("a").setLocation("a2").build(); + WorkspaceEvent d = WorkspaceEvent.newBuilder().setRule("b").setLocation("b2").build(); + WorkspaceEvent e = WorkspaceEvent.newBuilder().setRule("d").build(); + + ExcludingLogParser p = + new ExcludingLogParser( + toInputStream(Arrays.asList(a, b, c, d, e)), new HashSet<>(Arrays.asList("d"))); + assertThat(p.getNext()).isEqualTo(a); + assertThat(p.getNext()).isEqualTo(b); + assertThat(p.getNext()).isEqualTo(c); + assertThat(p.getNext()).isEqualTo(d); + assertThat(p.getNext()).isNull(); + } +}