workers: Put all tests in bazel_worker_test.sh. Add a testbed that makes it easy to test details of how the worker strategy behaves in corner cases.
--
MOS_MIGRATED_REVID=103541927
diff --git a/src/main/java/com/google/devtools/build/lib/worker/WorkerFactory.java b/src/main/java/com/google/devtools/build/lib/worker/WorkerFactory.java
index 208b88c..7da7014 100644
--- a/src/main/java/com/google/devtools/build/lib/worker/WorkerFactory.java
+++ b/src/main/java/com/google/devtools/build/lib/worker/WorkerFactory.java
@@ -76,15 +76,6 @@
*/
@Override
public boolean validateObject(WorkerKey key, PooledObject<Worker> p) {
- if (verbose) {
- reporter.handle(
- Event.info(
- "Validating "
- + key.getMnemonic()
- + " worker (id "
- + p.getObject().getWorkerId()
- + ")."));
- }
return p.getObject().isAlive();
}
}
diff --git a/src/test/java/BUILD b/src/test/java/BUILD
index 9f3ead7..da59f46 100644
--- a/src/test/java/BUILD
+++ b/src/test/java/BUILD
@@ -603,6 +603,30 @@
],
)
+java_library(
+ name = "worker-example-lib",
+ srcs = glob(["com/google/devtools/build/lib/worker/ExampleWorker*.java"]),
+ visibility = [
+ "//src/test/shell/bazel:__pkg__",
+ ],
+ deps = [
+ "//src/main/java:options",
+ "//src/main/protobuf:proto_worker_protocol",
+ "//third_party:guava",
+ ],
+)
+
+java_binary(
+ name = "worker-example",
+ main_class = "com.google.devtools.build.lib.worker.ExampleWorker",
+ visibility = [
+ "//src/test/shell/bazel:__pkg__",
+ ],
+ runtime_deps = [
+ ":worker-example-lib",
+ ],
+)
+
TEST_SUITES = [
"ziputils",
"rules",
diff --git a/src/test/java/com/google/devtools/build/lib/worker/ExampleWorker.java b/src/test/java/com/google/devtools/build/lib/worker/ExampleWorker.java
new file mode 100644
index 0000000..b9146b2
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/worker/ExampleWorker.java
@@ -0,0 +1,128 @@
+// Copyright 2015 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.worker;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import com.google.common.base.Joiner;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.devtools.build.lib.worker.ExampleWorkerOptions.ExampleWorkOptions;
+import com.google.devtools.build.lib.worker.WorkerProtocol.WorkRequest;
+import com.google.devtools.build.lib.worker.WorkerProtocol.WorkResponse;
+import com.google.devtools.common.options.OptionsParser;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * An example implementation of a worker process that is used for integration tests.
+ */
+public class ExampleWorker {
+
+ public static void main(String[] args) throws Exception {
+ if (ImmutableSet.copyOf(args).contains("--persistent_worker")) {
+ OptionsParser parser = OptionsParser.newOptionsParser(ExampleWorkerOptions.class);
+ parser.setAllowResidue(false);
+ parser.parse(args);
+ ExampleWorkerOptions workerOptions = parser.getOptions(ExampleWorkerOptions.class);
+ Preconditions.checkState(workerOptions.persistentWorker);
+
+ runPersistentWorker(workerOptions);
+ } else {
+ // This is a single invocation of the example that exits after it processed the request.
+ processRequest(ImmutableList.copyOf(args));
+ }
+ }
+
+ private static void runPersistentWorker(ExampleWorkerOptions workerOptions) throws IOException {
+ PrintStream originalStdOut = System.out;
+ PrintStream originalStdErr = System.err;
+
+ while (true) {
+ try {
+ WorkRequest request = WorkRequest.parseDelimitedFrom(System.in);
+ if (request == null) {
+ break;
+ }
+
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ int exitCode = 0;
+
+ try (PrintStream ps = new PrintStream(baos)) {
+ System.setOut(ps);
+ System.setErr(ps);
+
+ try {
+ processRequest(request.getArgumentsList());
+ } catch (Exception e) {
+ e.printStackTrace();
+ exitCode = 1;
+ }
+ } finally {
+ System.setOut(originalStdOut);
+ System.setErr(originalStdErr);
+ }
+
+ WorkResponse.newBuilder()
+ .setOutput(baos.toString())
+ .setExitCode(exitCode)
+ .build()
+ .writeDelimitedTo(System.out);
+ System.out.flush();
+ } finally {
+ // Be a good worker process and consume less memory when idle.
+ System.gc();
+ }
+ }
+ }
+
+ private static void processRequest(List<String> args) throws Exception {
+ if (args.size() == 1 && args.get(0).startsWith("@")) {
+ args = Files.readAllLines(Paths.get(args.get(0).substring(1)), UTF_8);
+ }
+
+ OptionsParser parser = OptionsParser.newOptionsParser(ExampleWorkOptions.class);
+ parser.setAllowResidue(true);
+ parser.parse(args);
+ ExampleWorkOptions workOptions = parser.getOptions(ExampleWorkOptions.class);
+
+ List<String> residue = parser.getResidue();
+ List<String> outputs = new ArrayList(residue.size());
+ for (String arg : residue) {
+ String output = arg;
+ if (workOptions.uppercase) {
+ output = output.toUpperCase();
+ }
+ outputs.add(output);
+ }
+
+ String outputStr = Joiner.on(' ').join(outputs);
+ if (workOptions.outputFile.isEmpty()) {
+ System.err.println("ExampleWorker: Writing to stdout!");
+ System.out.println(outputStr);
+ } else {
+ System.err.println("ExampleWorker: Writing to file " + workOptions.outputFile);
+ try (PrintStream outputFile = new PrintStream(workOptions.outputFile)) {
+ outputFile.println(outputStr);
+ }
+ }
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/worker/ExampleWorkerOptions.java b/src/test/java/com/google/devtools/build/lib/worker/ExampleWorkerOptions.java
new file mode 100644
index 0000000..3b5878c
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/worker/ExampleWorkerOptions.java
@@ -0,0 +1,41 @@
+// Copyright 2015 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.worker;
+
+import com.google.devtools.common.options.Option;
+import com.google.devtools.common.options.OptionsBase;
+
+/**
+ * Options for the example worker itself.
+ */
+public class ExampleWorkerOptions extends OptionsBase {
+
+ /**
+ * Options for the example worker concerning single units of work.
+ */
+ public static class ExampleWorkOptions extends OptionsBase {
+ @Option(
+ name = "output_file",
+ defaultValue = "",
+ help = "Write the output to a file instead of stdout."
+ )
+ public String outputFile;
+
+ @Option(name = "uppercase", defaultValue = "false", help = "Uppercase the input.")
+ public boolean uppercase;
+ }
+
+ @Option(name = "persistent_worker", defaultValue = "false")
+ public boolean persistentWorker;
+}
diff --git a/src/test/shell/bazel/BUILD b/src/test/shell/bazel/BUILD
index 1b606b7..4df03b7 100644
--- a/src/test/shell/bazel/BUILD
+++ b/src/test/shell/bazel/BUILD
@@ -213,7 +213,10 @@
name = "bazel_worker_test",
size = "large",
srcs = ["bazel_worker_test.sh"],
- data = [":test-deps"],
+ data = [
+ ":test-deps",
+ "//src/test/java:worker-example_deploy.jar",
+ ],
)
sh_test(
diff --git a/src/test/shell/bazel/bazel_worker_test.sh b/src/test/shell/bazel/bazel_worker_test.sh
index 1652fe6..9367bb2 100755
--- a/src/test/shell/bazel/bazel_worker_test.sh
+++ b/src/test/shell/bazel/bazel_worker_test.sh
@@ -17,6 +17,11 @@
# Test rules provided in Bazel not tested by examples
#
+# TODO(philwo): Change this so the path to the custom worker gets passed in as an argument to the
+# test, once the bug that makes using the "args" attribute with sh_tests in Bazel impossible is
+# fixed.
+example_worker=$(find $PWD -name worker-example_deploy.jar)
+
# Load test environment
source $(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/test-setup.sh \
|| { echo "test-setup.sh not found!" >&2; exit 1; }
@@ -58,22 +63,164 @@
EOF
}
+function print_workers() {
+ pid=$(bazel info | fgrep server_pid | cut -d' ' -f2)
+ # DANGER. This contains arcane shell wizardry that was carefully crafted to be compatible with
+ # both BSD and GNU tools so that this works under Linux and OS X.
+ ps ax -o ppid,pid | awk '{$1=$1};1' | egrep "^${pid} " | cut -d' ' -f2
+}
+
+function shutdown_and_print_unkilled_workers() {
+ workers=$(print_workers)
+ bazel shutdown || fail "shutdown failed"
+ # Wait at most 10 seconds for all workers to shut down, then print the remaining (if any).
+ for i in 0 1 2 3 4 5 6 7 8 9; do
+ still_running_workers=$(for pid in $workers; do ps -p $pid | sed 1d; done)
+ if [[ ! -z "${still_running_workers}" ]]; then
+ sleep 1
+ fi
+ done
+}
+
+function assert_workers_running() {
+ workers=$(print_workers)
+ if [[ -z "${workers}" ]]; then
+ fail "Expected workers to be running, but found none"
+ fi
+}
+
+function assert_workers_not_running() {
+ workers=$(print_workers)
+ if [[ ! -z "${workers}" ]]; then
+ fail "Expected no workers, but found some running: ${workers}"
+ fi
+}
+
function test_compiles_hello_library_using_persistent_javac() {
write_hello_library_files
bazel --batch clean
- bazel build --experimental_persistent_javac //java/main:main || fail "build failed"
+
+ bazel build --strategy=Javac=worker //java/main:main || fail "build failed"
bazel-bin/java/main/main | grep -q "Hello, Library!;Hello, World!" \
|| fail "comparison failed"
- bazel_pid=$(bazel info | fgrep server_pid | cut -d' ' -f2)
- # DANGER. This contains arcane shell wizardry that was carefully crafted to be compatible with
- # both BSD and GNU tools so that this works under Linux and OS X.
- bazel_children=$(ps ax -o ppid,pid | awk '{$1=$1};1' | egrep "^${bazel_pid} " | cut -d' ' -f2)
- bazel shutdown || fail "shutdown failed"
- sleep 10
- unkilled_children=$(for pid in $bazel_children; do ps -p $pid | sed 1d; done)
- if [ ! -z "$unkilled_children" ]; then
- fail "Worker processes were still running: ${unkilled_children}"
+ assert_workers_running
+ unkilled_workers=$(shutdown_and_print_unkilled_workers)
+ if [ ! -z "$unkilled_workers" ]; then
+ fail "Worker processes were still running after shutdown: ${unkilled_workers}"
fi
}
+function test_incremental_heuristic() {
+ write_hello_library_files
+ bazel --batch clean
+
+ # Default strategy is assumed to not use workers.
+ bazel build //java/main:main || fail "build failed"
+ assert_workers_not_running
+
+ # No workers used, because too many files changed.
+ echo '// hello '>> java/hello_library/HelloLibrary.java
+ echo '// hello' >> java/main/Main.java
+ bazel build --worker_max_changed_files=1 --strategy=Javac=worker //java/main:main \
+ || fail "build failed"
+ assert_workers_not_running
+
+ # Workers used, because changed number of files is less-or-equal to --worker_max_changed_files=2.
+ echo '// again '>> java/hello_library/HelloLibrary.java
+ echo '// again' >> java/main/Main.java
+ bazel build --worker_max_changed_files=2 --strategy=Javac=worker //java/main:main \
+ || fail "build failed"
+ assert_workers_running
+}
+
+function test_workers_quit_after_build() {
+ write_hello_library_files
+ bazel --batch clean
+
+ bazel build --worker_quit_after_build --strategy=Javac=worker //java/main:main \
+ || fail "build failed"
+ assert_workers_not_running
+}
+
+function prepare_example_worker() {
+ cp -v ${example_worker} worker_lib.jar
+
+ cat >work.bzl <<'EOF'
+def _impl(ctx):
+ worker = ctx.executable.worker
+ output = ctx.outputs.out
+
+ # Generate the "@"-file containing the command-line args for the unit of work.
+ argfile = ctx.new_file(ctx.configuration.bin_dir, "worker_input")
+ ctx.file_action(output=argfile, content="\n".join(["--output_file=" + output.path] + ctx.attr.args))
+
+ ctx.action(
+ inputs=[argfile],
+ outputs=[output],
+ executable=worker,
+ progress_message="Working on %s" % ctx.label.name,
+ mnemonic="Work",
+ arguments=ctx.attr.worker_args + ["@" + argfile.path],
+ )
+
+work = rule(
+ implementation=_impl,
+ attrs={
+ "worker": attr.label(cfg=HOST_CFG, mandatory=True, allow_files=True, executable=True),
+ "worker_args": attr.string_list(),
+ "args": attr.string_list(),
+ },
+ outputs = {"out": "%{name}.out"},
+)
+EOF
+ cat >BUILD <<EOF
+load("work", "work")
+
+java_import(
+ name = "worker_lib",
+ jars = ["worker_lib.jar"],
+)
+
+java_binary(
+ name = "worker",
+ main_class = "com.google.devtools.build.lib.worker.ExampleWorker",
+ runtime_deps = [
+ ":worker_lib",
+ ],
+)
+
+EOF
+}
+
+function test_example_worker() {
+ prepare_example_worker
+
+ cat >>BUILD <<EOF
+work(
+ name = "hello_world",
+ worker = ":worker",
+ args = ["hello world"],
+)
+
+work(
+ name = "hello_world_uppercase",
+ worker = ":worker",
+ args = ["--uppercase", "hello world"],
+)
+EOF
+
+ bazel --batch clean
+ assert_workers_not_running
+
+ bazel build --strategy=Work=worker :hello_world \
+ || fail "build failed"
+ assert_equals "hello world" "$(cat bazel-bin/hello_world.out)"
+ assert_workers_running
+
+ bazel build --worker_quit_after_build --strategy=Work=worker :hello_world_uppercase \
+ || fail "build failed"
+ assert_equals "HELLO WORLD" "$(cat bazel-bin/hello_world_uppercase.out)"
+ assert_workers_not_running
+}
+
run_suite "Worker integration tests"
