| // Copyright 2020 The Bazel Authors. All rights reserved. |
| // |
| // Licensed under the Apache License, Version 2.0 (the "License"); |
| // you may not use this file except in compliance with the License. |
| // You may obtain a copy of the License at |
| // |
| // http://www.apache.org/licenses/LICENSE-2.0 |
| // |
| // Unless required by applicable law or agreed to in writing, software |
| // distributed under the License is distributed on an "AS IS" BASIS, |
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| // See the License for the specific language governing permissions and |
| // limitations under the License. |
| |
| package com.google.devtools.build.lib.worker; |
| |
| import static com.google.common.collect.ImmutableSet.toImmutableSet; |
| import static com.google.common.truth.Truth.assertThat; |
| import static org.mockito.ArgumentMatchers.any; |
| import static org.mockito.Mockito.doReturn; |
| import static org.mockito.Mockito.spy; |
| |
| import com.google.common.collect.ImmutableList; |
| import com.google.common.collect.ImmutableMap; |
| import com.google.devtools.build.lib.buildeventstream.BuildEventStreamProtos.BuildMetrics.WorkerMetrics; |
| import com.google.devtools.build.lib.buildeventstream.BuildEventStreamProtos.BuildMetrics.WorkerMetrics.WorkerStatus; |
| import com.google.devtools.build.lib.clock.Clock; |
| import com.google.devtools.build.lib.metrics.PsInfoCollector; |
| import com.google.devtools.build.lib.worker.WorkerProcessMetricsCollector.WorkerMetricsPublishComparator; |
| import com.google.devtools.build.lib.worker.WorkerProcessStatus.Status; |
| import java.time.Instant; |
| import org.junit.Before; |
| import org.junit.Test; |
| import org.junit.runner.RunWith; |
| import org.junit.runners.JUnit4; |
| |
| /** Unit tests for the WorkerSpawnRunner. */ |
| @RunWith(JUnit4.class) |
| public class WorkerProcessMetricsCollectorTest { |
| |
| private final WorkerProcessMetricsCollector spyCollector = |
| spy(WorkerProcessMetricsCollector.instance()); |
| ManualClock clock = new ManualClock(); |
| |
| @Before |
| public void setUp() { |
| spyCollector.clear(); |
| spyCollector.setClock(clock); |
| } |
| |
| private static final int WORKER_ID_1 = 1; |
| private static final int WORKER_ID_2 = 2; |
| private static final int WORKER_ID_3 = 3; |
| private static final int WORKER_ID_4 = 4; |
| private static final int WORKER_ID_5 = 5; |
| private static final long PROCESS_ID_1 = 100L; |
| private static final long PROCESS_ID_2 = 200L; |
| private static final long PROCESS_ID_3 = 300L; |
| private static final long PROCESS_ID_4 = 400L; |
| private static final long PROCESS_ID_5 = 500L; |
| private static final int WORKER_KEY_HASH_1 = 1; |
| private static final int WORKER_KEY_HASH_2 = 2; |
| private static final int WORKER_KEY_HASH_3 = 3; |
| private static final int WORKER_KEY_HASH_4 = 4; |
| private static final int WORKER_KEY_HASH_5 = 5; |
| private static final String JAVAC_MNEMONIC = "Javac"; |
| private static final String CPP_COMPILE_MNEMONIC = "CppCompile"; |
| private static final String PROTO_MNEMONIC = "Proto"; |
| |
| private void assertWorkerMetricContains( |
| WorkerProcessMetrics workerMetric, |
| ImmutableList<Integer> expectedWorkerIds, |
| Long expectedProcessId, |
| String expectedMnemonic, |
| boolean expectedIsMultiplex, |
| boolean expectedIsSandboxed, |
| int expectedWorkerKeyHash, |
| int expectedActionsExecuted, |
| boolean expectedIsMeasurable, |
| Instant expectedLastCallTime, |
| Instant expectedCollectedTime) { |
| assertThat(workerMetric).isNotNull(); |
| assertThat(workerMetric.getWorkerIds()).containsExactlyElementsIn(expectedWorkerIds); |
| assertThat(workerMetric.getProcessId()).isEqualTo(expectedProcessId); |
| assertThat(workerMetric.getMnemonic()).isEqualTo(expectedMnemonic); |
| assertThat(workerMetric.isMultiplex()).isEqualTo(expectedIsMultiplex); |
| assertThat(workerMetric.isSandboxed()).isEqualTo(expectedIsSandboxed); |
| assertThat(workerMetric.getWorkerKeyHash()).isEqualTo(expectedWorkerKeyHash); |
| assertThat(workerMetric.getActionsExecuted()).isEqualTo(expectedActionsExecuted); |
| assertThat(workerMetric.isMeasurable()).isEqualTo(expectedIsMeasurable); |
| assertThat(workerMetric.getLastCallTime().get()).isEqualTo(expectedLastCallTime); |
| if (expectedCollectedTime == null) { |
| assertThat(workerMetric.getLastCollectedTime().isEmpty()).isTrue(); |
| } else { |
| assertThat(workerMetric.getLastCollectedTime().isPresent()).isTrue(); |
| assertThat(workerMetric.getLastCollectedTime().get()).isEqualTo(expectedCollectedTime); |
| } |
| } |
| |
| @Test |
| public void testRegisterWorker_insertDifferent() throws Exception { |
| spyCollector.registerWorker( |
| WORKER_ID_1, |
| PROCESS_ID_1, |
| new WorkerProcessStatus(), |
| JAVAC_MNEMONIC, |
| /* isMultiplex= */ true, |
| /* isSandboxed= */ false, |
| WORKER_KEY_HASH_1); |
| assertThat(spyCollector.getProcessIdToWorkerProcessMetrics().keySet()) |
| .containsExactly(PROCESS_ID_1); |
| spyCollector.registerWorker( |
| WORKER_ID_2, |
| PROCESS_ID_2, |
| new WorkerProcessStatus(), |
| CPP_COMPILE_MNEMONIC, |
| /* isMultiplex= */ false, |
| /* isSandboxed= */ true, |
| WORKER_KEY_HASH_2); |
| assertThat(spyCollector.getProcessIdToWorkerProcessMetrics().keySet()) |
| .containsExactly(PROCESS_ID_1, PROCESS_ID_2); |
| assertWorkerMetricContains( |
| spyCollector.getProcessIdToWorkerProcessMetrics().get(PROCESS_ID_1), |
| ImmutableList.of(WORKER_ID_1), |
| PROCESS_ID_1, |
| JAVAC_MNEMONIC, |
| /* expectedIsMultiplex= */ true, |
| /* expectedIsSandboxed= */ false, |
| WORKER_KEY_HASH_1, |
| /* expectedActionsExecuted= */ 0, |
| /* expectedIsMeasurable= */ false, |
| /* expectedLastCallTime= */ DEFAULT_CLOCK_START_INSTANT, |
| /* expectedCollectedTime= */ null); |
| assertWorkerMetricContains( |
| spyCollector.getProcessIdToWorkerProcessMetrics().get(PROCESS_ID_2), |
| ImmutableList.of(WORKER_ID_2), |
| PROCESS_ID_2, |
| CPP_COMPILE_MNEMONIC, |
| /* expectedIsMultiplex= */ false, |
| /* expectedIsSandboxed= */ true, |
| WORKER_KEY_HASH_2, |
| /* expectedActionsExecuted= */ 0, |
| /* expectedIsMeasurable= */ false, |
| /* expectedLastCallTime= */ DEFAULT_CLOCK_START_INSTANT, |
| /* expectedCollectedTime= */ null); |
| } |
| |
| @Test |
| public void testRegisterWorker_insertMultiplex() throws Exception { |
| spyCollector.registerWorker( |
| WORKER_ID_1, |
| PROCESS_ID_1, |
| new WorkerProcessStatus(), |
| JAVAC_MNEMONIC, |
| /* isMultiplex= */ true, |
| /* isSandboxed= */ true, |
| WORKER_KEY_HASH_1); |
| assertThat(spyCollector.getProcessIdToWorkerProcessMetrics().keySet()) |
| .containsExactly(PROCESS_ID_1); |
| assertWorkerMetricContains( |
| spyCollector.getProcessIdToWorkerProcessMetrics().get(PROCESS_ID_1), |
| ImmutableList.of(WORKER_ID_1), |
| PROCESS_ID_1, |
| JAVAC_MNEMONIC, |
| /* expectedIsMultiplex= */ true, |
| /* expectedIsSandboxed= */ true, |
| WORKER_KEY_HASH_1, |
| /* expectedActionsExecuted= */ 0, |
| /* expectedIsMeasurable= */ false, |
| /* expectedLastCallTime= */ DEFAULT_CLOCK_START_INSTANT, |
| /* expectedCollectedTime= */ null); |
| |
| Instant secondTime = DEFAULT_CLOCK_START_INSTANT.plusSeconds(10); |
| clock.setTime(secondTime.toEpochMilli()); |
| |
| spyCollector.registerWorker( |
| WORKER_ID_2, |
| PROCESS_ID_1, |
| new WorkerProcessStatus(), |
| JAVAC_MNEMONIC, |
| /* isMultiplex= */ true, |
| /* isSandboxed= */ true, |
| WORKER_KEY_HASH_1); |
| assertThat(spyCollector.getProcessIdToWorkerProcessMetrics().keySet()) |
| .containsExactly(PROCESS_ID_1); |
| assertWorkerMetricContains( |
| spyCollector.getProcessIdToWorkerProcessMetrics().get(PROCESS_ID_1), |
| ImmutableList.of(WORKER_ID_1, WORKER_ID_2), |
| PROCESS_ID_1, |
| JAVAC_MNEMONIC, |
| /* expectedIsMultiplex= */ true, |
| /* expectedIsSandboxed= */ true, |
| WORKER_KEY_HASH_1, |
| /* expectedActionsExecuted= */ 0, |
| /* expectedIsMeasurable= */ false, |
| /* expectedLastCallTime= */ secondTime, |
| /* expectedCollectedTime= */ null); |
| } |
| |
| @Test |
| public void testRegisterWorker_insertSame() throws Exception { |
| spyCollector.registerWorker( |
| WORKER_ID_1, |
| PROCESS_ID_1, |
| new WorkerProcessStatus(), |
| JAVAC_MNEMONIC, |
| /* isMultiplex= */ true, |
| /* isSandboxed= */ true, |
| WORKER_KEY_HASH_1); |
| assertThat(spyCollector.getProcessIdToWorkerProcessMetrics().keySet()) |
| .containsExactly(PROCESS_ID_1); |
| assertWorkerMetricContains( |
| spyCollector.getProcessIdToWorkerProcessMetrics().get(PROCESS_ID_1), |
| ImmutableList.of(WORKER_ID_1), |
| PROCESS_ID_1, |
| JAVAC_MNEMONIC, |
| /* expectedIsMultiplex= */ true, |
| /* expectedIsSandboxed= */ true, |
| WORKER_KEY_HASH_1, |
| /* expectedActionsExecuted= */ 0, |
| /* expectedIsMeasurable= */ false, |
| /* expectedLastCallTime= */ DEFAULT_CLOCK_START_INSTANT, |
| /* expectedCollectedTime= */ null); |
| |
| Instant secondTime = DEFAULT_CLOCK_START_INSTANT.plusSeconds(10); |
| clock.setTime(secondTime.toEpochMilli()); |
| |
| // When it is the same worker, it should only update the last call time. |
| spyCollector.registerWorker( |
| WORKER_ID_1, |
| PROCESS_ID_1, |
| new WorkerProcessStatus(), |
| JAVAC_MNEMONIC, |
| /* isMultiplex= */ true, |
| /* isSandboxed= */ true, |
| WORKER_KEY_HASH_1); |
| assertThat(spyCollector.getProcessIdToWorkerProcessMetrics().keySet()) |
| .containsExactly(PROCESS_ID_1); |
| assertWorkerMetricContains( |
| spyCollector.getProcessIdToWorkerProcessMetrics().get(PROCESS_ID_1), |
| ImmutableList.of(WORKER_ID_1), |
| PROCESS_ID_1, |
| JAVAC_MNEMONIC, |
| /* expectedIsMultiplex= */ true, |
| /* expectedIsSandboxed= */ true, |
| WORKER_KEY_HASH_1, |
| /* expectedActionsExecuted= */ 0, |
| /* expectedIsMeasurable= */ false, |
| /* expectedLastCallTime= */ secondTime, |
| /* expectedCollectedTime= */ null); |
| } |
| |
| @Test |
| public void testCollectMetrics() throws Exception { |
| // Worker 1 simulates a measurable worker processes has executed some actions. |
| spyCollector.registerWorker( |
| WORKER_ID_1, |
| PROCESS_ID_1, |
| new WorkerProcessStatus(), |
| JAVAC_MNEMONIC, |
| /* isMultiplex= */ true, |
| /* isSandboxed= */ false, |
| WORKER_KEY_HASH_1); |
| WorkerProcessMetricsCollector.instance().onWorkerFinishExecution(PROCESS_ID_1); |
| // Worker 2 simulates a measurable worker process that has not yet completed execution of any |
| // actions. |
| spyCollector.registerWorker( |
| WORKER_ID_2, |
| PROCESS_ID_2, |
| new WorkerProcessStatus(), |
| CPP_COMPILE_MNEMONIC, |
| /* isMultiplex= */ false, |
| /* isSandboxed= */ true, |
| WORKER_KEY_HASH_2); |
| // Worker 3 simulates a non-measurable worker that has not executed any actions. |
| spyCollector.registerWorker( |
| WORKER_ID_3, |
| PROCESS_ID_3, |
| new WorkerProcessStatus(), |
| PROTO_MNEMONIC, |
| /* isMultiplex= */ true, |
| /* isSandboxed= */ true, |
| WORKER_KEY_HASH_3); |
| // Worker 4 simulates a non-measurable worker that has executed an action and was killed. |
| WorkerProcessStatus s4 = new WorkerProcessStatus(); |
| spyCollector.registerWorker( |
| WORKER_ID_4, |
| PROCESS_ID_4, |
| /* status= */ s4, |
| PROTO_MNEMONIC, |
| /* isMultiplex= */ true, |
| /* isSandboxed= */ true, |
| WORKER_KEY_HASH_4); |
| WorkerProcessMetricsCollector.instance().onWorkerFinishExecution(PROCESS_ID_4); |
| s4.maybeUpdateStatus(Status.KILLED_DUE_TO_MEMORY_PRESSURE); |
| // Worker 5 simulates a non-measurable worker that has executed an action, but was not killed. |
| WorkerProcessStatus s5 = new WorkerProcessStatus(); |
| spyCollector.registerWorker( |
| WORKER_ID_5, |
| PROCESS_ID_5, |
| /* status= */ s5, |
| PROTO_MNEMONIC, |
| /* isMultiplex= */ true, |
| /* isSandboxed= */ true, |
| WORKER_KEY_HASH_5); |
| WorkerProcessMetricsCollector.instance().onWorkerFinishExecution(PROCESS_ID_5); |
| |
| ImmutableMap<Long, Integer> memoryUsageMap = |
| ImmutableMap.of( |
| PROCESS_ID_1, 1234, |
| PROCESS_ID_2, 2345); |
| Instant collectionTime = DEFAULT_CLOCK_START_INSTANT.plusSeconds(10); |
| PsInfoCollector.ResourceSnapshot resourceSnapshot = |
| PsInfoCollector.ResourceSnapshot.create(memoryUsageMap, collectionTime); |
| doReturn(resourceSnapshot).when(spyCollector).collectMemoryUsageByPid(any(), any()); |
| clock.setTime(collectionTime.toEpochMilli()); |
| |
| ImmutableList<WorkerProcessMetrics> metrics = spyCollector.collectMetrics(); |
| |
| // All workers measurable or non-measurable should be reported. |
| assertThat(metrics.stream().flatMap(m -> m.getWorkerIds().stream()).collect(toImmutableSet())) |
| .containsExactly(WORKER_ID_1, WORKER_ID_2, WORKER_ID_3, WORKER_ID_4, WORKER_ID_5); |
| assertWorkerMetricContains( |
| metrics.stream().filter(wm -> wm.getWorkerIds().contains(WORKER_ID_1)).findFirst().get(), |
| ImmutableList.of(WORKER_ID_1), |
| PROCESS_ID_1, |
| JAVAC_MNEMONIC, |
| /* expectedIsMultiplex= */ true, |
| /* expectedIsSandboxed= */ false, |
| WORKER_KEY_HASH_1, |
| /* expectedActionsExecuted= */ 1, |
| /* expectedIsMeasurable= */ true, |
| /* expectedLastCallTime= */ DEFAULT_CLOCK_START_INSTANT, |
| /* expectedCollectedTime= */ collectionTime); |
| assertWorkerMetricContains( |
| metrics.stream().filter(wm -> wm.getWorkerIds().contains(WORKER_ID_2)).findFirst().get(), |
| ImmutableList.of(WORKER_ID_2), |
| PROCESS_ID_2, |
| CPP_COMPILE_MNEMONIC, |
| /* expectedIsMultiplex= */ false, |
| /* expectedIsSandboxed= */ true, |
| WORKER_KEY_HASH_2, |
| /* expectedActionsExecuted= */ 0, |
| /* expectedIsMeasurable= */ true, |
| /* expectedLastCallTime= */ DEFAULT_CLOCK_START_INSTANT, |
| /* expectedCollectedTime= */ collectionTime); |
| // Worker 3's metrics should not be included since it is both non-measurable and did not execute |
| // any actions. It's status shouldn't be unknown because it is possible that |
| assertWorkerMetricContains( |
| metrics.stream().filter(wm -> wm.getWorkerIds().contains(WORKER_ID_3)).findFirst().get(), |
| ImmutableList.of(WORKER_ID_3), |
| PROCESS_ID_3, |
| PROTO_MNEMONIC, |
| /* expectedIsMultiplex= */ true, |
| /* expectedIsSandboxed= */ true, |
| WORKER_KEY_HASH_3, |
| /* expectedActionsExecuted= */ 0, |
| /* expectedIsMeasurable= */ false, |
| /* expectedLastCallTime= */ DEFAULT_CLOCK_START_INSTANT, |
| /* expectedCollectedTime= */ null); |
| assertWorkerMetricContains( |
| metrics.stream().filter(wm -> wm.getWorkerIds().contains(WORKER_ID_4)).findFirst().get(), |
| ImmutableList.of(WORKER_ID_4), |
| PROCESS_ID_4, |
| PROTO_MNEMONIC, |
| /* expectedIsMultiplex= */ true, |
| /* expectedIsSandboxed= */ true, |
| WORKER_KEY_HASH_4, |
| /* expectedActionsExecuted= */ 1, |
| /* expectedIsMeasurable= */ false, |
| /* expectedLastCallTime= */ DEFAULT_CLOCK_START_INSTANT, |
| /* expectedCollectedTime= */ null); |
| assertWorkerMetricContains( |
| metrics.stream().filter(wm -> wm.getWorkerIds().contains(WORKER_ID_5)).findFirst().get(), |
| ImmutableList.of(WORKER_ID_5), |
| PROCESS_ID_5, |
| PROTO_MNEMONIC, |
| /* expectedIsMultiplex= */ true, |
| /* expectedIsSandboxed= */ true, |
| WORKER_KEY_HASH_5, |
| /* expectedActionsExecuted= */ 1, |
| /* expectedIsMeasurable= */ false, |
| /* expectedLastCallTime= */ DEFAULT_CLOCK_START_INSTANT, |
| /* expectedCollectedTime= */ null); |
| // Worker 5's status should have been updated to killed_unknown, because it had executed actions |
| // but is now non-measurable. |
| assertThat(s5.get()).isEqualTo(Status.KILLED_UNKNOWN); |
| } |
| |
| @Test |
| public void testWorkerMetricsPublishComparator_compare() { |
| WorkerMetrics alive1 = newWorkerMetrics(1, WorkerStatus.ALIVE, 100); |
| WorkerMetrics alive2 = newWorkerMetrics(2, WorkerStatus.ALIVE, 200); |
| WorkerMetrics evicted1 = newWorkerMetrics(3, WorkerStatus.KILLED_DUE_TO_MEMORY_PRESSURE, 100); |
| WorkerMetrics evicted2 = newWorkerMetrics(4, WorkerStatus.KILLED_DUE_TO_MEMORY_PRESSURE, 200); |
| WorkerMetrics others1 = newWorkerMetrics(5, WorkerStatus.KILLED_UNKNOWN, 100); |
| WorkerMetrics others2 = |
| newWorkerMetrics(6, WorkerStatus.KILLED_DUE_TO_USER_EXEC_EXCEPTION, 200); |
| |
| WorkerMetricsPublishComparator comparator = new WorkerMetricsPublishComparator(); |
| // WorkerMetrics of the same status priority should be compared by their memory usage (higher |
| // gets prioritized). |
| assertThat(comparator.compare(alive1, alive2)).isEqualTo(1); |
| assertThat(comparator.compare(evicted1, evicted2)).isEqualTo(1); |
| assertThat(comparator.compare(others1, others2)).isEqualTo(1); |
| |
| // WorkerMetrics should be first compared by their status priorities rather than their memory |
| // usage. |
| assertThat(comparator.compare(alive1, evicted2)).isEqualTo(-1); |
| assertThat(comparator.compare(evicted1, others2)).isEqualTo(-1); |
| assertThat(comparator.compare(others2, alive1)).isEqualTo(1); |
| } |
| |
| @Test |
| public void testLimitWorkerMetricsToPublish() { |
| WorkerMetrics alive1 = newWorkerMetrics(1, WorkerStatus.ALIVE, 200); |
| WorkerMetrics alive2 = newWorkerMetrics(2, WorkerStatus.ALIVE, 100); |
| WorkerMetrics evicted3 = newWorkerMetrics(3, WorkerStatus.KILLED_DUE_TO_MEMORY_PRESSURE, 100); |
| WorkerMetrics evicted4 = newWorkerMetrics(4, WorkerStatus.KILLED_DUE_TO_MEMORY_PRESSURE, 200); |
| WorkerMetrics others5 = newWorkerMetrics(5, WorkerStatus.KILLED_UNKNOWN, 200); |
| WorkerMetrics others6 = |
| newWorkerMetrics(6, WorkerStatus.KILLED_DUE_TO_USER_EXEC_EXCEPTION, 100); |
| |
| // Based on prioritization and then sorted by worker id. |
| assertThat( |
| WorkerProcessMetricsCollector.limitWorkerMetricsToPublish( |
| ImmutableList.of(alive1, alive2, evicted3, evicted4, others5, others6), 3)) |
| .containsExactly(alive1, alive2, evicted4); |
| assertThat( |
| WorkerProcessMetricsCollector.limitWorkerMetricsToPublish( |
| ImmutableList.of(alive1, evicted4, others5, others6), 3)) |
| .containsExactly(alive1, evicted4, others5); |
| // If under the limit, it should just report everything. |
| assertThat( |
| WorkerProcessMetricsCollector.limitWorkerMetricsToPublish( |
| ImmutableList.of(alive1, alive2, evicted4, others6), 10)) |
| .containsExactly(alive1, alive2, evicted4, others6); |
| } |
| |
| private WorkerMetrics newWorkerMetrics(int id, WorkerStatus status, int memoryInKb) { |
| return WorkerMetrics.newBuilder() |
| .addWorkerIds(id) |
| .setWorkerStatus(status) |
| .addWorkerStats( |
| WorkerMetrics.WorkerStats.newBuilder().setWorkerMemoryInKb(memoryInKb).build()) |
| .build(); |
| } |
| |
| private static final long DEFAULT_CLOCK_START_TIME = 0L; |
| private static final Instant DEFAULT_CLOCK_START_INSTANT = |
| Instant.ofEpochMilli(DEFAULT_CLOCK_START_TIME); |
| |
| private static class ManualClock implements Clock { |
| private long currentTime = DEFAULT_CLOCK_START_TIME; |
| |
| ManualClock() {} |
| |
| @Override |
| public long nanoTime() { |
| throw new AssertionError("unexpected method call"); |
| } |
| |
| @Override |
| public long currentTimeMillis() { |
| return currentTime; |
| } |
| |
| void setTime(long currentTime) { |
| this.currentTime = currentTime; |
| } |
| } |
| } |