| // Copyright 2022 The Bazel Authors. All rights reserved. |
| // |
| // Licensed under the Apache License, Version 2.0 (the "License"); |
| // you may not use this file except in compliance with the License. |
| // You may obtain a copy of the License at |
| // |
| // http://www.apache.org/licenses/LICENSE-2.0 |
| // |
| // Unless required by applicable law or agreed to in writing, software |
| // distributed under the License is distributed on an "AS IS" BASIS, |
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| // See the License for the specific language governing permissions and |
| // limitations under the License. |
| package com.google.devtools.build.lib.worker; |
| |
| import static com.google.common.truth.Truth.assertThat; |
| import static com.google.devtools.build.lib.worker.TestUtils.createWorkerKey; |
| import static org.mockito.ArgumentMatchers.any; |
| import static org.mockito.Mockito.doAnswer; |
| import static org.mockito.Mockito.when; |
| |
| import com.google.common.collect.ImmutableList; |
| import com.google.common.collect.Maps; |
| import com.google.common.collect.Sets; |
| import com.google.common.eventbus.EventBus; |
| import com.google.common.eventbus.Subscribe; |
| import com.google.devtools.build.lib.clock.BlazeClock; |
| import com.google.devtools.build.lib.vfs.DigestHashFunction; |
| import com.google.devtools.build.lib.vfs.FileSystem; |
| import com.google.devtools.build.lib.vfs.inmemoryfs.InMemoryFileSystem; |
| import com.google.devtools.build.lib.worker.WorkerPoolImpl.WorkerPoolConfig; |
| import java.time.Instant; |
| import java.util.ArrayList; |
| import java.util.List; |
| import java.util.Map.Entry; |
| import org.apache.commons.pool2.impl.DefaultPooledObject; |
| import org.junit.Before; |
| import org.junit.Rule; |
| import org.junit.Test; |
| import org.junit.runner.RunWith; |
| import org.junit.runners.JUnit4; |
| import org.mockito.Mock; |
| import org.mockito.junit.MockitoJUnit; |
| import org.mockito.junit.MockitoRule; |
| |
| @RunWith(JUnit4.class) |
| public final class WorkerLifecycleManagerTest { |
| |
| @Rule public final MockitoRule mockito = MockitoJUnit.rule(); |
| private final EventBus eventBus = new EventBus(); |
| |
| @Mock WorkerFactory factoryMock; |
| private FileSystem fileSystem; |
| private int workerIds = 1; |
| private EventRecorder eventRecorder = new EventRecorder(); |
| |
| private static class EventRecorder { |
| public List<WorkerEvictedEvent> workerEvictedEvents; |
| |
| public EventRecorder() { |
| workerEvictedEvents = new ArrayList<>(); |
| } |
| |
| @Subscribe |
| public synchronized void setXcodeConfigInfo(WorkerEvictedEvent workerEvictedEvent) { |
| this.workerEvictedEvents.add(workerEvictedEvent); |
| } |
| } |
| |
| @Before |
| public void setUp() throws Exception { |
| fileSystem = new InMemoryFileSystem(BlazeClock.instance(), DigestHashFunction.SHA256); |
| doAnswer( |
| arg -> |
| new DefaultPooledObject<>( |
| new SingleplexWorker( |
| arg.getArgument(0), |
| workerIds++, |
| fileSystem.getPath("/workDir"), |
| fileSystem.getPath("/logDir")))) |
| .when(factoryMock) |
| .makeObject(any()); |
| when(factoryMock.validateObject(any(), any())).thenReturn(true); |
| eventRecorder = new EventRecorder(); |
| eventBus.register(eventRecorder); |
| } |
| |
| @Test |
| public void testEvictWorkers_doNothing_lowMemoryUsage() throws Exception { |
| WorkerPoolImpl workerPool = |
| new WorkerPoolImpl( |
| new WorkerPoolConfig(factoryMock, entryList("dummy", 1), emptyEntryList())); |
| WorkerKey key = createWorkerKey("dummy", fileSystem); |
| Worker w1 = workerPool.borrowObject(key); |
| workerPool.returnObject(key, w1); |
| |
| ImmutableList<WorkerMetric> workerMetrics = |
| ImmutableList.of( |
| WorkerMetric.create( |
| createWorkerProperties(w1.getWorkerId(), 1L, "dummy"), |
| createWorkerStat(1024), |
| true)); |
| WorkerOptions options = new WorkerOptions(); |
| options.totalWorkerMemoryLimitMb = 1024 * 100; |
| |
| WorkerLifecycleManager manager = new WorkerLifecycleManager(workerPool, options); |
| manager.setEventBus(eventBus); |
| |
| assertThat(workerPool.getNumIdlePerKey(key)).isEqualTo(1); |
| assertThat(workerPool.getNumActive(key)).isEqualTo(0); |
| |
| manager.evictWorkers(workerMetrics); |
| |
| assertThat(workerPool.getNumIdlePerKey(key)).isEqualTo(1); |
| assertThat(workerPool.getNumActive(key)).isEqualTo(0); |
| assertThat(eventRecorder.workerEvictedEvents).isEmpty(); |
| } |
| |
| @Test |
| public void testEvictWorkers_doNothing_zeroThreshold() throws Exception { |
| WorkerPoolImpl workerPool = |
| new WorkerPoolImpl( |
| new WorkerPoolConfig(factoryMock, entryList("dummy", 1), emptyEntryList())); |
| WorkerKey key = createWorkerKey("dummy", fileSystem); |
| Worker w1 = workerPool.borrowObject(key); |
| workerPool.returnObject(key, w1); |
| |
| ImmutableList<WorkerMetric> workerMetrics = |
| ImmutableList.of( |
| WorkerMetric.create( |
| createWorkerProperties(w1.getWorkerId(), 1L, "dummy"), |
| createWorkerStat(1024), |
| true)); |
| WorkerOptions options = new WorkerOptions(); |
| options.totalWorkerMemoryLimitMb = 0; |
| |
| WorkerLifecycleManager manager = new WorkerLifecycleManager(workerPool, options); |
| |
| assertThat(workerPool.getNumIdlePerKey(key)).isEqualTo(1); |
| assertThat(workerPool.getNumActive(key)).isEqualTo(0); |
| |
| manager.evictWorkers(workerMetrics); |
| |
| assertThat(workerPool.getNumIdlePerKey(key)).isEqualTo(1); |
| assertThat(workerPool.getNumActive(key)).isEqualTo(0); |
| } |
| |
| @Test |
| public void testEvictWorkers_doNothing_emptyMetrics() throws Exception { |
| WorkerPoolImpl workerPool = |
| new WorkerPoolImpl( |
| new WorkerPoolConfig(factoryMock, entryList("dummy", 1), emptyEntryList())); |
| WorkerKey key = createWorkerKey("dummy", fileSystem); |
| Worker w1 = workerPool.borrowObject(key); |
| workerPool.returnObject(key, w1); |
| |
| ImmutableList<WorkerMetric> workerMetrics = ImmutableList.of(); |
| WorkerOptions options = new WorkerOptions(); |
| options.totalWorkerMemoryLimitMb = 1; |
| |
| WorkerLifecycleManager manager = new WorkerLifecycleManager(workerPool, options); |
| |
| assertThat(workerPool.getNumIdlePerKey(key)).isEqualTo(1); |
| assertThat(workerPool.getNumActive(key)).isEqualTo(0); |
| |
| manager.evictWorkers(workerMetrics); |
| |
| assertThat(workerPool.getNumIdlePerKey(key)).isEqualTo(1); |
| assertThat(workerPool.getNumActive(key)).isEqualTo(0); |
| } |
| |
| @Test |
| public void testGetEvictionCandidates_selectOnlyWorker() throws Exception { |
| WorkerPoolImpl workerPool = |
| new WorkerPoolImpl( |
| new WorkerPoolConfig(factoryMock, entryList("dummy", 1), emptyEntryList())); |
| WorkerKey key = createWorkerKey("dummy", fileSystem); |
| Worker w1 = workerPool.borrowObject(key); |
| workerPool.returnObject(key, w1); |
| ImmutableList<WorkerMetric> workerMetrics = |
| ImmutableList.of( |
| WorkerMetric.create( |
| createWorkerProperties(w1.getWorkerId(), 1L, "dummy", w1.getWorkerKey().hashCode()), |
| createWorkerStat(2000), |
| true)); |
| WorkerOptions options = new WorkerOptions(); |
| options.totalWorkerMemoryLimitMb = 1; |
| WorkerLifecycleManager manager = new WorkerLifecycleManager(workerPool, options); |
| manager.setEventBus(eventBus); |
| WorkerEvictedEvent event = |
| new WorkerEvictedEvent(w1.getWorkerKey().hashCode(), w1.getWorkerKey().getMnemonic()); |
| |
| assertThat(workerPool.getNumIdlePerKey(key)).isEqualTo(1); |
| assertThat(workerPool.getNumActive(key)).isEqualTo(0); |
| |
| manager.evictWorkers(workerMetrics); |
| |
| assertThat(workerPool.getNumIdlePerKey(key)).isEqualTo(0); |
| assertThat(workerPool.getNumActive(key)).isEqualTo(0); |
| assertThat(eventRecorder.workerEvictedEvents).containsExactly(event); |
| } |
| |
| @Test |
| public void testGetEvictionCandidates_evictLargestWorkers() throws Exception { |
| WorkerPoolImpl workerPool = |
| new WorkerPoolImpl( |
| new WorkerPoolConfig(factoryMock, entryList("dummy", 3), emptyEntryList())); |
| WorkerKey key = createWorkerKey("dummy", fileSystem); |
| Worker w1 = workerPool.borrowObject(key); |
| Worker w2 = workerPool.borrowObject(key); |
| Worker w3 = workerPool.borrowObject(key); |
| workerPool.returnObject(key, w1); |
| workerPool.returnObject(key, w2); |
| workerPool.returnObject(key, w3); |
| |
| ImmutableList<WorkerMetric> workerMetrics = |
| ImmutableList.of( |
| WorkerMetric.create( |
| createWorkerProperties(w1.getWorkerId(), 1L, "dummy"), |
| createWorkerStat(2000), |
| true), |
| WorkerMetric.create( |
| createWorkerProperties(w2.getWorkerId(), 2L, "dummy"), |
| createWorkerStat(1000), |
| true), |
| WorkerMetric.create( |
| createWorkerProperties(w3.getWorkerId(), 3L, "dummy"), |
| createWorkerStat(4000), |
| true)); |
| |
| WorkerOptions options = new WorkerOptions(); |
| options.totalWorkerMemoryLimitMb = 2; |
| WorkerLifecycleManager manager = new WorkerLifecycleManager(workerPool, options); |
| |
| assertThat(workerPool.getNumIdlePerKey(key)).isEqualTo(3); |
| assertThat(workerPool.getNumActive(key)).isEqualTo(0); |
| |
| manager.evictWorkers(workerMetrics); |
| |
| assertThat(workerPool.getNumIdlePerKey(key)).isEqualTo(1); |
| assertThat(workerPool.getNumActive(key)).isEqualTo(0); |
| assertThat(workerPool.borrowObject(key).getWorkerId()).isEqualTo(w2.getWorkerId()); |
| } |
| |
| @Test |
| public void testGetEvictionCandidates_numberOfWorkersIsMoreThanDeafaultNumTests() |
| throws Exception { |
| WorkerPoolImpl workerPool = |
| new WorkerPoolImpl( |
| new WorkerPoolConfig(factoryMock, entryList("dummy", 4), emptyEntryList())); |
| WorkerKey key = createWorkerKey("dummy", fileSystem); |
| Worker w1 = workerPool.borrowObject(key); |
| Worker w2 = workerPool.borrowObject(key); |
| Worker w3 = workerPool.borrowObject(key); |
| Worker w4 = workerPool.borrowObject(key); |
| workerPool.returnObject(key, w1); |
| workerPool.returnObject(key, w2); |
| workerPool.returnObject(key, w3); |
| workerPool.returnObject(key, w4); |
| |
| ImmutableList<WorkerMetric> workerMetrics = |
| ImmutableList.of( |
| WorkerMetric.create( |
| createWorkerProperties(w1.getWorkerId(), 1L, "dummy"), |
| createWorkerStat(2000), |
| true), |
| WorkerMetric.create( |
| createWorkerProperties(w2.getWorkerId(), 2L, "dummy"), |
| createWorkerStat(2000), |
| true), |
| WorkerMetric.create( |
| createWorkerProperties(w3.getWorkerId(), 3L, "dummy"), |
| createWorkerStat(4000), |
| true), |
| WorkerMetric.create( |
| createWorkerProperties(w4.getWorkerId(), 4L, "dummy"), |
| createWorkerStat(4000), |
| true)); |
| |
| WorkerOptions options = new WorkerOptions(); |
| options.totalWorkerMemoryLimitMb = 1; |
| WorkerLifecycleManager manager = new WorkerLifecycleManager(workerPool, options); |
| |
| assertThat(workerPool.getNumIdlePerKey(key)).isEqualTo(4); |
| assertThat(workerPool.getNumActive(key)).isEqualTo(0); |
| |
| manager.evictWorkers(workerMetrics); |
| |
| assertThat(workerPool.getNumIdlePerKey(key)).isEqualTo(0); |
| assertThat(workerPool.getNumActive(key)).isEqualTo(0); |
| } |
| |
| @Test |
| public void testGetEvictionCandidates_evictWorkerWithSameMenmonicButDifferentKeys() |
| throws Exception { |
| WorkerPoolImpl workerPool = |
| new WorkerPoolImpl( |
| new WorkerPoolConfig(factoryMock, entryList("dummy", 3), emptyEntryList())); |
| WorkerKey key1 = createWorkerKey("dummy", fileSystem); |
| WorkerKey key2 = createWorkerKey("dummy", fileSystem, true); |
| |
| Worker w1 = workerPool.borrowObject(key1); |
| Worker w2 = workerPool.borrowObject(key2); |
| Worker w3 = workerPool.borrowObject(key2); |
| workerPool.returnObject(key1, w1); |
| workerPool.returnObject(key2, w2); |
| workerPool.returnObject(key2, w3); |
| |
| ImmutableList<WorkerMetric> workerMetrics = |
| ImmutableList.of( |
| WorkerMetric.create( |
| createWorkerProperties(w1.getWorkerId(), 1L, "dummy"), |
| createWorkerStat(3000), |
| true), |
| WorkerMetric.create( |
| createWorkerProperties(w2.getWorkerId(), 2L, "dummy"), |
| createWorkerStat(3000), |
| true), |
| WorkerMetric.create( |
| createWorkerProperties(w3.getWorkerId(), 3L, "dummy"), |
| createWorkerStat(1000), |
| true)); |
| |
| WorkerOptions options = new WorkerOptions(); |
| options.totalWorkerMemoryLimitMb = 2; |
| options.workerVerbose = true; |
| WorkerLifecycleManager manager = new WorkerLifecycleManager(workerPool, options); |
| |
| assertThat(workerPool.getNumIdlePerKey(key1)).isEqualTo(1); |
| assertThat(workerPool.getNumIdlePerKey(key2)).isEqualTo(2); |
| |
| manager.evictWorkers(workerMetrics); |
| |
| assertThat(workerPool.getNumIdlePerKey(key1)).isEqualTo(0); |
| assertThat(workerPool.getNumActive(key1)).isEqualTo(0); |
| |
| assertThat(workerPool.getNumIdlePerKey(key2)).isEqualTo(1); |
| assertThat(workerPool.getNumActive(key2)).isEqualTo(0); |
| } |
| |
| @Test |
| public void testGetEvictionCandidates_evictOnlyIdleWorkers() throws Exception { |
| WorkerPoolImpl workerPool = |
| new WorkerPoolImpl( |
| new WorkerPoolConfig(factoryMock, entryList("dummy", 3), emptyEntryList())); |
| WorkerKey key = createWorkerKey("dummy", fileSystem); |
| Worker w1 = workerPool.borrowObject(key); |
| Worker w2 = workerPool.borrowObject(key); |
| Worker w3 = workerPool.borrowObject(key); |
| workerPool.returnObject(key, w1); |
| workerPool.returnObject(key, w2); |
| |
| ImmutableList<WorkerMetric> workerMetrics = |
| ImmutableList.of( |
| WorkerMetric.create( |
| createWorkerProperties(w1.getWorkerId(), 1L, "dummy"), |
| createWorkerStat(2000), |
| true), |
| WorkerMetric.create( |
| createWorkerProperties(w2.getWorkerId(), 2L, "dummy"), |
| createWorkerStat(1000), |
| true), |
| WorkerMetric.create( |
| createWorkerProperties(w3.getWorkerId(), 3L, "dummy"), |
| createWorkerStat(4000), |
| true)); |
| |
| WorkerOptions options = new WorkerOptions(); |
| options.totalWorkerMemoryLimitMb = 2; |
| |
| WorkerLifecycleManager manager = new WorkerLifecycleManager(workerPool, options); |
| |
| assertThat(workerPool.getNumIdlePerKey(key)).isEqualTo(2); |
| assertThat(workerPool.getNumActive(key)).isEqualTo(1); |
| |
| manager.evictWorkers(workerMetrics); |
| |
| assertThat(workerPool.getNumIdlePerKey(key)).isEqualTo(0); |
| assertThat(workerPool.getNumActive(key)).isEqualTo(1); |
| } |
| |
| @Test |
| public void testGetEvictionCandidates_evictDifferentWorkerKeys() throws Exception { |
| WorkerPoolImpl workerPool = |
| new WorkerPoolImpl( |
| new WorkerPoolConfig(factoryMock, entryList("dummy", 2, "smart", 2), emptyEntryList())); |
| WorkerKey key1 = createWorkerKey("dummy", fileSystem); |
| WorkerKey key2 = createWorkerKey("smart", fileSystem); |
| Worker w1 = workerPool.borrowObject(key1); |
| Worker w2 = workerPool.borrowObject(key1); |
| Worker w3 = workerPool.borrowObject(key2); |
| Worker w4 = workerPool.borrowObject(key2); |
| workerPool.returnObject(key1, w1); |
| workerPool.returnObject(key1, w2); |
| workerPool.returnObject(key2, w3); |
| workerPool.returnObject(key2, w4); |
| |
| ImmutableList<WorkerMetric> workerMetrics = |
| ImmutableList.of( |
| WorkerMetric.create( |
| createWorkerProperties(w1.getWorkerId(), 1L, "dummy"), |
| createWorkerStat(1000), |
| true), |
| WorkerMetric.create( |
| createWorkerProperties(w2.getWorkerId(), 2L, "dummy"), |
| createWorkerStat(4000), |
| true), |
| WorkerMetric.create( |
| createWorkerProperties(w3.getWorkerId(), 3L, "smart"), |
| createWorkerStat(3000), |
| true), |
| WorkerMetric.create( |
| createWorkerProperties(w4.getWorkerId(), 4L, "smart"), |
| createWorkerStat(1000), |
| true)); |
| |
| WorkerOptions options = new WorkerOptions(); |
| options.totalWorkerMemoryLimitMb = 2; |
| |
| WorkerLifecycleManager manager = new WorkerLifecycleManager(workerPool, options); |
| |
| assertThat(workerPool.getNumIdlePerKey(key1)).isEqualTo(2); |
| assertThat(workerPool.getNumActive(key1)).isEqualTo(0); |
| assertThat(workerPool.getNumIdlePerKey(key2)).isEqualTo(2); |
| assertThat(workerPool.getNumActive(key2)).isEqualTo(0); |
| |
| manager.evictWorkers(workerMetrics); |
| |
| assertThat(workerPool.getNumIdlePerKey(key1)).isEqualTo(1); |
| assertThat(workerPool.getNumActive(key1)).isEqualTo(0); |
| assertThat(workerPool.borrowObject(key1).getWorkerId()).isEqualTo(w1.getWorkerId()); |
| |
| assertThat(workerPool.getNumIdlePerKey(key2)).isEqualTo(1); |
| assertThat(workerPool.getNumActive(key2)).isEqualTo(0); |
| assertThat(workerPool.borrowObject(key2).getWorkerId()).isEqualTo(w4.getWorkerId()); |
| } |
| |
| @Test |
| public void testGetEvictionCandidates_testDoomedWorkers() throws Exception { |
| WorkerPoolImpl workerPool = |
| new WorkerPoolImpl( |
| new WorkerPoolConfig(factoryMock, entryList("dummy", 2), emptyEntryList())); |
| WorkerKey key = createWorkerKey("dummy", fileSystem); |
| Worker w1 = workerPool.borrowObject(key); |
| Worker w2 = workerPool.borrowObject(key); |
| |
| ImmutableList<WorkerMetric> workerMetrics = |
| ImmutableList.of( |
| WorkerMetric.create( |
| createWorkerProperties(w1.getWorkerId(), 1L, "dummy"), |
| createWorkerStat(2000), |
| true), |
| WorkerMetric.create( |
| createWorkerProperties(w2.getWorkerId(), 2L, "dummy"), |
| createWorkerStat(2000), |
| true)); |
| |
| WorkerOptions options = new WorkerOptions(); |
| options.totalWorkerMemoryLimitMb = 1; |
| options.shrinkWorkerPool = true; |
| |
| WorkerLifecycleManager manager = new WorkerLifecycleManager(workerPool, options); |
| |
| assertThat(workerPool.getNumIdlePerKey(key)).isEqualTo(0); |
| assertThat(workerPool.getNumActive(key)).isEqualTo(2); |
| |
| manager.evictWorkers(workerMetrics); |
| |
| assertThat(workerPool.getNumIdlePerKey(key)).isEqualTo(0); |
| assertThat(workerPool.getNumActive(key)).isEqualTo(2); |
| assertThat(workerPool.getDoomedWorkers()) |
| .isEqualTo(Sets.newHashSet(w1.getWorkerId(), w2.getWorkerId())); |
| } |
| |
| @Test |
| public void testGetEvictionCandidates_testDoomedAndIdleWorkers() throws Exception { |
| WorkerPoolImpl workerPool = |
| new WorkerPoolImpl( |
| new WorkerPoolConfig(factoryMock, entryList("dummy", 5), emptyEntryList())); |
| WorkerKey key = createWorkerKey("dummy", fileSystem); |
| Worker w1 = workerPool.borrowObject(key); |
| Worker w2 = workerPool.borrowObject(key); |
| Worker w3 = workerPool.borrowObject(key); |
| Worker w4 = workerPool.borrowObject(key); |
| Worker w5 = workerPool.borrowObject(key); |
| workerPool.returnObject(key, w1); |
| workerPool.returnObject(key, w2); |
| |
| ImmutableList<WorkerMetric> workerMetrics = |
| ImmutableList.of( |
| WorkerMetric.create( |
| createWorkerProperties(w1.getWorkerId(), 1L, "dummy"), |
| createWorkerStat(2000), |
| true), |
| WorkerMetric.create( |
| createWorkerProperties(w2.getWorkerId(), 2L, "dummy"), |
| createWorkerStat(1000), |
| true), |
| WorkerMetric.create( |
| createWorkerProperties(w3.getWorkerId(), 3L, "dummy"), |
| createWorkerStat(4000), |
| true), |
| WorkerMetric.create( |
| createWorkerProperties(w4.getWorkerId(), 4L, "dummy"), |
| createWorkerStat(5000), |
| true), |
| WorkerMetric.create( |
| createWorkerProperties(w5.getWorkerId(), 5L, "dummy"), |
| createWorkerStat(1000), |
| true)); |
| |
| WorkerOptions options = new WorkerOptions(); |
| options.totalWorkerMemoryLimitMb = 2; |
| options.shrinkWorkerPool = true; |
| |
| WorkerLifecycleManager manager = new WorkerLifecycleManager(workerPool, options); |
| |
| assertThat(workerPool.getNumIdlePerKey(key)).isEqualTo(2); |
| assertThat(workerPool.getNumActive(key)).isEqualTo(3); |
| |
| manager.evictWorkers(workerMetrics); |
| |
| assertThat(workerPool.getNumIdlePerKey(key)).isEqualTo(0); |
| assertThat(workerPool.getNumActive(key)).isEqualTo(3); |
| assertThat(workerPool.getDoomedWorkers()) |
| .isEqualTo(Sets.newHashSet(w3.getWorkerId(), w4.getWorkerId())); |
| } |
| |
| @Test |
| public void evictWorkers_testMultiplexWorkers() throws Exception { |
| WorkerPoolImpl workerPool = |
| new WorkerPoolImpl( |
| new WorkerPoolConfig(factoryMock, emptyEntryList(), entryList("dummy", 2))); |
| WorkerKey key = |
| createWorkerKey("dummy", fileSystem, /* multiplex= */ true, /* sandboxed= */ false); |
| Worker w1 = workerPool.borrowObject(key); |
| Worker w2 = workerPool.borrowObject(key); |
| workerPool.returnObject(key, w1); |
| workerPool.returnObject(key, w2); |
| ImmutableList<WorkerMetric> workerMetrics = |
| ImmutableList.of( |
| WorkerMetric.create( |
| createMultiplexWorkerProperties( |
| ImmutableList.of(w1.getWorkerId(), w2.getWorkerId()), 1L, "dummy"), |
| createWorkerStat(4000), |
| true)); |
| WorkerOptions options = new WorkerOptions(); |
| options.totalWorkerMemoryLimitMb = 1; |
| WorkerLifecycleManager manager = new WorkerLifecycleManager(workerPool, options); |
| |
| manager.evictWorkers(workerMetrics); |
| |
| assertThat(workerPool.getNumIdlePerKey(key)).isEqualTo(0); |
| assertThat(workerPool.getNumActive(key)).isEqualTo(0); |
| } |
| |
| @Test |
| public void evictWorkers_doomMultiplexWorker() throws Exception { |
| WorkerPoolImpl workerPool = |
| new WorkerPoolImpl( |
| new WorkerPoolConfig(factoryMock, emptyEntryList(), entryList("dummy", 2))); |
| WorkerKey key = |
| createWorkerKey("dummy", fileSystem, /* multiplex= */ true, /* sandboxed= */ false); |
| Worker w1 = workerPool.borrowObject(key); |
| Worker w2 = workerPool.borrowObject(key); |
| workerPool.returnObject(key, w1); |
| ImmutableList<WorkerMetric> workerMetrics = |
| ImmutableList.of( |
| WorkerMetric.create( |
| createMultiplexWorkerProperties( |
| ImmutableList.of(w1.getWorkerId(), w2.getWorkerId()), 1L, "dummy"), |
| createWorkerStat(4000), |
| true)); |
| WorkerOptions options = new WorkerOptions(); |
| options.totalWorkerMemoryLimitMb = 1; |
| options.shrinkWorkerPool = true; |
| WorkerLifecycleManager manager = new WorkerLifecycleManager(workerPool, options); |
| |
| manager.evictWorkers(workerMetrics); |
| |
| assertThat(workerPool.getNumIdlePerKey(key)).isEqualTo(1); |
| assertThat(workerPool.getNumActive(key)).isEqualTo(1); |
| assertThat(workerPool.getDoomedWorkers()) |
| .isEqualTo(Sets.newHashSet(w1.getWorkerId(), w2.getWorkerId())); |
| } |
| |
| private static WorkerMetric.WorkerProperties createWorkerProperties( |
| int workerId, long processId, String mnemonic, int workerKeyHash) { |
| return WorkerMetric.WorkerProperties.create( |
| ImmutableList.of(workerId), |
| processId, |
| mnemonic, |
| /* isMultiplex= */ false, |
| /* isSandboxed= */ false, |
| workerKeyHash); |
| } |
| |
| private static WorkerMetric.WorkerProperties createWorkerProperties( |
| int workerId, long processId, String mnemonic) { |
| return createWorkerProperties(workerId, processId, mnemonic, /* workerKeyHash= */ 0); |
| } |
| |
| private static WorkerMetric.WorkerProperties createMultiplexWorkerProperties( |
| ImmutableList<Integer> workerIds, long processId, String mnemonic) { |
| return WorkerMetric.WorkerProperties.create( |
| workerIds, processId, mnemonic, true, /* isSandboxed= */ false, /* workerKeyHash= */ 0); |
| } |
| |
| private static WorkerMetric.WorkerStat createWorkerStat(int memoryUsage) { |
| return WorkerMetric.WorkerStat.create( |
| memoryUsage, /*lastCallTimestamp */ Instant.now(), /* timestamp*/ Instant.now()); |
| } |
| |
| private static ImmutableList<Entry<String, Integer>> emptyEntryList() { |
| return ImmutableList.of(); |
| } |
| |
| private static ImmutableList<Entry<String, Integer>> entryList(String key1, int value1) { |
| return ImmutableList.of(Maps.immutableEntry(key1, value1)); |
| } |
| |
| private static ImmutableList<Entry<String, Integer>> entryList( |
| String key1, int value1, String key2, int value2) { |
| return ImmutableList.of(Maps.immutableEntry(key1, value1), Maps.immutableEntry(key2, value2)); |
| } |
| } |
