blob: 617ec047c523be0cbea6b028b08fdac13f555cbf [file] [log] [blame]
shreyaxc1832a42018-10-18 08:54:49 -07001// Copyright 2018 The Bazel Authors. All rights reserved.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15package com.google.devtools.build.lib.server;
16
17import com.google.common.base.Preconditions;
18import com.google.common.collect.ImmutableSet;
janakrc3bcb982020-04-14 06:50:08 -070019import com.google.common.flogger.GoogleLogger;
shreyaxc1832a42018-10-18 08:54:49 -070020import com.google.devtools.build.lib.server.CommandProtos.CancelRequest;
21import com.google.devtools.build.lib.util.ThreadUtils;
22import java.util.Collections;
23import java.util.HashMap;
24import java.util.Map;
25import java.util.UUID;
26import java.util.concurrent.atomic.AtomicLong;
janakr7c015cd2021-08-24 17:57:21 -070027import javax.annotation.Nullable;
shreyaxc1832a42018-10-18 08:54:49 -070028import javax.annotation.concurrent.GuardedBy;
29
30/** Helper class for commands that are currently running on the server. */
31class CommandManager {
janakrc3bcb982020-04-14 06:50:08 -070032 private static final GoogleLogger logger = GoogleLogger.forEnclosingClass();
shreyaxc1832a42018-10-18 08:54:49 -070033
34 @GuardedBy("runningCommandsMap")
35 private final Map<String, RunningCommand> runningCommandsMap = new HashMap<>();
36
37 private final AtomicLong interruptCounter = new AtomicLong(0);
38 private final boolean doIdleServerTasks;
39
40 private IdleServerTasks idleServerTasks;
janakr7c015cd2021-08-24 17:57:21 -070041 @Nullable private final String slowInterruptMessageSuffix;
shreyaxc1832a42018-10-18 08:54:49 -070042
janakr7c015cd2021-08-24 17:57:21 -070043 CommandManager(boolean doIdleServerTasks, @Nullable String slowInterruptMessageSuffix) {
shreyaxc1832a42018-10-18 08:54:49 -070044 this.doIdleServerTasks = doIdleServerTasks;
janakr7c015cd2021-08-24 17:57:21 -070045 this.slowInterruptMessageSuffix = slowInterruptMessageSuffix;
shreyaxc1832a42018-10-18 08:54:49 -070046 idle();
47 }
48
David Cummings65d18a52020-12-09 15:37:38 -080049 void preemptEligibleCommands() {
50 synchronized (runningCommandsMap) {
51 ImmutableSet.Builder<String> commandsToInterruptBuilder = new ImmutableSet.Builder<>();
52
53 for (RunningCommand command : runningCommandsMap.values()) {
54 if (command.isPreemptible()) {
55 command.thread.interrupt();
56 commandsToInterruptBuilder.add(command.id);
57 }
58 }
59
60 ImmutableSet<String> commandsToInterrupt = commandsToInterruptBuilder.build();
61 if (!commandsToInterrupt.isEmpty()) {
62 startSlowInterruptWatcher(commandsToInterrupt);
63 }
64 }
65 }
66
shreyaxc1832a42018-10-18 08:54:49 -070067 void interruptInflightCommands() {
68 synchronized (runningCommandsMap) {
69 for (RunningCommand command : runningCommandsMap.values()) {
70 command.thread.interrupt();
71 }
72
73 startSlowInterruptWatcher(ImmutableSet.copyOf(runningCommandsMap.keySet()));
74 }
75 }
76
77 void doCancel(CancelRequest request) {
David Cummings65d18a52020-12-09 15:37:38 -080078 try (RunningCommand cancelCommand = createCommand()) {
shreyaxc1832a42018-10-18 08:54:49 -070079 synchronized (runningCommandsMap) {
80 RunningCommand pendingCommand = runningCommandsMap.get(request.getCommandId());
81 if (pendingCommand != null) {
janakrc3bcb982020-04-14 06:50:08 -070082 logger.atInfo().log(
83 "Interrupting command %s on thread %s",
84 request.getCommandId(), pendingCommand.thread.getName());
shreyaxc1832a42018-10-18 08:54:49 -070085 pendingCommand.thread.interrupt();
86 startSlowInterruptWatcher(ImmutableSet.of(request.getCommandId()));
87 } else {
janakrc3bcb982020-04-14 06:50:08 -070088 logger.atInfo().log("Cannot find command %s to interrupt", request.getCommandId());
shreyaxc1832a42018-10-18 08:54:49 -070089 }
90 }
91 }
92 }
93
94 boolean isEmpty() {
95 synchronized (runningCommandsMap) {
96 return runningCommandsMap.isEmpty();
97 }
98 }
99
100 void waitForChange() throws InterruptedException {
101 synchronized (runningCommandsMap) {
102 runningCommandsMap.wait();
103 }
104 }
105
106 void waitForChange(long timeout) throws InterruptedException {
107 synchronized (runningCommandsMap) {
108 runningCommandsMap.wait(timeout);
109 }
110 }
111
David Cummings65d18a52020-12-09 15:37:38 -0800112 RunningCommand createPreemptibleCommand() {
113 RunningCommand command = new RunningCommand(true);
114 registerCommand(command);
115 return command;
116 }
117
118 RunningCommand createCommand() {
119 RunningCommand command = new RunningCommand(false);
120 registerCommand(command);
121 return command;
122 }
123
124 private void registerCommand(RunningCommand command) {
shreyaxc1832a42018-10-18 08:54:49 -0700125 synchronized (runningCommandsMap) {
126 if (runningCommandsMap.isEmpty()) {
127 busy();
128 }
129 runningCommandsMap.put(command.id, command);
130 runningCommandsMap.notify();
131 }
janakrc3bcb982020-04-14 06:50:08 -0700132 logger.atInfo().log("Starting command %s on thread %s", command.id, command.thread.getName());
shreyaxc1832a42018-10-18 08:54:49 -0700133 }
134
135 private void idle() {
136 Preconditions.checkState(idleServerTasks == null);
137 if (doIdleServerTasks) {
138 idleServerTasks = new IdleServerTasks();
139 idleServerTasks.idle();
140 }
141 }
142
143 private void busy() {
144 if (doIdleServerTasks) {
145 Preconditions.checkState(idleServerTasks != null);
146 idleServerTasks.busy();
147 idleServerTasks = null;
148 }
149 }
150
151 private void startSlowInterruptWatcher(final ImmutableSet<String> commandIds) {
152 if (commandIds.isEmpty()) {
153 return;
154 }
155
156 Runnable interruptWatcher =
157 () -> {
158 try {
159 Thread.sleep(10 * 1000);
160 boolean ok;
161 synchronized (runningCommandsMap) {
162 ok = Collections.disjoint(commandIds, runningCommandsMap.keySet());
163 }
164 if (!ok) {
165 // At least one command was not interrupted. Interrupt took too long.
janakr7c015cd2021-08-24 17:57:21 -0700166 ThreadUtils.warnAboutSlowInterrupt(slowInterruptMessageSuffix);
shreyaxc1832a42018-10-18 08:54:49 -0700167 }
168 } catch (InterruptedException e) {
169 // Ignore.
170 }
171 };
172
173 Thread interruptWatcherThread =
174 new Thread(interruptWatcher, "interrupt-watcher-" + interruptCounter.incrementAndGet());
175 interruptWatcherThread.setDaemon(true);
176 interruptWatcherThread.start();
177 }
178
179 class RunningCommand implements AutoCloseable {
180 private final Thread thread;
181 private final String id;
David Cummings65d18a52020-12-09 15:37:38 -0800182 private final boolean preemptible;
shreyaxc1832a42018-10-18 08:54:49 -0700183
David Cummings65d18a52020-12-09 15:37:38 -0800184 private RunningCommand(boolean preemptible) {
shreyaxc1832a42018-10-18 08:54:49 -0700185 thread = Thread.currentThread();
186 id = UUID.randomUUID().toString();
David Cummings65d18a52020-12-09 15:37:38 -0800187 this.preemptible = preemptible;
shreyaxc1832a42018-10-18 08:54:49 -0700188 }
189
190 @Override
191 public void close() {
192 synchronized (runningCommandsMap) {
193 runningCommandsMap.remove(id);
194 if (runningCommandsMap.isEmpty()) {
195 idle();
196 }
197 runningCommandsMap.notify();
198 }
199
janakrc3bcb982020-04-14 06:50:08 -0700200 logger.atInfo().log("Finished command %s on thread %s", id, thread.getName());
shreyaxc1832a42018-10-18 08:54:49 -0700201 }
202
203 String getId() {
204 return id;
205 }
David Cummings65d18a52020-12-09 15:37:38 -0800206
207 boolean isPreemptible() {
208 return this.preemptible;
209 }
shreyaxc1832a42018-10-18 08:54:49 -0700210 }
211}