| // Copyright 2022 The Bazel Authors. All rights reserved. |
| // |
| // Licensed under the Apache License, Version 2.0 (the "License"); |
| // you may not use this file except in compliance with the License. |
| // You may obtain a copy of the License at |
| // |
| // http://www.apache.org/licenses/LICENSE-2.0 |
| // |
| // Unless required by applicable law or agreed to in writing, software |
| // distributed under the License is distributed on an "AS IS" BASIS, |
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| // See the License for the specific language governing permissions and |
| // limitations under the License. |
| |
| package com.google.devtools.build.lib.sandbox; |
| |
| import static java.nio.charset.StandardCharsets.UTF_8; |
| |
| import com.google.common.annotations.VisibleForTesting; |
| import com.google.common.base.Joiner; |
| import com.google.common.base.Splitter; |
| import com.google.common.flogger.GoogleLogger; |
| import com.google.common.io.Files; |
| import com.google.devtools.build.lib.util.Pair; |
| import java.io.File; |
| import java.io.IOException; |
| import java.util.List; |
| import java.util.regex.Matcher; |
| import java.util.regex.Pattern; |
| import javax.annotation.Nullable; |
| |
| /** This class manages cgroups directories for memory-limiting sandboxed processes. */ |
| public class CgroupsInfo { |
| private static final GoogleLogger logger = GoogleLogger.forEnclosingClass(); |
| |
| /** |
| * A regexp that matches cgroups entries in {@code /proc/mounts}. |
| * |
| * <p>Group 1 is empty (cgroups v1) or '2' (cgroups v2) Group 2 is the mount point. Group 3 is the |
| * options, which for v1 includes which hierarchies are mounted here. |
| */ |
| private static final Pattern CGROUPS_MOUNT_PATTERN = |
| Pattern.compile("^cgroup(|2)\\s+(\\S*)\\s+cgroup2?\\s+(\\S*).*"); |
| |
| public static final String PROC_SELF_MOUNTS_PATH = "/proc/self/mounts"; |
| public static final String PROC_SELF_CGROUP_PATH = "/proc/self/cgroup"; |
| /** If non-null, this is a cgroups directory that sandboxes can put their directories into. */ |
| @Nullable private static volatile CgroupsInfo instance; |
| |
| private final boolean isCgroupsV2; |
| private final File blazeDir; |
| private final File mountPoint; |
| |
| private CgroupsInfo(boolean isCgroupsV2, File blazeDir, File mountPoint) { |
| this.isCgroupsV2 = isCgroupsV2; |
| this.blazeDir = blazeDir; |
| this.mountPoint = mountPoint; |
| } |
| |
| /** |
| * Creates a cgroups directory for Blaze to place sandboxes in. Figures out whether cgroups v1 or |
| * v2 is available, and for cgroups v2 sets subtree control for the <code>memory</code> and <code> |
| * pids</code> controllers. |
| * |
| * <p>The cgroups directory is created at most once per Blaze instance. |
| * |
| * @return A CgroupsInfo object that defines the cgroups directory that Blaze can use for |
| * sub-processes. The Blaze process itself is not moved into this directory. |
| * @throws IOException If there are errors reading any of the required files. |
| */ |
| public static CgroupsInfo create() throws IOException { |
| return create(PROC_SELF_MOUNTS_PATH, PROC_SELF_CGROUP_PATH); |
| } |
| |
| @VisibleForTesting |
| static CgroupsInfo create(String procSelfMountsPath, String procSelfCgroupPath) |
| throws IOException { |
| Pair<File, Boolean> cgroupsMount = getMemoryCgroupInfo(new File(procSelfMountsPath)); |
| File blazeDir; |
| File cgroupsMountPoint = cgroupsMount.first; |
| if (cgroupsMount.second) { |
| File cgroupsNode = getBlazeMemoryCgroup(cgroupsMountPoint, 0, procSelfCgroupPath); |
| // In cgroups v2, we need to step back from the leaf node to make a further hierarchy. |
| blazeDir = |
| new File( |
| cgroupsNode.getParentFile(), |
| "blaze_" + ProcessHandle.current().pid() + "_spawns.slice"); |
| blazeDir.mkdirs(); |
| blazeDir.deleteOnExit(); |
| setSubtreeControllers(blazeDir); |
| logger.atInfo().log("Creating cgroups v2 node at %s", blazeDir); |
| return new CgroupsInfo(true, blazeDir, cgroupsMountPoint); |
| } else { |
| int memoryHierarchy = getMemoryHierarchy(new File(procSelfCgroupPath)); |
| File cgroupsNode = |
| getBlazeMemoryCgroup(cgroupsMountPoint, memoryHierarchy, procSelfCgroupPath); |
| blazeDir = new File(cgroupsNode, "blaze_" + ProcessHandle.current().pid() + "_spawns"); |
| blazeDir.mkdirs(); |
| blazeDir.deleteOnExit(); |
| logger.atInfo().log("Creating cgroups v1 node at %s", blazeDir); |
| return new CgroupsInfo(false, blazeDir, cgroupsMountPoint); |
| } |
| } |
| |
| /** |
| * Sets the subtree controllers we need. This also checks that the controllers are available. |
| * |
| * @param blazeDir A directory in the cgroups hierarchy. |
| * @throws IOException If reading or writing the {@code cgroup.controllers} or {@code |
| * cgroup.subtree_control} file fails. |
| * @throws IllegalStateException if the {@code memory} and {code pids} controllers are either not |
| * available or cannot be set for subtrees. |
| */ |
| private static void setSubtreeControllers(File blazeDir) throws IOException { |
| var controllers = |
| Joiner.on(' ').join(Files.readLines(new File(blazeDir, "cgroup.controllers"), UTF_8)); |
| if (!(controllers.contains("memory") && controllers.contains("pids"))) { |
| throw new IllegalStateException( |
| String.format( |
| "Required controllers 'memory' and 'pids' not found in %s/cgroup.controllers", |
| blazeDir)); |
| } |
| var subtreeControllers = |
| Joiner.on(' ').join(Files.readLines(new File(blazeDir, "cgroup.subtree_control"), UTF_8)); |
| if (!subtreeControllers.contains("memory") || !subtreeControllers.contains("pids")) { |
| Files.asCharSink(new File(blazeDir, "cgroup.subtree_control"), UTF_8) |
| .write("+memory +pids\n"); |
| } |
| } |
| |
| public boolean isCgroupsV2() { |
| return isCgroupsV2; |
| } |
| |
| /** A cgroups directory for this Blaze instance to put sandboxes in. */ |
| public File getBlazeDir() { |
| return blazeDir; |
| } |
| |
| /** The place where the cgroups (memory) file system is mounted. */ |
| public File getMountPoint() { |
| return mountPoint; |
| } |
| |
| /** |
| * Reads from the given file (e.g. /proc/mounts) where cgroups are mounted. If both cgroups v1 and |
| * cgroups v2 are mounted, the one that has the memory controller is used. |
| * |
| * <p>In cgroups v1, a typical mount line looks like this (note {@code memory} in the options): |
| * |
| * <pre> |
| * cgroup /dev/cgroup/memory cgroup rw,memory,hugetlb 0 0 |
| * </pre> |
| * |
| * In cgroups v2, there is only one relevant line, and it looks like this: |
| * |
| * <pre> |
| * cgroup2 /sys/fs/cgroup cgroup2 rw,[...] 0 0 |
| * </pre> |
| * |
| * @param procMountsPath Paths of the mounts file, e.g. /proc/mounts |
| * @return Pair of |
| * <ol> |
| * <li>the path of the cgroups mount (for cgroups v1, this is the memory hierarchy) and |
| * <li>whether this is cgroups v2. |
| * </ol> |
| * |
| * @throws IOException If there are errors reading the given file. |
| */ |
| @VisibleForTesting |
| static Pair<File, Boolean> getMemoryCgroupInfo(File procMountsPath) throws IOException { |
| var procMountContents = Files.readLines(procMountsPath, UTF_8); |
| Pair<File, Boolean> v1 = null; |
| Pair<File, Boolean> v2 = null; |
| for (String s : procMountContents) { |
| Matcher m = CGROUPS_MOUNT_PATTERN.matcher(s); |
| if (m.matches()) { |
| if (m.group(1).isEmpty()) { |
| // v1 |
| if (m.group(3).contains("memory")) { |
| // For now, we only care about the memory cgroup |
| v1 = Pair.of(new File(m.group(2)), false); |
| } |
| } else { |
| // v2 |
| v2 = Pair.of(new File(m.group(2)), true); |
| } |
| } |
| } |
| // If we found the memory controller in v1, we use that, just in case we have a hybrid system |
| // where some controllers are v1 and some are v2. It would be harder to detect if v2 has the |
| // memory controller |
| if (v1 != null) { |
| return v1; |
| } |
| if (v2 != null) { |
| return v2; |
| } |
| throw new IllegalStateException( |
| "Cgroups requested, but no applicable cgroups are mounted on this machine"); |
| } |
| |
| /** |
| * Returns the number of the memory cgroups v1 hierarchy. |
| * |
| * <p>The <code>/proc/self/cgroup</code> file look like this in v1: |
| * |
| * <pre> |
| * 8:net:/some/path |
| * 7:memory,hugetlb:/some/other/path |
| * ... |
| * </pre> |
| * |
| * In v2, there is only one entry, and it looks something like |
| * |
| * <pre> |
| * 0::/user.slice/user-123.slice/session-1.scope |
| * </pre> |
| * |
| * @param procSelfCgroupPath Path for the <code>/proc/self/cgroup</code> file. |
| * @return The hierarchy number for the cgroups v1 hierarchy that contains the memory controller. |
| * @throws IOException If there are errors reading the file. |
| */ |
| @VisibleForTesting |
| static int getMemoryHierarchy(File procSelfCgroupPath) throws IOException { |
| List<String> devCgroupContents = Files.readLines(procSelfCgroupPath, UTF_8); |
| for (String s : devCgroupContents) { |
| if (s.contains("memory")) { |
| return Integer.parseInt(Splitter.on(":").split(s).iterator().next()); |
| } |
| } |
| throw new IllegalStateException( |
| String.format( |
| "Cgroups v1 requested, but no memory cgroup found in %s", procSelfCgroupPath)); |
| } |
| |
| /** |
| * Returns the path of the memory cgroups node that Blaze itself runs inside. |
| * |
| * @param mountPoint Where the cgroups hierarchy (with the memory controller for v1) is mounted. |
| * @param memoryHierarchyId The v1 hierarchy that contains the memory controller, or 0 for v2. |
| * @param procSelfPath The path of the <code>/proc/self/cgroup</code> file. |
| * @return A <code>File</code> object of the cgroup directory of the current process. |
| * @throws IOException If the given file cannot be read. |
| */ |
| @VisibleForTesting |
| static File getBlazeMemoryCgroup(File mountPoint, int memoryHierarchyId, String procSelfPath) |
| throws IOException { |
| var procSelfCgroupContents = Files.readLines(new File(procSelfPath), UTF_8); |
| if (procSelfCgroupContents.isEmpty()) { |
| throw new IOException("Cgroups requested, but /proc/self/cgroup is empty"); |
| } |
| File cgroupsNode = null; |
| for (String s : procSelfCgroupContents) { |
| List<String> parts = Splitter.on(":").limit(3).splitToList(s); |
| if (parts.size() == 3 && Integer.parseInt(parts.get(0)) == memoryHierarchyId) { |
| String path = parts.get(2); |
| if (path.startsWith(File.pathSeparator)) { |
| path = path.substring(1); |
| } |
| cgroupsNode = new File(mountPoint, path); |
| break; |
| } |
| } |
| if (cgroupsNode == null) { |
| throw new IllegalStateException("Found no memory cgroups entries in '" + procSelfPath + "'"); |
| } |
| if (!cgroupsNode.exists()) { |
| throw new IllegalStateException("Cgroups node '" + cgroupsNode + "' does not exist"); |
| } |
| if (!cgroupsNode.isDirectory()) { |
| throw new IllegalStateException("Cgroups node " + cgroupsNode + " is not a directory"); |
| } |
| if (!cgroupsNode.canWrite()) { |
| throw new IllegalStateException("Cgroups node " + cgroupsNode + " is not writable"); |
| } |
| return cgroupsNode; |
| } |
| |
| public static CgroupsInfo getInstance() throws IOException { |
| if (instance == null) { |
| synchronized (CgroupsInfo.class) { |
| if (instance == null) { |
| instance = create(); |
| } |
| } |
| } |
| return instance; |
| } |
| |
| /** |
| * Creates a cgroups directory with the given memory limit. |
| * |
| * @param memoryLimit Memory limit in megabytes (MiB). |
| * @param dirName Base name of the directory created. In cgroups v2, <code>.scope</code> gets |
| * appended. |
| */ |
| public String createMemoryLimitCgroupDir(String dirName, int memoryLimit) throws IOException { |
| File cgroupsDir; |
| if (isCgroupsV2) { |
| cgroupsDir = new File(blazeDir, dirName + ".scope"); |
| cgroupsDir.mkdirs(); |
| cgroupsDir.deleteOnExit(); |
| // In cgroups v2, we need to propagate the controllers into new subdirs. |
| Files.asCharSink(new File(cgroupsDir, "memory.oom.group"), UTF_8).write("1\n"); |
| Files.asCharSink(new File(cgroupsDir, "memory.max"), UTF_8) |
| .write(Long.toString(memoryLimit * 1024L * 1024L)); |
| } else { |
| cgroupsDir = new File(getBlazeDir(), dirName); |
| cgroupsDir.mkdirs(); |
| cgroupsDir.deleteOnExit(); |
| Files.asCharSink(new File(cgroupsDir, "memory.limit_in_bytes"), UTF_8) |
| .write(Long.toString(memoryLimit * 1024L * 1024L)); |
| } |
| return cgroupsDir.toString(); |
| } |
| } |