Add SMT detection for mac. Fixes #963.
--
Change-Id: Ib83af0d0a04dc6b173bef1df28d17abc7a3c824d
Reviewed-on: https://bazel-review.googlesource.com/#/c/3120/
MOS_MIGRATED_REVID=119027507
diff --git a/src/main/java/com/google/devtools/build/lib/actions/BUILD b/src/main/java/com/google/devtools/build/lib/actions/BUILD
index e51fbea..8f717e7 100644
--- a/src/main/java/com/google/devtools/build/lib/actions/BUILD
+++ b/src/main/java/com/google/devtools/build/lib/actions/BUILD
@@ -18,6 +18,7 @@
"//src/main/java/com/google/devtools/build/lib:packages-internal",
"//src/main/java/com/google/devtools/build/lib:shell",
"//src/main/java/com/google/devtools/build/lib:skylarkinterface",
+ "//src/main/java/com/google/devtools/build/lib:unix",
"//src/main/java/com/google/devtools/build/lib:util",
"//src/main/java/com/google/devtools/build/lib:vfs",
"//src/main/java/com/google/devtools/build/skyframe",
diff --git a/src/main/java/com/google/devtools/build/lib/actions/LocalHostCapacity.java b/src/main/java/com/google/devtools/build/lib/actions/LocalHostCapacity.java
index 43509b5..74ef683 100644
--- a/src/main/java/com/google/devtools/build/lib/actions/LocalHostCapacity.java
+++ b/src/main/java/com/google/devtools/build/lib/actions/LocalHostCapacity.java
@@ -14,17 +14,8 @@
package com.google.devtools.build.lib.actions;
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Splitter;
-import com.google.common.io.Files;
import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadCompatible;
-import com.google.devtools.build.lib.util.ProcMeminfoParser;
-
-import java.io.File;
-import java.io.IOException;
-import java.nio.charset.Charset;
-import java.util.HashSet;
-import java.util.Set;
+import com.google.devtools.build.lib.util.OS;
/**
* This class estimates the local host's resource capacity.
@@ -32,115 +23,34 @@
@ThreadCompatible
public final class LocalHostCapacity {
- /* If /proc/* information is not available, guess based on what the JVM thinks. Anecdotally,
- * the JVM picks 0.22 the total available memory as maxMemory (tested on a standard Mac), so
- * multiply by 3, and divide by 2^20 because we want megabytes.
- */
- private static final ResourceSet DEFAULT_RESOURCES = ResourceSet.create(
- 3.0 * (Runtime.getRuntime().maxMemory() >> 20),
- Runtime.getRuntime().availableProcessors(), 1.0,
- Integer.MAX_VALUE);
+ private static final OS currentOS = OS.getCurrent();
+ private static ResourceSet localHostCapacity;
private LocalHostCapacity() {}
- /**
- * Estimates of the local host's resource capacity,
- * obtained by reading /proc/cpuinfo and /proc/meminfo.
- */
- private static ResourceSet localHostCapacity;
-
- /**
- * Estimates of the local host's resource capacity,
- * obtained by reading /proc/cpuinfo and /proc/meminfo.
- */
public static ResourceSet getLocalHostCapacity() {
if (localHostCapacity == null) {
- localHostCapacity = getLocalHostCapacity("/proc/cpuinfo", "/proc/meminfo");
+ localHostCapacity = getNewLocalHostCapacity();
}
return localHostCapacity;
}
- private static final Splitter NEWLINE_SPLITTER = Splitter.on('\n').omitEmptyStrings();
-
- @VisibleForTesting
- static int getLogicalCpuCount(String cpuinfoContent) {
- Iterable<String> lines = NEWLINE_SPLITTER.split(cpuinfoContent);
- int count = 0;
- for (String line : lines) {
- if(line.startsWith("processor")) {
- count++;
- }
+ private static ResourceSet getNewLocalHostCapacity() {
+ ResourceSet localResources = null;
+ switch (currentOS) {
+ case DARWIN:
+ localResources = LocalHostResourceManagerDarwin.getLocalHostResources();
+ break;
+ case LINUX:
+ localResources = LocalHostResourceManagerLinux.getLocalHostResources();
+ break;
+ default:
+ break;
}
- if (count == 0) {
- throw new IllegalArgumentException("Can't locate processor in the /proc/cpuinfo");
+ if (localResources == null) {
+ localResources = LocalHostResourceFallback.getLocalHostResources();
}
- return count;
- }
-
- @VisibleForTesting
- static int getPhysicalCpuCount(String cpuinfoContent, int logicalCpuCount) {
- Iterable<String> lines = NEWLINE_SPLITTER.split(cpuinfoContent);
- Set<String> uniq = new HashSet<>();
- for (String line : lines) {
- if(line.startsWith("physical id")) {
- uniq.add(line);
- }
- }
- int physicalCpuCount = uniq.size();
- if (physicalCpuCount == 0) {
- physicalCpuCount = logicalCpuCount;
- }
- return physicalCpuCount;
- }
-
- @VisibleForTesting
- static int getCoresPerCpu(String cpuinfoFileContent) {
- Iterable<String> lines = NEWLINE_SPLITTER.split(cpuinfoFileContent);
- Set<String> uniq = new HashSet<>();
- for (String line : lines) {
- if(line.startsWith("core id")) {
- uniq.add(line);
- }
- }
- int coresPerCpu = uniq.size();
- if (coresPerCpu == 0) {
- coresPerCpu = 1;
- }
- return coresPerCpu;
- }
-
- @VisibleForTesting
- static ResourceSet getLocalHostCapacity(String cpuinfoFile, String meminfoFile) {
- try {
- String cpuinfoContent = readContent(cpuinfoFile);
- ProcMeminfoParser memInfo = new ProcMeminfoParser(meminfoFile);
- int logicalCpuCount = getLogicalCpuCount(cpuinfoContent);
- int physicalCpuCount = getPhysicalCpuCount(cpuinfoContent, logicalCpuCount);
- int coresPerCpu = getCoresPerCpu(cpuinfoContent);
- int totalCores = coresPerCpu * physicalCpuCount;
- boolean hyperthreading = (logicalCpuCount != totalCores);
- double ramMb = ProcMeminfoParser.kbToMb(memInfo.getTotalKb());
- final double EFFECTIVE_CPUS_PER_HYPERTHREADED_CPU = 0.6;
- return ResourceSet.create(
- ramMb,
- logicalCpuCount * (hyperthreading ? EFFECTIVE_CPUS_PER_HYPERTHREADED_CPU : 1.0),
- 1.0,
- Integer.MAX_VALUE);
- } catch (IOException | IllegalArgumentException e) {
- return DEFAULT_RESOURCES;
- }
- }
-
- /**
- * For testing purposes only. Do not use it.
- */
- @VisibleForTesting
- static void setLocalHostCapacity(ResourceSet resources) {
- localHostCapacity = resources;
- }
-
- private static String readContent(String filename) throws IOException {
- return Files.toString(new File(filename), Charset.defaultCharset());
+ return localResources;
}
}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/LocalHostResourceFallback.java b/src/main/java/com/google/devtools/build/lib/actions/LocalHostResourceFallback.java
new file mode 100644
index 0000000..2829cb4
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/actions/LocalHostResourceFallback.java
@@ -0,0 +1,36 @@
+// Copyright 2016 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.actions;
+
+/**
+ * This class provide a fallback of the local host's resource capacity.
+ */
+public class LocalHostResourceFallback {
+
+ /* If /proc/* information is not available, guess based on what the JVM thinks. Anecdotally,
+ * the JVM picks 0.22 the total available memory as maxMemory (tested on a standard Mac), so
+ * multiply by 3, and divide by 2^20 because we want megabytes.
+ */
+ private static final ResourceSet DEFAULT_RESOURCES =
+ ResourceSet.create(
+ 3.0 * (Runtime.getRuntime().maxMemory() >> 20),
+ Runtime.getRuntime().availableProcessors(),
+ 1.0,
+ Integer.MAX_VALUE);
+
+ public static ResourceSet getLocalHostResources() {
+ return DEFAULT_RESOURCES;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/LocalHostResourceManagerDarwin.java b/src/main/java/com/google/devtools/build/lib/actions/LocalHostResourceManagerDarwin.java
new file mode 100644
index 0000000..1ada1bf
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/actions/LocalHostResourceManagerDarwin.java
@@ -0,0 +1,60 @@
+// Copyright 2016 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.actions;
+
+import com.google.devtools.build.lib.unix.NativePosixSystem;
+
+import java.io.IOException;
+
+/**
+ * This class estimates the local host's resource capacity for Darwin.
+ */
+public class LocalHostResourceManagerDarwin {
+ private static final Boolean JNI_UNAVAILABLE =
+ "0".equals(System.getProperty("io.bazel.UnixFileSystem"));
+ private static final double EFFECTIVE_CPUS_PER_HYPERTHREADED_CPU = 0.6;
+
+ private static int getLogicalCpuCount() throws IOException {
+ return (int) NativePosixSystem.sysctlbynameGetLong("hw.logicalcpu");
+ }
+
+ private static int getPhysicalCpuCount() throws IOException {
+ return (int) NativePosixSystem.sysctlbynameGetLong("hw.physicalcpu");
+ }
+
+ private static double getMemoryInMb() throws IOException {
+ return NativePosixSystem.sysctlbynameGetLong("hw.memsize") / 1E6;
+ }
+
+ public static ResourceSet getLocalHostResources() {
+ if (JNI_UNAVAILABLE) {
+ return null;
+ }
+ try {
+ int logicalCpuCount = getLogicalCpuCount();
+ int physicalCpuCount = getPhysicalCpuCount();
+ double ramMb = getMemoryInMb();
+ boolean hyperthreading = (logicalCpuCount != physicalCpuCount);
+
+ return ResourceSet.create(
+ ramMb,
+ logicalCpuCount * (hyperthreading ? EFFECTIVE_CPUS_PER_HYPERTHREADED_CPU : 1.0),
+ 1.0,
+ Integer.MAX_VALUE);
+ } catch (IOException e) {
+ return null;
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/LocalHostResourceManagerLinux.java b/src/main/java/com/google/devtools/build/lib/actions/LocalHostResourceManagerLinux.java
new file mode 100644
index 0000000..a9b50f9
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/actions/LocalHostResourceManagerLinux.java
@@ -0,0 +1,132 @@
+// Copyright 2016 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.actions;
+
+import com.google.common.base.Splitter;
+import com.google.common.io.Files;
+import com.google.devtools.build.lib.util.ProcMeminfoParser;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.Charset;
+import java.util.HashSet;
+import java.util.Set;
+
+/**
+ * This class estimates the local host's resource capacity for Linux.
+ */
+public class LocalHostResourceManagerLinux {
+ private static String cpuInfoContent = null;
+
+ private static final Splitter NEWLINE_SPLITTER = Splitter.on('\n').omitEmptyStrings();
+ private static final String CPU_INFO_FILE = "/proc/cpuinfo";
+ private static final String MEM_INFO_FILE = "/proc/meminfo";
+
+ private static int getLogicalCpuCount() throws IOException {
+ String content = getCpuInfoContent();
+ return getLogicalCpuCountHelper(content);
+ }
+
+ private static int getPhysicalCpuCount(int logicalCpuCount) throws IOException {
+ String content = getCpuInfoContent();
+ return getPhysicalCpuCountHelper(logicalCpuCount, content);
+ }
+
+ private static double getMemoryInMb() throws IOException {
+ return getMemoryInMbHelper(MEM_INFO_FILE);
+ }
+
+ public static ResourceSet getLocalHostResources() {
+ try {
+ int logicalCpuCount = getLogicalCpuCount();
+ int physicalCpuCount = getPhysicalCpuCount(logicalCpuCount);
+ double ramMb = getMemoryInMb();
+
+ boolean hyperthreading = (logicalCpuCount != physicalCpuCount);
+ final double EFFECTIVE_CPUS_PER_HYPERTHREADED_CPU = 0.6;
+ return ResourceSet.create(
+ ramMb,
+ logicalCpuCount * (hyperthreading ? EFFECTIVE_CPUS_PER_HYPERTHREADED_CPU : 1.0),
+ 1.0,
+ Integer.MAX_VALUE);
+ } catch (IOException | IllegalArgumentException e) {
+ return null;
+ }
+ }
+
+ private static String getCpuInfoContent() throws IOException {
+ if (cpuInfoContent == null) {
+ cpuInfoContent = readContent(CPU_INFO_FILE);
+ }
+ return cpuInfoContent;
+ }
+
+ private static String readContent(String filename) throws IOException {
+ return Files.toString(new File(filename), Charset.defaultCharset());
+ }
+
+ /**
+ * For testing purposes only. Do not use it.
+ */
+ public static int getLogicalCpuCountHelper(String content) throws IOException {
+ int count = 0;
+ Iterable<String> lines = NEWLINE_SPLITTER.split(content);
+ for (String line : lines) {
+ if (line.startsWith("processor")) {
+ count++;
+ }
+ }
+ if (count == 0) {
+ throw new IllegalArgumentException("Can't get logical CPU count");
+ }
+ return count;
+ }
+
+ public static int getPhysicalCpuCountHelper(int logicalCpuCount, String content)
+ throws IOException {
+ // CPU count
+ Iterable<String> lines = NEWLINE_SPLITTER.split(content);
+ Set<String> uniq = new HashSet<>();
+ for (String line : lines) {
+ if (line.startsWith("physical id")) {
+ uniq.add(line);
+ }
+ }
+ int cpuCount = uniq.size();
+ if (cpuCount == 0) {
+ cpuCount = logicalCpuCount;
+ }
+
+ // core per CPU
+ uniq = new HashSet<>();
+ for (String line : lines) {
+ if (line.startsWith("core id")) {
+ uniq.add(line);
+ }
+ }
+ int coresPerCpu = uniq.size();
+ if (coresPerCpu == 0) {
+ coresPerCpu = 1;
+ }
+
+ return cpuCount * coresPerCpu;
+ }
+
+ public static double getMemoryInMbHelper(String memInfoFileName) throws IOException {
+ ProcMeminfoParser memInfo = new ProcMeminfoParser(memInfoFileName);
+ double ramMb = ProcMeminfoParser.kbToMb(memInfo.getTotalKb());
+ return ramMb;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/unix/NativePosixSystem.java b/src/main/java/com/google/devtools/build/lib/unix/NativePosixSystem.java
new file mode 100644
index 0000000..9834f24
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/unix/NativePosixSystem.java
@@ -0,0 +1,43 @@
+// Copyright 2016 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.unix;
+
+import com.google.devtools.build.lib.UnixJniLoader;
+
+import java.io.IOException;
+
+/**
+ * Utility methods for access to UNIX system calls not exposed by the Java
+ * SDK. Exception messages are selected to be consistent with those generated
+ * by the java.io package where appropriate--see package javadoc for details.
+ */
+public class NativePosixSystem {
+
+ private NativePosixSystem() {}
+
+ static {
+ if (!"0".equals(System.getProperty("io.bazel.UnixFileSystem"))) {
+ UnixJniLoader.loadJni();
+ }
+ }
+
+ /**
+ * Native wrapper around POSIX sysctlbyname(3) syscall.
+ *
+ * @param name the name for value to get from sysctl
+ * @throws IOException iff the sysctlbyname() syscall failed.
+ */
+ public static native long sysctlbynameGetLong(String name) throws IOException;
+}
