Add memory profiler.
This adds two dump command, bazel dump --rules and bazel dump --skylark_memory.
dump --rules outputs a summary of the count, action count, and memory consumption of each rule and aspect class.
dump --skylark_memory outputs a pprof-compatible file with all Skylark analysis allocations. Users can then use pprof as per normal to analyse their builds.
RELNOTES: Add memory profiler.
PiperOrigin-RevId: 172558600
diff --git a/src/test/java/com/google/devtools/build/lib/BUILD b/src/test/java/com/google/devtools/build/lib/BUILD
index bbca953..e16db89 100644
--- a/src/test/java/com/google/devtools/build/lib/BUILD
+++ b/src/test/java/com/google/devtools/build/lib/BUILD
@@ -45,6 +45,7 @@
"//src/test/java/com/google/devtools/build/lib/buildeventstream/transports:srcs",
"//src/test/java/com/google/devtools/build/lib/buildtool:srcs",
"//src/test/java/com/google/devtools/build/lib/profiler/callcounts:srcs",
+ "//src/test/java/com/google/devtools/build/lib/profiler/memory:srcs",
"//src/test/java/com/google/devtools/build/lib/rules/android:srcs",
"//src/test/java/com/google/devtools/build/lib/rules/apple:srcs",
"//src/test/java/com/google/devtools/build/lib/rules/config:srcs",
diff --git a/src/test/java/com/google/devtools/build/lib/packages/RuleClassTest.java b/src/test/java/com/google/devtools/build/lib/packages/RuleClassTest.java
index 6fad206..dd2c54c 100644
--- a/src/test/java/com/google/devtools/build/lib/packages/RuleClassTest.java
+++ b/src/test/java/com/google/devtools/build/lib/packages/RuleClassTest.java
@@ -862,6 +862,7 @@
: ruleDefinitionEnvironment.getTransitiveContentHashCode();
return new RuleClass(
name,
+ name,
/*isSkylark=*/ skylarkExecutable,
skylarkExecutable,
/*skylarkTestable=*/ false,
diff --git a/src/test/java/com/google/devtools/build/lib/profiler/memory/AllocationTrackerTest.java b/src/test/java/com/google/devtools/build/lib/profiler/memory/AllocationTrackerTest.java
new file mode 100644
index 0000000..9f35a70
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/profiler/memory/AllocationTrackerTest.java
@@ -0,0 +1,210 @@
+// Copyright 2017 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.profiler.memory;
+
+import static com.google.common.truth.Truth.assertThat;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import com.google.devtools.build.lib.events.Location;
+import com.google.devtools.build.lib.events.Location.LineAndColumn;
+import com.google.devtools.build.lib.packages.RuleClass;
+import com.google.devtools.build.lib.packages.RuleFunction;
+import com.google.devtools.build.lib.profiler.memory.AllocationTracker.RuleBytes;
+import com.google.devtools.build.lib.syntax.ASTNode;
+import com.google.devtools.build.lib.syntax.BaseFunction;
+import com.google.devtools.build.lib.syntax.Callstack;
+import com.google.devtools.build.lib.syntax.SyntaxTreeVisitor;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.perftools.profiles.ProfileProto.Function;
+import com.google.perftools.profiles.ProfileProto.Profile;
+import com.google.perftools.profiles.ProfileProto.Sample;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/** Tests for {@link AllocationTracker}. */
+@RunWith(JUnit4.class)
+public class AllocationTrackerTest {
+
+ private AllocationTracker allocationTracker;
+
+ static class TestNode extends ASTNode {
+ TestNode(String file, int line) {
+ setLocation(location(file, line));
+ }
+
+ @Override
+ public void prettyPrint(Appendable buffer, int indentLevel) throws IOException {}
+
+ @Override
+ public void accept(SyntaxTreeVisitor visitor) {}
+ }
+
+ static class TestFunction extends BaseFunction {
+ TestFunction(String file, String name, int line) {
+ super(name);
+ this.location = location(file, line);
+ }
+ }
+
+ static class TestRuleFunction extends TestFunction implements RuleFunction {
+
+ private final RuleClass ruleClass;
+
+ TestRuleFunction(String file, String name, int line) {
+ super(file, name, line);
+ this.ruleClass = mock(RuleClass.class);
+ when(ruleClass.getName()).thenReturn(name);
+ when(ruleClass.getKey()).thenReturn(name);
+ }
+
+ @Override
+ public RuleClass getRuleClass() {
+ return ruleClass;
+ }
+ }
+
+ @Before
+ public void setup() {
+ Callstack.setEnabled(true);
+ CurrentRuleTracker.setEnabled(true);
+ allocationTracker = new AllocationTracker(1, 0);
+ }
+
+ @After
+ public void tearDown() {
+ Callstack.resetStateForTest();
+ CurrentRuleTracker.setEnabled(false);
+ }
+
+ @Test
+ public void testSimpleMemoryProfile() {
+ Object allocation = new Object();
+ Callstack.push(new TestFunction("fileA", "fn", 120));
+ Callstack.push(new TestNode("fileA", 10));
+ allocationTracker.sampleAllocation(1, "", allocation, 12);
+ Callstack.pop();
+ Callstack.pop();
+
+ Map<String, RuleBytes> rules = new HashMap<>();
+ Map<String, RuleBytes> aspects = new HashMap<>();
+ allocationTracker.getRuleMemoryConsumption(rules, aspects);
+ assertThat(rules).isEmpty();
+ assertThat(aspects).isEmpty();
+
+ Profile profile = allocationTracker.buildMemoryProfile();
+ assertThat(profile.getSampleList()).hasSize(1);
+ assertThat(sampleToCallstack(profile, profile.getSample(0))).containsExactly("fileA:fn:10");
+ }
+
+ @Test
+ public void testLongerCallstack() {
+ Object allocation = new Object();
+ Callstack.push(new TestFunction("fileB", "fnB", 120));
+ Callstack.push(new TestNode("fileB", 10));
+ Callstack.push(new TestNode("fileB", 12));
+ Callstack.push(new TestNode("fileB", 14));
+ Callstack.push(new TestNode("fileB", 18));
+ Callstack.push(new TestFunction("fileA", "fnA", 120));
+ Callstack.push(new TestNode("fileA", 10));
+ allocationTracker.sampleAllocation(1, "", allocation, 12);
+ for (int i = 0; i < 7; ++i) {
+ Callstack.pop();
+ }
+
+ Profile profile = allocationTracker.buildMemoryProfile();
+ assertThat(profile.getSampleList()).hasSize(1);
+ assertThat(sampleToCallstack(profile, profile.getSample(0)))
+ .containsExactly("fileB:fnB:18", "fileA:fnA:10");
+ }
+
+ @Test
+ public void testConfiguredTargetsMemoryAllocation() {
+ RuleClass ruleClass = mock(RuleClass.class);
+ when(ruleClass.getName()).thenReturn("rule");
+ when(ruleClass.getKey()).thenReturn("rule");
+ CurrentRuleTracker.beginConfiguredTarget(ruleClass);
+ Object ruleAllocation0 = new Object();
+ Object ruleAllocation1 = new Object();
+ allocationTracker.sampleAllocation(1, "", ruleAllocation0, 10);
+ allocationTracker.sampleAllocation(1, "", ruleAllocation1, 20);
+ CurrentRuleTracker.endConfiguredTarget();
+
+ CurrentRuleTracker.beginConfiguredAspect(() -> "aspect");
+ Object aspectAllocation = new Object();
+ allocationTracker.sampleAllocation(1, "", aspectAllocation, 12);
+ CurrentRuleTracker.endConfiguredAspect();
+
+ Map<String, RuleBytes> rules = new HashMap<>();
+ Map<String, RuleBytes> aspects = new HashMap<>();
+ allocationTracker.getRuleMemoryConsumption(rules, aspects);
+ assertThat(rules).containsExactly("rule", new RuleBytes("rule").addBytes(30L));
+ assertThat(aspects).containsExactly("aspect", new RuleBytes("aspect").addBytes(12L));
+
+ Profile profile = allocationTracker.buildMemoryProfile();
+ assertThat(profile.getSampleList()).isEmpty(); // No callstacks
+ }
+
+ @Test
+ public void testLoadingPhaseRuleAllocations() {
+ Object allocation = new Object();
+ Callstack.push(new TestFunction("fileB", "fnB", 120));
+ Callstack.push(new TestNode("fileB", 18));
+ Callstack.push(new TestFunction("fileA", "fnA", 120));
+ Callstack.push(new TestNode("fileA", 10));
+ Callstack.push(new TestRuleFunction("<native>", "proto_library", -1));
+ allocationTracker.sampleAllocation(1, "", allocation, 128);
+ for (int i = 0; i < 5; ++i) {
+ Callstack.pop();
+ }
+
+ Map<String, RuleBytes> rules = new HashMap<>();
+ Map<String, RuleBytes> aspects = new HashMap<>();
+ allocationTracker.getRuleMemoryConsumption(rules, aspects);
+ assertThat(rules)
+ .containsExactly("proto_library", new RuleBytes("proto_library").addBytes(128L));
+ }
+
+ /** Formats a callstack as (file):(method name):(line) */
+ private List<String> sampleToCallstack(Profile profile, Sample sample) {
+ List<String> result = new ArrayList<>();
+ for (long locationId : sample.getLocationIdList()) {
+ com.google.perftools.profiles.ProfileProto.Location location =
+ profile.getLocation((int) locationId - 1);
+ assertThat(location.getLineList()).hasSize(1);
+ long functionId = location.getLine(0).getFunctionId();
+ long line = location.getLine(0).getLine();
+ Function function = profile.getFunction((int) functionId - 1);
+ long fileId = function.getFilename();
+ long methodId = function.getName();
+ String file = profile.getStringTable((int) fileId);
+ String method = profile.getStringTable((int) methodId);
+ result.add(String.format("%s:%s:%d", file, method, line));
+ }
+ return result;
+ }
+
+ private static Location location(String path, int line) {
+ return Location.fromPathAndStartColumn(
+ PathFragment.create(path), 0, 0, new LineAndColumn(line, 0));
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/profiler/memory/BUILD b/src/test/java/com/google/devtools/build/lib/profiler/memory/BUILD
new file mode 100644
index 0000000..3bfc630
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/profiler/memory/BUILD
@@ -0,0 +1,25 @@
+licenses(["notice"]) # Apache 2.0
+
+filegroup(
+ name = "srcs",
+ srcs = glob(["**"]),
+ visibility = ["//src/test/java/com/google/devtools/build/lib:__pkg__"],
+)
+
+java_test(
+ name = "AllocationTrackerTest",
+ srcs = ["AllocationTrackerTest.java"],
+ deps = [
+ "//src/main/java/com/google/devtools/build/lib:events",
+ "//src/main/java/com/google/devtools/build/lib:packages",
+ "//src/main/java/com/google/devtools/build/lib:syntax",
+ "//src/main/java/com/google/devtools/build/lib/profiler/memory:allocationtracker",
+ "//src/main/java/com/google/devtools/build/lib/profiler/memory:current_rule_tracker",
+ "//src/main/java/com/google/devtools/build/lib/vfs",
+ "//third_party:guava",
+ "//third_party:junit4",
+ "//third_party:mockito",
+ "//third_party:truth",
+ "//third_party/pprof:profile_java_proto",
+ ],
+)
diff --git a/src/test/java/com/google/devtools/build/lib/skylark/SkylarkRuleClassFunctionsTest.java b/src/test/java/com/google/devtools/build/lib/skylark/SkylarkRuleClassFunctionsTest.java
index 855162d..bbf3c5b 100644
--- a/src/test/java/com/google/devtools/build/lib/skylark/SkylarkRuleClassFunctionsTest.java
+++ b/src/test/java/com/google/devtools/build/lib/skylark/SkylarkRuleClassFunctionsTest.java
@@ -25,7 +25,7 @@
import com.google.devtools.build.lib.analysis.skylark.SkylarkAttr;
import com.google.devtools.build.lib.analysis.skylark.SkylarkAttr.Descriptor;
import com.google.devtools.build.lib.analysis.skylark.SkylarkFileType;
-import com.google.devtools.build.lib.analysis.skylark.SkylarkRuleClassFunctions.RuleFunction;
+import com.google.devtools.build.lib.analysis.skylark.SkylarkRuleClassFunctions.SkylarkRuleFunction;
import com.google.devtools.build.lib.analysis.skylark.SkylarkRuleContext;
import com.google.devtools.build.lib.cmdline.Label;
import com.google.devtools.build.lib.packages.AdvertisedProviderSet;
@@ -141,7 +141,7 @@
}
private RuleClass getRuleClass(String name) throws Exception {
- return ((RuleFunction) lookup(name)).getRuleClass();
+ return ((SkylarkRuleFunction) lookup(name)).getRuleClass();
}
private void registerDummyUserDefinedFunction() throws Exception {
@@ -605,7 +605,7 @@
@Test
public void testRuleImplementation() throws Exception {
evalAndExport("def impl(ctx): return None", "rule1 = rule(impl)");
- RuleClass c = ((RuleFunction) lookup("rule1")).getRuleClass();
+ RuleClass c = ((SkylarkRuleFunction) lookup("rule1")).getRuleClass();
assertThat(c.getConfiguredTargetFunction().getName()).isEqualTo("impl");
}
@@ -628,7 +628,7 @@
@Test
public void testRuleAddAttribute() throws Exception {
evalAndExport("def impl(ctx): return None", "r1 = rule(impl, attrs={'a1': attr.string()})");
- RuleClass c = ((RuleFunction) lookup("r1")).getRuleClass();
+ RuleClass c = ((SkylarkRuleFunction) lookup("r1")).getRuleClass();
assertThat(c.hasAttr("a1", Type.STRING)).isTrue();
}
@@ -659,8 +659,8 @@
"x = d",
"y = d",
"z = d");
- String dName = ((RuleFunction) lookup("d")).getRuleClass().getName();
- String fooName = ((RuleFunction) lookup("foo")).getRuleClass().getName();
+ String dName = ((SkylarkRuleFunction) lookup("d")).getRuleClass().getName();
+ String fooName = ((SkylarkRuleFunction) lookup("foo")).getRuleClass().getName();
assertThat(dName).isEqualTo("d");
assertThat(fooName).isEqualTo("d");
}
@@ -668,7 +668,7 @@
@Test
public void testOutputToGenfiles() throws Exception {
evalAndExport("def impl(ctx): pass", "r1 = rule(impl, output_to_genfiles=True)");
- RuleClass c = ((RuleFunction) lookup("r1")).getRuleClass();
+ RuleClass c = ((SkylarkRuleFunction) lookup("r1")).getRuleClass();
assertThat(c.hasBinaryOutput()).isFalse();
}
@@ -681,7 +681,7 @@
" 'a1': attr.label_list(allow_files=True),",
" 'a2': attr.int()",
"})");
- RuleClass c = ((RuleFunction) lookup("r1")).getRuleClass();
+ RuleClass c = ((SkylarkRuleFunction) lookup("r1")).getRuleClass();
assertThat(c.hasAttr("a1", BuildType.LABEL_LIST)).isTrue();
assertThat(c.hasAttr("a2", Type.INTEGER)).isTrue();
}
@@ -690,7 +690,7 @@
evalAndExport(
"def impl(ctx): return None",
"r1 = rule(impl, attrs = {'a1': attr.string(mandatory=True)})");
- RuleClass c = ((RuleFunction) lookup("r1")).getRuleClass();
+ RuleClass c = ((SkylarkRuleFunction) lookup("r1")).getRuleClass();
assertThat(c.getAttributeByName("a1").isMandatory()).isTrue();
}
@@ -699,7 +699,7 @@
evalAndExport(
"def impl(ctx): return None",
"r1 = rule(impl, outputs = {'a': 'a.txt'})");
- RuleClass c = ((RuleFunction) lookup("r1")).getRuleClass();
+ RuleClass c = ((SkylarkRuleFunction) lookup("r1")).getRuleClass();
ImplicitOutputsFunction function = c.getDefaultImplicitOutputsFunction();
assertThat(function.getImplicitOutputs(null)).containsExactly("a.txt");
}
@@ -775,7 +775,7 @@
"def impl(ctx): return None\n"
+ "r1 = rule(impl, attrs = {'a1': "
+ "attr.label(default = Label('//foo:foo'), allow_files=True)})");
- RuleClass c = ((RuleFunction) lookup("r1")).getRuleClass();
+ RuleClass c = ((SkylarkRuleFunction) lookup("r1")).getRuleClass();
Attribute a = c.getAttributeByName("a1");
assertThat(a.getDefaultValueForTesting()).isInstanceOf(Label.class);
assertThat(a.getDefaultValueForTesting().toString()).isEqualTo("//foo:foo");
@@ -786,7 +786,7 @@
evalAndExport(
"def impl(ctx): return None",
"r1 = rule(impl, attrs = {'a1': attr.int(default = 40+2)})");
- RuleClass c = ((RuleFunction) lookup("r1")).getRuleClass();
+ RuleClass c = ((SkylarkRuleFunction) lookup("r1")).getRuleClass();
Attribute a = c.getAttributeByName("a1");
assertThat(a.getDefaultValueForTesting()).isEqualTo(42);
}
@@ -801,7 +801,7 @@
@Test
public void testRuleInheritsBaseRuleAttributes() throws Exception {
evalAndExport("def impl(ctx): return None", "r1 = rule(impl)");
- RuleClass c = ((RuleFunction) lookup("r1")).getRuleClass();
+ RuleClass c = ((SkylarkRuleFunction) lookup("r1")).getRuleClass();
assertThat(c.hasAttr("tags", Type.STRING_LIST)).isTrue();
assertThat(c.hasAttr("visibility", BuildType.NODEP_LABEL_LIST)).isTrue();
assertThat(c.hasAttr("deprecation", Type.STRING)).isTrue();
@@ -1590,7 +1590,7 @@
scratch.file("test/BUILD", "toolchain_type(name = 'my_toolchain_type')");
evalAndExport(
"def impl(ctx): return None", "r1 = rule(impl, toolchains=['//test:my_toolchain_type'])");
- RuleClass c = ((RuleFunction) lookup("r1")).getRuleClass();
+ RuleClass c = ((SkylarkRuleFunction) lookup("r1")).getRuleClass();
assertThat(c.getRequiredToolchains()).containsExactly(makeLabel("//test:my_toolchain_type"));
}
