Remove aquery_differ for v1.
RELNOTES: None
PiperOrigin-RevId: 347378393
diff --git a/tools/BUILD b/tools/BUILD
index 803dc05..6df838e 100644
--- a/tools/BUILD
+++ b/tools/BUILD
@@ -89,7 +89,6 @@
tests = [
"//tools/android:all_windows_tests",
"//tools/aquery_differ:aquery_differ_test",
- "//tools/aquery_differ:aquery_differ_v2_test",
"//tools/bash:all_windows_tests",
"//tools/build_defs:all_windows_tests",
"//tools/cpp/runfiles:all_windows_tests",
diff --git a/tools/aquery_differ/BUILD b/tools/aquery_differ/BUILD
index 7f686bb..b58288a 100644
--- a/tools/aquery_differ/BUILD
+++ b/tools/aquery_differ/BUILD
@@ -46,7 +46,7 @@
srcs = ["aquery_differ.py"],
deps = [
":aquery_differ_resolvers",
- "//src/main/protobuf:analysis_py_proto",
+ "//src/main/protobuf:analysis_v2_py_proto",
"//third_party/py/abseil",
"//third_party/py/six",
],
@@ -57,38 +57,6 @@
srcs = ["aquery_differ_test.py"],
deps = [
":aquery_differ_main_lib",
- "//src/main/protobuf:analysis_py_proto",
- "//third_party/py/mock",
- "//third_party/py/six",
- ],
-)
-
-py_binary(
- name = "aquery_differ_v2",
- srcs = ["aquery_differ_v2.py"],
- python_version = "PY3",
- deps = [
- ":aquery_differ_main_lib_v2",
- "//third_party/py/six",
- ],
-)
-
-py_library(
- name = "aquery_differ_main_lib_v2",
- srcs = ["aquery_differ_v2.py"],
- deps = [
- ":aquery_differ_resolvers",
- "//src/main/protobuf:analysis_v2_py_proto",
- "//third_party/py/abseil",
- "//third_party/py/six",
- ],
-)
-
-py_test_alias(
- name = "aquery_differ_v2_test",
- srcs = ["aquery_differ_v2_test.py"],
- deps = [
- ":aquery_differ_main_lib_v2",
"//src/main/protobuf:analysis_v2_py_proto",
"//third_party/py/mock",
"//third_party/py/six",
diff --git a/tools/aquery_differ/aquery_differ.py b/tools/aquery_differ/aquery_differ.py
index 7776b98..f135c96 100644
--- a/tools/aquery_differ/aquery_differ.py
+++ b/tools/aquery_differ/aquery_differ.py
@@ -1,5 +1,5 @@
# Lint as: python2, python3
-# Copyright 2018 The Bazel Authors. All rights reserved.
+# Copyright 2020 The Bazel Authors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -14,17 +14,19 @@
# limitations under the License.
r"""Command line diffing tool that compares two bazel aquery invocations.
-This script compares the proto output of two bazel aquery invocations. For
-each set of output files of an action, it compares the command lines that
-generated the files.
+This script compares the proto or textproto output of two bazel aquery
+invocations. For each set of output files of an action, it compares the command
+lines that generated the files.
Example usage:
+
+1. Prepare 2 aquery output files:
bazel aquery //path/to:target_one --output=textproto > \
/path/to/output_one.textproto
bazel aquery //path/to:target_two --output=textproto > \
/path/to/output_two.textproto
-From a bazel repo:
+2. Run the differ from a bazel repo:
bazel run //tools/aquery_differ:aquery_differ -- \
--before=/path/to/output_one.textproto \
--after=/path/to/output_two.textproto \
@@ -45,8 +47,14 @@
from absl import flags
from six.moves import map
from google.protobuf import text_format
-from src.main.protobuf import analysis_pb2
+from src.main.protobuf import analysis_v2_pb2
from tools.aquery_differ.resolvers.dep_set_resolver import DepSetResolver
+from tools.aquery_differ.resolvers.path_fragment_resolver import PathFragmentResolver
+# pylint: disable=g-import-not-at-top
+# resource lib isn't available on Windows.
+if os.name != "nt":
+ import resource
+# pylint: enable=g-import-not-at-top
flags.DEFINE_string("before", None, "Aquery output before the change")
flags.DEFINE_string("after", None, "Aquery output after the change")
@@ -55,6 +63,9 @@
"The format of the aquery proto input. One of 'proto' and 'textproto.")
flags.DEFINE_multi_enum("attrs", ["cmdline"], ["inputs", "cmdline"],
"Attributes of the actions to be compared.")
+flags.DEFINE_integer(
+ "max_mem_alloc_mb", 3072,
+ "Amount of max memory available for aquery_differ, in MB.")
flags.mark_flag_as_required("before")
flags.mark_flag_as_required("after")
@@ -96,8 +107,12 @@
"\n\t%s\n%s\n") % (attr, "\n\t".join(output_files.split()), diff))
-def _map_artifact_id_to_path(artifacts):
- return {artifact.id: artifact.exec_path for artifact in artifacts}
+def _map_artifact_id_to_path(artifacts, path_fragments):
+ path_fragment_resolver = PathFragmentResolver(path_fragments)
+ return {
+ artifact.id: path_fragment_resolver.resolve(artifact.path_fragment_id)
+ for artifact in artifacts
+ }
def _map_action_index_to_output_files(actions, artifacts):
@@ -120,8 +135,9 @@
# output files -> input artifacts
-def _map_output_files_to_input_artifacts(
- action_graph_container, artifact_id_to_path, action_index_to_output_files):
+def _map_output_files_to_input_artifacts(action_graph_container,
+ artifact_id_to_path,
+ action_index_to_output_files):
"""Constructs a map from output files to input artifacts.
Args:
@@ -176,8 +192,10 @@
def _aquery_diff(before_proto, after_proto, attrs, before_file, after_file):
"""Returns differences between command lines that generate same outputs."""
found_difference = False
- artifacts_before = _map_artifact_id_to_path(before_proto.artifacts)
- artifacts_after = _map_artifact_id_to_path(after_proto.artifacts)
+ artifacts_before = _map_artifact_id_to_path(before_proto.artifacts,
+ before_proto.path_fragments)
+ artifacts_after = _map_artifact_id_to_path(after_proto.artifacts,
+ after_proto.path_fragments)
action_to_output_files_before = _map_action_index_to_output_files(
before_proto.actions, artifacts_before)
@@ -248,23 +266,38 @@
after_file = to_absolute_path(flags.FLAGS.after)
input_type = flags.FLAGS.input_type
attrs = flags.FLAGS.attrs
+ max_mem_alloc_mb = flags.FLAGS.max_mem_alloc_mb
- before_proto = analysis_pb2.ActionGraphContainer()
- after_proto = analysis_pb2.ActionGraphContainer()
- if input_type == "proto":
- with open(before_file, "rb") as f:
- before_proto.ParseFromString(f.read())
- with open(after_file, "rb") as f:
- after_proto.ParseFromString(f.read())
- else:
- with open(before_file, "r") as f:
- before_text = f.read()
- text_format.Merge(before_text, before_proto)
- with open(after_file, "r") as f:
- after_text = f.read()
- text_format.Merge(after_text, after_proto)
+ # resource lib isn't available on Windows.
+ if os.name != "nt":
+ max_heap_bytes = max_mem_alloc_mb * 1024 * 1024
+ resource.setrlimit(resource.RLIMIT_AS, (max_heap_bytes, max_heap_bytes))
- _aquery_diff(before_proto, after_proto, attrs, before_file, after_file)
+ before_proto = analysis_v2_pb2.ActionGraphContainer()
+ after_proto = analysis_v2_pb2.ActionGraphContainer()
+ try:
+ if input_type == "proto":
+ with open(before_file, "rb") as f:
+ before_proto.ParseFromString(f.read())
+ with open(after_file, "rb") as f:
+ after_proto.ParseFromString(f.read())
+ else:
+ with open(before_file, "r") as f:
+ before_text = f.read()
+ text_format.Merge(before_text, before_proto)
+ with open(after_file, "r") as f:
+ after_text = f.read()
+ text_format.Merge(after_text, after_proto)
+
+ _aquery_diff(before_proto, after_proto, attrs, before_file, after_file)
+ except MemoryError:
+ print(
+ "aquery_differ is known to cause OOM issue with large inputs. More details: b/154620006.",
+ file=sys.stderr)
+ print(
+ "Max mem space of {}MB exceeded".format(max_mem_alloc_mb),
+ file=sys.stderr)
+ sys.exit(1)
if __name__ == "__main__":
diff --git a/tools/aquery_differ/aquery_differ_test.py b/tools/aquery_differ/aquery_differ_test.py
index b5b8bec..a214890 100644
--- a/tools/aquery_differ/aquery_differ_test.py
+++ b/tools/aquery_differ/aquery_differ_test.py
@@ -14,47 +14,56 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+import os
import unittest
-
# Do not edit this line. Copybara replaces it with PY2 migration helper.
from third_party.py import mock
import six
+from src.main.protobuf import analysis_v2_pb2
+from tools.aquery_differ import aquery_differ
+
+# pylint: disable=g-import-not-at-top
if six.PY2:
from cStringIO import StringIO
else:
from io import StringIO
-
-from src.main.protobuf import analysis_pb2
-from tools.aquery_differ import aquery_differ
+# pylint: enable=g-import-not-at-top
-def make_aquery_output(actions, artifact_paths):
- action_graph = analysis_pb2.ActionGraphContainer()
+def make_aquery_output(action_objs, artifact_objs, path_fragment_objs):
+ action_graph = analysis_v2_pb2.ActionGraphContainer()
- for artifact_path in artifact_paths:
- next_id = len(action_graph.artifacts)
+ for path_fragment_obj in path_fragment_objs:
+ path_fragment = action_graph.path_fragments.add()
+ path_fragment.id = path_fragment_obj["id"]
+ path_fragment.label = path_fragment_obj["label"]
+ if "parent_id" in path_fragment_obj:
+ path_fragment.parent_id = path_fragment_obj["parent_id"]
+
+ for artifact_obj in artifact_objs:
artifact = action_graph.artifacts.add()
- artifact.id = str(next_id)
- artifact.exec_path = artifact_path
+ artifact.id = artifact_obj["id"]
+ artifact.path_fragment_id = artifact_obj["path_fragment_id"]
- for next_action in actions:
+ for action_obj in action_objs:
action = action_graph.actions.add()
- action.output_ids.extend(next_action["output_ids"])
- action.arguments.extend(next_action["arguments"])
+ action.output_ids.extend(action_obj["output_ids"])
+ action.arguments.extend(action_obj["arguments"])
- if "input_dep_set_ids" in next_action:
- action.input_dep_set_ids.extend(next_action["input_dep_set_ids"])
+ if "input_dep_set_ids" in action_obj:
+ action.input_dep_set_ids.extend(action_obj["input_dep_set_ids"])
return action_graph
-def make_aquery_output_with_dep_set(actions, artifact_paths, dep_sets):
- action_graph = make_aquery_output(actions, artifact_paths)
+def make_aquery_output_with_dep_set(action_objs, artifact_objs,
+ path_fragment_objs, dep_set_objs):
+ action_graph = make_aquery_output(action_objs, artifact_objs,
+ path_fragment_objs)
- for ds in dep_sets:
- next_id = len(action_graph.dep_set_of_files)
+ for ds in dep_set_objs:
dep_set = action_graph.dep_set_of_files.add()
- dep_set.id = str(next_id)
+ dep_set.id = ds["id"]
dep_set.direct_artifact_ids.extend(ds["direct_artifact_ids"])
dep_set.transitive_dep_set_ids.extend(ds["transitive_dep_set_ids"])
@@ -65,14 +74,47 @@
def test_no_difference(self):
action_graph = make_aquery_output(
- actions=[{
- "arguments": ["-a", "-b"],
- "output_ids": ["0", "1"]
+ action_objs=[
+ {
+ "arguments": ["-a", "-b"],
+ "output_ids": [1, 2]
+ },
+ {
+ "arguments": ["-c"],
+ "output_ids": [3]
+ },
+ ],
+ artifact_objs=[{
+ "id": 1,
+ "path_fragment_id": 2
}, {
- "arguments": ["-c"],
- "output_ids": ["2"]
+ "id": 2,
+ "path_fragment_id": 3
+ }, {
+ "id": 3,
+ "path_fragment_id": 4
}],
- artifact_paths=["exec/path/zero", "exec/path/one", "exec/path/two"])
+ path_fragment_objs=[
+ {
+ "id": 1,
+ "label": "root"
+ },
+ {
+ "id": 2,
+ "label": "foo",
+ "parent_id": 1
+ },
+ {
+ "id": 3,
+ "label": "bar",
+ "parent_id": 1
+ },
+ {
+ "id": 4,
+ "label": "baz",
+ "parent_id": 1
+ },
+ ])
mock_stdout = StringIO()
attrs = ["cmdline"]
with mock.patch("sys.stdout", mock_stdout):
@@ -82,21 +124,66 @@
def test_no_difference_different_output_files_order(self):
first = make_aquery_output(
- actions=[
+ action_objs=[
{
"arguments": ["-a", "-b"],
- "output_ids": ["0", "1"]
+ "output_ids": [1, 2]
},
],
- artifact_paths=["exec/path/zero", "exec/path/one"])
+ artifact_objs=[{
+ "id": 1,
+ "path_fragment_id": 2
+ }, {
+ "id": 2,
+ "path_fragment_id": 3
+ }],
+ path_fragment_objs=[
+ {
+ "id": 1,
+ "label": "root"
+ },
+ {
+ "id": 2,
+ "label": "foo",
+ "parent_id": 1
+ },
+ {
+ "id": 3,
+ "label": "bar",
+ "parent_id": 1
+ },
+ ])
+
second = make_aquery_output(
- actions=[
+ action_objs=[
{
"arguments": ["-a", "-b"],
- "output_ids": ["1", "0"]
+ "output_ids": [2, 1]
},
],
- artifact_paths=["exec/path/zero", "exec/path/one"])
+ artifact_objs=[{
+ "id": 1,
+ "path_fragment_id": 2
+ }, {
+ "id": 2,
+ "path_fragment_id": 3
+ }],
+ path_fragment_objs=[
+ {
+ "id": 1,
+ "label": "root"
+ },
+ {
+ "id": 2,
+ "label": "foo",
+ "parent_id": 1
+ },
+ {
+ "id": 3,
+ "label": "bar",
+ "parent_id": 1
+ },
+ ])
mock_stdout = StringIO()
attrs = ["cmdline"]
@@ -106,64 +193,78 @@
def test_first_has_extra_output_files(self):
first = make_aquery_output(
- actions=[
+ action_objs=[
{
"arguments": ["-a", "-b"],
- "output_ids": ["0", "1"]
+ "output_ids": [1, 2]
},
{
"arguments": ["-c"],
- "output_ids": ["2"]
+ "output_ids": [3]
},
],
- artifact_paths=["exec/path/zero", "exec/path/one", "exec/path/two"],
- )
+ artifact_objs=[{
+ "id": 1,
+ "path_fragment_id": 2
+ }, {
+ "id": 2,
+ "path_fragment_id": 3
+ }, {
+ "id": 3,
+ "path_fragment_id": 4
+ }],
+ path_fragment_objs=[
+ {
+ "id": 1,
+ "label": "root"
+ },
+ {
+ "id": 2,
+ "label": "foo",
+ "parent_id": 1
+ },
+ {
+ "id": 3,
+ "label": "bar",
+ "parent_id": 1
+ },
+ {
+ "id": 4,
+ "label": "baz",
+ "parent_id": 1
+ },
+ ])
second = make_aquery_output(
- actions=[
+ action_objs=[
{
"arguments": ["-a", "-b"],
- "output_ids": ["1", "0"]
+ "output_ids": [1, 2]
},
],
- artifact_paths=["exec/path/zero", "exec/path/one", "exec/path/two"],
- )
+ artifact_objs=[{
+ "id": 1,
+ "path_fragment_id": 2
+ }, {
+ "id": 2,
+ "path_fragment_id": 3
+ }],
+ path_fragment_objs=[{
+ "id": 1,
+ "label": "root"
+ }, {
+ "id": 2,
+ "label": "foo",
+ "parent_id": 1
+ }, {
+ "id": 3,
+ "label": "bar",
+ "parent_id": 1
+ }])
+ baz_path = os.path.join("root", "baz")
expected_error = ("Aquery output 'before' change contains an action "
"that generates the following outputs that aquery "
- "output 'after' change doesn't:\nexec/path/two\n\n")
- mock_stdout = StringIO()
- attrs = ["cmdline"]
- with mock.patch("sys.stdout", mock_stdout):
- aquery_differ._aquery_diff(first, second, attrs, "before", "after")
- self.assertEqual(mock_stdout.getvalue(), expected_error)
-
- def test_second_has_extra_output_files(self):
- first = make_aquery_output(
- actions=[
- {
- "arguments": ["-a", "-b"],
- "output_ids": ["0", "1"]
- },
- ],
- artifact_paths=["exec/path/zero", "exec/path/one", "exec/path/two"],
- )
- second = make_aquery_output(
- actions=[
- {
- "arguments": ["-a", "-b"],
- "output_ids": ["0", "1"]
- },
- {
- "arguments": ["-c"],
- "output_ids": ["2"]
- },
- ],
- artifact_paths=["exec/path/zero", "exec/path/one", "exec/path/two"],
- )
-
- expected_error = ("Aquery output 'after' change contains an action that"
- " generates the following outputs that aquery"
- " output 'before' change doesn't:\nexec/path/two\n\n")
+ "output 'after' change doesn't:\n{}\n\n".format(baz_path))
mock_stdout = StringIO()
attrs = ["cmdline"]
with mock.patch("sys.stdout", mock_stdout):
@@ -172,41 +273,103 @@
def test_different_command_lines(self):
first = make_aquery_output(
- actions=[
+ action_objs=[
{
"arguments": ["-a", "-d"],
- "output_ids": ["0", "1"]
+ "output_ids": [1, 2]
},
{
"arguments": ["-c"],
- "output_ids": ["2"]
+ "output_ids": [3]
},
],
- artifact_paths=["exec/path/zero", "exec/path/one", "exec/path/two"],
- )
+ artifact_objs=[{
+ "id": 1,
+ "path_fragment_id": 2
+ }, {
+ "id": 2,
+ "path_fragment_id": 3
+ }, {
+ "id": 3,
+ "path_fragment_id": 4
+ }],
+ path_fragment_objs=[
+ {
+ "id": 1,
+ "label": "root"
+ },
+ {
+ "id": 2,
+ "label": "foo",
+ "parent_id": 1
+ },
+ {
+ "id": 3,
+ "label": "bar",
+ "parent_id": 1
+ },
+ {
+ "id": 4,
+ "label": "baz",
+ "parent_id": 1
+ },
+ ])
second = make_aquery_output(
- actions=[
+ action_objs=[
{
"arguments": ["-a", "-b"],
- "output_ids": ["0", "1"]
+ "output_ids": [1, 2]
},
{
"arguments": ["-c", "-d"],
- "output_ids": ["2"]
+ "output_ids": [3]
},
],
- artifact_paths=["exec/path/zero", "exec/path/one", "exec/path/two"],
- )
+ artifact_objs=[{
+ "id": 1,
+ "path_fragment_id": 2
+ }, {
+ "id": 2,
+ "path_fragment_id": 3
+ }, {
+ "id": 3,
+ "path_fragment_id": 4
+ }],
+ path_fragment_objs=[
+ {
+ "id": 1,
+ "label": "root"
+ },
+ {
+ "id": 2,
+ "label": "foo",
+ "parent_id": 1
+ },
+ {
+ "id": 3,
+ "label": "bar",
+ "parent_id": 1
+ },
+ {
+ "id": 4,
+ "label": "baz",
+ "parent_id": 1
+ },
+ ])
+
+ foo_path = os.path.join("root", "foo")
+ bar_path = os.path.join("root", "bar")
+ baz_path = os.path.join("root", "baz")
expected_error_one = "\n".join([
"Difference in the action that generates the following output(s):",
- "\texec/path/two", "--- before", "+++ after", "@@ -1 +1,2 @@", " -c",
- "+-d", "\n"
+ "\t{}".format(baz_path), "--- before", "+++ after", "@@ -1 +1,2 @@",
+ " -c", "+-d", "\n"
])
expected_error_two = "\n".join([
"Difference in the action that generates the following output(s):",
- "\texec/path/one", "\texec/path/zero", "--- before", "+++ after",
- "@@ -1,2 +1,2 @@", " -a", "--d", "+-b", "\n"
+ "\t{}".format(bar_path), "\t{}".format(foo_path), "--- before",
+ "+++ after", "@@ -1,2 +1,2 @@", " -a", "--d", "+-b", "\n"
])
attrs = ["cmdline"]
@@ -218,37 +381,87 @@
def test_different_inputs(self):
first = make_aquery_output_with_dep_set(
- actions=[{
+ action_objs=[{
"arguments": [],
- "output_ids": ["0", "1"],
- "input_dep_set_ids": ["1"]
+ "output_ids": [1, 2],
+ "input_dep_set_ids": [2]
}],
- artifact_paths=["exec/path/zero", "exec/path/one"],
- dep_sets=[{
- "transitive_dep_set_ids": [],
- "direct_artifact_ids": ["0"]
+ artifact_objs=[{
+ "id": 1,
+ "path_fragment_id": 2
}, {
- "transitive_dep_set_ids": ["0"],
- "direct_artifact_ids": ["1"]
- }])
- second = make_aquery_output_with_dep_set(
- actions=[
+ "id": 2,
+ "path_fragment_id": 3
+ }],
+ path_fragment_objs=[
{
- "arguments": [],
- "output_ids": ["0", "1"],
- "input_dep_set_ids": ["0"]
+ "id": 1,
+ "label": "root"
+ },
+ {
+ "id": 2,
+ "label": "foo",
+ "parent_id": 1
+ },
+ {
+ "id": 3,
+ "label": "bar",
+ "parent_id": 1
},
],
- artifact_paths=["exec/path/zero", "exec/path/one"],
- dep_sets=[{
+ dep_set_objs=[{
+ "id": 1,
"transitive_dep_set_ids": [],
- "direct_artifact_ids": ["0"]
+ "direct_artifact_ids": [1]
+ }, {
+ "id": 2,
+ "transitive_dep_set_ids": [1],
+ "direct_artifact_ids": [2]
+ }])
+ second = make_aquery_output_with_dep_set(
+ action_objs=[
+ {
+ "arguments": [],
+ "output_ids": [1, 2],
+ "input_dep_set_ids": [1]
+ },
+ ],
+ artifact_objs=[{
+ "id": 1,
+ "path_fragment_id": 2
+ }, {
+ "id": 2,
+ "path_fragment_id": 3
+ }],
+ path_fragment_objs=[
+ {
+ "id": 1,
+ "label": "root"
+ },
+ {
+ "id": 2,
+ "label": "foo",
+ "parent_id": 1
+ },
+ {
+ "id": 3,
+ "label": "bar",
+ "parent_id": 1
+ },
+ ],
+ dep_set_objs=[{
+ "id": 1,
+ "transitive_dep_set_ids": [],
+ "direct_artifact_ids": [1]
}])
+ foo_path = os.path.join("root", "foo")
+ bar_path = os.path.join("root", "bar")
expected_error_one = "\n".join([
"Difference in the action that generates the following output(s):",
- "\texec/path/one", "\texec/path/zero", "--- before", "+++ after",
- "@@ -1,2 +1 @@", "-exec/path/one", " exec/path/zero", "\n"
+ "\t{}".format(bar_path), "\t{}".format(foo_path), "--- before",
+ "+++ after", "@@ -1,2 +1 @@", "-{}".format(bar_path),
+ " {}".format(foo_path), "\n"
])
attrs = ["inputs"]
diff --git a/tools/aquery_differ/aquery_differ_v2.py b/tools/aquery_differ/aquery_differ_v2.py
deleted file mode 100644
index b3fd92e..0000000
--- a/tools/aquery_differ/aquery_differ_v2.py
+++ /dev/null
@@ -1,306 +0,0 @@
-# Lint as: python2, python3
-# Copyright 2020 The Bazel Authors. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-r"""Command line diffing tool that compares two bazel aquery invocations.
-
-This makes use of aquery v2 proto format.
-
-This script compares the proto or textproto output of two bazel aquery
-invocations. For each set of output files of an action, it compares the command
-lines that generated the files.
-
-Example usage:
-
-1. Prepare 2 aquery output files:
-bazel aquery //path/to:target_one --output=textproto > \
- /path/to/output_one.textproto
-bazel aquery //path/to:target_two --output=textproto > \
- /path/to/output_two.textproto
-
-2. Run the differ from a bazel repo:
-bazel run //tools/aquery_differ:aquery_differ_v2 -- \
---before=/path/to/output_one.textproto \
---after=/path/to/output_two.textproto \
---input_type=textproto \
---attrs=cmdline \
---attrs=inputs
-"""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-import difflib
-import os
-import sys
-
-# Do not edit this line. Copybara replaces it with PY2 migration helper.
-from absl import app
-from absl import flags
-from six.moves import map
-from google.protobuf import text_format
-from src.main.protobuf import analysis_v2_pb2
-from tools.aquery_differ.resolvers.dep_set_resolver import DepSetResolver
-from tools.aquery_differ.resolvers.path_fragment_resolver import PathFragmentResolver
-# pylint: disable=g-import-not-at-top
-# resource lib isn't available on Windows.
-if os.name != "nt":
- import resource
-# pylint: enable=g-import-not-at-top
-
-flags.DEFINE_string("before", None, "Aquery output before the change")
-flags.DEFINE_string("after", None, "Aquery output after the change")
-flags.DEFINE_enum(
- "input_type", "proto", ["proto", "textproto"],
- "The format of the aquery proto input. One of 'proto' and 'textproto.")
-flags.DEFINE_multi_enum("attrs", ["cmdline"], ["inputs", "cmdline"],
- "Attributes of the actions to be compared.")
-flags.DEFINE_integer(
- "max_mem_alloc_mb", 3072,
- "Amount of max memory available for aquery_differ, in MB.")
-flags.mark_flag_as_required("before")
-flags.mark_flag_as_required("after")
-
-WHITE = "\033[37m%s\033[0m"
-CYAN = "\033[36m%s\033[0m"
-RED = "\033[31m%s\033[0m"
-GREEN = "\033[32m%s\033[0m"
-
-
-def _colorize(line):
- """Add color to the input string."""
- if not sys.stdout.isatty():
- return line
-
- if line.startswith("+++") or line.startswith("---"):
- return WHITE % line
-
- if line.startswith("@@"):
- return CYAN % line
-
- if line.startswith("+"):
- return GREEN % line
-
- if line.startswith("-"):
- return RED % line
-
- return line
-
-
-def _print_diff(output_files, before_val, after_val, attr, before_file,
- after_file):
- diff = "\n".join(
- map(_colorize, [
- s.strip("\n") for s in difflib.unified_diff(before_val, after_val,
- before_file, after_file)
- ]))
- print(("[%s]\n"
- "Difference in the action that generates the following output(s):"
- "\n\t%s\n%s\n") % (attr, "\n\t".join(output_files.split()), diff))
-
-
-def _map_artifact_id_to_path(artifacts, path_fragments):
- path_fragment_resolver = PathFragmentResolver(path_fragments)
- return {
- artifact.id: path_fragment_resolver.resolve(artifact.path_fragment_id)
- for artifact in artifacts
- }
-
-
-def _map_action_index_to_output_files(actions, artifacts):
- """Constructs a map from action index to output files.
-
- Args:
- actions: a list of actions from the action graph container
- artifacts: a map {artifact_id: artifact path}
-
- Returns:
- A map from action index (in action graph container) to a string of
- concatenated output artifacts paths.
- """
- action_index_to_output_files = {}
- for i, action in enumerate(actions):
- output_files = " ".join(
- sorted([artifacts[output_id] for output_id in action.output_ids]))
- action_index_to_output_files[i] = output_files
- return action_index_to_output_files
-
-
-# output files -> input artifacts
-def _map_output_files_to_input_artifacts(action_graph_container,
- artifact_id_to_path,
- action_index_to_output_files):
- """Constructs a map from output files to input artifacts.
-
- Args:
- action_graph_container: the full action graph container object
- artifact_id_to_path: a map {artifact_id: artifact path}
- action_index_to_output_files: a map from action index (in action graph
- container) to a string of concatenated output artifacts paths.
-
- Returns:
- A map from output files (string of concatenated output artifacts paths) to a
- list of input artifacts.
- """
- actions = action_graph_container.actions
- dep_set_of_files = action_graph_container.dep_set_of_files
- id_to_dep_set = {dep_set.id: dep_set for dep_set in dep_set_of_files}
- dep_set_resolver = DepSetResolver(dep_set_of_files, artifact_id_to_path)
-
- output_files_to_input_artifacts = {}
- for i, action in enumerate(actions):
- input_artifacts = []
-
- for dep_set_id in action.input_dep_set_ids:
- input_artifacts.extend(
- dep_set_resolver.resolve(id_to_dep_set[dep_set_id]))
-
- output_files_to_input_artifacts[action_index_to_output_files[i]] = list(
- sorted(input_artifacts))
-
- return output_files_to_input_artifacts
-
-
-# output files -> command line
-def _map_output_files_to_command_line(actions, action_index_to_output_files):
- """Constructs a map from output files to command line.
-
- Args:
- actions: a list of actions from the action graph container
- action_index_to_output_files: a map from action index (in action graph
- container) to a string of concatenated output artifacts paths.
-
- Returns:
- A map from output files (string of concatenated output artifacts paths)
- to the command line (a list of arguments).
- """
- output_files_to_command_line = {}
- for i, action in enumerate(actions):
- output_files_to_command_line[
- action_index_to_output_files[i]] = action.arguments
- return output_files_to_command_line
-
-
-def _aquery_diff(before_proto, after_proto, attrs, before_file, after_file):
- """Returns differences between command lines that generate same outputs."""
- found_difference = False
- artifacts_before = _map_artifact_id_to_path(before_proto.artifacts,
- before_proto.path_fragments)
- artifacts_after = _map_artifact_id_to_path(after_proto.artifacts,
- after_proto.path_fragments)
-
- action_to_output_files_before = _map_action_index_to_output_files(
- before_proto.actions, artifacts_before)
- action_to_output_files_after = _map_action_index_to_output_files(
- after_proto.actions, artifacts_after)
-
- # There's a 1-to-1 mapping between action and outputs
- output_files_before = set(action_to_output_files_before.values())
- output_files_after = set(action_to_output_files_after.values())
-
- before_after_diff = output_files_before - output_files_after
- after_before_diff = output_files_after - output_files_before
-
- if before_after_diff:
- print(("Aquery output 'before' change contains an action that generates "
- "the following outputs that aquery output 'after' change doesn't:"
- "\n%s\n") % "\n".join(before_after_diff))
- found_difference = True
- if after_before_diff:
- print(("Aquery output 'after' change contains an action that generates "
- "the following outputs that aquery output 'before' change doesn't:"
- "\n%s\n") % "\n".join(after_before_diff))
- found_difference = True
-
- if "cmdline" in attrs:
- output_to_command_line_before = _map_output_files_to_command_line(
- before_proto.actions, action_to_output_files_before)
- output_to_command_line_after = _map_output_files_to_command_line(
- after_proto.actions, action_to_output_files_after)
- for output_files in output_to_command_line_before:
- arguments = output_to_command_line_before[output_files]
- after_arguments = output_to_command_line_after.get(output_files, None)
- if after_arguments and arguments != after_arguments:
- _print_diff(output_files, arguments, after_arguments, "cmdline",
- before_file, after_file)
- found_difference = True
-
- if "inputs" in attrs:
- output_to_input_files_before = _map_output_files_to_input_artifacts(
- before_proto, artifacts_before, action_to_output_files_before)
- output_to_input_files_after = _map_output_files_to_input_artifacts(
- after_proto, artifacts_after, action_to_output_files_after)
- for output_files in output_to_input_files_before:
- before_inputs = output_to_input_files_before[output_files]
- after_inputs = output_to_input_files_after.get(output_files, None)
- if after_inputs and before_inputs != after_inputs:
- _print_diff(output_files, before_inputs, after_inputs, "inputs",
- before_file, after_file)
- found_difference = True
-
- if not found_difference:
- print("No difference")
-
-
-def to_absolute_path(path):
- path = os.path.expanduser(path)
- if os.path.isabs(path):
- return path
- else:
- if "BUILD_WORKING_DIRECTORY" in os.environ:
- return os.path.join(os.environ["BUILD_WORKING_DIRECTORY"], path)
- else:
- return path
-
-
-def main(unused_argv):
- before_file = to_absolute_path(flags.FLAGS.before)
- after_file = to_absolute_path(flags.FLAGS.after)
- input_type = flags.FLAGS.input_type
- attrs = flags.FLAGS.attrs
- max_mem_alloc_mb = flags.FLAGS.max_mem_alloc_mb
-
- # resource lib isn't available on Windows.
- if os.name != "nt":
- max_heap_bytes = max_mem_alloc_mb * 1024 * 1024
- resource.setrlimit(resource.RLIMIT_AS, (max_heap_bytes, max_heap_bytes))
-
- before_proto = analysis_v2_pb2.ActionGraphContainer()
- after_proto = analysis_v2_pb2.ActionGraphContainer()
- try:
- if input_type == "proto":
- with open(before_file, "rb") as f:
- before_proto.ParseFromString(f.read())
- with open(after_file, "rb") as f:
- after_proto.ParseFromString(f.read())
- else:
- with open(before_file, "r") as f:
- before_text = f.read()
- text_format.Merge(before_text, before_proto)
- with open(after_file, "r") as f:
- after_text = f.read()
- text_format.Merge(after_text, after_proto)
-
- _aquery_diff(before_proto, after_proto, attrs, before_file, after_file)
- except MemoryError:
- print(
- "aquery_differ is known to cause OOM issue with large inputs. More details: b/154620006.",
- file=sys.stderr)
- print(
- "Max mem space of {}MB exceeded".format(max_mem_alloc_mb),
- file=sys.stderr)
- sys.exit(1)
-
-
-if __name__ == "__main__":
- app.run(main)
diff --git a/tools/aquery_differ/aquery_differ_v2_test.py b/tools/aquery_differ/aquery_differ_v2_test.py
deleted file mode 100644
index 6a92b3b..0000000
--- a/tools/aquery_differ/aquery_differ_v2_test.py
+++ /dev/null
@@ -1,472 +0,0 @@
-# Lint as: python2, python3
-# pylint: disable=g-direct-third-party-import
-# Copyright 2018 The Bazel Authors. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import os
-import unittest
-# Do not edit this line. Copybara replaces it with PY2 migration helper.
-from third_party.py import mock
-import six
-from src.main.protobuf import analysis_v2_pb2
-from tools.aquery_differ import aquery_differ_v2 as aquery_differ
-if six.PY2:
- from cStringIO import StringIO
-else:
- from io import StringIO
-
-
-def make_aquery_output(action_objs, artifact_objs, path_fragment_objs):
- action_graph = analysis_v2_pb2.ActionGraphContainer()
-
- for path_fragment_obj in path_fragment_objs:
- path_fragment = action_graph.path_fragments.add()
- path_fragment.id = path_fragment_obj["id"]
- path_fragment.label = path_fragment_obj["label"]
- if "parent_id" in path_fragment_obj:
- path_fragment.parent_id = path_fragment_obj["parent_id"]
-
- for artifact_obj in artifact_objs:
- artifact = action_graph.artifacts.add()
- artifact.id = artifact_obj["id"]
- artifact.path_fragment_id = artifact_obj["path_fragment_id"]
-
- for action_obj in action_objs:
- action = action_graph.actions.add()
- action.output_ids.extend(action_obj["output_ids"])
- action.arguments.extend(action_obj["arguments"])
-
- if "input_dep_set_ids" in action_obj:
- action.input_dep_set_ids.extend(action_obj["input_dep_set_ids"])
-
- return action_graph
-
-
-def make_aquery_output_with_dep_set(action_objs, artifact_objs,
- path_fragment_objs, dep_set_objs):
- action_graph = make_aquery_output(action_objs, artifact_objs,
- path_fragment_objs)
-
- for ds in dep_set_objs:
- dep_set = action_graph.dep_set_of_files.add()
- dep_set.id = ds["id"]
- dep_set.direct_artifact_ids.extend(ds["direct_artifact_ids"])
- dep_set.transitive_dep_set_ids.extend(ds["transitive_dep_set_ids"])
-
- return action_graph
-
-
-class CmdLineDifferTest(unittest.TestCase):
-
- def test_no_difference(self):
- action_graph = make_aquery_output(
- action_objs=[
- {
- "arguments": ["-a", "-b"],
- "output_ids": [1, 2]
- },
- {
- "arguments": ["-c"],
- "output_ids": [3]
- },
- ],
- artifact_objs=[{
- "id": 1,
- "path_fragment_id": 2
- }, {
- "id": 2,
- "path_fragment_id": 3
- }, {
- "id": 3,
- "path_fragment_id": 4
- }],
- path_fragment_objs=[
- {
- "id": 1,
- "label": "root"
- },
- {
- "id": 2,
- "label": "foo",
- "parent_id": 1
- },
- {
- "id": 3,
- "label": "bar",
- "parent_id": 1
- },
- {
- "id": 4,
- "label": "baz",
- "parent_id": 1
- },
- ])
- mock_stdout = StringIO()
- attrs = ["cmdline"]
- with mock.patch("sys.stdout", mock_stdout):
- aquery_differ._aquery_diff(action_graph, action_graph, attrs, "before",
- "after")
- self.assertEqual(mock_stdout.getvalue(), "No difference\n")
-
- def test_no_difference_different_output_files_order(self):
- first = make_aquery_output(
- action_objs=[
- {
- "arguments": ["-a", "-b"],
- "output_ids": [1, 2]
- },
- ],
- artifact_objs=[{
- "id": 1,
- "path_fragment_id": 2
- }, {
- "id": 2,
- "path_fragment_id": 3
- }],
- path_fragment_objs=[
- {
- "id": 1,
- "label": "root"
- },
- {
- "id": 2,
- "label": "foo",
- "parent_id": 1
- },
- {
- "id": 3,
- "label": "bar",
- "parent_id": 1
- },
- ])
-
- second = make_aquery_output(
- action_objs=[
- {
- "arguments": ["-a", "-b"],
- "output_ids": [2, 1]
- },
- ],
- artifact_objs=[{
- "id": 1,
- "path_fragment_id": 2
- }, {
- "id": 2,
- "path_fragment_id": 3
- }],
- path_fragment_objs=[
- {
- "id": 1,
- "label": "root"
- },
- {
- "id": 2,
- "label": "foo",
- "parent_id": 1
- },
- {
- "id": 3,
- "label": "bar",
- "parent_id": 1
- },
- ])
-
- mock_stdout = StringIO()
- attrs = ["cmdline"]
- with mock.patch("sys.stdout", mock_stdout):
- aquery_differ._aquery_diff(first, second, attrs, "before", "after")
- self.assertEqual(mock_stdout.getvalue(), "No difference\n")
-
- def test_first_has_extra_output_files(self):
- first = make_aquery_output(
- action_objs=[
- {
- "arguments": ["-a", "-b"],
- "output_ids": [1, 2]
- },
- {
- "arguments": ["-c"],
- "output_ids": [3]
- },
- ],
- artifact_objs=[{
- "id": 1,
- "path_fragment_id": 2
- }, {
- "id": 2,
- "path_fragment_id": 3
- }, {
- "id": 3,
- "path_fragment_id": 4
- }],
- path_fragment_objs=[
- {
- "id": 1,
- "label": "root"
- },
- {
- "id": 2,
- "label": "foo",
- "parent_id": 1
- },
- {
- "id": 3,
- "label": "bar",
- "parent_id": 1
- },
- {
- "id": 4,
- "label": "baz",
- "parent_id": 1
- },
- ])
- second = make_aquery_output(
- action_objs=[
- {
- "arguments": ["-a", "-b"],
- "output_ids": [1, 2]
- },
- ],
- artifact_objs=[{
- "id": 1,
- "path_fragment_id": 2
- }, {
- "id": 2,
- "path_fragment_id": 3
- }],
- path_fragment_objs=[{
- "id": 1,
- "label": "root"
- }, {
- "id": 2,
- "label": "foo",
- "parent_id": 1
- }, {
- "id": 3,
- "label": "bar",
- "parent_id": 1
- }])
-
- baz_path = os.path.join("root", "baz")
- expected_error = ("Aquery output 'before' change contains an action "
- "that generates the following outputs that aquery "
- "output 'after' change doesn't:\n{}\n\n".format(baz_path))
- mock_stdout = StringIO()
- attrs = ["cmdline"]
- with mock.patch("sys.stdout", mock_stdout):
- aquery_differ._aquery_diff(first, second, attrs, "before", "after")
- self.assertEqual(mock_stdout.getvalue(), expected_error)
-
- def test_different_command_lines(self):
- first = make_aquery_output(
- action_objs=[
- {
- "arguments": ["-a", "-d"],
- "output_ids": [1, 2]
- },
- {
- "arguments": ["-c"],
- "output_ids": [3]
- },
- ],
- artifact_objs=[{
- "id": 1,
- "path_fragment_id": 2
- }, {
- "id": 2,
- "path_fragment_id": 3
- }, {
- "id": 3,
- "path_fragment_id": 4
- }],
- path_fragment_objs=[
- {
- "id": 1,
- "label": "root"
- },
- {
- "id": 2,
- "label": "foo",
- "parent_id": 1
- },
- {
- "id": 3,
- "label": "bar",
- "parent_id": 1
- },
- {
- "id": 4,
- "label": "baz",
- "parent_id": 1
- },
- ])
- second = make_aquery_output(
- action_objs=[
- {
- "arguments": ["-a", "-b"],
- "output_ids": [1, 2]
- },
- {
- "arguments": ["-c", "-d"],
- "output_ids": [3]
- },
- ],
- artifact_objs=[{
- "id": 1,
- "path_fragment_id": 2
- }, {
- "id": 2,
- "path_fragment_id": 3
- }, {
- "id": 3,
- "path_fragment_id": 4
- }],
- path_fragment_objs=[
- {
- "id": 1,
- "label": "root"
- },
- {
- "id": 2,
- "label": "foo",
- "parent_id": 1
- },
- {
- "id": 3,
- "label": "bar",
- "parent_id": 1
- },
- {
- "id": 4,
- "label": "baz",
- "parent_id": 1
- },
- ])
-
- foo_path = os.path.join("root", "foo")
- bar_path = os.path.join("root", "bar")
- baz_path = os.path.join("root", "baz")
-
- expected_error_one = "\n".join([
- "Difference in the action that generates the following output(s):",
- "\t{}".format(baz_path), "--- before", "+++ after", "@@ -1 +1,2 @@",
- " -c", "+-d", "\n"
- ])
- expected_error_two = "\n".join([
- "Difference in the action that generates the following output(s):",
- "\t{}".format(bar_path), "\t{}".format(foo_path), "--- before",
- "+++ after", "@@ -1,2 +1,2 @@", " -a", "--d", "+-b", "\n"
- ])
- attrs = ["cmdline"]
-
- mock_stdout = StringIO()
- with mock.patch("sys.stdout", mock_stdout):
- aquery_differ._aquery_diff(first, second, attrs, "before", "after")
- self.assertIn(expected_error_one, mock_stdout.getvalue())
- self.assertIn(expected_error_two, mock_stdout.getvalue())
-
- def test_different_inputs(self):
- first = make_aquery_output_with_dep_set(
- action_objs=[{
- "arguments": [],
- "output_ids": [1, 2],
- "input_dep_set_ids": [2]
- }],
- artifact_objs=[{
- "id": 1,
- "path_fragment_id": 2
- }, {
- "id": 2,
- "path_fragment_id": 3
- }],
- path_fragment_objs=[
- {
- "id": 1,
- "label": "root"
- },
- {
- "id": 2,
- "label": "foo",
- "parent_id": 1
- },
- {
- "id": 3,
- "label": "bar",
- "parent_id": 1
- },
- ],
- dep_set_objs=[{
- "id": 1,
- "transitive_dep_set_ids": [],
- "direct_artifact_ids": [1]
- }, {
- "id": 2,
- "transitive_dep_set_ids": [1],
- "direct_artifact_ids": [2]
- }])
- second = make_aquery_output_with_dep_set(
- action_objs=[
- {
- "arguments": [],
- "output_ids": [1, 2],
- "input_dep_set_ids": [1]
- },
- ],
- artifact_objs=[{
- "id": 1,
- "path_fragment_id": 2
- }, {
- "id": 2,
- "path_fragment_id": 3
- }],
- path_fragment_objs=[
- {
- "id": 1,
- "label": "root"
- },
- {
- "id": 2,
- "label": "foo",
- "parent_id": 1
- },
- {
- "id": 3,
- "label": "bar",
- "parent_id": 1
- },
- ],
- dep_set_objs=[{
- "id": 1,
- "transitive_dep_set_ids": [],
- "direct_artifact_ids": [1]
- }])
-
- foo_path = os.path.join("root", "foo")
- bar_path = os.path.join("root", "bar")
- expected_error_one = "\n".join([
- "Difference in the action that generates the following output(s):",
- "\t{}".format(bar_path), "\t{}".format(foo_path), "--- before",
- "+++ after", "@@ -1,2 +1 @@", "-{}".format(bar_path),
- " {}".format(foo_path), "\n"
- ])
- attrs = ["inputs"]
-
- mock_stdout = StringIO()
- with mock.patch("sys.stdout", mock_stdout):
- aquery_differ._aquery_diff(first, second, attrs, "before", "after")
- self.assertIn(expected_error_one, mock_stdout.getvalue())
-
-
-if __name__ == "__main__":
- unittest.main()
