Remove bazel-diff annotation if failed (#1644)

Belongs to #1637
diff --git a/buildkite/bazelci.py b/buildkite/bazelci.py
index 7cb10b6..0b1f31d 100755
--- a/buildkite/bazelci.py
+++ b/buildkite/bazelci.py
@@ -638,6 +638,8 @@
 # TODO: change to USE_BAZEL_DIFF once the feature has been tested in QA
 USE_BAZEL_DIFF_ENV_VAR = "USE_BAZEL_DIFF"
 
+BAZEL_DIFF_ANNOTATION_CTX = "'diff'"
+
 # TODO(fweikert): Install bazel-diff on the Docker images and on the Mac machines
 BAZEL_DIFF_URL = (
     "https://github.com/Tinder/bazel-diff/releases/download/4.5.0/bazel-diff_deploy.jar"
@@ -2287,6 +2289,9 @@
                     ),
                 ]
             )
+            execute_command(
+                ["buildkite-agent", "annotation", "remove", "--context", BAZEL_DIFF_ANNOTATION_CTX]
+            )
         finally:
             return expanded_test_targets
     finally:
@@ -2310,7 +2315,7 @@
                 "annotate",
                 "--style=info",
                 "--context",
-                "'diff'",
+                BAZEL_DIFF_ANNOTATION_CTX,
                 "This run only contains test targets that have been changed since "
                 "{} due to the `{}` env variable".format(resolved_diffbase, USE_BAZEL_DIFF_ENV_VAR),
             ]