Extend http downloader to add support for non-standard authorization headers
In order to leverage downloading from certain cloud storage providers that don't support basic authentication, this PR extends the existing authentication mechanism in bazel to forward an oauth2 token to the request header.
Sample netrc
```
machine storage.googleapis.com
oauth2-token aaabbbbccccdddd
```
See https://github.com/bazelbuild/bazel/issues/10442
Closes #10445.
PiperOrigin-RevId: 299885980
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/skylark/SkylarkRepositoryContext.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/skylark/SkylarkRepositoryContext.java
index 0311248..eeb4f77 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/repository/skylark/SkylarkRepositoryContext.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/skylark/SkylarkRepositoryContext.java
@@ -81,6 +81,7 @@
import java.nio.charset.StandardCharsets;
import java.time.Duration;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Base64;
import java.util.List;
import java.util.Map;
@@ -1102,6 +1103,37 @@
"Basic "
+ Base64.getEncoder()
.encodeToString(credentials.getBytes(StandardCharsets.UTF_8))));
+ } else if ("pattern".equals(authMap.get("type"))) {
+ if (!authMap.containsKey("pattern")) {
+ throw new EvalException(
+ null,
+ "Found request to do pattern auth for "
+ + entry.getKey()
+ + " without a pattern being provided");
+ }
+
+ String result = (String) authMap.get("pattern");
+
+ for (String component : Arrays.asList("password", "login")) {
+ String demarcatedComponent = "<" + component + ">";
+
+ if (result.contains(demarcatedComponent)) {
+ if (!authMap.containsKey(component)) {
+ throw new EvalException(
+ null,
+ "Auth pattern contains "
+ + demarcatedComponent
+ + " but it was not provided in auth dict.");
+ }
+ } else {
+ // component isn't in the pattern, ignore it
+ continue;
+ }
+
+ result = result.replaceAll(demarcatedComponent, (String) authMap.get(component));
+ }
+
+ headers.put(url.toURI(), ImmutableMap.<String, String>of("Authorization", result));
}
}
} catch (MalformedURLException e) {
diff --git a/src/test/shell/bazel/skylark_repository_test.sh b/src/test/shell/bazel/skylark_repository_test.sh
index 782fafa..3bf128d 100755
--- a/src/test/shell/bazel/skylark_repository_test.sh
+++ b/src/test/shell/bazel/skylark_repository_test.sh
@@ -1779,6 +1779,9 @@
machine bar.example.org
login barusername
password passbar
+
+machine oauthlife.com
+password TOKEN
EOF
# Read a given .netrc file and combine it with a list of URL,
# and write the obtained authentication dicionary to disk; this
@@ -1787,7 +1790,7 @@
load("@bazel_tools//tools/build_defs/repo:utils.bzl", "read_netrc", "use_netrc")
def _impl(ctx):
rc = read_netrc(ctx, ctx.attr.path)
- auth = use_netrc(rc, ctx.attr.urls)
+ auth = use_netrc(rc, ctx.attr.urls, {"oauthlife.com": "Bearer <password>",})
ctx.file("data.bzl", "auth = %s" % (auth,))
ctx.file("BUILD", "")
ctx.file("WORKSPACE", "")
@@ -1811,6 +1814,7 @@
"https://foo.example.org:8080/file2.tar",
"https://bar.example.org/file3.tar",
"https://evil.com/bar.example.org/file4.tar",
+ "https://oauthlife.com/fizz/buzz/file5.tar",
],
)
EOF
@@ -1833,6 +1837,11 @@
"login": "barusername",
"password" : "passbar",
},
+ "https://oauthlife.com/fizz/buzz/file5.tar": {
+ "type" : "pattern",
+ "pattern" : "Bearer <password>",
+ "password" : "TOKEN",
+ },
}
EOF
cat > verify.bzl <<'EOF'
@@ -1939,6 +1948,41 @@
|| fail "Expected success despite needing a file behind basic auth"
}
+function test_http_archive_auth_patterns() {
+ mkdir x
+ echo 'exports_files(["file.txt"])' > x/BUILD
+ echo 'Hello World' > x/file.txt
+ tar cvf x.tar x
+ sha256=$(sha256sum x.tar | head -c 64)
+ serve_file_auth x.tar
+ cat > WORKSPACE <<EOF
+load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
+http_archive(
+ name="ext",
+ url = "http://127.0.0.1:$nc_port/x.tar",
+ netrc = "$(pwd)/.netrc",
+ sha256="$sha256",
+ auth_patterns = {
+ "127.0.0.1": "Bearer <password>"
+ }
+)
+EOF
+ cat > .netrc <<'EOF'
+machine 127.0.0.1
+password TOKEN
+EOF
+ cat > BUILD <<'EOF'
+genrule(
+ name = "it",
+ srcs = ["@ext//x:file.txt"],
+ outs = ["it.txt"],
+ cmd = "cp $< $@",
+)
+EOF
+ bazel build //:it \
+ || fail "Expected success despite needing a file behind bearer auth"
+}
+
function test_implicit_netrc() {
mkdir x
echo 'exports_files(["file.txt"])' > x/BUILD
diff --git a/src/test/shell/bazel/testing_server.py b/src/test/shell/bazel/testing_server.py
index e78c49a..b84f100 100644
--- a/src/test/shell/bazel/testing_server.py
+++ b/src/test/shell/bazel/testing_server.py
@@ -44,7 +44,9 @@
simulate_timeout = False
filename = None
redirect = None
- valid_header = b'Basic ' + base64.b64encode('foo:bar'.encode('ascii'))
+ valid_headers = [
+ b'Basic ' + base64.b64encode('foo:bar'.encode('ascii')), b'Bearer TOKEN'
+ ]
def do_HEAD(self): # pylint: disable=invalid-name
self.send_response(200)
@@ -84,12 +86,13 @@
return
auth_header = self.headers.get('Authorization', '').encode('ascii')
- if auth_header == self.valid_header:
+
+ if auth_header in self.valid_headers:
self.do_HEAD()
self.serve_file()
else:
self.do_AUTHHEAD()
- self.wfile.write(b'Login required.')
+ self.wfile.write(b'Login required.' + str(auth_header))
def serve_file(self):
path_to_serve = self.path[1:]
diff --git a/tools/build_defs/repo/http.bzl b/tools/build_defs/repo/http.bzl
index a77f4ed..587dff0 100644
--- a/tools/build_defs/repo/http.bzl
+++ b/tools/build_defs/repo/http.bzl
@@ -43,16 +43,16 @@
"""Given the list of URLs obtain the correct auth dict."""
if ctx.attr.netrc:
netrc = read_netrc(ctx, ctx.attr.netrc)
- return use_netrc(netrc, urls)
+ return use_netrc(netrc, urls, ctx.attr.auth_patterns)
if "HOME" in ctx.os.environ:
if not ctx.os.name.startswith("windows"):
netrcfile = "%s/.netrc" % (ctx.os.environ["HOME"],)
if ctx.execute(["test", "-f", netrcfile]).return_code == 0:
netrc = read_netrc(ctx, netrcfile)
- return use_netrc(netrc, urls)
+ return use_netrc(netrc, urls, ctx.attr.auth_patterns)
- # TODO: Search at a similarly canonical place for Windows as well
+ # TODO: Search at a similarly canonical place for Windows as well
return {}
@@ -192,6 +192,45 @@
"netrc": attr.string(
doc = "Location of the .netrc file to use for authentication",
),
+ "auth_patterns": attr.string_dict(
+ doc = """An optional dict mapping host names to custom authorization patterns.
+
+If a URL's host name is present in this dict the value will be used as a pattern when
+generating the authorization header for the http request. This enables the use of custom
+authorization schemes used in a lot of common cloud storage providers.
+
+The pattern currently supports 2 tokens: `<login>` and `<password>`, which are replaced with
+their equivalent value in the netrc file for the same host name. After formatting, the result
+is set as the value for the `Authorization` field of the HTTP request.
+
+Example WORKSPACE and netrc for a http download to an oauth2 enabled API using a bearer token:
+
+```python
+http_jar(
+ name = "custom-artifact",
+ url = "https://storage.cloudprovider.com/custom-bucket/custom-artifact.jar",
+ sha256 = "...",
+ netrc = "/home/testuser/workspace/netrc",
+ auth_patterns = {
+ "storage.cloudprovider.com": "Bearer <password>"
+ }
+)
+
+netrc:
+
+```
+machine storage.cloudprovider.com
+ password RANDOM-TOKEN
+```
+
+The final HTTP request would have the following header:
+
+```
+Authorization: Bearer RANDOM-TOKEN
+```
+
+""",
+ ),
"canonical_id": attr.string(
doc = """A canonical id of the archive downloaded.
@@ -378,6 +417,45 @@
"netrc": attr.string(
doc = "Location of the .netrc file to use for authentication",
),
+ "auth_patterns": attr.string_dict(
+ doc = """An optional dict mapping host names to custom authorization patterns.
+
+If a URL's host name is present in this dict the value will be used as a pattern when
+generating the authorization header for the http request. This enables the use of custom
+authorization schemes used in a lot of common cloud storage providers.
+
+The pattern currently supports 2 tokens: `<login>` and `<password>`, which are replaced with
+their equivalent value in the netrc file for the same host name. After formatting, the result
+is set as the value for the `Authorization` field of the HTTP request.
+
+Example WORKSPACE and netrc for a http download to an oauth2 enabled API using a bearer token:
+
+```python
+http_jar(
+ name = "custom-artifact",
+ url = "https://storage.cloudprovider.com/custom-bucket/custom-artifact.jar",
+ sha256 = "...",
+ netrc = "/home/testuser/workspace/netrc",
+ auth_patterns = {
+ "storage.cloudprovider.com": "Bearer <password>"
+ }
+)
+
+netrc:
+
+```
+machine storage.cloudprovider.com
+ password RANDOM-TOKEN
+```
+
+The final HTTP request would have the following header:
+
+```
+Authorization: Bearer RANDOM-TOKEN
+```
+
+""",
+ ),
}
http_file = repository_rule(
@@ -429,6 +507,44 @@
"netrc": attr.string(
doc = "Location of the .netrc file to use for authentication",
),
+ "auth_patterns": attr.string_dict(
+ doc = """An optional dict mapping host names to custom authorization patterns.
+
+If a URL's host name is present in this dict the value will be used as a pattern when
+generating the authorization header for the http request. This enables the use of custom
+authorization schemes used in a lot of common cloud storage providers.
+
+The pattern currently supports 2 tokens: `<login>` and `<password>`, which are replaced with
+their equivalent value in the netrc file for the same host name. After formatting, the result
+is set as the value for the `Authorization` field of the HTTP request.
+
+Example WORKSPACE and netrc for a http download to an oauth2 enabled API using a bearer token:
+
+```python
+http_jar(
+ name = "custom-artifact",
+ url = "https://storage.cloudprovider.com/custom-bucket/custom-artifact.jar",
+ sha256 = "...",
+ netrc = "/home/testuser/workspace/netrc",
+ auth_patterns = {
+ "storage.cloudprovider.com": "Bearer <password>"
+ }
+)
+
+netrc:
+
+```
+machine storage.cloudprovider.com
+ password RANDOM-TOKEN
+```
+
+The final HTTP request would have the following header:
+
+```
+Authorization: Bearer RANDOM-TOKEN
+```
+""",
+ ),
}
http_jar = repository_rule(
diff --git a/tools/build_defs/repo/utils.bzl b/tools/build_defs/repo/utils.bzl
index 6c01a47..0ee6681 100644
--- a/tools/build_defs/repo/utils.bzl
+++ b/tools/build_defs/repo/utils.bzl
@@ -289,19 +289,20 @@
netrc[currentmachinename] = currentmachine
return netrc
-def use_netrc(netrc, urls):
+def use_netrc(netrc, urls, patterns):
"""compute an auth dict from a parsed netrc file and a list of URLs
Args:
netrc: a netrc file already parsed to a dict, e.g., as obtained from
read_netrc
urls: a list of URLs.
+ patterns: optional dict of url to authorization patterns
Returns:
dict suitable as auth argument for ctx.download; more precisely, the dict
will map all URLs where the netrc file provides login and password to a
- dict containing the corresponding login and passwored, as well as the
- mapping of "type" to "basic"
+ dict containing the corresponding login, password and optional authorization pattern,
+ as well as the mapping of "type" to "basic" or "pattern".
"""
auth = {}
for url in urls:
@@ -316,10 +317,24 @@
if not host in netrc:
continue
authforhost = netrc[host]
- if "login" in authforhost and "password" in authforhost:
+ if host in patterns:
+ auth_dict = {
+ "type": "pattern",
+ "pattern": patterns[host],
+ }
+
+ if "login" in authforhost:
+ auth_dict["login"] = authforhost["login"]
+
+ if "password" in authforhost:
+ auth_dict["password"] = authforhost["password"]
+
+ auth[url] = auth_dict
+ elif "login" in authforhost and "password" in authforhost:
auth[url] = {
"type": "basic",
"login": authforhost["login"],
"password": authforhost["password"],
}
+
return auth
