Pull maven -src jars when available alongside jars re:
https://github.com/bazelbuild/bazel/issues/308
RELNOTES: None.
PiperOrigin-RevId: 166219871
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/JarDecompressor.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/JarDecompressor.java
index 703c4a3..26286ad 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/repository/JarDecompressor.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/JarDecompressor.java
@@ -15,6 +15,7 @@
package com.google.devtools.build.lib.bazel.repository;
import com.google.common.base.Joiner;
+import com.google.common.base.Optional;
import com.google.devtools.build.lib.bazel.repository.DecompressorValue.Decompressor;
import com.google.devtools.build.lib.rules.repository.RepositoryFunction;
import com.google.devtools.build.lib.rules.repository.RepositoryFunction.RepositoryFunctionException;
@@ -34,34 +35,46 @@
protected JarDecompressor() {
}
- /**
- * The .jar can be used compressed, so this just exposes it in a way Bazel can use.
- *
- * <p>It moves the jar from some-name/x/y/z/foo.jar to some-name/jar/foo.jar and creates a
- * BUILD.bazel file containing one entry: the .jar.
- */
@Override
@Nullable
public Path decompress(DecompressorDescriptor descriptor) throws RepositoryFunctionException {
- // Example: archiveFile is external/some-name/foo.jar.
- String baseName = descriptor.archivePath().getBaseName();
+ return decompressWithSrcjar(descriptor, Optional.absent());
+ }
+ /**
+ * The jar (and srcjar if available) can be used compressed, so this just exposes them in a way
+ * Bazel can use.
+ *
+ * <p>It moves the jar(s) from some-name/x/y/z/foo.jar to some-name/jar/foo.jar and creates a
+ * BUILD.bazel file containing only the jar(s).
+ */
+ public Path decompressWithSrcjar(
+ DecompressorDescriptor descriptor, Optional<DecompressorDescriptor> srcjarDescriptor)
+ throws RepositoryFunctionException {
try {
+ // external/some-name/
FileSystemUtils.createDirectoryAndParents(descriptor.repositoryPath());
- // external/some-name/WORKSPACE.
+ // external/some-name/WORKSPACE
RepositoryFunction.createWorkspaceFile(
descriptor.repositoryPath(), descriptor.targetKind(), descriptor.targetName());
- // external/some-name/jar.
+ // external/some-name/jar/
Path jarDirectory = descriptor.repositoryPath().getRelative(getPackageName());
FileSystemUtils.createDirectoryAndParents(jarDirectory);
- // external/some-name/repository/jar/foo.jar is a symbolic link to the jar in
- // external/some-name.
- Path jarSymlink = jarDirectory.getRelative(baseName);
- if (!jarSymlink.exists()) {
- jarSymlink.createSymbolicLink(descriptor.archivePath());
- }
- // external/some-name/repository/jar/BUILD.bazel defines the //jar target.
+ // external/some-name/jar/BUILD.bazel defines the //jar target.
Path buildFile = jarDirectory.getRelative("BUILD.bazel");
+
+ // Example: get foo.jar from external/some-name/foo.jar.
+ String baseName = descriptor.archivePath().getBaseName();
+ makeSymlink(descriptor);
+ String buildFileContents;
+ if (srcjarDescriptor.isPresent()) {
+ String srcjarBaseName = srcjarDescriptor.get().archivePath().getBaseName();
+ makeSymlink(srcjarDescriptor.get());
+ buildFileContents = createBuildFileWithSrcjar(baseName, srcjarBaseName);
+ } else {
+ buildFileContents = createBuildFile(baseName);
+ }
+
FileSystemUtils.writeLinesAs(
buildFile,
Charset.forName("UTF-8"),
@@ -69,7 +82,7 @@
+ descriptor.targetKind()
+ " rule "
+ descriptor.targetName(),
- createBuildFile(baseName));
+ buildFileContents);
if (descriptor.executable()) {
descriptor.archivePath().chmod(0755);
}
@@ -80,6 +93,25 @@
return descriptor.repositoryPath();
}
+ /*
+ * Links some-name/x/y/z/foo.jar to some-name/jar/foo.jar (represented by {@code descriptor}
+ */
+ protected void makeSymlink(DecompressorDescriptor descriptor) throws RepositoryFunctionException {
+ try {
+ Path jarDirectory = descriptor.repositoryPath().getRelative(getPackageName());
+ // external/some-name/jar/foo.jar is a symbolic link to the jar in
+ // external/some-name.
+ Path jarSymlink = jarDirectory.getRelative(descriptor.archivePath().getBaseName());
+ if (!jarSymlink.exists()) {
+ jarSymlink.createSymbolicLink(descriptor.archivePath());
+ }
+ } catch (IOException e) {
+ throw new RepositoryFunctionException(
+ new IOException("Error auto-creating jar repo structure: " + e.getMessage()),
+ Transience.TRANSIENT);
+ }
+ }
+
protected String getPackageName() {
return "jar";
}
@@ -99,4 +131,24 @@
" visibility = ['//visibility:public']",
")");
}
+
+ protected String createBuildFileWithSrcjar(String baseName, String srcjarBaseName) {
+ return Joiner.on("\n")
+ .join(
+ "java_import(",
+ " name = 'jar',",
+ " jars = ['" + baseName + "'],",
+ " srcjar = '" + srcjarBaseName + "',",
+ " visibility = ['//visibility:public']",
+ ")",
+ "",
+ "filegroup(",
+ " name = 'file',",
+ " srcs = [",
+ " '" + baseName + "',",
+ " '" + srcjarBaseName + "',",
+ " ],",
+ " visibility = ['//visibility:public']",
+ ")");
+ }
}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/MavenDownloader.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/MavenDownloader.java
index 264173e..e663a80 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/repository/MavenDownloader.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/MavenDownloader.java
@@ -14,6 +14,7 @@
package com.google.devtools.build.lib.bazel.repository;
+import com.google.common.base.Optional;
import com.google.common.base.Strings;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
@@ -72,9 +73,10 @@
}
/**
- * Download the Maven artifact to the output directory. Returns the path to the jar.
+ * Download the Maven artifact to the output directory. Returns the path to the jar (and the
+ * srcjar if available).
*/
- public Path download(String name, WorkspaceAttributeMapper mapper, Path outputDirectory,
+ public JarPaths download(String name, WorkspaceAttributeMapper mapper, Path outputDirectory,
MavenServerValue serverValue) throws IOException, EvalException {
this.name = name;
this.outputDirectory = outputDirectory;
@@ -82,59 +84,73 @@
String url = serverValue.getUrl();
Server server = serverValue.getServer();
+ // Initialize maven artifacts
Artifact artifact;
- String artifactId = mapper.get("artifact", Type.STRING);
- String sha1 = mapper.isAttributeValueExplicitlySpecified("sha1")
- ? mapper.get("sha1", Type.STRING) : null;
- if (sha1 != null && !KeyType.SHA1.isValid(sha1)) {
- throw new IOException("Invalid SHA-1 for maven_jar " + name + ": '" + sha1 + "'");
- }
+ String artifactCoords = mapper.get("artifact", Type.STRING);
+ String sha1 =
+ mapper.isAttributeValueExplicitlySpecified("sha1") ? mapper.get("sha1", Type.STRING) : null;
+ if (sha1 != null && !KeyType.SHA1.isValid(sha1)) {
+ throw new IOException("Invalid SHA-1 for maven_jar " + name + ": '" + sha1 + "'");
+ }
+
try {
- artifact = new DefaultArtifact(artifactId);
+ artifact = new DefaultArtifact(artifactCoords);
} catch (IllegalArgumentException e) {
throw new IOException(e.getMessage());
}
-
+
boolean isCaching = repositoryCache.isEnabled() && KeyType.SHA1.isValid(sha1);
if (isCaching) {
Path downloadPath = getDownloadDestination(artifact);
Path cachedDestination = repositoryCache.get(sha1, downloadPath, KeyType.SHA1);
if (cachedDestination != null) {
- return cachedDestination;
+ return new JarPaths(cachedDestination, Optional.absent());
}
}
+ // Setup env for fetching jars
MavenConnector connector = new MavenConnector(outputDirectory.getPathString());
RepositorySystem system = connector.newRepositorySystem();
RepositorySystemSession session = connector.newRepositorySystemSession(system);
+ RemoteRepository repository =
+ new RemoteRepository.Builder(name, MavenServerValue.DEFAULT_ID, url)
+ .setAuthentication(new MavenAuthentication(server))
+ .build();
- RemoteRepository repository = new RemoteRepository.Builder(
- name, MavenServerValue.DEFAULT_ID, url)
- .setAuthentication(new MavenAuthentication(server))
- .build();
- ArtifactRequest artifactRequest = new ArtifactRequest();
-
- artifactRequest.setArtifact(artifact);
- artifactRequest.setRepositories(ImmutableList.of(repository));
-
+ // Try fetching jar.
+ final Path jarDownload;
try {
- ArtifactResult artifactResult = system.resolveArtifact(session, artifactRequest);
- artifact = artifactResult.getArtifact();
+ artifact = downloadArtifact(artifact, repository, session, system);
} catch (ArtifactResolutionException e) {
throw new IOException("Failed to fetch Maven dependency: " + e.getMessage());
}
- Path downloadPath = outputDirectory.getRelative(artifact.getFile().getAbsolutePath());
+ // Try also fetching srcjar.
+ Artifact artifactWithSrcs = srcjarCoords(artifact);
+ try {
+ artifactWithSrcs = downloadArtifact(artifactWithSrcs, repository, session, system);
+ } catch (ArtifactResolutionException e) {
+ // Intentionally ignored - missing srcjar is not an error.
+ }
+
+ jarDownload = outputDirectory.getRelative(artifact.getFile().getAbsolutePath());
// Verify checksum.
if (!Strings.isNullOrEmpty(sha1)) {
- RepositoryCache.assertFileChecksum(sha1, downloadPath, KeyType.SHA1);
+ RepositoryCache.assertFileChecksum(sha1, jarDownload, KeyType.SHA1);
}
if (isCaching) {
- repositoryCache.put(sha1, downloadPath, KeyType.SHA1);
+ repositoryCache.put(sha1, jarDownload, KeyType.SHA1);
}
- return downloadPath;
+
+ if (artifactWithSrcs.getFile() != null) {
+ Path srcjarDownload =
+ outputDirectory.getRelative(artifactWithSrcs.getFile().getAbsolutePath());
+ return new JarPaths(jarDownload, Optional.fromNullable(srcjarDownload));
+ } else {
+ return new JarPaths(jarDownload, Optional.absent());
+ }
}
private Path getDownloadDestination(Artifact artifact) {
@@ -149,6 +165,40 @@
return outputDirectory.getRelative(joiner.toString());
}
+ private Artifact srcjarCoords(Artifact jar) {
+ return new DefaultArtifact(
+ jar.getGroupId(), jar.getArtifactId(), "sources", jar.getExtension(), jar.getVersion());
+ }
+
+ /*
+ * Set up request for and resolve (retrieve to local repo) artifact
+ */
+ private Artifact downloadArtifact(
+ Artifact artifact,
+ RemoteRepository repository,
+ RepositorySystemSession session,
+ RepositorySystem system)
+ throws ArtifactResolutionException {
+ ArtifactRequest artifactRequest = new ArtifactRequest();
+ artifactRequest.setArtifact(artifact);
+ artifactRequest.setRepositories(ImmutableList.of(repository));
+ ArtifactResult artifactResult = system.resolveArtifact(session, artifactRequest);
+ return artifactResult.getArtifact();
+ }
+
+ /*
+ * Class for packaging srcjar and jar paths together when srcjar is available.
+ */
+ static class JarPaths {
+ final Path jar;
+ @Nullable final Optional<Path> srcjar;
+
+ private JarPaths(Path jar, Optional<Path> srcjar) {
+ this.jar = jar;
+ this.srcjar = srcjar;
+ }
+ }
+
private static class MavenAuthentication implements Authentication {
private final Map<String, String> authenticationInfo;
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/MavenJarFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/MavenJarFunction.java
index 907227e..45aa2ba 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/repository/MavenJarFunction.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/MavenJarFunction.java
@@ -14,9 +14,11 @@
package com.google.devtools.build.lib.bazel.repository;
+import com.google.common.base.Optional;
import com.google.common.base.Preconditions;
import com.google.devtools.build.lib.analysis.BlazeDirectories;
import com.google.devtools.build.lib.analysis.RuleDefinition;
+import com.google.devtools.build.lib.bazel.repository.MavenDownloader.JarPaths;
import com.google.devtools.build.lib.bazel.rules.workspace.MavenJarRule;
import com.google.devtools.build.lib.packages.Rule;
import com.google.devtools.build.lib.rules.repository.RepositoryDirectoryValue;
@@ -98,7 +100,7 @@
if (env.valuesMissing()) {
return null;
}
-
+
Path outputDir = getExternalRepositoryDirectory(directories).getRelative(rule.getName());
return createOutputTree(rule, outputDir, serverValue);
}
@@ -110,10 +112,11 @@
createDirectory(outputDirectory);
String name = rule.getName();
- Path repositoryJar;
+ final JarPaths repositoryJars;
try {
- repositoryJar = mavenDownloader.download(
- name, WorkspaceAttributeMapper.of(rule), outputDirectory, serverValue);
+ repositoryJars =
+ mavenDownloader.download(
+ name, WorkspaceAttributeMapper.of(rule), outputDirectory, serverValue);
} catch (IOException e) {
throw new RepositoryFunctionException(e, Transience.TRANSIENT);
} catch (EvalException e) {
@@ -121,13 +124,25 @@
}
// Add a WORKSPACE file & BUILD file to the Maven jar.
- Path result = DecompressorValue.decompress(DecompressorDescriptor.builder()
+ DecompressorDescriptor jar = getDescriptorBuilder(name, repositoryJars.jar, outputDirectory);
+ DecompressorDescriptor srcjar =
+ repositoryJars.srcjar.isPresent()
+ ? getDescriptorBuilder(name, repositoryJars.srcjar.get(), outputDirectory)
+ : null;
+ JarDecompressor decompressor = (JarDecompressor) jar.getDecompressor();
+ Path result = decompressor.decompressWithSrcjar(jar, Optional.fromNullable(srcjar));
+ return RepositoryDirectoryValue.builder().setPath(result);
+ }
+
+ private DecompressorDescriptor getDescriptorBuilder(String name, Path jar, Path outputDirectory)
+ throws RepositoryFunctionException, InterruptedException {
+ return DecompressorDescriptor.builder()
.setDecompressor(JarDecompressor.INSTANCE)
.setTargetKind(MavenJarRule.NAME)
.setTargetName(name)
- .setArchivePath(repositoryJar)
- .setRepositoryPath(outputDirectory).build());
- return RepositoryDirectoryValue.builder().setPath(result);
+ .setArchivePath(jar)
+ .setRepositoryPath(outputDirectory)
+ .build();
}
/**
diff --git a/src/test/java/com/google/devtools/build/lib/rules/repository/JarDecompressorTest.java b/src/test/java/com/google/devtools/build/lib/rules/repository/JarDecompressorTest.java
index 37cfa8d..11dfdbb 100644
--- a/src/test/java/com/google/devtools/build/lib/rules/repository/JarDecompressorTest.java
+++ b/src/test/java/com/google/devtools/build/lib/rules/repository/JarDecompressorTest.java
@@ -16,9 +16,10 @@
import static com.google.common.truth.Truth.assertThat;
+import com.google.common.base.Optional;
import com.google.devtools.build.lib.bazel.repository.DecompressorDescriptor;
-import com.google.devtools.build.lib.bazel.repository.DecompressorValue;
import com.google.devtools.build.lib.bazel.repository.JarDecompressor;
+import com.google.devtools.build.lib.bazel.rules.workspace.MavenJarRule;
import com.google.devtools.build.lib.testutil.Scratch;
import com.google.devtools.build.lib.vfs.FileSystemUtils;
import com.google.devtools.build.lib.vfs.Path;
@@ -32,35 +33,69 @@
*/
@RunWith(JUnit4.class)
public class JarDecompressorTest {
- private DecompressorDescriptor.Builder descriptorBuilder;
+ private DecompressorDescriptor.Builder jarDescriptorBuilder;
+ private DecompressorDescriptor.Builder srcjarDescriptorBuilder;
+ private JarDecompressor decompressor;
@Before
public void setUpFs() throws Exception {
Scratch fs = new Scratch();
Path dir = fs.dir("/whatever/external/tester");
Path jar = fs.file("/foo.jar", "I'm a jar");
+ Path srcjar = fs.file("/foo-sources.jar", "I'm a source jar");
FileSystemUtils.createDirectoryAndParents(dir);
- descriptorBuilder = DecompressorDescriptor.builder()
- .setDecompressor(JarDecompressor.INSTANCE)
- .setTargetName("tester")
- .setTargetKind("http_jar rule")
- .setRepositoryPath(dir)
- .setArchivePath(jar);
+ jarDescriptorBuilder =
+ DecompressorDescriptor.builder()
+ .setDecompressor(JarDecompressor.INSTANCE)
+ .setTargetName("tester")
+ .setTargetKind(MavenJarRule.NAME)
+ .setRepositoryPath(dir)
+ .setArchivePath(jar);
+ srcjarDescriptorBuilder =
+ DecompressorDescriptor.builder()
+ .setDecompressor(JarDecompressor.INSTANCE)
+ .setTargetName("tester")
+ .setTargetKind(MavenJarRule.NAME)
+ .setRepositoryPath(dir)
+ .setArchivePath(srcjar);
+ decompressor = (JarDecompressor) jarDescriptorBuilder.build().getDecompressor();
}
@Test
- public void testTargets() throws Exception {
- Path outputDir = DecompressorValue.decompress(descriptorBuilder.build());
+ public void testTargetsWithSources() throws Exception {
+ Path outputDir =
+ decompressor.decompressWithSrcjar(
+ jarDescriptorBuilder.build(), Optional.fromNullable(srcjarDescriptorBuilder.build()));
assertThat(outputDir.exists()).isTrue();
+ assertThat(outputDir.getRelative("jar/foo.jar").exists()).isTrue();
+ assertThat(outputDir.getRelative("jar/foo-sources.jar").exists()).isTrue();
String buildContent =
new String(FileSystemUtils.readContentAsLatin1(outputDir.getRelative("jar/BUILD.bazel")));
assertThat(buildContent).contains("java_import");
+ assertThat(buildContent).contains("srcjar = 'foo-sources.jar'");
assertThat(buildContent).contains("filegroup");
}
@Test
+ public void testTargetsWithoutSources() throws Exception {
+ Path outputDir =
+ decompressor.decompressWithSrcjar(jarDescriptorBuilder.build(), Optional.absent());
+ assertThat(outputDir.exists()).isTrue();
+ assertThat(outputDir.getRelative("jar/foo.jar").exists()).isTrue();
+ assertThat(outputDir.getRelative("jar/foo-sources.jar").exists()).isFalse();
+ String buildContent =
+ new String(FileSystemUtils.readContentAsLatin1(outputDir.getRelative("jar/BUILD.bazel")));
+ assertThat(buildContent).contains("java_import");
+ assertThat(buildContent).doesNotContain("srcjar = 'foo-sources.jar'");
+ assertThat(buildContent).contains("filegroup");
+ }
+
+ @Test
+ // Note: WORKSPACE gen is not affected by presence or absence of Optional arg to
+ // decompressWithSrcjar
public void testWorkspaceGen() throws Exception {
- Path outputDir = DecompressorValue.decompress(descriptorBuilder.build());
+ Path outputDir =
+ decompressor.decompressWithSrcjar(jarDescriptorBuilder.build(), Optional.absent());
assertThat(outputDir.exists()).isTrue();
String workspaceContent = new String(
FileSystemUtils.readContentAsLatin1(outputDir.getRelative("WORKSPACE")));
diff --git a/src/test/shell/bazel/maven_test.sh b/src/test/shell/bazel/maven_test.sh
index 04e217c..7a14c20 100755
--- a/src/test/shell/bazel/maven_test.sh
+++ b/src/test/shell/bazel/maven_test.sh
@@ -17,6 +17,8 @@
# Test //external mechanisms
#
+set -euo pipefail
+
# Load the test setup defined in the parent directory
CURRENT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "${CURRENT_DIR}/../integration_test_setup.sh" \
@@ -85,6 +87,27 @@
expect_log "Tra-la!"
}
+# makes sure both jar and srcjar are downloaded
+function test_maven_jar_downloads() {
+ serve_artifact com.example.carnivore carnivore 1.23
+
+ cat > WORKSPACE <<EOF
+maven_jar(
+ name = 'endangered',
+ artifact = "com.example.carnivore:carnivore:1.23",
+ repository = 'http://127.0.0.1:$fileserver_port/',
+)
+bind(name = 'mongoose', actual = '@endangered//jar')
+EOF
+
+ bazel run //zoo:ball-pit >& $TEST_log || fail "Expected run to succeed"
+ output_base="$(bazel info output_base)"
+ test -e "${output_base}/external/endangered/jar/carnivore-1.23.jar" \
+ || fail "jar not downloaded to expected place"
+ test -e "${output_base}/external/endangered/jar/carnivore-1.23-sources.jar" \
+ || fail "srcjar not downloaded to expected place"
+}
+
function test_maven_jar_404() {
setup_zoo
serve_not_found
diff --git a/src/test/shell/bazel/remote_helpers.sh b/src/test/shell/bazel/remote_helpers.sh
index 7903063..ff76585 100755
--- a/src/test/shell/bazel/remote_helpers.sh
+++ b/src/test/shell/bazel/remote_helpers.sh
@@ -14,6 +14,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+set -euo pipefail
+
case "${PLATFORM}" in
darwin|freebsd)
function nc_l() {
@@ -68,8 +70,10 @@
EOF
${bazel_javabase}/bin/javac $pkg_dir/Mongoose.java
test_jar=$TEST_TMPDIR/libcarnivore.jar
+ test_srcjar=$TEST_TMPDIR/libcarnivore-sources.jar
cd ${TEST_TMPDIR}
${bazel_javabase}/bin/jar cf $test_jar carnivore/Mongoose.class
+ ${bazel_javabase}/bin/jar cf $test_srcjar carnivore/Mongoose.java
sha256=$(sha256sum $test_jar | cut -f 1 -d ' ')
# OS X doesn't have sha1sum, so use openssl.
sha1=$(openssl sha1 $test_jar | cut -f 2 -d ' ')
@@ -168,11 +172,14 @@
else
make_test_jar
local artifact=$test_jar
+ local srcjar_artifact=$test_srcjar
fi
maven_path=$PWD/$(echo $group_id | sed 's/\./\//g')/$artifact_id/$version
mkdir -p $maven_path
openssl sha1 $artifact > $maven_path/$artifact_id-$version.$packaging.sha1
+ openssl sha1 $srcjar_artifact > $maven_path/$artifact_id-$version-sources.$packaging.sha1
mv $artifact $maven_path/$artifact_id-$version.$packaging
+ mv $srcjar_artifact $maven_path/$artifact_id-$version-sources.$packaging
}
function serve_artifact() {
