Remove jar tool usage and add jar normalisation. (#94)
* remove jar tool with a port of JarCreator
* ensure jar normalization
* make the hashing test local and add ij runconfig
diff --git a/.bazelproject b/.bazelproject
index 73d0819..142e24b 100644
--- a/.bazelproject
+++ b/.bazelproject
@@ -27,4 +27,7 @@
*/integrationtests/*
additional_languages:
- kotlin
\ No newline at end of file
+ kotlin
+
+import_run_configurations:
+ tests/Bazel_all_local_tests.xml
\ No newline at end of file
diff --git a/.gitignore b/.gitignore
index d1607bf..101fc88 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1 +1,2 @@
-**/bazel-*
\ No newline at end of file
+**/bazel-*
+.project/**
diff --git a/BUILD b/BUILD
index b80e4d9..886f2de 100644
--- a/BUILD
+++ b/BUILD
@@ -11,6 +11,8 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+
+# The entire test suite excluding local tests.
test_suite(
name = "all_tests",
tests = [
@@ -18,4 +20,13 @@
"//kotlin/builder:integrationtests",
"//tests/integrationtests"
]
+)
+
+# Local tests
+test_suite(
+ name = "all_local_tests",
+ tests = [
+ ":all_tests",
+ "//tests/integrationtests:integrationtests_local"
+ ]
)
\ No newline at end of file
diff --git a/kotlin/builder/src/io/bazel/kotlin/builder/KotlinBuilder.kt b/kotlin/builder/src/io/bazel/kotlin/builder/KotlinBuilder.kt
index 61431ba..0b0e262 100644
--- a/kotlin/builder/src/io/bazel/kotlin/builder/KotlinBuilder.kt
+++ b/kotlin/builder/src/io/bazel/kotlin/builder/KotlinBuilder.kt
@@ -21,7 +21,9 @@
import io.bazel.kotlin.builder.mode.jvm.KotlinJvmCompilationExecutor
import io.bazel.kotlin.builder.utils.ArgMap
import io.bazel.kotlin.builder.utils.ArgMaps
+import io.bazel.kotlin.builder.utils.IS_JVM_SOURCE_FILE
import io.bazel.kotlin.builder.utils.ensureDirectories
+import io.bazel.kotlin.builder.utils.jars.SourceJarExtractor
import io.bazel.kotlin.model.KotlinModel
import java.nio.file.Paths
@@ -29,7 +31,6 @@
@Suppress("MemberVisibilityCanBePrivate")
class KotlinBuilder @Inject internal constructor(
private val commandBuilder: BuildCommandBuilder,
- private val jarToolInvoker: KotlinToolchain.JarToolInvoker,
private val compilationExector: KotlinJvmCompilationExecutor
) : CommandLineProgram {
fun execute(args: List<String>): Int =
@@ -62,30 +63,15 @@
if (command.inputs.sourceJarsList.isEmpty()) {
command
} else {
- val sourceUnpackDirectory =
- Paths.get(command.directories.temp).let {
- it.resolve("_srcjars").toFile().let {
- try {
- it.mkdirs(); it
- } catch (ex: Exception) {
- throw RuntimeException("could not create unpack directory at $it", ex)
- }
- }
- }
- for (sourceJar in command.inputs.sourceJarsList) {
- jarToolInvoker.invoke(
- listOf("xf", Paths.get(sourceJar).toAbsolutePath().toString()), sourceUnpackDirectory
- )
+ SourceJarExtractor(
+ destDir = Paths.get(command.directories.temp).resolve("_srcjars"),
+ fileMatcher = IS_JVM_SOURCE_FILE
+ ).also {
+ it.jarFiles.addAll(command.inputs.sourceJarsList.map { Paths.get(it) })
+ it.execute()
+ }.let {
+ commandBuilder.withSources(command, it.sourcesList.iterator())
}
-
- commandBuilder.withSources(
- command,
- sourceUnpackDirectory
- .walk()
- .filter { it.name.endsWith(".kt") || it.name.endsWith(".java") }
- .map { it.toString() }
- .iterator()
- )
}
override fun apply(args: List<String>): Int {
diff --git a/kotlin/builder/src/io/bazel/kotlin/builder/KotlinToolchain.kt b/kotlin/builder/src/io/bazel/kotlin/builder/KotlinToolchain.kt
index 8efabd0..84766d3 100644
--- a/kotlin/builder/src/io/bazel/kotlin/builder/KotlinToolchain.kt
+++ b/kotlin/builder/src/io/bazel/kotlin/builder/KotlinToolchain.kt
@@ -107,10 +107,6 @@
interface KotlincInvoker {
fun compile(args: Array<String>, out: PrintStream): Int
}
-
- interface JarToolInvoker {
- fun invoke(args: List<String>, directory: File? = null)
- }
}
diff --git a/kotlin/builder/src/io/bazel/kotlin/builder/KotlinToolchainModule.kt b/kotlin/builder/src/io/bazel/kotlin/builder/KotlinToolchainModule.kt
index 9ede01a..5bb5cc1 100644
--- a/kotlin/builder/src/io/bazel/kotlin/builder/KotlinToolchainModule.kt
+++ b/kotlin/builder/src/io/bazel/kotlin/builder/KotlinToolchainModule.kt
@@ -18,27 +18,12 @@
import com.google.inject.AbstractModule
import com.google.inject.Provides
import io.bazel.kotlin.builder.KotlinToolchain.Companion.NO_ARGS
-import io.bazel.kotlin.builder.utils.executeAndAwait
import io.bazel.kotlin.builder.utils.resolveVerified
-import io.bazel.kotlin.builder.utils.resolveVerifiedToAbsoluteString
-import java.io.File
import java.io.PrintStream
import java.io.PrintWriter
internal object KotlinToolchainModule : AbstractModule() {
@Provides
- fun jarToolInvoker(toolchain: KotlinToolchain): KotlinToolchain.JarToolInvoker =
- object : KotlinToolchain.JarToolInvoker {
- override fun invoke(args: List<String>, directory: File?) {
- val jarTool = toolchain.javaHome.resolveVerifiedToAbsoluteString("bin", "jar")
- val command = mutableListOf(jarTool).also { it.addAll(args) }
- executeAndAwait(10, directory, command).takeIf { it != 0 }?.also {
- throw CompilationStatusException("error running jar command ${command.joinToString(" ")}", it)
- }
- }
- }
-
- @Provides
fun javacInvoker(toolchain: KotlinToolchain): KotlinToolchain.JavacInvoker = object : KotlinToolchain.JavacInvoker {
val c = toolchain.classLoader.loadClass("com.sun.tools.javac.Main")
val m = c.getMethod("compile", Array<String>::class.java)
diff --git a/kotlin/builder/src/io/bazel/kotlin/builder/mode/jvm/actions/OutputJarCreator.kt b/kotlin/builder/src/io/bazel/kotlin/builder/mode/jvm/actions/OutputJarCreator.kt
index e46147b..88c98af 100644
--- a/kotlin/builder/src/io/bazel/kotlin/builder/mode/jvm/actions/OutputJarCreator.kt
+++ b/kotlin/builder/src/io/bazel/kotlin/builder/mode/jvm/actions/OutputJarCreator.kt
@@ -16,10 +16,8 @@
package io.bazel.kotlin.builder.mode.jvm.actions
import com.google.inject.ImplementedBy
-import com.google.inject.Inject
-import io.bazel.kotlin.builder.KotlinToolchain
+import io.bazel.kotlin.builder.utils.jars.JarCreator
import io.bazel.kotlin.model.KotlinModel
-import java.nio.file.Path
import java.nio.file.Paths
@ImplementedBy(DefaultOutputJarCreator::class)
@@ -27,16 +25,17 @@
fun createOutputJar(command: KotlinModel.BuilderCommand)
}
-private class DefaultOutputJarCreator @Inject constructor(
- val toolInvoker: KotlinToolchain.JarToolInvoker
-) : OutputJarCreator {
- private fun MutableList<String>.addAllFrom(dir: Path) = addAll(arrayOf("-C", dir.toString(), "."))
+private class DefaultOutputJarCreator : OutputJarCreator {
override fun createOutputJar(command: KotlinModel.BuilderCommand) {
- mutableListOf(
- "cf", command.outputs.jar
- ).also { args ->
- args.addAllFrom(Paths.get(command.directories.classes))
- args.addAllFrom(Paths.get(command.directories.generatedClasses))
- }.let { toolInvoker.invoke(it) }
+ JarCreator(
+ path = Paths.get(command.outputs.jar),
+ normalize = true,
+ verbose = false
+ ).also {
+ it.addDirectory(Paths.get(command.directories.classes))
+ it.addDirectory(Paths.get(command.directories.generatedClasses))
+ it.setJarOwner(command.info.label, command.info.ruleKind)
+ it.execute()
+ }
}
}
diff --git a/kotlin/builder/src/io/bazel/kotlin/builder/utils/MiscUtils.kt b/kotlin/builder/src/io/bazel/kotlin/builder/utils/MiscUtils.kt
index bbbf19f..fcfd90a 100644
--- a/kotlin/builder/src/io/bazel/kotlin/builder/utils/MiscUtils.kt
+++ b/kotlin/builder/src/io/bazel/kotlin/builder/utils/MiscUtils.kt
@@ -16,5 +16,12 @@
package io.bazel.kotlin.builder.utils
+import java.util.function.Predicate
+import java.util.regex.Pattern
+
fun <T, C : MutableCollection<T>> C.addAll(vararg entries: T): C = this.also { addAll(entries) }
+private fun extensionMatcher(vararg ext: String): Predicate<String> =
+ Pattern.compile("^(.+?)${ext.joinToString("|\\.", prefix = "(\\.",postfix = ")$")}").asPredicate()
+
+val IS_JVM_SOURCE_FILE = extensionMatcher("kt", "java")
diff --git a/kotlin/builder/src/io/bazel/kotlin/builder/utils/jars/JarCreator.kt b/kotlin/builder/src/io/bazel/kotlin/builder/utils/jars/JarCreator.kt
new file mode 100644
index 0000000..7127ad1
--- /dev/null
+++ b/kotlin/builder/src/io/bazel/kotlin/builder/utils/jars/JarCreator.kt
@@ -0,0 +1,225 @@
+/*
+ * Copyright 2018 The Bazel Authors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.bazel.kotlin.builder.utils.jars
+
+import java.io.*
+import java.nio.file.*
+import java.nio.file.Paths.*
+import java.nio.file.attribute.BasicFileAttributes
+import java.util.*
+import java.util.jar.Attributes
+import java.util.jar.JarOutputStream
+import java.util.jar.Manifest
+
+@Suppress("unused")
+/**
+ * A class for creating Jar files. Allows normalization of Jar entries by setting their timestamp to
+ * the DOS epoch. All Jar entries are sorted alphabetically.
+ */
+class JarCreator(
+ path: Path,
+ normalize: Boolean = true,
+ verbose: Boolean = false
+) : JarHelper(path, normalize, verbose) {
+ // Map from Jar entry names to files. Use TreeMap so we can establish a canonical order for the
+ // entries regardless in what order they get added.
+ private val jarEntries = TreeMap<String, Path>()
+ private var manifestFile: String? = null
+ private var mainClass: String? = null
+ private var targetLabel: String? = null
+ private var injectingRuleKind: String? = null
+
+ /**
+ * Adds an entry to the Jar file, normalizing the name.
+ *
+ * @param entryName the name of the entry in the Jar file
+ * @param path the path of the input for the entry
+ * @return true iff a new entry was added
+ */
+ private fun addEntry(entryName: String, path: Path): Boolean {
+ var normalizedEntryName = entryName
+ if (normalizedEntryName.startsWith("/")) {
+ normalizedEntryName = normalizedEntryName.substring(1)
+ } else if (normalizedEntryName.length >= 3
+ && Character.isLetter(normalizedEntryName[0])
+ && normalizedEntryName[1] == ':'
+ && (normalizedEntryName[2] == '\\' || normalizedEntryName[2] == '/')
+ ) {
+ // Windows absolute path, e.g. "D:\foo" or "e:/blah".
+ // Windows paths are case-insensitive, and support both backslashes and forward slashes.
+ normalizedEntryName = normalizedEntryName.substring(3)
+ } else if (normalizedEntryName.startsWith("./")) {
+ normalizedEntryName = normalizedEntryName.substring(2)
+ }
+ return jarEntries.put(normalizedEntryName, path) == null
+ }
+
+ /**
+ * Adds an entry to the Jar file, normalizing the name.
+ *
+ * @param entryName the name of the entry in the Jar file
+ * @param fileName the name of the input file for the entry
+ * @return true iff a new entry was added
+ */
+ fun addEntry(entryName: String, fileName: String): Boolean {
+ return addEntry(entryName, get(fileName))
+ }
+
+ /**
+ * Adds the contents of a directory to the Jar file. All files below this directory will be added
+ * to the Jar file using the name relative to the directory as the name for the Jar entry.
+ *
+ * @param directory the directory to add to the jar
+ */
+ fun addDirectory(directory: Path) {
+ if (!Files.exists(directory)) {
+ throw IllegalArgumentException("directory does not exist: $directory")
+ }
+ try {
+ Files.walkFileTree(
+ directory,
+ object : SimpleFileVisitor<Path>() {
+
+ @Throws(IOException::class)
+ override fun preVisitDirectory(path: Path, attrs: BasicFileAttributes): FileVisitResult {
+ if (path != directory) {
+ // For consistency with legacy behaviour, include entries for directories except for
+ // the root.
+ addEntry(path, /* isDirectory= */ true)
+ }
+ return FileVisitResult.CONTINUE
+ }
+
+ @Throws(IOException::class)
+ override fun visitFile(path: Path, attrs: BasicFileAttributes): FileVisitResult {
+ addEntry(path, /* isDirectory= */ false)
+ return FileVisitResult.CONTINUE
+ }
+
+ fun addEntry(path: Path, isDirectory: Boolean) {
+ val sb = StringBuilder()
+ var first = true
+ for (entry in directory.relativize(path)) {
+ if (!first) {
+ // use `/` as the directory separator for jar paths, even on Windows
+ sb.append('/')
+ }
+ sb.append(entry.fileName)
+ first = false
+ }
+ if (isDirectory) {
+ sb.append('/')
+ }
+ jarEntries[sb.toString()] = path
+ }
+ })
+ } catch (e: IOException) {
+ throw UncheckedIOException(e)
+ }
+
+ }
+
+ /**
+ * Adds a collection of entries to the jar, each with a given source path, and with the resulting
+ * file in the root of the jar.
+ *
+ * <pre>
+ * some/long/path.foo => (path.foo, some/long/path.foo)
+ </pre> *
+ */
+ fun addRootEntries(entries: Collection<String>) {
+ for (entry in entries) {
+ val path = get(entry)
+ jarEntries[path.fileName.toString()] = path
+ }
+ }
+
+ /**
+ * Sets the main.class entry for the manifest. A value of `null` (the default) will
+ * omit the entry.
+ *
+ * @param mainClass the fully qualified name of the main class
+ */
+ fun setMainClass(mainClass: String) {
+ this.mainClass = mainClass
+ }
+
+ fun setJarOwner(targetLabel: String, injectingRuleKind: String) {
+ this.targetLabel = targetLabel
+ this.injectingRuleKind = injectingRuleKind
+ }
+
+ /**
+ * Sets filename for the manifest content. If this is set the manifest will be read from this file
+ * otherwise the manifest content will get generated on the fly.
+ *
+ * @param manifestFile the filename of the manifest file.
+ */
+ fun setManifestFile(manifestFile: String) {
+ this.manifestFile = manifestFile
+ }
+
+ @Throws(IOException::class)
+ private fun manifestContent(): ByteArray {
+ if (manifestFile != null) {
+ FileInputStream(manifestFile!!).use { `in` -> return manifestContentImpl(Manifest(`in`)) }
+ } else {
+ return manifestContentImpl(Manifest())
+ }
+ }
+
+ @Throws(IOException::class)
+ private fun manifestContentImpl(manifest: Manifest): ByteArray {
+ val attributes = manifest.mainAttributes
+ attributes[Attributes.Name.MANIFEST_VERSION] = "1.0"
+ val createdBy = Attributes.Name("Created-By")
+ if (attributes.getValue(createdBy) == null) {
+ attributes[createdBy] = "io.bazel.rules.kotlin"
+ }
+ if (mainClass != null) {
+ attributes[Attributes.Name.MAIN_CLASS] = mainClass
+ }
+ if (targetLabel != null) {
+ attributes[TARGET_LABEL] = targetLabel
+ }
+ if (injectingRuleKind != null) {
+ attributes[INJECTING_RULE_KIND] = injectingRuleKind
+ }
+ val out = ByteArrayOutputStream()
+ manifest.write(out)
+ return out.toByteArray()
+ }
+
+ /**
+ * Executes the creation of the Jar file.
+ *
+ * @throws IOException if the Jar cannot be written or any of the entries cannot be read.
+ */
+ @Throws(IOException::class)
+ fun execute() {
+ Files.newOutputStream(jarPath).use { os ->
+ BufferedOutputStream(os).use { bos ->
+ JarOutputStream(bos).use { out ->
+ // Create the manifest entry in the Jar file
+ writeManifestEntry(out, manifestContent())
+ for ((key, value) in jarEntries) {
+ out.copyEntry(key, value)
+ }
+ }
+ }
+ }
+ }
+}
\ No newline at end of file
diff --git a/kotlin/builder/src/io/bazel/kotlin/builder/utils/jars/JarExtractor.kt b/kotlin/builder/src/io/bazel/kotlin/builder/utils/jars/JarExtractor.kt
new file mode 100644
index 0000000..13574ba
--- /dev/null
+++ b/kotlin/builder/src/io/bazel/kotlin/builder/utils/jars/JarExtractor.kt
@@ -0,0 +1,57 @@
+/*
+ * Copyright 2018 The Bazel Authors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.bazel.kotlin.builder.utils.jars
+
+import java.nio.file.Files
+import java.nio.file.Path
+import java.util.jar.JarFile
+
+
+open class JarExtractor protected constructor(
+ protected var destDir: Path
+) {
+
+ /**
+ * @param isDirectory is the target a directory.
+ * @param target path the operation will apply to.
+ * @return weather the operation should be applied or not.
+ */
+ internal open fun preWrite(isDirectory: Boolean, target: Path): Boolean = true
+
+ protected fun extract(jarFile: Path) {
+ JarFile(jarFile.toFile()).use { jar ->
+ jar.entries().also { entries ->
+ while (entries.hasMoreElements()) {
+ (entries.nextElement() as java.util.jar.JarEntry).also { entry ->
+ destDir.resolve(entry.name).also { target ->
+ if (preWrite(entry.isDirectory, target)) {
+ when {
+ entry.isDirectory ->
+ Files.createDirectories(target)
+ else -> jar.getInputStream(entry).use {
+ Files.createDirectories(target.parent)
+ Files.copy(it, target)
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+}
+
diff --git a/kotlin/builder/src/io/bazel/kotlin/builder/utils/jars/JarHelper.kt b/kotlin/builder/src/io/bazel/kotlin/builder/utils/jars/JarHelper.kt
new file mode 100644
index 0000000..2fa0a13
--- /dev/null
+++ b/kotlin/builder/src/io/bazel/kotlin/builder/utils/jars/JarHelper.kt
@@ -0,0 +1,254 @@
+// Copyright 2014 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Copied from bazel core and there is some code in other branches which will use the some of the unused elements. Fix
+// this later on.
+@file:Suppress("unused","MemberVisibilityCanBePrivate")
+package io.bazel.kotlin.builder.utils.jars
+
+import java.io.FileNotFoundException
+import java.io.IOException
+import java.nio.file.Files
+import java.nio.file.Path
+import java.time.LocalDateTime
+import java.time.ZoneId
+import java.util.*
+import java.util.jar.Attributes
+import java.util.jar.JarEntry
+import java.util.jar.JarFile
+import java.util.jar.JarOutputStream
+import java.util.zip.CRC32
+
+/**
+ * A simple helper class for creating Jar files. All Jar entries are sorted alphabetically. Allows
+ * normalization of Jar entries by setting the timestamp of non-.class files to the DOS epoch.
+ * Timestamps of .class files are set to the DOS epoch + 2 seconds (The zip timestamp granularity)
+ * Adjusting the timestamp for .class files is necessary since otherwise javac will recompile java
+ * files if both the java file and its .class file are present.
+ */
+open class JarHelper internal constructor (
+ // The path to the Jar we want to create
+ protected val jarPath: Path,
+ // The properties to describe how to create the Jar
+ protected val normalize: Boolean = true,
+ protected val verbose: Boolean = false,
+ compression: Boolean = true
+) {
+ private var storageMethod: Int = JarEntry.DEFLATED
+ // The state needed to create the Jar
+ private val names: MutableSet<String> = HashSet()
+
+ init {
+ setCompression(compression)
+ }
+
+ /**
+ * Enables or disables compression for the Jar file entries.
+ *
+ * @param compression if true enables compressions for the Jar file entries.
+ */
+ private fun setCompression(compression: Boolean) {
+ storageMethod = if (compression) JarEntry.DEFLATED else JarEntry.STORED
+ }
+
+ /**
+ * Returns the normalized timestamp for a jar entry based on its name. This is necessary since
+ * javac will, when loading a class X, prefer a source file to a class file, if both files have
+ * the same timestamp. Therefore, we need to adjust the timestamp for class files to slightly
+ * after the normalized time.
+ *
+ * @param name The name of the file for which we should return the normalized timestamp.
+ * @return the time for a new Jar file entry in milliseconds since the epoch.
+ */
+ private fun normalizedTimestamp(name: String): Long {
+ return if (name.endsWith(".class")) {
+ DEFAULT_TIMESTAMP + MINIMUM_TIMESTAMP_INCREMENT
+ } else {
+ DEFAULT_TIMESTAMP
+ }
+ }
+
+ /**
+ * Returns the time for a new Jar file entry in milliseconds since the epoch. Uses JarCreator.DEFAULT_TIMESTAMP]
+ * for normalized entries, [System.currentTimeMillis] otherwise.
+ *
+ * @param filename The name of the file for which we are entering the time
+ * @return the time for a new Jar file entry in milliseconds since the epoch.
+ */
+ private fun newEntryTimeMillis(filename: String): Long {
+ return if (normalize) normalizedTimestamp(filename) else System.currentTimeMillis()
+ }
+
+ /**
+ * Writes an entry with specific contents to the jar. Directory entries must include the trailing
+ * '/'.
+ */
+ @Throws(IOException::class)
+ private fun writeEntry(out: JarOutputStream, name: String, content: ByteArray) {
+ if (names.add(name)) {
+ // Create a new entry
+ val entry = JarEntry(name)
+ entry.time = newEntryTimeMillis(name)
+ val size = content.size
+ entry.size = size.toLong()
+ if (size == 0) {
+ entry.method = JarEntry.STORED
+ entry.crc = 0
+ out.putNextEntry(entry)
+ } else {
+ entry.method = storageMethod
+ if (storageMethod == JarEntry.STORED) {
+ val crc = CRC32()
+ crc.update(content)
+ entry.crc = crc.value
+ }
+ out.putNextEntry(entry)
+ out.write(content)
+ }
+ out.closeEntry()
+ }
+ }
+
+ /**
+ * Writes a standard Java manifest entry into the JarOutputStream. This includes the directory
+ * entry for the "META-INF" directory
+ *
+ * @param content the Manifest content to write to the manifest entry.
+ * @throws IOException
+ */
+ @Throws(IOException::class)
+ protected fun writeManifestEntry(out: JarOutputStream, content: ByteArray) {
+ val oldStorageMethod = storageMethod
+ // Do not compress small manifest files, the compressed one is frequently
+ // larger than the original. The threshold of 256 bytes is somewhat arbitrary.
+ if (content.size < 256) {
+ storageMethod = JarEntry.STORED
+ }
+ try {
+ writeEntry(out, MANIFEST_DIR, byteArrayOf())
+ writeEntry(out, MANIFEST_NAME, content)
+ } finally {
+ storageMethod = oldStorageMethod
+ }
+ }
+
+ /**
+ * Copies file or directory entries from the file system into the jar. Directory entries will be
+ * detected and their names automatically '/' suffixed.
+ */
+ @Throws(IOException::class)
+ protected fun JarOutputStream.copyEntry(name: String, path: Path) {
+ var normalizedName = name
+ if (!names.contains(normalizedName)) {
+ if (!Files.exists(path)) {
+ throw FileNotFoundException("${path.toAbsolutePath()} (No such file or directory)")
+ }
+ val isDirectory = Files.isDirectory(path)
+ if (isDirectory && !normalizedName.endsWith("/")) {
+ normalizedName = "$normalizedName/" // always normalize directory names before checking set
+ }
+ if (names.add(normalizedName)) {
+ if (verbose) {
+ System.err.println("adding $path")
+ }
+ // Create a new entry
+ val size = if (isDirectory) 0 else Files.size(path)
+ val outEntry = JarEntry(normalizedName)
+ val newtime = if (normalize) normalizedTimestamp(normalizedName) else Files.getLastModifiedTime(path).toMillis()
+ outEntry.time = newtime
+ outEntry.size = size
+ if (size == 0L) {
+ outEntry.method = JarEntry.STORED
+ outEntry.crc = 0
+ putNextEntry(outEntry)
+ } else {
+ outEntry.method = storageMethod
+ if (storageMethod == JarEntry.STORED) {
+ // ZipFile requires us to calculate the CRC-32 for any STORED entry.
+ // It would be nicer to do this via DigestInputStream, but
+ // the architecture of ZipOutputStream requires us to know the CRC-32
+ // before we write the data to the stream.
+ val bytes = Files.readAllBytes(path)
+ val crc = CRC32()
+ crc.update(bytes)
+ outEntry.crc = crc.value
+ putNextEntry(outEntry)
+ write(bytes)
+ } else {
+ putNextEntry(outEntry)
+ Files.copy(path, this)
+ }
+ }
+ closeEntry()
+ }
+ }
+ }
+
+ /**
+ * Copies a a single entry into the jar. This variant differs from the other [copyEntry] in two ways. Firstly the
+ * jar contents are already loaded in memory and Secondly the [name] and [path] entries don't necessarily have a
+ * correspondence.
+ *
+ * @param path the path used to retrieve the timestamp in case normalize is disabled.
+ * @param data if this is empty array then the entry is a directory.
+ */
+ protected fun JarOutputStream.copyEntry(name: String, path: Path? = null, data: ByteArray = EMPTY_BYTEARRAY) {
+ val outEntry = JarEntry(name)
+ outEntry.time = when {
+ normalize -> normalizedTimestamp(name)
+ else -> Files.getLastModifiedTime(checkNotNull(path)).toMillis()
+ }
+ outEntry.size = data.size.toLong()
+
+ if (data.isEmpty()) {
+ outEntry.method = JarEntry.STORED
+ outEntry.crc = 0
+ putNextEntry(outEntry)
+ } else {
+ outEntry.method = storageMethod
+ if (storageMethod == JarEntry.STORED) {
+ val crc = CRC32()
+ crc.update(data)
+ outEntry.crc = crc.value
+ putNextEntry(outEntry)
+ write(data)
+ } else {
+ putNextEntry(outEntry)
+ write(data)
+ }
+ }
+ closeEntry()
+ }
+
+ companion object {
+ const val MANIFEST_DIR = "META-INF/"
+ const val MANIFEST_NAME = JarFile.MANIFEST_NAME
+ const val SERVICES_DIR = "META-INF/services/"
+ internal val EMPTY_BYTEARRAY = ByteArray(0)
+ /** Normalize timestamps. */
+ val DEFAULT_TIMESTAMP = LocalDateTime.of(1980, 1, 1, 0, 0, 0)
+ .atZone(ZoneId.systemDefault())
+ .toInstant()
+ .toEpochMilli()
+
+ // These attributes are used by JavaBuilder, Turbine, and ijar.
+ // They must all be kept in sync.
+ val TARGET_LABEL = Attributes.Name("Target-Label")
+ val INJECTING_RULE_KIND = Attributes.Name("Injecting-Rule-Kind")
+
+ // ZIP timestamps have a resolution of 2 seconds.
+ // see http://www.info-zip.org/FAQ.html#limits
+ const val MINIMUM_TIMESTAMP_INCREMENT = 2000L
+ }
+}
diff --git a/kotlin/builder/src/io/bazel/kotlin/builder/utils/jars/SourceJarExtractor.kt b/kotlin/builder/src/io/bazel/kotlin/builder/utils/jars/SourceJarExtractor.kt
new file mode 100644
index 0000000..29eeccf
--- /dev/null
+++ b/kotlin/builder/src/io/bazel/kotlin/builder/utils/jars/SourceJarExtractor.kt
@@ -0,0 +1,48 @@
+/*
+ * Copyright 2018 The Bazel Authors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.bazel.kotlin.builder.utils.jars
+
+import java.nio.file.Path
+import java.util.function.Predicate
+
+class SourceJarExtractor(destDir: Path, val fileMatcher: Predicate<String> = Predicate { true }) : JarExtractor(destDir) {
+ val jarFiles = mutableListOf<Path>()
+ val sourcesList = mutableListOf<String>()
+
+ override fun preWrite(isDirectory: Boolean, target: Path): Boolean {
+ if (!isDirectory && fileMatcher.test(target.toString())) {
+ sourcesList.add(target.toString())
+ }
+ return true
+ }
+
+ fun execute() {
+ destDir.also {
+ try {
+ it.toFile().mkdirs()
+ } catch (ex: Exception) {
+ throw RuntimeException("could not create unpack directory at $it", ex)
+ }
+ }
+ jarFiles.forEach {
+ try {
+ extract(it)
+ } catch (ex: Throwable) {
+ throw RuntimeException("error extracting source jar $it", ex)
+ }
+ }
+ }
+}
\ No newline at end of file
diff --git a/kotlin/internal/compile.bzl b/kotlin/internal/compile.bzl
index e08429a..c07fe77 100644
--- a/kotlin/internal/compile.bzl
+++ b/kotlin/internal/compile.bzl
@@ -221,7 +221,7 @@
# setup the merge action if needed.
if len(output_merge_list) > 0:
- utils.actions.fold_jars(ctx, output_jar, output_merge_list)
+ utils.actions.fold_jars(ctx, rule_kind, output_jar, output_merge_list)
# create the java provider but the kotlin and default provider cannot be created here.
return _make_java_provider(ctx, srcs, deps, kotlin_auto_deps)
diff --git a/kotlin/internal/utils.bzl b/kotlin/internal/utils.bzl
index fd34a66..4830f6d 100644
--- a/kotlin/internal/utils.bzl
+++ b/kotlin/internal/utils.bzl
@@ -187,11 +187,17 @@
# PACKAGE JARS #################################################################################################################################################
-def _fold_jars_action(ctx, output_jar, input_jars):
- args=["--output", output_jar.path]
+def _fold_jars_action(ctx, rule_kind, output_jar, input_jars):
+ args=[
+ "--normalize",
+ "--compression",
+ "--deploy_manifest_lines",
+ "Target-Label: %s" % str(ctx.label),
+ "Injecting-Rule-Kind: %s" % rule_kind,
+ "--output", output_jar.path
+ ]
for i in input_jars:
args += ["--sources", i.path]
-
ctx.action(
mnemonic = "KotlinFoldOutput",
inputs = input_jars,
diff --git a/tests/Bazel_all_local_tests.xml b/tests/Bazel_all_local_tests.xml
new file mode 100644
index 0000000..c2aeb5e
--- /dev/null
+++ b/tests/Bazel_all_local_tests.xml
@@ -0,0 +1,5 @@
+<configuration name="Bazel test :all_local_tests" type="BlazeCommandRunConfigurationType" factoryName="Bazel Command" nameIsGenerated="true">
+ <blaze-settings handler-id="BlazeCommandGenericRunConfigurationHandlerProvider" blaze-command="test">
+ <blaze-target>//:all_local_tests</blaze-target>
+ </blaze-settings>
+</configuration>
\ No newline at end of file
diff --git a/tests/integrationtests/BUILD b/tests/integrationtests/BUILD
index 0d16629..ee2b8bc 100644
--- a/tests/integrationtests/BUILD
+++ b/tests/integrationtests/BUILD
@@ -18,4 +18,11 @@
tests=[
"//tests/integrationtests/jvm"
]
+)
+
+test_suite(
+ name = "integrationtests_local",
+ tests = [
+ "//tests/integrationtests/jvm:jvm_local"
+ ]
)
\ No newline at end of file
diff --git a/tests/integrationtests/jvm/BUILD b/tests/integrationtests/jvm/BUILD
index 70bd38a..1b4a427 100644
--- a/tests/integrationtests/jvm/BUILD
+++ b/tests/integrationtests/jvm/BUILD
@@ -22,6 +22,12 @@
)
kt_it_assertion_test(
+ name ="basic_local_tests",
+ cases="//tests/integrationtests/jvm/basic:cases",
+ test_class = "io.bazel.kotlin.testing.jvm.JvmBasicLocalTests",
+)
+
+kt_it_assertion_test(
name = "kapt_tests",
cases = "//tests/integrationtests/jvm/kapt:cases",
test_class="io.bazel.kotlin.testing.jvm.KaptTests",
@@ -42,3 +48,10 @@
"//tests/integrationtests/jvm/basic:friends_tests"
]
)
+
+test_suite(
+ name = "jvm_local",
+ tests = [
+ ":basic_local_tests"
+ ]
+)
diff --git a/tests/integrationtests/jvm/JvmBasicLocalTests.kt b/tests/integrationtests/jvm/JvmBasicLocalTests.kt
new file mode 100644
index 0000000..ccb018e
--- /dev/null
+++ b/tests/integrationtests/jvm/JvmBasicLocalTests.kt
@@ -0,0 +1,47 @@
+/*
+ * Copyright 2018 The Bazel Authors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.bazel.kotlin.testing.jvm
+
+import io.bazel.kotlin.testing.AssertionTestCase
+import org.junit.Test
+
+
+class JvmBasicLocalTests : AssertionTestCase("tests/integrationtests/jvm/basic") {
+ /*
+ * (hsyed) This test is running locally because things hash differently on the ci servers. Don't have the time to
+ * look into it.
+ */
+ @Test
+ fun testJarNormalization() {
+ jarTestCase(
+ name = "test_module_name_lib.jar",
+ description = "Builder jars should be normalized with the same timestamps as singlejar and including stamp data"
+ ) {
+ validateFileSha256("513d14b29eb1b95b97bf7d34e2126a716c7d1012b259b5021c16b99ca82feeb5")
+ assertManifestStamped()
+ assertEntryCompressedAndNormalizedTimestampYear("helloworld/Main.class")
+ }
+ jarTestCase(
+ name = "test_embed_resources.jar",
+ description = "Merging resources into the main output jar should still result in a normalized jar"
+ ) {
+ validateFileSha256("2d9175e9ecc6b9bc62f59ce861e9b67c6f64dd581f6cbd986c0a694b89e310b1")
+ assertManifestStamped()
+ assertEntryCompressedAndNormalizedTimestampYear("testresources/AClass.class")
+ assertEntryCompressedAndNormalizedTimestampYear("tests/integrationtests/jvm/basic/testresources/resources/one/two/aFile.txt")
+ }
+ }
+}
\ No newline at end of file
diff --git a/tests/integrationtests/jvm/basic/testresources/resources/one/alsoAFile.txt b/tests/integrationtests/jvm/basic/testresources/resources/one/alsoAFile.txt
index e69de29..1be511f 100644
--- a/tests/integrationtests/jvm/basic/testresources/resources/one/alsoAFile.txt
+++ b/tests/integrationtests/jvm/basic/testresources/resources/one/alsoAFile.txt
@@ -0,0 +1 @@
+other data
\ No newline at end of file
diff --git a/tests/integrationtests/jvm/basic/testresources/resources/one/two/aFile.txt b/tests/integrationtests/jvm/basic/testresources/resources/one/two/aFile.txt
index e69de29..e036be5 100644
--- a/tests/integrationtests/jvm/basic/testresources/resources/one/two/aFile.txt
+++ b/tests/integrationtests/jvm/basic/testresources/resources/one/two/aFile.txt
@@ -0,0 +1 @@
+Enough data to cause compression to kick in. Sometimes an extra sentence is needed.
\ No newline at end of file
diff --git a/tests/rules/AssertionTestCase.kt b/tests/rules/AssertionTestCase.kt
index 0e44ea7..2b6cf2d 100644
--- a/tests/rules/AssertionTestCase.kt
+++ b/tests/rules/AssertionTestCase.kt
@@ -15,17 +15,28 @@
*/
package io.bazel.kotlin.testing
+import com.google.common.hash.Hashing
import java.io.File
+import java.nio.file.Files
import java.nio.file.Path
import java.nio.file.Paths
+import java.time.Instant
+import java.time.LocalDateTime
+import java.time.ZoneId
import java.util.concurrent.TimeUnit
+import java.util.jar.JarEntry
import java.util.jar.JarFile
+import kotlin.test.assertEquals
+import kotlin.test.assertNotNull
+import kotlin.test.assertTrue
import kotlin.test.fail
-class TestCaseFailedException(description: String? = null, ex: Throwable) :
- AssertionError(""""$description" failed, error: ${ex.message}""", ex)
+class TestCaseFailedException(name: String? = null, description: String? = null, cause: Throwable) :
+ AssertionError(""""${name?.let { "jar: $it " } ?: ""} "$description" failed, error: ${cause.message}""", cause)
abstract class AssertionTestCase(root: String) : BasicAssertionTestCase() {
+ private lateinit var currentFile: File
+
private val testRunfileRoot: Path = Paths.get(root).also {
it.toFile().also {
assert(it.exists()) { "runfile directory $root does not exist" }
@@ -33,31 +44,23 @@
}
}
- private inline fun runTestCase(description: String? = null, op: () -> Unit) =
+ private inline fun runTestCase(name: String, description: String? = null, op: () -> Unit) =
try {
op()
} catch (t: Throwable) {
when (t) {
- is AssertionError -> throw TestCaseFailedException(description, t)
- is Exception -> throw TestCaseFailedException(description, t)
+ is AssertionError -> throw TestCaseFailedException(name, description, t)
+ is Exception -> throw TestCaseFailedException(name, description, t)
else -> throw t
}
}
- private fun testCaseJar(jarName: String) = testRunfileRoot.resolve(jarName).toFile().let {
- check(it.exists()) { "jar $jarName did not exist in test case root $testRunfileRoot" }
- JarFile(it)
- }
-
- private fun jarTestCase(name: String, op: JarFile.() -> Unit) {
- testCaseJar(name).also { op(it) }
- }
-
protected fun jarTestCase(name: String, description: String? = null, op: JarFile.() -> Unit) {
- runTestCase(description, { jarTestCase(name, op) })
+ currentFile = testRunfileRoot.resolve(name).toFile()
+ check(currentFile.exists()) { "testFile $name did not exist in test case root $testRunfileRoot" }
+ runTestCase(name, description) { JarFile(currentFile).op() }
}
-
protected fun JarFile.assertContainsEntries(vararg entries: String) {
entries.forEach {
if (this.getJarEntry(it) == null) {
@@ -66,6 +69,37 @@
}
}
+ /**
+ * Validated the entry is compressed and has the DOS epoch for it's timestamp.
+ */
+ protected fun JarFile.assertEntryCompressedAndNormalizedTimestampYear(entry: String) {
+ checkNotNull(this.getJarEntry(entry)).also {
+ check(!it.isDirectory)
+ assertTrue("$entry is not compressed") { JarEntry.DEFLATED == it.method }
+ val modifiedTimestamp = LocalDateTime.ofInstant(
+ Instant.ofEpochMilli(it.lastModifiedTime.toMillis()), ZoneId.systemDefault()
+ )
+ assertTrue("normalized modification time stamps should have year 1980") { modifiedTimestamp.year == 1980 }
+ }
+ }
+
+ /**
+ * Assert the manifest in the jar is stamped.
+ */
+ protected fun JarFile.assertManifestStamped() {
+ assertNotNull(
+ manifest.mainAttributes.getValue("Target-Label"), "missing manifest entry Target-Label"
+ )
+ assertNotNull(
+ manifest.mainAttributes.getValue("Injecting-Rule-Kind"), "missing manifest entry Injecting-Rule-Kind"
+ )
+ }
+
+ protected fun validateFileSha256(expected: String) {
+ val result = Hashing.sha256().hashBytes(Files.readAllBytes(currentFile.toPath())).toString()
+ assertEquals(expected, result, "files did not hash as expected")
+ }
+
protected fun JarFile.assertDoesNotContainEntries(vararg entries: String) {
entries.forEach {
if (this.getJarEntry(it) != null) {
@@ -88,7 +122,10 @@
.start().let {
it.waitFor(5, TimeUnit.SECONDS)
assert(it.exitValue() == 0) {
- throw TestCaseFailedException(description, RuntimeException("non-zero return code: ${it.exitValue()}"))
+ throw TestCaseFailedException(
+ description = description,
+ cause = RuntimeException("non-zero return code: ${it.exitValue()}")
+ )
}
}
}
diff --git a/tests/rules/BUILD b/tests/rules/BUILD
index 704c08e..f223bb6 100644
--- a/tests/rules/BUILD
+++ b/tests/rules/BUILD
@@ -18,6 +18,9 @@
name = "assertion_test_case",
srcs = ["AssertionTestCase.kt"],
visibility=["//tests:__subpackages__"],
- deps=["@com_github_jetbrains_kotlin//:kotlin-test"],
+ deps=[
+ "@com_github_jetbrains_kotlin//:kotlin-test",
+ "@io_bazel_rules_kotlin_com_google_guava_guava//jar",
+ ],
testonly=1
)
\ No newline at end of file