Remove jar tool usage and add jar normalisation. (#94) * remove jar tool with a port of JarCreator * ensure jar normalization * make the hashing test local and add ij runconfig
diff --git a/.bazelproject b/.bazelproject index 73d0819..142e24b 100644 --- a/.bazelproject +++ b/.bazelproject
@@ -27,4 +27,7 @@ */integrationtests/* additional_languages: - kotlin \ No newline at end of file + kotlin + +import_run_configurations: + tests/Bazel_all_local_tests.xml \ No newline at end of file
diff --git a/.gitignore b/.gitignore index d1607bf..101fc88 100644 --- a/.gitignore +++ b/.gitignore
@@ -1 +1,2 @@ -**/bazel-* \ No newline at end of file +**/bazel-* +.project/**
diff --git a/BUILD b/BUILD index b80e4d9..886f2de 100644 --- a/BUILD +++ b/BUILD
@@ -11,6 +11,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +# The entire test suite excluding local tests. test_suite( name = "all_tests", tests = [ @@ -18,4 +20,13 @@ "//kotlin/builder:integrationtests", "//tests/integrationtests" ] +) + +# Local tests +test_suite( + name = "all_local_tests", + tests = [ + ":all_tests", + "//tests/integrationtests:integrationtests_local" + ] ) \ No newline at end of file
diff --git a/kotlin/builder/src/io/bazel/kotlin/builder/KotlinBuilder.kt b/kotlin/builder/src/io/bazel/kotlin/builder/KotlinBuilder.kt index 61431ba..0b0e262 100644 --- a/kotlin/builder/src/io/bazel/kotlin/builder/KotlinBuilder.kt +++ b/kotlin/builder/src/io/bazel/kotlin/builder/KotlinBuilder.kt
@@ -21,7 +21,9 @@ import io.bazel.kotlin.builder.mode.jvm.KotlinJvmCompilationExecutor import io.bazel.kotlin.builder.utils.ArgMap import io.bazel.kotlin.builder.utils.ArgMaps +import io.bazel.kotlin.builder.utils.IS_JVM_SOURCE_FILE import io.bazel.kotlin.builder.utils.ensureDirectories +import io.bazel.kotlin.builder.utils.jars.SourceJarExtractor import io.bazel.kotlin.model.KotlinModel import java.nio.file.Paths @@ -29,7 +31,6 @@ @Suppress("MemberVisibilityCanBePrivate") class KotlinBuilder @Inject internal constructor( private val commandBuilder: BuildCommandBuilder, - private val jarToolInvoker: KotlinToolchain.JarToolInvoker, private val compilationExector: KotlinJvmCompilationExecutor ) : CommandLineProgram { fun execute(args: List<String>): Int = @@ -62,30 +63,15 @@ if (command.inputs.sourceJarsList.isEmpty()) { command } else { - val sourceUnpackDirectory = - Paths.get(command.directories.temp).let { - it.resolve("_srcjars").toFile().let { - try { - it.mkdirs(); it - } catch (ex: Exception) { - throw RuntimeException("could not create unpack directory at $it", ex) - } - } - } - for (sourceJar in command.inputs.sourceJarsList) { - jarToolInvoker.invoke( - listOf("xf", Paths.get(sourceJar).toAbsolutePath().toString()), sourceUnpackDirectory - ) + SourceJarExtractor( + destDir = Paths.get(command.directories.temp).resolve("_srcjars"), + fileMatcher = IS_JVM_SOURCE_FILE + ).also { + it.jarFiles.addAll(command.inputs.sourceJarsList.map { Paths.get(it) }) + it.execute() + }.let { + commandBuilder.withSources(command, it.sourcesList.iterator()) } - - commandBuilder.withSources( - command, - sourceUnpackDirectory - .walk() - .filter { it.name.endsWith(".kt") || it.name.endsWith(".java") } - .map { it.toString() } - .iterator() - ) } override fun apply(args: List<String>): Int {
diff --git a/kotlin/builder/src/io/bazel/kotlin/builder/KotlinToolchain.kt b/kotlin/builder/src/io/bazel/kotlin/builder/KotlinToolchain.kt index 8efabd0..84766d3 100644 --- a/kotlin/builder/src/io/bazel/kotlin/builder/KotlinToolchain.kt +++ b/kotlin/builder/src/io/bazel/kotlin/builder/KotlinToolchain.kt
@@ -107,10 +107,6 @@ interface KotlincInvoker { fun compile(args: Array<String>, out: PrintStream): Int } - - interface JarToolInvoker { - fun invoke(args: List<String>, directory: File? = null) - } }
diff --git a/kotlin/builder/src/io/bazel/kotlin/builder/KotlinToolchainModule.kt b/kotlin/builder/src/io/bazel/kotlin/builder/KotlinToolchainModule.kt index 9ede01a..5bb5cc1 100644 --- a/kotlin/builder/src/io/bazel/kotlin/builder/KotlinToolchainModule.kt +++ b/kotlin/builder/src/io/bazel/kotlin/builder/KotlinToolchainModule.kt
@@ -18,27 +18,12 @@ import com.google.inject.AbstractModule import com.google.inject.Provides import io.bazel.kotlin.builder.KotlinToolchain.Companion.NO_ARGS -import io.bazel.kotlin.builder.utils.executeAndAwait import io.bazel.kotlin.builder.utils.resolveVerified -import io.bazel.kotlin.builder.utils.resolveVerifiedToAbsoluteString -import java.io.File import java.io.PrintStream import java.io.PrintWriter internal object KotlinToolchainModule : AbstractModule() { @Provides - fun jarToolInvoker(toolchain: KotlinToolchain): KotlinToolchain.JarToolInvoker = - object : KotlinToolchain.JarToolInvoker { - override fun invoke(args: List<String>, directory: File?) { - val jarTool = toolchain.javaHome.resolveVerifiedToAbsoluteString("bin", "jar") - val command = mutableListOf(jarTool).also { it.addAll(args) } - executeAndAwait(10, directory, command).takeIf { it != 0 }?.also { - throw CompilationStatusException("error running jar command ${command.joinToString(" ")}", it) - } - } - } - - @Provides fun javacInvoker(toolchain: KotlinToolchain): KotlinToolchain.JavacInvoker = object : KotlinToolchain.JavacInvoker { val c = toolchain.classLoader.loadClass("com.sun.tools.javac.Main") val m = c.getMethod("compile", Array<String>::class.java)
diff --git a/kotlin/builder/src/io/bazel/kotlin/builder/mode/jvm/actions/OutputJarCreator.kt b/kotlin/builder/src/io/bazel/kotlin/builder/mode/jvm/actions/OutputJarCreator.kt index e46147b..88c98af 100644 --- a/kotlin/builder/src/io/bazel/kotlin/builder/mode/jvm/actions/OutputJarCreator.kt +++ b/kotlin/builder/src/io/bazel/kotlin/builder/mode/jvm/actions/OutputJarCreator.kt
@@ -16,10 +16,8 @@ package io.bazel.kotlin.builder.mode.jvm.actions import com.google.inject.ImplementedBy -import com.google.inject.Inject -import io.bazel.kotlin.builder.KotlinToolchain +import io.bazel.kotlin.builder.utils.jars.JarCreator import io.bazel.kotlin.model.KotlinModel -import java.nio.file.Path import java.nio.file.Paths @ImplementedBy(DefaultOutputJarCreator::class) @@ -27,16 +25,17 @@ fun createOutputJar(command: KotlinModel.BuilderCommand) } -private class DefaultOutputJarCreator @Inject constructor( - val toolInvoker: KotlinToolchain.JarToolInvoker -) : OutputJarCreator { - private fun MutableList<String>.addAllFrom(dir: Path) = addAll(arrayOf("-C", dir.toString(), ".")) +private class DefaultOutputJarCreator : OutputJarCreator { override fun createOutputJar(command: KotlinModel.BuilderCommand) { - mutableListOf( - "cf", command.outputs.jar - ).also { args -> - args.addAllFrom(Paths.get(command.directories.classes)) - args.addAllFrom(Paths.get(command.directories.generatedClasses)) - }.let { toolInvoker.invoke(it) } + JarCreator( + path = Paths.get(command.outputs.jar), + normalize = true, + verbose = false + ).also { + it.addDirectory(Paths.get(command.directories.classes)) + it.addDirectory(Paths.get(command.directories.generatedClasses)) + it.setJarOwner(command.info.label, command.info.ruleKind) + it.execute() + } } }
diff --git a/kotlin/builder/src/io/bazel/kotlin/builder/utils/MiscUtils.kt b/kotlin/builder/src/io/bazel/kotlin/builder/utils/MiscUtils.kt index bbbf19f..fcfd90a 100644 --- a/kotlin/builder/src/io/bazel/kotlin/builder/utils/MiscUtils.kt +++ b/kotlin/builder/src/io/bazel/kotlin/builder/utils/MiscUtils.kt
@@ -16,5 +16,12 @@ package io.bazel.kotlin.builder.utils +import java.util.function.Predicate +import java.util.regex.Pattern + fun <T, C : MutableCollection<T>> C.addAll(vararg entries: T): C = this.also { addAll(entries) } +private fun extensionMatcher(vararg ext: String): Predicate<String> = + Pattern.compile("^(.+?)${ext.joinToString("|\\.", prefix = "(\\.",postfix = ")$")}").asPredicate() + +val IS_JVM_SOURCE_FILE = extensionMatcher("kt", "java")
diff --git a/kotlin/builder/src/io/bazel/kotlin/builder/utils/jars/JarCreator.kt b/kotlin/builder/src/io/bazel/kotlin/builder/utils/jars/JarCreator.kt new file mode 100644 index 0000000..7127ad1 --- /dev/null +++ b/kotlin/builder/src/io/bazel/kotlin/builder/utils/jars/JarCreator.kt
@@ -0,0 +1,225 @@ +/* + * Copyright 2018 The Bazel Authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.bazel.kotlin.builder.utils.jars + +import java.io.* +import java.nio.file.* +import java.nio.file.Paths.* +import java.nio.file.attribute.BasicFileAttributes +import java.util.* +import java.util.jar.Attributes +import java.util.jar.JarOutputStream +import java.util.jar.Manifest + +@Suppress("unused") +/** + * A class for creating Jar files. Allows normalization of Jar entries by setting their timestamp to + * the DOS epoch. All Jar entries are sorted alphabetically. + */ +class JarCreator( + path: Path, + normalize: Boolean = true, + verbose: Boolean = false +) : JarHelper(path, normalize, verbose) { + // Map from Jar entry names to files. Use TreeMap so we can establish a canonical order for the + // entries regardless in what order they get added. + private val jarEntries = TreeMap<String, Path>() + private var manifestFile: String? = null + private var mainClass: String? = null + private var targetLabel: String? = null + private var injectingRuleKind: String? = null + + /** + * Adds an entry to the Jar file, normalizing the name. + * + * @param entryName the name of the entry in the Jar file + * @param path the path of the input for the entry + * @return true iff a new entry was added + */ + private fun addEntry(entryName: String, path: Path): Boolean { + var normalizedEntryName = entryName + if (normalizedEntryName.startsWith("/")) { + normalizedEntryName = normalizedEntryName.substring(1) + } else if (normalizedEntryName.length >= 3 + && Character.isLetter(normalizedEntryName[0]) + && normalizedEntryName[1] == ':' + && (normalizedEntryName[2] == '\\' || normalizedEntryName[2] == '/') + ) { + // Windows absolute path, e.g. "D:\foo" or "e:/blah". + // Windows paths are case-insensitive, and support both backslashes and forward slashes. + normalizedEntryName = normalizedEntryName.substring(3) + } else if (normalizedEntryName.startsWith("./")) { + normalizedEntryName = normalizedEntryName.substring(2) + } + return jarEntries.put(normalizedEntryName, path) == null + } + + /** + * Adds an entry to the Jar file, normalizing the name. + * + * @param entryName the name of the entry in the Jar file + * @param fileName the name of the input file for the entry + * @return true iff a new entry was added + */ + fun addEntry(entryName: String, fileName: String): Boolean { + return addEntry(entryName, get(fileName)) + } + + /** + * Adds the contents of a directory to the Jar file. All files below this directory will be added + * to the Jar file using the name relative to the directory as the name for the Jar entry. + * + * @param directory the directory to add to the jar + */ + fun addDirectory(directory: Path) { + if (!Files.exists(directory)) { + throw IllegalArgumentException("directory does not exist: $directory") + } + try { + Files.walkFileTree( + directory, + object : SimpleFileVisitor<Path>() { + + @Throws(IOException::class) + override fun preVisitDirectory(path: Path, attrs: BasicFileAttributes): FileVisitResult { + if (path != directory) { + // For consistency with legacy behaviour, include entries for directories except for + // the root. + addEntry(path, /* isDirectory= */ true) + } + return FileVisitResult.CONTINUE + } + + @Throws(IOException::class) + override fun visitFile(path: Path, attrs: BasicFileAttributes): FileVisitResult { + addEntry(path, /* isDirectory= */ false) + return FileVisitResult.CONTINUE + } + + fun addEntry(path: Path, isDirectory: Boolean) { + val sb = StringBuilder() + var first = true + for (entry in directory.relativize(path)) { + if (!first) { + // use `/` as the directory separator for jar paths, even on Windows + sb.append('/') + } + sb.append(entry.fileName) + first = false + } + if (isDirectory) { + sb.append('/') + } + jarEntries[sb.toString()] = path + } + }) + } catch (e: IOException) { + throw UncheckedIOException(e) + } + + } + + /** + * Adds a collection of entries to the jar, each with a given source path, and with the resulting + * file in the root of the jar. + * + * <pre> + * some/long/path.foo => (path.foo, some/long/path.foo) + </pre> * + */ + fun addRootEntries(entries: Collection<String>) { + for (entry in entries) { + val path = get(entry) + jarEntries[path.fileName.toString()] = path + } + } + + /** + * Sets the main.class entry for the manifest. A value of `null` (the default) will + * omit the entry. + * + * @param mainClass the fully qualified name of the main class + */ + fun setMainClass(mainClass: String) { + this.mainClass = mainClass + } + + fun setJarOwner(targetLabel: String, injectingRuleKind: String) { + this.targetLabel = targetLabel + this.injectingRuleKind = injectingRuleKind + } + + /** + * Sets filename for the manifest content. If this is set the manifest will be read from this file + * otherwise the manifest content will get generated on the fly. + * + * @param manifestFile the filename of the manifest file. + */ + fun setManifestFile(manifestFile: String) { + this.manifestFile = manifestFile + } + + @Throws(IOException::class) + private fun manifestContent(): ByteArray { + if (manifestFile != null) { + FileInputStream(manifestFile!!).use { `in` -> return manifestContentImpl(Manifest(`in`)) } + } else { + return manifestContentImpl(Manifest()) + } + } + + @Throws(IOException::class) + private fun manifestContentImpl(manifest: Manifest): ByteArray { + val attributes = manifest.mainAttributes + attributes[Attributes.Name.MANIFEST_VERSION] = "1.0" + val createdBy = Attributes.Name("Created-By") + if (attributes.getValue(createdBy) == null) { + attributes[createdBy] = "io.bazel.rules.kotlin" + } + if (mainClass != null) { + attributes[Attributes.Name.MAIN_CLASS] = mainClass + } + if (targetLabel != null) { + attributes[TARGET_LABEL] = targetLabel + } + if (injectingRuleKind != null) { + attributes[INJECTING_RULE_KIND] = injectingRuleKind + } + val out = ByteArrayOutputStream() + manifest.write(out) + return out.toByteArray() + } + + /** + * Executes the creation of the Jar file. + * + * @throws IOException if the Jar cannot be written or any of the entries cannot be read. + */ + @Throws(IOException::class) + fun execute() { + Files.newOutputStream(jarPath).use { os -> + BufferedOutputStream(os).use { bos -> + JarOutputStream(bos).use { out -> + // Create the manifest entry in the Jar file + writeManifestEntry(out, manifestContent()) + for ((key, value) in jarEntries) { + out.copyEntry(key, value) + } + } + } + } + } +} \ No newline at end of file
diff --git a/kotlin/builder/src/io/bazel/kotlin/builder/utils/jars/JarExtractor.kt b/kotlin/builder/src/io/bazel/kotlin/builder/utils/jars/JarExtractor.kt new file mode 100644 index 0000000..13574ba --- /dev/null +++ b/kotlin/builder/src/io/bazel/kotlin/builder/utils/jars/JarExtractor.kt
@@ -0,0 +1,57 @@ +/* + * Copyright 2018 The Bazel Authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.bazel.kotlin.builder.utils.jars + +import java.nio.file.Files +import java.nio.file.Path +import java.util.jar.JarFile + + +open class JarExtractor protected constructor( + protected var destDir: Path +) { + + /** + * @param isDirectory is the target a directory. + * @param target path the operation will apply to. + * @return weather the operation should be applied or not. + */ + internal open fun preWrite(isDirectory: Boolean, target: Path): Boolean = true + + protected fun extract(jarFile: Path) { + JarFile(jarFile.toFile()).use { jar -> + jar.entries().also { entries -> + while (entries.hasMoreElements()) { + (entries.nextElement() as java.util.jar.JarEntry).also { entry -> + destDir.resolve(entry.name).also { target -> + if (preWrite(entry.isDirectory, target)) { + when { + entry.isDirectory -> + Files.createDirectories(target) + else -> jar.getInputStream(entry).use { + Files.createDirectories(target.parent) + Files.copy(it, target) + } + } + } + } + } + } + } + } + } +} +
diff --git a/kotlin/builder/src/io/bazel/kotlin/builder/utils/jars/JarHelper.kt b/kotlin/builder/src/io/bazel/kotlin/builder/utils/jars/JarHelper.kt new file mode 100644 index 0000000..2fa0a13 --- /dev/null +++ b/kotlin/builder/src/io/bazel/kotlin/builder/utils/jars/JarHelper.kt
@@ -0,0 +1,254 @@ +// Copyright 2014 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Copied from bazel core and there is some code in other branches which will use the some of the unused elements. Fix +// this later on. +@file:Suppress("unused","MemberVisibilityCanBePrivate") +package io.bazel.kotlin.builder.utils.jars + +import java.io.FileNotFoundException +import java.io.IOException +import java.nio.file.Files +import java.nio.file.Path +import java.time.LocalDateTime +import java.time.ZoneId +import java.util.* +import java.util.jar.Attributes +import java.util.jar.JarEntry +import java.util.jar.JarFile +import java.util.jar.JarOutputStream +import java.util.zip.CRC32 + +/** + * A simple helper class for creating Jar files. All Jar entries are sorted alphabetically. Allows + * normalization of Jar entries by setting the timestamp of non-.class files to the DOS epoch. + * Timestamps of .class files are set to the DOS epoch + 2 seconds (The zip timestamp granularity) + * Adjusting the timestamp for .class files is necessary since otherwise javac will recompile java + * files if both the java file and its .class file are present. + */ +open class JarHelper internal constructor ( + // The path to the Jar we want to create + protected val jarPath: Path, + // The properties to describe how to create the Jar + protected val normalize: Boolean = true, + protected val verbose: Boolean = false, + compression: Boolean = true +) { + private var storageMethod: Int = JarEntry.DEFLATED + // The state needed to create the Jar + private val names: MutableSet<String> = HashSet() + + init { + setCompression(compression) + } + + /** + * Enables or disables compression for the Jar file entries. + * + * @param compression if true enables compressions for the Jar file entries. + */ + private fun setCompression(compression: Boolean) { + storageMethod = if (compression) JarEntry.DEFLATED else JarEntry.STORED + } + + /** + * Returns the normalized timestamp for a jar entry based on its name. This is necessary since + * javac will, when loading a class X, prefer a source file to a class file, if both files have + * the same timestamp. Therefore, we need to adjust the timestamp for class files to slightly + * after the normalized time. + * + * @param name The name of the file for which we should return the normalized timestamp. + * @return the time for a new Jar file entry in milliseconds since the epoch. + */ + private fun normalizedTimestamp(name: String): Long { + return if (name.endsWith(".class")) { + DEFAULT_TIMESTAMP + MINIMUM_TIMESTAMP_INCREMENT + } else { + DEFAULT_TIMESTAMP + } + } + + /** + * Returns the time for a new Jar file entry in milliseconds since the epoch. Uses JarCreator.DEFAULT_TIMESTAMP] + * for normalized entries, [System.currentTimeMillis] otherwise. + * + * @param filename The name of the file for which we are entering the time + * @return the time for a new Jar file entry in milliseconds since the epoch. + */ + private fun newEntryTimeMillis(filename: String): Long { + return if (normalize) normalizedTimestamp(filename) else System.currentTimeMillis() + } + + /** + * Writes an entry with specific contents to the jar. Directory entries must include the trailing + * '/'. + */ + @Throws(IOException::class) + private fun writeEntry(out: JarOutputStream, name: String, content: ByteArray) { + if (names.add(name)) { + // Create a new entry + val entry = JarEntry(name) + entry.time = newEntryTimeMillis(name) + val size = content.size + entry.size = size.toLong() + if (size == 0) { + entry.method = JarEntry.STORED + entry.crc = 0 + out.putNextEntry(entry) + } else { + entry.method = storageMethod + if (storageMethod == JarEntry.STORED) { + val crc = CRC32() + crc.update(content) + entry.crc = crc.value + } + out.putNextEntry(entry) + out.write(content) + } + out.closeEntry() + } + } + + /** + * Writes a standard Java manifest entry into the JarOutputStream. This includes the directory + * entry for the "META-INF" directory + * + * @param content the Manifest content to write to the manifest entry. + * @throws IOException + */ + @Throws(IOException::class) + protected fun writeManifestEntry(out: JarOutputStream, content: ByteArray) { + val oldStorageMethod = storageMethod + // Do not compress small manifest files, the compressed one is frequently + // larger than the original. The threshold of 256 bytes is somewhat arbitrary. + if (content.size < 256) { + storageMethod = JarEntry.STORED + } + try { + writeEntry(out, MANIFEST_DIR, byteArrayOf()) + writeEntry(out, MANIFEST_NAME, content) + } finally { + storageMethod = oldStorageMethod + } + } + + /** + * Copies file or directory entries from the file system into the jar. Directory entries will be + * detected and their names automatically '/' suffixed. + */ + @Throws(IOException::class) + protected fun JarOutputStream.copyEntry(name: String, path: Path) { + var normalizedName = name + if (!names.contains(normalizedName)) { + if (!Files.exists(path)) { + throw FileNotFoundException("${path.toAbsolutePath()} (No such file or directory)") + } + val isDirectory = Files.isDirectory(path) + if (isDirectory && !normalizedName.endsWith("/")) { + normalizedName = "$normalizedName/" // always normalize directory names before checking set + } + if (names.add(normalizedName)) { + if (verbose) { + System.err.println("adding $path") + } + // Create a new entry + val size = if (isDirectory) 0 else Files.size(path) + val outEntry = JarEntry(normalizedName) + val newtime = if (normalize) normalizedTimestamp(normalizedName) else Files.getLastModifiedTime(path).toMillis() + outEntry.time = newtime + outEntry.size = size + if (size == 0L) { + outEntry.method = JarEntry.STORED + outEntry.crc = 0 + putNextEntry(outEntry) + } else { + outEntry.method = storageMethod + if (storageMethod == JarEntry.STORED) { + // ZipFile requires us to calculate the CRC-32 for any STORED entry. + // It would be nicer to do this via DigestInputStream, but + // the architecture of ZipOutputStream requires us to know the CRC-32 + // before we write the data to the stream. + val bytes = Files.readAllBytes(path) + val crc = CRC32() + crc.update(bytes) + outEntry.crc = crc.value + putNextEntry(outEntry) + write(bytes) + } else { + putNextEntry(outEntry) + Files.copy(path, this) + } + } + closeEntry() + } + } + } + + /** + * Copies a a single entry into the jar. This variant differs from the other [copyEntry] in two ways. Firstly the + * jar contents are already loaded in memory and Secondly the [name] and [path] entries don't necessarily have a + * correspondence. + * + * @param path the path used to retrieve the timestamp in case normalize is disabled. + * @param data if this is empty array then the entry is a directory. + */ + protected fun JarOutputStream.copyEntry(name: String, path: Path? = null, data: ByteArray = EMPTY_BYTEARRAY) { + val outEntry = JarEntry(name) + outEntry.time = when { + normalize -> normalizedTimestamp(name) + else -> Files.getLastModifiedTime(checkNotNull(path)).toMillis() + } + outEntry.size = data.size.toLong() + + if (data.isEmpty()) { + outEntry.method = JarEntry.STORED + outEntry.crc = 0 + putNextEntry(outEntry) + } else { + outEntry.method = storageMethod + if (storageMethod == JarEntry.STORED) { + val crc = CRC32() + crc.update(data) + outEntry.crc = crc.value + putNextEntry(outEntry) + write(data) + } else { + putNextEntry(outEntry) + write(data) + } + } + closeEntry() + } + + companion object { + const val MANIFEST_DIR = "META-INF/" + const val MANIFEST_NAME = JarFile.MANIFEST_NAME + const val SERVICES_DIR = "META-INF/services/" + internal val EMPTY_BYTEARRAY = ByteArray(0) + /** Normalize timestamps. */ + val DEFAULT_TIMESTAMP = LocalDateTime.of(1980, 1, 1, 0, 0, 0) + .atZone(ZoneId.systemDefault()) + .toInstant() + .toEpochMilli() + + // These attributes are used by JavaBuilder, Turbine, and ijar. + // They must all be kept in sync. + val TARGET_LABEL = Attributes.Name("Target-Label") + val INJECTING_RULE_KIND = Attributes.Name("Injecting-Rule-Kind") + + // ZIP timestamps have a resolution of 2 seconds. + // see http://www.info-zip.org/FAQ.html#limits + const val MINIMUM_TIMESTAMP_INCREMENT = 2000L + } +}
diff --git a/kotlin/builder/src/io/bazel/kotlin/builder/utils/jars/SourceJarExtractor.kt b/kotlin/builder/src/io/bazel/kotlin/builder/utils/jars/SourceJarExtractor.kt new file mode 100644 index 0000000..29eeccf --- /dev/null +++ b/kotlin/builder/src/io/bazel/kotlin/builder/utils/jars/SourceJarExtractor.kt
@@ -0,0 +1,48 @@ +/* + * Copyright 2018 The Bazel Authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.bazel.kotlin.builder.utils.jars + +import java.nio.file.Path +import java.util.function.Predicate + +class SourceJarExtractor(destDir: Path, val fileMatcher: Predicate<String> = Predicate { true }) : JarExtractor(destDir) { + val jarFiles = mutableListOf<Path>() + val sourcesList = mutableListOf<String>() + + override fun preWrite(isDirectory: Boolean, target: Path): Boolean { + if (!isDirectory && fileMatcher.test(target.toString())) { + sourcesList.add(target.toString()) + } + return true + } + + fun execute() { + destDir.also { + try { + it.toFile().mkdirs() + } catch (ex: Exception) { + throw RuntimeException("could not create unpack directory at $it", ex) + } + } + jarFiles.forEach { + try { + extract(it) + } catch (ex: Throwable) { + throw RuntimeException("error extracting source jar $it", ex) + } + } + } +} \ No newline at end of file
diff --git a/kotlin/internal/compile.bzl b/kotlin/internal/compile.bzl index e08429a..c07fe77 100644 --- a/kotlin/internal/compile.bzl +++ b/kotlin/internal/compile.bzl
@@ -221,7 +221,7 @@ # setup the merge action if needed. if len(output_merge_list) > 0: - utils.actions.fold_jars(ctx, output_jar, output_merge_list) + utils.actions.fold_jars(ctx, rule_kind, output_jar, output_merge_list) # create the java provider but the kotlin and default provider cannot be created here. return _make_java_provider(ctx, srcs, deps, kotlin_auto_deps)
diff --git a/kotlin/internal/utils.bzl b/kotlin/internal/utils.bzl index fd34a66..4830f6d 100644 --- a/kotlin/internal/utils.bzl +++ b/kotlin/internal/utils.bzl
@@ -187,11 +187,17 @@ # PACKAGE JARS ################################################################################################################################################# -def _fold_jars_action(ctx, output_jar, input_jars): - args=["--output", output_jar.path] +def _fold_jars_action(ctx, rule_kind, output_jar, input_jars): + args=[ + "--normalize", + "--compression", + "--deploy_manifest_lines", + "Target-Label: %s" % str(ctx.label), + "Injecting-Rule-Kind: %s" % rule_kind, + "--output", output_jar.path + ] for i in input_jars: args += ["--sources", i.path] - ctx.action( mnemonic = "KotlinFoldOutput", inputs = input_jars,
diff --git a/tests/Bazel_all_local_tests.xml b/tests/Bazel_all_local_tests.xml new file mode 100644 index 0000000..c2aeb5e --- /dev/null +++ b/tests/Bazel_all_local_tests.xml
@@ -0,0 +1,5 @@ +<configuration name="Bazel test :all_local_tests" type="BlazeCommandRunConfigurationType" factoryName="Bazel Command" nameIsGenerated="true"> + <blaze-settings handler-id="BlazeCommandGenericRunConfigurationHandlerProvider" blaze-command="test"> + <blaze-target>//:all_local_tests</blaze-target> + </blaze-settings> +</configuration> \ No newline at end of file
diff --git a/tests/integrationtests/BUILD b/tests/integrationtests/BUILD index 0d16629..ee2b8bc 100644 --- a/tests/integrationtests/BUILD +++ b/tests/integrationtests/BUILD
@@ -18,4 +18,11 @@ tests=[ "//tests/integrationtests/jvm" ] +) + +test_suite( + name = "integrationtests_local", + tests = [ + "//tests/integrationtests/jvm:jvm_local" + ] ) \ No newline at end of file
diff --git a/tests/integrationtests/jvm/BUILD b/tests/integrationtests/jvm/BUILD index 70bd38a..1b4a427 100644 --- a/tests/integrationtests/jvm/BUILD +++ b/tests/integrationtests/jvm/BUILD
@@ -22,6 +22,12 @@ ) kt_it_assertion_test( + name ="basic_local_tests", + cases="//tests/integrationtests/jvm/basic:cases", + test_class = "io.bazel.kotlin.testing.jvm.JvmBasicLocalTests", +) + +kt_it_assertion_test( name = "kapt_tests", cases = "//tests/integrationtests/jvm/kapt:cases", test_class="io.bazel.kotlin.testing.jvm.KaptTests", @@ -42,3 +48,10 @@ "//tests/integrationtests/jvm/basic:friends_tests" ] ) + +test_suite( + name = "jvm_local", + tests = [ + ":basic_local_tests" + ] +)
diff --git a/tests/integrationtests/jvm/JvmBasicLocalTests.kt b/tests/integrationtests/jvm/JvmBasicLocalTests.kt new file mode 100644 index 0000000..ccb018e --- /dev/null +++ b/tests/integrationtests/jvm/JvmBasicLocalTests.kt
@@ -0,0 +1,47 @@ +/* + * Copyright 2018 The Bazel Authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.bazel.kotlin.testing.jvm + +import io.bazel.kotlin.testing.AssertionTestCase +import org.junit.Test + + +class JvmBasicLocalTests : AssertionTestCase("tests/integrationtests/jvm/basic") { + /* + * (hsyed) This test is running locally because things hash differently on the ci servers. Don't have the time to + * look into it. + */ + @Test + fun testJarNormalization() { + jarTestCase( + name = "test_module_name_lib.jar", + description = "Builder jars should be normalized with the same timestamps as singlejar and including stamp data" + ) { + validateFileSha256("513d14b29eb1b95b97bf7d34e2126a716c7d1012b259b5021c16b99ca82feeb5") + assertManifestStamped() + assertEntryCompressedAndNormalizedTimestampYear("helloworld/Main.class") + } + jarTestCase( + name = "test_embed_resources.jar", + description = "Merging resources into the main output jar should still result in a normalized jar" + ) { + validateFileSha256("2d9175e9ecc6b9bc62f59ce861e9b67c6f64dd581f6cbd986c0a694b89e310b1") + assertManifestStamped() + assertEntryCompressedAndNormalizedTimestampYear("testresources/AClass.class") + assertEntryCompressedAndNormalizedTimestampYear("tests/integrationtests/jvm/basic/testresources/resources/one/two/aFile.txt") + } + } +} \ No newline at end of file
diff --git a/tests/integrationtests/jvm/basic/testresources/resources/one/alsoAFile.txt b/tests/integrationtests/jvm/basic/testresources/resources/one/alsoAFile.txt index e69de29..1be511f 100644 --- a/tests/integrationtests/jvm/basic/testresources/resources/one/alsoAFile.txt +++ b/tests/integrationtests/jvm/basic/testresources/resources/one/alsoAFile.txt
@@ -0,0 +1 @@ +other data \ No newline at end of file
diff --git a/tests/integrationtests/jvm/basic/testresources/resources/one/two/aFile.txt b/tests/integrationtests/jvm/basic/testresources/resources/one/two/aFile.txt index e69de29..e036be5 100644 --- a/tests/integrationtests/jvm/basic/testresources/resources/one/two/aFile.txt +++ b/tests/integrationtests/jvm/basic/testresources/resources/one/two/aFile.txt
@@ -0,0 +1 @@ +Enough data to cause compression to kick in. Sometimes an extra sentence is needed. \ No newline at end of file
diff --git a/tests/rules/AssertionTestCase.kt b/tests/rules/AssertionTestCase.kt index 0e44ea7..2b6cf2d 100644 --- a/tests/rules/AssertionTestCase.kt +++ b/tests/rules/AssertionTestCase.kt
@@ -15,17 +15,28 @@ */ package io.bazel.kotlin.testing +import com.google.common.hash.Hashing import java.io.File +import java.nio.file.Files import java.nio.file.Path import java.nio.file.Paths +import java.time.Instant +import java.time.LocalDateTime +import java.time.ZoneId import java.util.concurrent.TimeUnit +import java.util.jar.JarEntry import java.util.jar.JarFile +import kotlin.test.assertEquals +import kotlin.test.assertNotNull +import kotlin.test.assertTrue import kotlin.test.fail -class TestCaseFailedException(description: String? = null, ex: Throwable) : - AssertionError(""""$description" failed, error: ${ex.message}""", ex) +class TestCaseFailedException(name: String? = null, description: String? = null, cause: Throwable) : + AssertionError(""""${name?.let { "jar: $it " } ?: ""} "$description" failed, error: ${cause.message}""", cause) abstract class AssertionTestCase(root: String) : BasicAssertionTestCase() { + private lateinit var currentFile: File + private val testRunfileRoot: Path = Paths.get(root).also { it.toFile().also { assert(it.exists()) { "runfile directory $root does not exist" } @@ -33,31 +44,23 @@ } } - private inline fun runTestCase(description: String? = null, op: () -> Unit) = + private inline fun runTestCase(name: String, description: String? = null, op: () -> Unit) = try { op() } catch (t: Throwable) { when (t) { - is AssertionError -> throw TestCaseFailedException(description, t) - is Exception -> throw TestCaseFailedException(description, t) + is AssertionError -> throw TestCaseFailedException(name, description, t) + is Exception -> throw TestCaseFailedException(name, description, t) else -> throw t } } - private fun testCaseJar(jarName: String) = testRunfileRoot.resolve(jarName).toFile().let { - check(it.exists()) { "jar $jarName did not exist in test case root $testRunfileRoot" } - JarFile(it) - } - - private fun jarTestCase(name: String, op: JarFile.() -> Unit) { - testCaseJar(name).also { op(it) } - } - protected fun jarTestCase(name: String, description: String? = null, op: JarFile.() -> Unit) { - runTestCase(description, { jarTestCase(name, op) }) + currentFile = testRunfileRoot.resolve(name).toFile() + check(currentFile.exists()) { "testFile $name did not exist in test case root $testRunfileRoot" } + runTestCase(name, description) { JarFile(currentFile).op() } } - protected fun JarFile.assertContainsEntries(vararg entries: String) { entries.forEach { if (this.getJarEntry(it) == null) { @@ -66,6 +69,37 @@ } } + /** + * Validated the entry is compressed and has the DOS epoch for it's timestamp. + */ + protected fun JarFile.assertEntryCompressedAndNormalizedTimestampYear(entry: String) { + checkNotNull(this.getJarEntry(entry)).also { + check(!it.isDirectory) + assertTrue("$entry is not compressed") { JarEntry.DEFLATED == it.method } + val modifiedTimestamp = LocalDateTime.ofInstant( + Instant.ofEpochMilli(it.lastModifiedTime.toMillis()), ZoneId.systemDefault() + ) + assertTrue("normalized modification time stamps should have year 1980") { modifiedTimestamp.year == 1980 } + } + } + + /** + * Assert the manifest in the jar is stamped. + */ + protected fun JarFile.assertManifestStamped() { + assertNotNull( + manifest.mainAttributes.getValue("Target-Label"), "missing manifest entry Target-Label" + ) + assertNotNull( + manifest.mainAttributes.getValue("Injecting-Rule-Kind"), "missing manifest entry Injecting-Rule-Kind" + ) + } + + protected fun validateFileSha256(expected: String) { + val result = Hashing.sha256().hashBytes(Files.readAllBytes(currentFile.toPath())).toString() + assertEquals(expected, result, "files did not hash as expected") + } + protected fun JarFile.assertDoesNotContainEntries(vararg entries: String) { entries.forEach { if (this.getJarEntry(it) != null) { @@ -88,7 +122,10 @@ .start().let { it.waitFor(5, TimeUnit.SECONDS) assert(it.exitValue() == 0) { - throw TestCaseFailedException(description, RuntimeException("non-zero return code: ${it.exitValue()}")) + throw TestCaseFailedException( + description = description, + cause = RuntimeException("non-zero return code: ${it.exitValue()}") + ) } } }
diff --git a/tests/rules/BUILD b/tests/rules/BUILD index 704c08e..f223bb6 100644 --- a/tests/rules/BUILD +++ b/tests/rules/BUILD
@@ -18,6 +18,9 @@ name = "assertion_test_case", srcs = ["AssertionTestCase.kt"], visibility=["//tests:__subpackages__"], - deps=["@com_github_jetbrains_kotlin//:kotlin-test"], + deps=[ + "@com_github_jetbrains_kotlin//:kotlin-test", + "@io_bazel_rules_kotlin_com_google_guava_guava//jar", + ], testonly=1 ) \ No newline at end of file