Source jar (#100) add source jar generation to the builder and remove skylark version
diff --git a/examples/dagger/BUILD b/examples/dagger/BUILD index 94ccdc9..fedb1ae 100644 --- a/examples/dagger/BUILD +++ b/examples/dagger/BUILD
@@ -39,7 +39,7 @@ # Generate a srcjar to validate intellij plugin correctly attaches it. genrule( name = "tea_lib_src", - outs = ["test.srcjar"], + outs = ["tea_lib_src.srcjar"], cmd = """ cat << EOF > TeaPot.kt package tea @@ -55,16 +55,13 @@ ) kt_jvm_library( - name = "tea_lib", - srcs = [":tea_lib_src"], -) - -kt_jvm_library( name = "coffee_lib", - srcs = glob(["src/**"]), + srcs = glob(["src/**"]) + [ + # Adding a file ending with .srcjar is how code generation patterns are implemented. + ":tea_lib_src" + ], deps = [ ":dagger_lib", - ":tea_lib", "//third_party:kotlinx_coroutines", ], )
diff --git a/kotlin/builder/integrationtests/KotlinBuilderTestCase.java b/kotlin/builder/integrationtests/KotlinBuilderTestCase.java index c4a7367..f285a8b 100644 --- a/kotlin/builder/integrationtests/KotlinBuilderTestCase.java +++ b/kotlin/builder/integrationtests/KotlinBuilderTestCase.java
@@ -113,7 +113,8 @@ builder .getOutputsBuilder() .setJar(prefixPath.resolve("jar_file.jar").toAbsolutePath().toString()) - .setJdeps(prefixPath.resolve("jdeps_files.jdeps").toAbsolutePath().toString()); + .setJdeps(prefixPath.resolve("jdeps_file.jdeps").toAbsolutePath().toString()) + .setSrcjar(prefixPath.resolve("jar_file-sources.jar").toAbsolutePath().toString()); } private static String createTestOuputDirectory(Path path) {
diff --git a/kotlin/builder/proto/jars/libkotlin_model_proto-speed.jar b/kotlin/builder/proto/jars/libkotlin_model_proto-speed.jar index 57e15f8..63d0189 100755 --- a/kotlin/builder/proto/jars/libkotlin_model_proto-speed.jar +++ b/kotlin/builder/proto/jars/libkotlin_model_proto-speed.jar Binary files differ
diff --git a/kotlin/builder/proto/kotlin_model.proto b/kotlin/builder/proto/kotlin_model.proto index f54fd54..ddea9f7 100644 --- a/kotlin/builder/proto/kotlin_model.proto +++ b/kotlin/builder/proto/kotlin_model.proto
@@ -116,6 +116,7 @@ string jar = 1; // The path to the jdeps file. string jdeps = 2; + string srcjar = 3; } message Inputs {
diff --git a/kotlin/builder/src/io/bazel/kotlin/builder/tasks/TaskBuilder.kt b/kotlin/builder/src/io/bazel/kotlin/builder/tasks/TaskBuilder.kt index 0ee3eff..1bc43ba 100644 --- a/kotlin/builder/src/io/bazel/kotlin/builder/tasks/TaskBuilder.kt +++ b/kotlin/builder/src/io/bazel/kotlin/builder/tasks/TaskBuilder.kt
@@ -78,6 +78,7 @@ with(root.outputsBuilder) { jar = argMap.mandatorySingle(JavaBuilderFlags.OUTPUT.flag) jdeps = argMap.mandatorySingle("--output_jdeps") + srcjar = argMap.mandatorySingle("--kotlin_output_srcjar") } with(root.directoriesBuilder) {
diff --git a/kotlin/builder/src/io/bazel/kotlin/builder/tasks/jvm/KotlinJvmTaskExecutor.kt b/kotlin/builder/src/io/bazel/kotlin/builder/tasks/jvm/KotlinJvmTaskExecutor.kt index 16b358a..978d465 100644 --- a/kotlin/builder/src/io/bazel/kotlin/builder/tasks/jvm/KotlinJvmTaskExecutor.kt +++ b/kotlin/builder/src/io/bazel/kotlin/builder/tasks/jvm/KotlinJvmTaskExecutor.kt
@@ -17,9 +17,12 @@ import io.bazel.kotlin.builder.toolchain.CompilationStatusException -import io.bazel.kotlin.model.KotlinModel.CompilationTask import io.bazel.kotlin.builder.utils.expandWithSources +import io.bazel.kotlin.builder.utils.jars.SourceJarCreator +import io.bazel.kotlin.model.KotlinModel.CompilationTask import java.io.File +import java.nio.file.Files +import java.nio.file.Paths import javax.inject.Inject import javax.inject.Singleton @@ -43,12 +46,31 @@ context.execute("create jar") { outputJarCreator.createOutputJar(commandWithApSources) } + produceSourceJar(commandWithApSources) context.execute("generate jdeps") { jDepsGenerator.generateJDeps(commandWithApSources) } return Result(context.timings, commandWithApSources) } + private fun produceSourceJar(command: CompilationTask) { + Paths.get(command.outputs.srcjar).also { sourceJarPath -> + Files.createFile(sourceJarPath) + SourceJarCreator( + sourceJarPath + ).also { creator -> + listOf( + command.inputs.javaSourcesList.stream(), + command.inputs.kotlinSourcesList.stream(), + command.inputs.sourceJarsList.stream() + ).stream().flatMap { it.map { Paths.get(it) } }.also { + creator.addSources(it) + } + creator.execute() + } + } + } + private fun runAnnotationProcessors(command: CompilationTask): CompilationTask = try { if (command.info.plugins.annotationProcessorsList.isNotEmpty()) {
diff --git a/kotlin/builder/src/io/bazel/kotlin/builder/utils/jars/SourceJarCreator.kt b/kotlin/builder/src/io/bazel/kotlin/builder/utils/jars/SourceJarCreator.kt new file mode 100644 index 0000000..a0a7c95 --- /dev/null +++ b/kotlin/builder/src/io/bazel/kotlin/builder/utils/jars/SourceJarCreator.kt
@@ -0,0 +1,206 @@ +/* + * Copyright 2018 The Bazel Authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.bazel.kotlin.builder.utils.jars + +import java.nio.charset.Charset +import java.nio.file.Files +import java.nio.file.Path +import java.util.* +import java.util.jar.JarFile +import java.util.jar.JarOutputStream +import java.util.regex.Pattern +import java.util.stream.Stream + +/** + * Source jar packager for JavaLike files. The placement is discovered from the package entry. + */ +class SourceJarCreator( + path: Path, + verbose: Boolean = false +) : JarHelper(path, normalize = true, verbose = verbose) { + companion object { + private const val BL = """\p{Blank}*""" + private const val COM_BL = """$BL(?:/\*[^\n]*\*/$BL)*""" + private val PKG_PATTERN: Pattern = + Pattern.compile("""^${COM_BL}package$COM_BL([a-zA-Z1-9.]+)$COM_BL(?:;?.*)$""") + + @JvmStatic + fun extractPackage(line: String): String? = PKG_PATTERN.matcher(line).takeIf { it.matches() }?.group(1) + + private fun isJavaSourceLike(name: String): Boolean { + return name.endsWith(".kt") || name.endsWith(".java") + } + } + + + sealed class Entry { + class File(val path: Path, val content: ByteArray) : Entry() { + override fun toString(): String = "File $path" + } + + object Directory : Entry() { + override fun toString(): String = "Directory" + } + } + + private class JarFilenameHelper { + /** + * A map from the directories in the underlying filesystem to jar directory names. + */ + private val directoryToPackageMap = mutableMapOf<Path, String>() + /** + * Entries for which packages could not be located. + */ + private val deferredEntries = mutableMapOf<Path, ByteArray>() + + /** + * Locate the directory name for the file as it should appear in the jar. + * + * If the directory could not be located add it to the deferred list and return null. + * + * Files like `package-info.java` could end up getting deferred if they have an annotation embedded on the same + * line or files that have entries such as `/* weird comment */package lala` + */ + fun getFilename(sourceFile: Path, bytes: ByteArray): String? { + return directoryToPackageMap[sourceFile.parent].let { existingPackageName -> + existingPackageName ?: locatePackageLineInBody(bytes).also { + if (it == null) { + deferredEntries[sourceFile] = bytes + } + }?.let { "$it/${sourceFile.fileName}" } + } + } + + /** + * Visit any deferred entries. + * + * @param block the visitor, the second param is the package name and may still be null. + */ + fun visitDeferredEntries(block: (Path, String?, ByteArray) -> Unit) { + deferredEntries.forEach { sourceFile, bytes -> + block(sourceFile, directoryToPackageMap[sourceFile.parent], bytes) + } + } + + private fun locatePackageLineInBody(bytes: ByteArray): String? = + bytes.inputStream().bufferedReader().use { + var res = it.readLine() + while (res != null) { + extractPackage(res)?.replace('.', '/')?.also { + return@use it + } + res = it.readLine() + } + null + } + } + + private val filenameHelper = JarFilenameHelper() + private val entries = TreeMap<String, Entry>() + + + /** + * Consume a stream of sources, this should contain valid source files and srcjar. + */ + fun addSources(sources: Stream<Path>) { + sources.forEach { path -> + val fileName = path.fileName.toString() + when { + isJavaSourceLike(fileName) -> addJavaLikeSourceFile(path) + fileName.endsWith(".srcjar") -> addSourceJar(path) + } + } + } + + /** + * Add a single source jar + */ + private fun addSourceJar(path: Path) { + if(verbose) { + System.err.println("adding source jar: $path") + } + JarFile(path.toFile()).use { jar -> + for (entry in jar.entries()) { + if (!entry.isDirectory) { + if (entry.name.endsWith(".kt") or entry.name.endsWith(".java")) { + jar.getInputStream(entry).readBytes(entry.size.toInt()).also { + addEntry(entry.name, path, it) + } + } + } + } + } + } + + /** + * Add a single source file. This method uses the [JarFilenameHelper] so it should be used when jar filename + * correction is desired. It should only be used for Java-Like source files. + */ + private fun addJavaLikeSourceFile(sourceFile: Path) { + val bytes = Files.readAllBytes(sourceFile) + filenameHelper.getFilename(sourceFile, bytes)?.also { + addEntry(it, sourceFile, bytes) + } + } + + fun execute() { + if (verbose) { + System.err.println("creating source jar file: $jarPath") + } + filenameHelper.visitDeferredEntries { path, jarFilename, bytes -> + if (jarFilename == null) { + System.err.println("could not determine jar entry name for $path. Body:\n${bytes.toString(Charset.defaultCharset())}}") + } else { + System.err.println("adding deferred source file $path -> $jarFilename") + addEntry(jarFilename, path, bytes) + } + } + Files.newOutputStream(jarPath).use { + JarOutputStream(it).use { out -> + for ((key, value) in entries) { + try { + when (value) { + is Entry.File -> out.copyEntry(key, value.path, value.content) + is Entry.Directory -> out.copyEntry(key) + } + } catch (throwable: Throwable) { + throw RuntimeException("could not copy JarEntry $key $value", throwable) + } + } + } + } + } + + private fun addEntry(name: String, path: Path, bytes: ByteArray) { + name.split("/").also { + if (it.size >= 2) { + for (i in ((it.size - 1) downTo 1)) { + val dirName = it.subList(0, i).joinToString("/", postfix = "/") + if (entries.putIfAbsent(dirName, Entry.Directory) != null) { + break + } else if (verbose) { + System.err.println("adding directory: $dirName") + } + } + } + } + + val result = entries.putIfAbsent(name, Entry.File(path, bytes)) + require(result as? Entry.Directory != null || result == null) { + "source entry jarName: $name from: $path collides with entry from: ${(result as Entry.File).path}" + } + } +} \ No newline at end of file
diff --git a/kotlin/builder/unittests/BUILD b/kotlin/builder/unittests/BUILD index 494de37..aeb66b8 100644 --- a/kotlin/builder/unittests/BUILD +++ b/kotlin/builder/unittests/BUILD
@@ -15,9 +15,10 @@ name = "unittests", size = "small", srcs = glob(["**/*.java"]), - test_class = "io.bazel.kotlin.builder.tasks.jvm.JdepsParserTest", + test_class = "io.bazel.kotlin.builder.BuilderUnitTestSuite", deps = [ "//kotlin/builder:builder_lib_for_tests", + "//third_party/jvm/com/google/truth:truth", "@com_github_jetbrains_kotlin//:kotlin-stdlib", "@io_bazel_rules_kotlin//kotlin/builder/proto:deps", "@io_bazel_rules_kotlin_com_google_protobuf_protobuf_java//jar",
diff --git a/kotlin/builder/unittests/io/bazel/kotlin/workers/BuilderUnitTestSuite.java b/kotlin/builder/unittests/io/bazel/kotlin/workers/BuilderUnitTestSuite.java new file mode 100644 index 0000000..0830110 --- /dev/null +++ b/kotlin/builder/unittests/io/bazel/kotlin/workers/BuilderUnitTestSuite.java
@@ -0,0 +1,25 @@ +/* + * Copyright 2018 The Bazel Authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.bazel.kotlin.builder; + +import io.bazel.kotlin.builder.tasks.jvm.JdepsParserTest; +import io.bazel.kotlin.builder.utils.jars.SourceJarCreatorUnitTests; +import org.junit.runner.RunWith; +import org.junit.runners.Suite; + +@RunWith(Suite.class) +@Suite.SuiteClasses({JdepsParserTest.class, SourceJarCreatorUnitTests.class}) +public class BuilderUnitTestSuite {}
diff --git a/kotlin/builder/unittests/io/bazel/kotlin/workers/utils/jars/SourceJarCreatorUnitTests.java b/kotlin/builder/unittests/io/bazel/kotlin/workers/utils/jars/SourceJarCreatorUnitTests.java new file mode 100644 index 0000000..c0ffa76 --- /dev/null +++ b/kotlin/builder/unittests/io/bazel/kotlin/workers/utils/jars/SourceJarCreatorUnitTests.java
@@ -0,0 +1,52 @@ +/* + * Copyright 2018 The Bazel Authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.bazel.kotlin.builder.utils.jars; + +import com.google.common.truth.StandardSubjectBuilder; +import org.junit.Test; + +import java.util.Arrays; +import java.util.List; + +import static com.google.common.truth.Truth.assertWithMessage; + +public class SourceJarCreatorUnitTests { + private final String expectedPackage = "iO.some1.package"; + + private final List<String> cases = + Arrays.asList( + "package iO.some1.package", + "package iO.some1.package ", + "package iO.some1.package;", + " package iO.some1.package; ", + " /* a comment*/ package iO.some1.package; ", + " /** a comment*/ package iO.some1.package; ", + " /* a comment*//*blah*/package iO.some1.package; ", + " /* a comment*//*blah*/ package iO.some1.package; ", + " /* a comment*/package /* blah */ iO.some1.package /*lala*/; ", + "/* a comment*/package/**/iO.some1.package/*b*/;"); + + @Test + public void testPackageNameRegex() { + cases.forEach( + (testCase) -> { + String pkg = SourceJarCreator.Companion.extractPackage(testCase); + StandardSubjectBuilder subj = assertWithMessage("positive test case: " + testCase); + subj.that(pkg).isNotNull(); + subj.that(pkg).isEqualTo(expectedPackage); + }); + } +}
diff --git a/kotlin/internal/compile.bzl b/kotlin/internal/compile.bzl index c07fe77..55ba157 100644 --- a/kotlin/internal/compile.bzl +++ b/kotlin/internal/compile.bzl
@@ -54,7 +54,8 @@ "--kotlin_api_version", toolchain.api_version, "--kotlin_language_version", toolchain.language_version, "--kotlin_module_name", module_name, - "--kotlin_passthrough_flags", "-Xcoroutines=%s" % toolchain.coroutines + "--kotlin_passthrough_flags", "-Xcoroutines=%s" % toolchain.coroutines, + "--kotlin_output_srcjar", ctx.outputs.srcjar.path, ] if (len(srcs.kt) + len(srcs.java)) > 0: @@ -80,12 +81,14 @@ ) inputs, _, input_manifests = ctx.resolve_command(tools = [toolchain.kotlinbuilder]) + ctx.actions.run( mnemonic = "KotlinCompile", inputs = depset([args_file]) + inputs + ctx.files.srcs + compile_jars, outputs = [ output_jar, ctx.outputs.jdeps, + ctx.outputs.srcjar, sourcegen_directory, classes_directory, temp_directory, @@ -135,7 +138,7 @@ use_ijar = False, # A list or set of output source jars that contain the uncompiled source files including the source files # generated by annotation processors if the case. - source_jars = utils.actions.maybe_make_srcsjar(ctx, srcs), + source_jars = [ctx.outputs.srcjar], # A list or a set of jars that should be used at compilation for a given target. compile_time_jars = my_compile_jars, # A list or a set of jars that should be used at runtime for a given target.
diff --git a/kotlin/internal/utils.bzl b/kotlin/internal/utils.bzl index 4830f6d..b8c0305 100644 --- a/kotlin/internal/utils.bzl +++ b/kotlin/internal/utils.bzl
@@ -158,34 +158,6 @@ arguments=[] ) return resources_jar_output - -# SRC JARS ##################################################################################################################################################### -def _maybe_make_srcsjar_action(ctx, srcs): - source_files = srcs.kt + srcs.java - if (len(source_files) + len(srcs.src_jars)) > 0: - output_srcjar = ctx.actions.declare_file(ctx.label.name + "-sources.jar") - - args = ["--output", output_srcjar.path] - - for sj in srcs.src_jars: - args += ["--sources", sj.path] - - for sf in source_files: - args += ["--resources", sf.path] - - ctx.action( - mnemonic = "KotlinPackageSources", - inputs = source_files + srcs.src_jars, - outputs = [output_srcjar], - executable = ctx.executable._singlejar, - arguments = args, - progress_message="Creating Kotlin srcjar from %d srcs" % len(source_files), - ) - return [output_srcjar] - else: - return [] - - # PACKAGE JARS ################################################################################################################################################# def _fold_jars_action(ctx, rule_kind, output_jar, input_jars): args=[ @@ -239,7 +211,6 @@ actions = struct( build_resourcejar = _build_resourcejar_action, fold_jars = _fold_jars_action, - maybe_make_srcsjar = _maybe_make_srcsjar_action, write_launcher = _write_launcher_action, ), collect_all_jars = _collect_all_jars,
diff --git a/tests/integrationtests/jvm/JvmExampleTests.kt b/tests/integrationtests/jvm/JvmExampleTests.kt index 2129824..0f7f7c2 100644 --- a/tests/integrationtests/jvm/JvmExampleTests.kt +++ b/tests/integrationtests/jvm/JvmExampleTests.kt
@@ -21,10 +21,12 @@ /** * These tests verify properties of the example. */ -class JvmExampleTests: BasicAssertionTestCase() { +class JvmExampleTests : BasicAssertionTestCase() { @Test fun daggerExampleIsRunnable() { - assertExecutableRunfileSucceeds("//examples/dagger/coffee_app", - description = "the dagger coffee_app should execute succesfully") + assertExecutableRunfileSucceeds( + "//examples/dagger/coffee_app", + description = "the dagger coffee_app should execute successfully" + ) } } \ No newline at end of file