split timing and trace into seperate flags and tidy up
diff --git a/kotlin/BUILD b/kotlin/BUILD index b69cba7..e7ab37c 100644 --- a/kotlin/BUILD +++ b/kotlin/BUILD
@@ -17,10 +17,26 @@ load("//kotlin/internal:bootstrap.bzl", "kt_toolchain_ide_info") config_setting( - name = "builder_debug_setting", - values = { - "define": "kt_debug=1" - } + name = "builder_debug_timings", + values = { "define": "kt_timings=1" } +) + +config_setting( + name = "builder_debug_trace", + values = { "define": "kt_trace=1" } +) + +define_kt_toolchain( + name = "default_toolchain", + debug= + select({ + "//kotlin:builder_debug_trace": ["trace"], + "//conditions:default": [] + }) + + select({ + "//kotlin:builder_debug_timings": ["timings"], + "//conditions:default": [] + }) ) toolchain_type( @@ -28,14 +44,6 @@ visibility = ["//visibility:public"] ) -define_kt_toolchain( - name = "default_toolchain", - debug=select({ - "//kotlin:builder_debug_setting": 1, - "//conditions:default": 0 - }) -) - kt_toolchain_ide_info(name="kt_toolchain_ide_info") exports_files(["toolchains.bzl", "kotlin.bzl"], visibility=["//docs:__subpackages__"])
diff --git a/kotlin/builder/proto/jars/libkotlin_model_proto-speed.jar b/kotlin/builder/proto/jars/libkotlin_model_proto-speed.jar index 1411450..3e72540 100755 --- a/kotlin/builder/proto/jars/libkotlin_model_proto-speed.jar +++ b/kotlin/builder/proto/jars/libkotlin_model_proto-speed.jar Binary files differ
diff --git a/kotlin/builder/proto/kotlin_model.proto b/kotlin/builder/proto/kotlin_model.proto index 5344fed..8814ef6 100644 --- a/kotlin/builder/proto/kotlin_model.proto +++ b/kotlin/builder/proto/kotlin_model.proto
@@ -96,8 +96,10 @@ repeated string friend_paths = 8; // The path of the primary output for the task, this is derived from the flagfile. string primary_output_path = 9; - // If this is true the builder will print out debug messages. - int32 debug = 10; + // Strings to enable various debugging behaviours + // trace: enables trace logging. + // timings: causes timing information to be printed at the of an action. + repeated string debug = 10; } // Mested messages not marked with stable could be refactored.
diff --git a/kotlin/builder/src/io/bazel/kotlin/builder/tasks/KotlinBuilder.kt b/kotlin/builder/src/io/bazel/kotlin/builder/tasks/KotlinBuilder.kt index 8c6c973..c834790 100644 --- a/kotlin/builder/src/io/bazel/kotlin/builder/tasks/KotlinBuilder.kt +++ b/kotlin/builder/src/io/bazel/kotlin/builder/tasks/KotlinBuilder.kt
@@ -49,16 +49,24 @@ override fun apply(args: List<String>): Int { val (argMap, context) = buildContext(args) + var success = false + var status = 0 try { @Suppress("WHEN_ENUM_CAN_BE_NULL_IN_JAVA") when (context.info.platform) { Platform.JVM -> executeJvmTask(context, argMap) Platform.UNRECOGNIZED, Platform.JS -> throw IllegalStateException("unrecognized platform: ${context.info}") } + success = true } catch (ex: CompilationStatusException) { - return ex.status + status = ex.status + } catch(throwable: Throwable) { + context.reportUnhandledException(throwable) + throw throwable + } finally { + context.finalize(success) } - return 0 + return status } private fun buildContext(args: List<String>): Pair<ArgMap, CompilationTaskContext> { @@ -129,7 +137,8 @@ private fun buildTaskInfo(argMap: ArgMap): CompilationTaskInfo.Builder = with(CompilationTaskInfo.newBuilder()) { - debug = argMap.mandatorySingle(KotlinBuilderFlags.DEBUG).toInt() + addAllDebug(argMap.mandatory(KotlinBuilderFlags.DEBUG)) + label = argMap.mandatorySingle(JavaBuilderFlags.TARGET_LABEL) argMap.mandatorySingle(JavaBuilderFlags.RULE_KIND).split("_").also { check(it.size == 3 && it[0] == "kt") { "invalid rule kind $it" } @@ -152,7 +161,9 @@ private fun executeJvmTask(context: CompilationTaskContext, argMap: ArgMap) { val task = buildJvmTask(context.info, argMap) - context.debugPrintProto("jvm task message", task) + context.whenTracing { + printProto("jvm task message", task) + } jvmTaskExecutor.execute(context, task) } @@ -178,7 +189,7 @@ putAllIndirectDependencies(argMap.labelDepMap(JavaBuilderFlags.DIRECT_DEPENDENCY)) putAllIndirectDependencies(argMap.labelDepMap(JavaBuilderFlags.INDIRECT_DEPENDENCY)) - argMap.optional(JavaBuilderFlags.SOURCES)?.iterator()?.partitionSources( + argMap.optional(JavaBuilderFlags.SOURCES)?.iterator()?.partitionJvmSources( { addKotlinSources(it) }, { addJavaSources(it) } )
diff --git a/kotlin/builder/src/io/bazel/kotlin/builder/tasks/jvm/KotlinJvmTaskExecutor.kt b/kotlin/builder/src/io/bazel/kotlin/builder/tasks/jvm/KotlinJvmTaskExecutor.kt index caa3372..bac2bcd 100644 --- a/kotlin/builder/src/io/bazel/kotlin/builder/tasks/jvm/KotlinJvmTaskExecutor.kt +++ b/kotlin/builder/src/io/bazel/kotlin/builder/tasks/jvm/KotlinJvmTaskExecutor.kt
@@ -15,12 +15,9 @@ */ package io.bazel.kotlin.builder.tasks.jvm - import io.bazel.kotlin.builder.toolchain.CompilationStatusException -import io.bazel.kotlin.builder.utils.CompilationTaskContext -import io.bazel.kotlin.builder.utils.IS_JVM_SOURCE_FILE -import io.bazel.kotlin.builder.utils.ensureDirectories -import io.bazel.kotlin.builder.utils.expandWithSources +import io.bazel.kotlin.builder.utils.* +import io.bazel.kotlin.builder.utils.jars.JarCreator import io.bazel.kotlin.builder.utils.jars.SourceJarCreator import io.bazel.kotlin.builder.utils.jars.SourceJarExtractor import io.bazel.kotlin.model.JvmCompilationTask @@ -34,74 +31,50 @@ class KotlinJvmTaskExecutor @Inject internal constructor( private val kotlinCompiler: KotlinJvmCompiler, private val javaCompiler: JavaCompiler, - private val jDepsGenerator: JDepsGenerator, - private val outputJarCreator: OutputJarCreator + private val jDepsGenerator: JDepsGenerator ) { fun execute(context: CompilationTaskContext, task: JvmCompilationTask) { - val preprocessedTask = preprocessingSteps(task) - // fix error handling + // TODO fix error handling try { - val commandWithApSources = context.execute("kapt") { - runAnnotationProcessors(context, preprocessedTask) - } - compileClasses(context, commandWithApSources) - context.execute("create jar") { - outputJarCreator.createOutputJar(commandWithApSources) - } - produceSourceJar(commandWithApSources) - context.execute("generate jdeps") { - jDepsGenerator.generateJDeps(commandWithApSources) - } + val preprocessedTask = task.preprocessingSteps(context) + context.execute("compile classes") { preprocessedTask.compileClasses(context) } + context.execute("create jar") { preprocessedTask.createOutputJar() } + context.execute("produce src jar") { preprocessedTask.produceSourceJar() } + context.execute("generate jdeps") { preprocessedTask.generateJDeps() } } catch (ex: Throwable) { throw RuntimeException(ex) } } - - private fun preprocessingSteps(command: JvmCompilationTask): JvmCompilationTask { + private fun JvmCompilationTask.preprocessingSteps(context: CompilationTaskContext): JvmCompilationTask { ensureDirectories( - command.directories.classes, - command.directories.temp, - command.directories.generatedSources, - command.directories.generatedClasses + directories.temp, + directories.generatedSources, + directories.generatedClasses ) - return expandWithSourceJarSources(command) + val taskWithAdditionalSources = context.execute("expand sources") { expandWithSourceJarSources() } + return context.execute("kapt") { taskWithAdditionalSources.runAnnotationProcessors(context) } } - /** - * If any srcjars were provided expand the jars sources and create a new [JvmCompilationTask] with the - * Java and Kotlin sources merged in. - */ - private fun expandWithSourceJarSources(command: JvmCompilationTask): JvmCompilationTask = - if (command.inputs.sourceJarsList.isEmpty()) { - command - } else { - SourceJarExtractor( - destDir = Paths.get(command.directories.temp).resolve("_srcjars"), - fileMatcher = IS_JVM_SOURCE_FILE - ).also { - it.jarFiles.addAll(command.inputs.sourceJarsList.map { p -> Paths.get(p) }) - it.execute() - }.let { - command.expandWithSources(it.sourcesList.iterator()) - } - } + private fun JvmCompilationTask.generateJDeps() { + jDepsGenerator.generateJDeps(this) + } - private fun produceSourceJar(command: JvmCompilationTask) { - Paths.get(command.outputs.srcjar).also { sourceJarPath -> + private fun JvmCompilationTask.produceSourceJar() { + Paths.get(outputs.srcjar).also { sourceJarPath -> Files.createFile(sourceJarPath) SourceJarCreator( sourceJarPath ).also { creator -> // This check asserts that source jars were unpacked if present. check( - command.inputs.sourceJarsList.isEmpty() || - Files.exists(Paths.get(command.directories.temp).resolve("_srcjars")) + inputs.sourceJarsList.isEmpty() || + Files.exists(Paths.get(directories.temp).resolve("_srcjars")) ) listOf( // Any (input) source jars should already have been expanded so do not add them here. - command.inputs.javaSourcesList.stream(), - command.inputs.kotlinSourcesList.stream() + inputs.javaSourcesList.stream(), + inputs.kotlinSourcesList.stream() ).stream() .flatMap { it.map { p -> Paths.get(p) } } .also { creator.addSources(it) } @@ -110,35 +83,46 @@ } } - private fun runAnnotationProcessors( - context: CompilationTaskContext, - command: JvmCompilationTask + private fun JvmCompilationTask.runAnnotationProcessors( + context: CompilationTaskContext ): JvmCompilationTask = try { - if (command.info.plugins.annotationProcessorsList.isNotEmpty()) { - kotlinCompiler.runAnnotationProcessor(context, command) - File(command.directories.generatedSources).walkTopDown() - .filter { it.isFile } - .map { it.path } - .iterator() - .let { command.expandWithSources(it) } + if (info.plugins.annotationProcessorsList.isEmpty()) { + this } else { - command + val kaptOutput = kotlinCompiler.runAnnotationProcessor(context, this) + context.whenTracing { printLines("kapt output", kaptOutput) } + expandWithGeneratedSources() } } catch (ex: CompilationStatusException) { ex.lines.also(context::printCompilerOutput) throw ex } - private fun compileClasses( - context: CompilationTaskContext, - command: JvmCompilationTask - ) { + /** + * Produce the primary output jar. + */ + private fun JvmCompilationTask.createOutputJar() = + JarCreator( + path = Paths.get(outputs.jar), + normalize = true, + verbose = false + ).also { + it.addDirectory(Paths.get(directories.classes)) + it.addDirectory(Paths.get(directories.generatedClasses)) + it.setJarOwner(info.label, info.bazelRuleKind) + it.execute() + } + + private fun JvmCompilationTask.compileClasses(context: CompilationTaskContext) { + ensureDirectories( + directories.classes + ) var kotlinError: CompilationStatusException? = null var result: List<String>? = null context.execute("kotlinc") { result = try { - kotlinCompiler.compile(command) + kotlinCompiler.compile(this) } catch (ex: CompilationStatusException) { kotlinError = ex ex.lines @@ -146,11 +130,57 @@ } try { context.execute("javac") { - javaCompiler.compile(command) + javaCompiler.compile(this) } } finally { checkNotNull(result).also(context::printCompilerOutput) kotlinError?.also { throw it } } } + + /** + * If any srcjars were provided expand the jars sources and create a new [JvmCompilationTask] with the + * Java and Kotlin sources merged in. + */ + private fun JvmCompilationTask.expandWithSourceJarSources(): JvmCompilationTask = + if (inputs.sourceJarsList.isEmpty()) + this + else expandWithSources( + SourceJarExtractor( + destDir = Paths.get(directories.temp).resolve("_srcjars"), + fileMatcher = IS_JVM_SOURCE_FILE + ).also { + it.jarFiles.addAll(inputs.sourceJarsList.map { p -> Paths.get(p) }) + it.execute() + }.sourcesList.iterator() + ) + + /** + * Create a new [JvmCompilationTask] with sources found in the generatedSources directory. This should be run after + * annotation processors have been run. + */ + private fun JvmCompilationTask.expandWithGeneratedSources(): JvmCompilationTask = + expandWithSources( + File(directories.generatedSources).walkTopDown() + .filter { it.isFile } + .map { it.path } + .iterator() + ) + + private fun JvmCompilationTask.expandWithSources(sources: Iterator<String>): JvmCompilationTask = + updateBuilder { builder -> + sources.partitionJvmSources( + { builder.inputsBuilder.addKotlinSources(it) }, + { builder.inputsBuilder.addJavaSources(it) }) + } + + private fun JvmCompilationTask.updateBuilder( + block: (JvmCompilationTask.Builder) -> Unit + ): JvmCompilationTask = + toBuilder().let { + block(it) + it.build() + } } + +
diff --git a/kotlin/builder/src/io/bazel/kotlin/builder/tasks/jvm/OutputJarCreator.kt b/kotlin/builder/src/io/bazel/kotlin/builder/tasks/jvm/OutputJarCreator.kt deleted file mode 100644 index 0233dc5..0000000 --- a/kotlin/builder/src/io/bazel/kotlin/builder/tasks/jvm/OutputJarCreator.kt +++ /dev/null
@@ -1,39 +0,0 @@ -/* - * Copyright 2018 The Bazel Authors. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.bazel.kotlin.builder.tasks.jvm - -import io.bazel.kotlin.builder.utils.bazelRuleKind -import io.bazel.kotlin.builder.utils.jars.JarCreator -import io.bazel.kotlin.model.JvmCompilationTask -import java.nio.file.Paths -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -internal class OutputJarCreator @Inject constructor() { - fun createOutputJar(command: JvmCompilationTask) { - JarCreator( - path = Paths.get(command.outputs.jar), - normalize = true, - verbose = false - ).also { - it.addDirectory(Paths.get(command.directories.classes)) - it.addDirectory(Paths.get(command.directories.generatedClasses)) - it.setJarOwner(command.info.label, command.info.bazelRuleKind) - it.execute() - } - } -}
diff --git a/kotlin/builder/src/io/bazel/kotlin/builder/utils/CompilationTaskContext.kt b/kotlin/builder/src/io/bazel/kotlin/builder/utils/CompilationTaskContext.kt index 17946ae..fc85d0a 100644 --- a/kotlin/builder/src/io/bazel/kotlin/builder/utils/CompilationTaskContext.kt +++ b/kotlin/builder/src/io/bazel/kotlin/builder/utils/CompilationTaskContext.kt
@@ -25,39 +25,51 @@ class CompilationTaskContext(val info: CompilationTaskInfo, private val out: PrintStream) { private val executionRoot: String = Paths.get("").toAbsolutePath().toString() + File.separator - private val timings = mutableListOf<String>() - val isDebug: Boolean = info.debug > 0 + private val timings: MutableList<String>? + @PublishedApi + internal val isTracing: Boolean - /** - * Print debugging messages if it is enabled for the task. - */ - fun debugPrint(msg: String) { - if (info.debug > 0) { - out.println(msg) - } + init { + val debugging = info.debugList.toSet() + timings = if (debugging.contains("timings")) mutableListOf() else null + isTracing = debugging.contains("trace") } + fun reportUnhandledException(throwable: Throwable) { throwable.printStackTrace(out) } + /** - * Print a debugging message if it debugging is enabled for the task. The lines are tab seperated. + * Print a list of debugging lines. + * + * @param header a header string + * @param lines a list of lines to print out + * @param prefix a prefix to add to each line + * @param filterEmpty if empty lines should be discarded or not */ - private inline fun debugPrintHeadedLines(header: String, lines: () -> String) { - if (info.debug > 0) { - out.println(if (header.endsWith(":")) header else "$header:") - out.print("| ${lines().replace("\n", "\n| ")}") + fun printLines(header: String, lines: List<String>, prefix: String = "| ", filterEmpty: Boolean = false) { + check(header.isNotEmpty()) + out.println(if (header.endsWith(":")) header else "$header:") + lines.forEach { + if (it.isNotEmpty() || !filterEmpty) { + out.println("$prefix$it") + } } + out.println() + } + + inline fun <T> whenTracing(block: CompilationTaskContext.() -> T): T? { + return if(isTracing) { block() } else null } /** * Print a proto message if debugging is enabled for the task. */ - fun debugPrintProto(header: String, msg: MessageOrBuilder) { - debugPrintHeadedLines(header) { TextFormat.printToString(msg) } + fun printProto(header: String, msg: MessageOrBuilder) { + printLines(header, TextFormat.printToString(msg).split("\n"), filterEmpty = true) } - fun printCompilerOutput(line: String) { - out.println(trimExecutionRootPrefix(line)) - } - + /** + * This method normalizes and reports the output from the Kotlin compiler. + */ fun printCompilerOutput(lines: List<String>) { lines.map(::trimExecutionRootPrefix).forEach(out::println) } @@ -69,13 +81,31 @@ } else toPrint } + /** + * Runs a task and records the timings. + */ fun <T> execute(name: String, task: () -> T): T { - val start = System.currentTimeMillis() - return try { + return if (timings == null) { task() - } finally { - val stop = System.currentTimeMillis() - timings += "$name: ${stop - start} ms" + } else { + val start = System.currentTimeMillis() + try { + task() + } finally { + val stop = System.currentTimeMillis() + timings += "$name: ${stop - start} ms" + } + } + } + + /** + * This method should be called at the end of builder invocation. + * + * @param succesfull true if the task finished succesfully. + */ + fun finalize(succesfull: Boolean) { + if(succesfull) { + timings?.also { printLines("Task timings", it, prefix = " * ") } } } }
diff --git a/kotlin/builder/src/io/bazel/kotlin/builder/utils/KotlinCompilerPluginArgsEncoder.kt b/kotlin/builder/src/io/bazel/kotlin/builder/utils/KotlinCompilerPluginArgsEncoder.kt index 6974f0c..8e00e13 100644 --- a/kotlin/builder/src/io/bazel/kotlin/builder/utils/KotlinCompilerPluginArgsEncoder.kt +++ b/kotlin/builder/src/io/bazel/kotlin/builder/utils/KotlinCompilerPluginArgsEncoder.kt
@@ -99,7 +99,9 @@ arg["javacArguments"] = javacArgs.let(Companion::encodeMap) arg["aptMode"] = "stubsAndApt" arg["correctErrorTypes"] = "true" - arg["verbose"] = context.isDebug.toString() + context.whenTracing { + arg["verbose"] = "true" + } arg["processors"] = plugin.annotationProcessorsList .filter { it.processorClass.isNotEmpty() }
diff --git a/kotlin/builder/src/io/bazel/kotlin/builder/utils/TaskUtils.kt b/kotlin/builder/src/io/bazel/kotlin/builder/utils/TaskUtils.kt index 67c607a..2f91ca6 100644 --- a/kotlin/builder/src/io/bazel/kotlin/builder/utils/TaskUtils.kt +++ b/kotlin/builder/src/io/bazel/kotlin/builder/utils/TaskUtils.kt
@@ -19,28 +19,11 @@ import io.bazel.kotlin.model.JvmCompilationTask import java.io.File -fun JvmCompilationTask.expandWithSources( - sources: Iterator<String> -): JvmCompilationTask = - updateBuilder { builder -> - sources.partitionSources( - { builder.inputsBuilder.addKotlinSources(it) }, - { builder.inputsBuilder.addJavaSources(it) }) - } - val JvmCompilationTask.Inputs.joinedClasspath: String get() = this.classpathList.joinToString(File.pathSeparator) val CompilationTaskInfo.bazelRuleKind: String get() = "kt_${platform.name.toLowerCase()}_${ruleKind.name.toLowerCase()}" -private fun JvmCompilationTask.updateBuilder( - init: (JvmCompilationTask.Builder) -> Unit -): JvmCompilationTask = - toBuilder().let { - init(it) - it.build() - } - -fun Iterator<String>.partitionSources(kt: (String) -> Unit, java: (String) -> Unit) { +fun Iterator<String>.partitionJvmSources(kt: (String) -> Unit, java: (String) -> Unit) { forEach { when { it.endsWith(".kt") -> kt(it)
diff --git a/kotlin/internal/compile.bzl b/kotlin/internal/compile.bzl index 1d61f17..47e45a4 100644 --- a/kotlin/internal/compile.bzl +++ b/kotlin/internal/compile.bzl
@@ -33,7 +33,14 @@ args.add("--kotlin_api_version", toolchain.api_version) args.add("--kotlin_language_version", toolchain.language_version) args.add("--kotlin_passthrough_flags", "-Xcoroutines=%s" % toolchain.coroutines) - args.add("--kotlin_debug", toolchain.debug) + + debug = depset(toolchain.debug) + for tag in ctx.attr.tags: + if tag == "trace": + debug = debug + [tag] + if tag == "timings": + debug = debug + [tag] + args.add("--kotlin_debug", debug) return args
diff --git a/kotlin/toolchains.bzl b/kotlin/toolchains.bzl index 0e08ab8..c61c8ce 100644 --- a/kotlin/toolchains.bzl +++ b/kotlin/toolchains.bzl
@@ -71,8 +71,12 @@ "1.2", ], ), - "debug": attr.int( - doc="if this is non zero then the builder will produce debug logging." + "debug": attr.string_list( + allow_empty=True, + doc="""Debugging tags passed to the builder. Two tags are supported. `timings` will cause the builder to print +timing information. `trace` will cause the builder to print tracing messages. These tags can be enabled via the default +toolchain via the defines `kt_timings=1` and `kt_trace=1`. The tags may also be picked up via the `tags` attribute +defined directly on the rules.""" ), "coroutines": attr.string( default = "enable", @@ -151,7 +155,7 @@ api_version=None, jvm_target=None, coroutines=None, - debug=0 + debug=[] ): """Define a Kotlin JVM Toolchain, the name is used in the `toolchain` rule so can be used to register the toolchain in the WORKSPACE file. @@ -163,7 +167,7 @@ api_version = api_version, jvm_target = jvm_target, coroutines = coroutines, - debug=debug, + debug = debug, visibility = ["//visibility:public"] ) native.toolchain(