blob: 0e16ed50e6465fac1df1bd8c613f970663cc96a7 [file] [log] [blame]
// Copyright 2016 The Tulsi Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
import Foundation
// Provides methods utilizing Bazel aspects to extract information from a workspace.
final class BazelAspectInfoExtractor: QueuedLogging {
enum ExtractorError: Error {
/// Failed to build aspects.
case buildFailed
/// Parsing an aspect's output failed with the given debug info.
case parsingFailed(String)
}
/// Prefix to be used by Bazel for the output of the Tulsi aspect.
private static let SymlinkPrefix = "tulsigen-"
/// Suffixes used by Bazel when creating output symlinks.
private static let BazelOutputSymlinks = [
"bin", "genfiles", "out", "testlogs"].map({ SymlinkPrefix + $0 })
/// The location of the bazel binary.
var bazelURL: URL
/// The location of the Bazel workspace to be examined.
let workspaceRootURL: URL
private let bundle: Bundle
/// Absolute path to the workspace containing the Tulsi aspect bzl file.
private let aspectWorkspacePath: String
private let localizedMessageLogger: LocalizedMessageLogger
private var queuedInfoMessages: [String] = []
/// Path to the build events JSON file.
private let buildEventsFilePath: String
private typealias CompletionHandler = (Process, String) -> Void
init(bazelURL: URL,
workspaceRootURL: URL,
localizedMessageLogger: LocalizedMessageLogger) {
self.bazelURL = bazelURL
self.workspaceRootURL = workspaceRootURL
self.localizedMessageLogger = localizedMessageLogger
let buildEventsFileName = "build_events_\(getpid()).json"
self.buildEventsFilePath =
(NSTemporaryDirectory() as NSString).appendingPathComponent(buildEventsFileName)
bundle = Bundle(for: type(of: self))
let workspaceFilePath = bundle.path(forResource: "WORKSPACE", ofType: "")! as NSString
aspectWorkspacePath = workspaceFilePath.deletingLastPathComponent
}
/// Builds a map of RuleEntry instances keyed by their labels with information extracted from the
/// Bazel workspace for the given set of Bazel targets.
func extractRuleEntriesForLabels(_ targets: [BuildLabel],
startupOptions: [String] = [],
buildOptions: [String] = [],
useAspectForTestSuites: Bool = true) throws -> RuleEntryMap {
guard !targets.isEmpty else {
return RuleEntryMap()
}
return try extractRuleEntriesUsingBEP(targets,
startupOptions: startupOptions,
buildOptions: buildOptions,
useAspectForTestSuites: useAspectForTestSuites)
}
// MARK: - Private methods
private func extractRuleEntriesUsingBEP(_ targets: [BuildLabel],
startupOptions: [String],
buildOptions: [String],
useAspectForTestSuites: Bool) throws -> RuleEntryMap {
localizedMessageLogger.infoMessage("Build Events JSON file at \"\(buildEventsFilePath)\"")
let progressNotifier = ProgressNotifier(name: SourceFileExtraction,
maxValue: targets.count,
indeterminate: false,
suppressStart: true)
let profilingStart = localizedMessageLogger.startProfiling("extract_source_info",
message: "Extracting info for \(targets.count) rules")
let semaphore = DispatchSemaphore(value: 0)
var extractedEntries = RuleEntryMap()
var processDebugInfo: String? = nil
let process = bazelAspectProcessForTargets(targets.map({ $0.value }),
aspect: "tulsi_sources_aspect",
startupOptions: startupOptions,
buildOptions: buildOptions,
useAspectForTestSuites: useAspectForTestSuites,
progressNotifier: progressNotifier) {
(process: Process, debugInfo: String) -> Void in
defer { semaphore.signal() }
processDebugInfo = debugInfo
}
if let process = process {
process.currentDirectoryPath = workspaceRootURL.path
process.launch()
_ = semaphore.wait(timeout: DispatchTime.distantFuture)
guard process.terminationStatus == 0 else {
let debugInfo = processDebugInfo ?? "<No Debug Info>"
queuedInfoMessages.append(debugInfo)
localizedMessageLogger.error("BazelInfoExtractionFailed",
comment: "Error message for when a Bazel extractor did not complete successfully. Details are logged separately.",
details: BazelErrorExtractor.firstErrorLinesFromString(debugInfo))
throw ExtractorError.buildFailed
}
let reader = BazelBuildEventsReader(filePath: buildEventsFilePath,
localizedMessageLogger: localizedMessageLogger)
do {
let events = try reader.readAllEvents()
let artifacts = Set(events.flatMap { $0.files.lazy.filter { $0.hasSuffix(".tulsiinfo") } })
if !artifacts.isEmpty {
extractedEntries = self.extractRuleEntriesFromArtifacts(artifacts,
progressNotifier: progressNotifier)
try? FileManager.default.removeItem(atPath: buildEventsFilePath)
} else {
let debugInfo = processDebugInfo ?? "<No Debug Info>"
queuedInfoMessages.append(debugInfo)
self.localizedMessageLogger.error("BazelInfoExtractionFailed",
comment: "Error message for when a Bazel extractor did not complete successfully. Details are logged separately.",
details: BazelErrorExtractor.firstErrorLinesFromString(debugInfo))
throw ExtractorError.buildFailed
}
} catch let e as NSError {
self.localizedMessageLogger.error("BazelInfoExtractionFailed",
comment: "Error message for when a Bazel extractor did not complete successfully. Details are logged separately.",
details: "Failed to read all build events. Error: \(e.localizedDescription)")
throw ExtractorError.buildFailed
}
}
localizedMessageLogger.logProfilingEnd(profilingStart)
return extractedEntries
}
// Generates a Process that will run the given aspect against the given Bazel targets, capturing
// the output data and passing it to the terminationHandler.
private func bazelAspectProcessForTargets(_ targets: [String],
aspect: String,
startupOptions: [String] = [],
buildOptions: [String] = [],
useAspectForTestSuites: Bool = true,
progressNotifier: ProgressNotifier? = nil,
terminationHandler: @escaping CompletionHandler) -> Process? {
let infoExtractionNotifier = ProgressNotifier(name: WorkspaceInfoExtraction,
maxValue: 1,
indeterminate: true)
infoExtractionNotifier.incrementValue()
if let progressNotifier = progressNotifier {
progressNotifier.start()
}
var arguments = startupOptions
arguments.append(contentsOf: [
"build",
"-c",
"dbg", // The aspect is run in debug mode to match the default Xcode build configuration.
"--symlink_prefix", // Generate artifacts without overwriting the normal build symlinks.
BazelAspectInfoExtractor.SymlinkPrefix,
"--announce_rc", // Print the RC files used by this operation.
"--nocheck_visibility", // Don't do package visibility enforcement during aspect runs.
"--show_result=0", // Don't bother printing the build results.
"--noshow_loading_progress", // Don't show Bazel's loading progress.
"--noshow_progress", // Don't show Bazel's build progress.
"--override_repository=tulsi=\(aspectWorkspacePath)",
"--aspects",
"@tulsi//tulsi:tulsi_aspects.bzl%\(aspect)",
"--output_groups=tulsi-info,-_,-default", // Build only the aspect artifacts.
"--tool_tag=tulsi:generator", // Add a tag for tracking.
"--build_event_json_file=\(self.buildEventsFilePath)",
])
// Don't replace test_suites with their tests. This allows the Aspect to discover the structure
// of the test_suite.
if useAspectForTestSuites {
arguments.append("--noexpand_test_suites")
}
arguments.append(contentsOf: buildOptions)
arguments.append(contentsOf: targets)
let process = TulsiProcessRunner.createProcess(bazelURL.path,
arguments: arguments,
messageLogger: localizedMessageLogger,
loggingIdentifier: "bazel_extract_source_info") {
completionInfo in
let debugInfoFormatString = NSLocalizedString("DebugInfoForBazelCommand",
bundle: Bundle(for: type(of: self)),
comment: "Provides general information about a Bazel failure; a more detailed error may be reported elsewhere. The Bazel command is %1$@, exit code is %2$d, stderr %3$@.")
let stderr = NSString(data: completionInfo.stderr, encoding: String.Encoding.utf8.rawValue) ?? "<No STDERR>"
let debugInfo = String(format: debugInfoFormatString,
completionInfo.commandlineString,
completionInfo.terminationStatus,
stderr)
self.removeGeneratedSymlinks()
terminationHandler(completionInfo.process, debugInfo)
}
return process
}
private func removeGeneratedSymlinks() {
let fileManager = FileManager.default
for outputSymlink in BazelAspectInfoExtractor.BazelOutputSymlinks {
let symlinkURL = workspaceRootURL.appendingPathComponent(outputSymlink, isDirectory: true)
do {
let attributes = try fileManager.attributesOfItem(atPath: symlinkURL.path)
guard let type = attributes[FileAttributeKey.type] as? String, type == FileAttributeType.typeSymbolicLink.rawValue else {
continue
}
} catch {
// Any exceptions are expected to indicate that the file does not exist.
continue
}
do {
try fileManager.removeItem(at: symlinkURL)
} catch let e as NSError {
localizedMessageLogger.infoMessage("Failed to remove symlink at \(symlinkURL). \(e)")
}
}
}
/// Builds a list of RuleEntry instances using the data in the given set of .tulsiinfo files.
private func extractRuleEntriesFromArtifacts(_ files: Set<String>,
progressNotifier: ProgressNotifier? = nil) -> RuleEntryMap {
let fileManager = FileManager.default
func parseTulsiTargetFile(_ filename: String) throws -> RuleEntry {
return try autoreleasepool {
return try parseTulsiTargetFileImpl(filename)
}
}
func parseTulsiTargetFileImpl(_ filename: String) throws -> RuleEntry {
guard let data = fileManager.contents(atPath: filename) else {
throw ExtractorError.parsingFailed("The file could not be read")
}
guard let dict = try JSONSerialization.jsonObject(with: data, options: JSONSerialization.ReadingOptions()) as? [String: AnyObject] else {
throw ExtractorError.parsingFailed("Contents are not a dictionary")
}
func getRequiredField(_ field: String) throws -> String {
guard let value = dict[field] as? String else {
throw ExtractorError.parsingFailed("Missing required '\(field)' field")
}
return value
}
let ruleLabel = try getRequiredField("label")
let ruleType = try getRequiredField("type")
let attributes = dict["attr"] as? [String: AnyObject] ?? [:]
func MakeBazelFileInfos(_ attributeName: String) -> [BazelFileInfo] {
let infos = dict[attributeName] as? [[String: AnyObject]] ?? []
var bazelFileInfos = [BazelFileInfo]()
for info in infos {
if let pathInfo = BazelFileInfo(info: info as AnyObject?) {
bazelFileInfos.append(pathInfo)
}
}
return bazelFileInfos
}
let artifacts = MakeBazelFileInfos("artifacts")
var sources = MakeBazelFileInfos("srcs")
// Appends BazelFileInfo objects to the given array for any info dictionaries representing
// source code or (potential) source code containers. The directoryArtifacts set is also
// populated as a side effect.
var directoryArtifacts = Set<String>()
func appendGeneratedSourceArtifacts(_ infos: [[String: AnyObject]],
to artifacts: inout [BazelFileInfo]) {
for info in infos {
guard let pathInfo = BazelFileInfo(info: info as AnyObject?) else {
continue
}
if pathInfo.isDirectory {
directoryArtifacts.insert(pathInfo.fullPath)
} else {
guard let fileUTI = pathInfo.uti, fileUTI.hasPrefix("sourcecode.") else {
continue
}
}
artifacts.append(pathInfo)
}
}
let generatedSourceInfos = dict["generated_files"] as? [[String: AnyObject]] ?? []
appendGeneratedSourceArtifacts(generatedSourceInfos, to: &sources)
var nonARCSources = MakeBazelFileInfos("non_arc_srcs")
let generatedNonARCSourceInfos = dict["generated_non_arc_files"] as? [[String: AnyObject]] ?? []
appendGeneratedSourceArtifacts(generatedNonARCSourceInfos, to: &nonARCSources)
let includePaths: [RuleEntry.IncludePath]?
if let includes = dict["includes"] as? [String] {
includePaths = includes.map() {
RuleEntry.IncludePath($0, directoryArtifacts.contains($0))
}
} else {
includePaths = nil
}
let defines = dict["defines"] as? [String]
let deps = dict["deps"] as? [String] ?? []
let dependencyLabels = Set(deps.map({ BuildLabel($0) }))
let frameworkImports = MakeBazelFileInfos("framework_imports")
let buildFilePath = dict["build_file"] as? String
let osDeploymentTarget = dict["os_deployment_target"] as? String
let secondaryArtifacts = MakeBazelFileInfos("secondary_product_artifacts")
let swiftLanguageVersion = dict["swift_language_version"] as? String
let swiftToolchain = dict["swift_toolchain"] as? String
let swiftTransitiveModules = MakeBazelFileInfos("swift_transitive_modules")
let objCModuleMaps = MakeBazelFileInfos("objc_module_maps")
let extensions: Set<BuildLabel>?
if let extensionList = dict["extensions"] as? [String] {
extensions = Set(extensionList.map({ BuildLabel($0) }))
} else {
extensions = nil
}
let bundleID = dict["bundle_id"] as? String
let bundleName = dict["bundle_name"] as? String
let productType = dict["product_type"] as? String
let platformType = dict["platform_type"] as? String
let targetProductType: PBXTarget.ProductType?
if let productTypeStr = productType {
// Better be a type that we support, otherwise it's an error on our end.
if let actualProductType = PBXTarget.ProductType(rawValue: productTypeStr) {
targetProductType = actualProductType
} else {
throw ExtractorError.parsingFailed("Unsupported product type: \(productTypeStr)")
}
} else {
targetProductType = nil
}
var extensionType: String?
if targetProductType?.isiOSAppExtension ?? false, let infoplistPath = dict["infoplist"] as? String {
// TODO(b/73349137): This relies on the fact the Plist will be located next to the
// .tulsiinfo file for the same target. It would be better to get an absolute path to the
// plist from bazel.
let plistPath = URL(fileURLWithPath: filename)
.deletingLastPathComponent()
.appendingPathComponent(infoplistPath).path
guard let info = NSDictionary(contentsOfFile: plistPath) else {
throw ExtractorError.parsingFailed("Unable to load extension plist file: \(plistPath)")
}
guard let _extensionType = info.value(forKeyPath: "NSExtension.NSExtensionPointIdentifier") as? String else {
throw ExtractorError.parsingFailed("Missing NSExtensionPointIdentifier in extension plist: \(plistPath)")
}
extensionType = _extensionType
}
let ruleEntry = RuleEntry(label: ruleLabel,
type: ruleType,
attributes: attributes,
artifacts: artifacts,
sourceFiles: sources,
nonARCSourceFiles: nonARCSources,
dependencies: dependencyLabels,
frameworkImports: frameworkImports,
secondaryArtifacts: secondaryArtifacts,
extensions: extensions,
bundleID: bundleID,
bundleName: bundleName,
productType: targetProductType,
platformType: platformType,
osDeploymentTarget: osDeploymentTarget,
buildFilePath: buildFilePath,
defines: defines,
includePaths: includePaths,
swiftLanguageVersion: swiftLanguageVersion,
swiftToolchain: swiftToolchain,
swiftTransitiveModules: swiftTransitiveModules,
objCModuleMaps: objCModuleMaps,
extensionType: extensionType)
progressNotifier?.incrementValue()
return ruleEntry
}
let ruleEntryMap = RuleEntryMap(localizedMessageLogger: localizedMessageLogger)
let semaphore = DispatchSemaphore(value: 1)
let queue = DispatchQueue(label: "com.google.Tulsi.ruleEntryArtifactExtractor",
attributes: DispatchQueue.Attributes.concurrent)
var hasErrors = false
for filename in files {
queue.async {
let errorInfo: String
do {
let ruleEntry = try parseTulsiTargetFile(filename)
_ = semaphore.wait(timeout: DispatchTime.distantFuture)
ruleEntryMap.insert(ruleEntry: ruleEntry)
semaphore.signal()
return
} catch ExtractorError.parsingFailed(let info) {
errorInfo = info
} catch let e as NSError {
errorInfo = e.localizedDescription
} catch {
errorInfo = "Unexpected exception"
}
self.localizedMessageLogger.warning("BazelAspectSupportedTargetParsingFailed",
comment: "Error to show when tulsi_supported_targets_aspect produced data that could not be parsed. The artifact filename is in %1$@, additional information is in %2$@.",
values: filename, errorInfo)
hasErrors = true
}
}
// Wait for everything to be processed.
queue.sync(flags: .barrier, execute: {})
if hasErrors {
localizedMessageLogger.error("BazelAspectParsingFailedNotification",
comment: "Error to show as an alert when the output generated by an aspect failed in some way. Details about the failure are available in the message log.")
}
return ruleEntryMap
}
// MARK: - QueuedLogging
func logQueuedInfoMessages() {
guard !self.queuedInfoMessages.isEmpty else {
return
}
localizedMessageLogger.infoMessage("Log of Bazel aspect info output follows:")
for message in self.queuedInfoMessages {
localizedMessageLogger.infoMessage(message)
}
self.queuedInfoMessages.removeAll()
}
var hasQueuedInfoMessages: Bool {
return !self.queuedInfoMessages.isEmpty
}
}