| // Copyright 2016 The Tulsi Authors. All rights reserved. |
| // |
| // Licensed under the Apache License, Version 2.0 (the "License"); |
| // you may not use this file except in compliance with the License. |
| // You may obtain a copy of the License at |
| // |
| // http://www.apache.org/licenses/LICENSE-2.0 |
| // |
| // Unless required by applicable law or agreed to in writing, software |
| // distributed under the License is distributed on an "AS IS" BASIS, |
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| // See the License for the specific language governing permissions and |
| // limitations under the License. |
| |
| import Foundation |
| |
| |
| // Provides methods utilizing Bazel query (http://bazel.build/docs/query.html) to extract |
| // information from a workspace. |
| final class BazelQueryInfoExtractor: QueuedLogging { |
| |
| enum ExtractorError: Error { |
| /// A valid Bazel binary could not be located. |
| case invalidBazelPath |
| } |
| |
| /// The location of the bazel binary. |
| var bazelURL: URL |
| /// The location of the directory in which the workspace enclosing this BUILD file can be found. |
| let workspaceRootURL: URL |
| |
| private let localizedMessageLogger: LocalizedMessageLogger |
| private var queuedInfoMessages = [String]() |
| |
| private typealias CompletionHandler = (Process, Data, String?, String) -> Void |
| |
| init(bazelURL: URL, workspaceRootURL: URL, localizedMessageLogger: LocalizedMessageLogger) { |
| self.bazelURL = bazelURL |
| self.workspaceRootURL = workspaceRootURL |
| self.localizedMessageLogger = localizedMessageLogger |
| } |
| |
| func extractTargetRulesFromPackages(_ packages: [String]) -> [RuleInfo] { |
| guard !packages.isEmpty else { |
| return [] |
| } |
| |
| let profilingStart = localizedMessageLogger.startProfiling("fetch_rules", |
| message: "Fetching rules for packages \(packages)") |
| var infos = [RuleInfo]() |
| let query = packages.map({ "kind(rule, \($0):all)"}).joined(separator: "+") |
| do { |
| let (process, data, stderr, debugInfo) = |
| try self.bazelSynchronousQueryProcess(query, |
| outputKind: "xml", |
| loggingIdentifier: "bazel_query_fetch_rules") |
| if process.terminationStatus != 0 { |
| showExtractionError(debugInfo, stderr: stderr, displayLastLineIfNoErrorLines: true) |
| } else if let entries = self.extractRuleInfosFromBazelXMLOutput(data) { |
| infos = entries |
| } |
| } catch { |
| // The error has already been displayed to the user. |
| return [] |
| } |
| |
| localizedMessageLogger.logProfilingEnd(profilingStart) |
| return infos |
| } |
| |
| /// Extracts a map of RuleInfo to considered expansions for the given test_suite targets. |
| // The information provided represents the full possible set of tests for each test_suite; the |
| // actual expansion by Bazel may not include all of the returned labels and will be done |
| // recursively such that a test_suite whose expansion contains another test_suite would expand to |
| // the contents of the included suite. |
| func extractTestSuiteRules(_ testSuiteLabels: [BuildLabel]) -> [RuleInfo: Set<BuildLabel>] { |
| if testSuiteLabels.isEmpty { return [:] } |
| let profilingStart = localizedMessageLogger.startProfiling("expand_test_suite_rules", |
| message: "Expanding \(testSuiteLabels.count) test suites") |
| |
| var infos = [RuleInfo: Set<BuildLabel>]() |
| let labelDeps = testSuiteLabels.map {"deps(\($0.value))"} |
| let joinedLabelDeps = labelDeps.joined(separator: "+") |
| let query = "kind(\"test_suite rule\",\(joinedLabelDeps))" |
| do { |
| let (_, data, _, debugInfo) = try self.bazelSynchronousQueryProcess(query, |
| outputKind: "xml", |
| additionalArguments: ["--keep_going"], |
| loggingIdentifier: "bazel_query_expand_test_suite_rules") |
| if let entries = self.extractRuleInfosWithRuleInputsFromBazelXMLOutput(data) { |
| infos = entries |
| } |
| // Note that this query is expected to return a non-zero exit code on occasion, so messages |
| // should be handled as [Info]s, not errors. |
| self.queuedInfoMessages.append(debugInfo) |
| localizedMessageLogger.logProfilingEnd(profilingStart) |
| } catch { |
| // The error has already been displayed to the user. |
| return [:] |
| } |
| return infos |
| } |
| |
| /// Extracts all of the transitive BUILD and skylark (.bzl) files used by the given targets. |
| func extractBuildfiles<T: Collection>(_ targets: T) -> Set<BuildLabel> where T.Iterator.Element == BuildLabel { |
| if targets.isEmpty { return Set() } |
| |
| let profilingStart = localizedMessageLogger.startProfiling("extracting_skylark_files", |
| message: "Finding Skylark files for \(targets.count) rules") |
| |
| let labelDeps = targets.map {"deps(\($0.value))"} |
| let joinedLabelDeps = labelDeps.joined(separator: "+") |
| let query = "buildfiles(\(joinedLabelDeps))" |
| let buildFiles: Set<BuildLabel> |
| do { |
| // Errors in the BUILD structure being examined should not prevent partial extraction, so this |
| // command is considered successful if it returns any valid data at all. |
| let (_, data, _, debugInfo) = try self.bazelSynchronousQueryProcess(query, |
| outputKind: "xml", |
| additionalArguments: ["--keep_going"], |
| loggingIdentifier: "bazel_query_extracting_skylark_files") |
| self.queuedInfoMessages.append(debugInfo) |
| |
| if let labels = extractSourceFileLabelsFromBazelXMLOutput(data) { |
| buildFiles = Set(labels) |
| } else { |
| localizedMessageLogger.warning("BazelBuildfilesQueryFailed", |
| comment: "Bazel 'buildfiles' query failed to extract information.") |
| buildFiles = Set() |
| } |
| |
| localizedMessageLogger.logProfilingEnd(profilingStart) |
| } catch { |
| // Error will be displayed at the end of project generation. |
| return Set() |
| } |
| |
| return buildFiles |
| } |
| |
| // MARK: - Private methods |
| |
| private func showExtractionError(_ debugInfo: String, |
| stderr: String?, |
| displayLastLineIfNoErrorLines: Bool = false) { |
| localizedMessageLogger.infoMessage(debugInfo) |
| let details: String? |
| if let stderr = stderr { |
| if displayLastLineIfNoErrorLines { |
| details = BazelErrorExtractor.firstErrorLinesOrLastLinesFromString(stderr) |
| } else { |
| details = BazelErrorExtractor.firstErrorLinesFromString(stderr) |
| } |
| } else { |
| details = nil |
| } |
| localizedMessageLogger.error("BazelInfoExtractionFailed", |
| comment: "Error message for when a Bazel extractor did not complete successfully. Details are logged separately.", |
| details: details) |
| } |
| |
| // Generates a Process that will perform a bazel query, capturing the output data and passing it |
| // to the terminationHandler. |
| private func bazelQueryProcess(_ query: String, |
| outputKind: String? = nil, |
| additionalArguments: [String] = [], |
| message: String = "", |
| loggingIdentifier: String? = nil, |
| terminationHandler: @escaping CompletionHandler) throws -> Process { |
| guard FileManager.default.fileExists(atPath: bazelURL.path) else { |
| localizedMessageLogger.error("BazelBinaryNotFound", |
| comment: "Error to show when the bazel binary cannot be found at the previously saved location %1$@.", |
| values: bazelURL as NSURL) |
| throw ExtractorError.invalidBazelPath |
| } |
| |
| var arguments = [ |
| "--max_idle_secs=60", |
| "query", |
| "--announce_rc", // Print the RC files used by this operation. |
| "--noimplicit_deps", |
| "--order_output=no", |
| "--noshow_loading_progress", |
| "--noshow_progress", |
| query |
| ] |
| arguments.append(contentsOf: additionalArguments) |
| if let kind = outputKind { |
| arguments.append(contentsOf: ["--output", kind]) |
| } |
| |
| var message = message |
| if message != "" { |
| message = "\(message)\n" |
| } |
| |
| let process = TulsiProcessRunner.createProcess(bazelURL.path, |
| arguments: arguments, |
| messageLogger: localizedMessageLogger, |
| loggingIdentifier: loggingIdentifier) { |
| completionInfo in |
| let debugInfoFormatString = NSLocalizedString("DebugInfoForBazelCommand", |
| bundle: Bundle(for: type(of: self)), |
| comment: "Provides general information about a Bazel failure; a more detailed error may be reported elsewhere. The Bazel command is %1$@, exit code is %2$d, stderr %3$@.") |
| let stderr = NSString(data: completionInfo.stderr, encoding: String.Encoding.utf8.rawValue) |
| let debugInfo = String(format: debugInfoFormatString, |
| completionInfo.commandlineString, |
| completionInfo.terminationStatus, |
| stderr ?? "<No STDERR>") |
| |
| terminationHandler(completionInfo.process, |
| completionInfo.stdout, |
| stderr as String?, |
| debugInfo) |
| } |
| |
| return process |
| } |
| |
| /// Performs the given Bazel query synchronously in the workspaceRootURL directory. |
| private func bazelSynchronousQueryProcess(_ query: String, |
| outputKind: String? = nil, |
| additionalArguments: [String] = [], |
| message: String = "", |
| loggingIdentifier: String? = nil) throws -> (bazelProcess: Process, |
| returnedData: Data, |
| stderrString: String?, |
| debugInfo: String) { |
| let semaphore = DispatchSemaphore(value: 0) |
| var data: Data! = nil |
| var stderr: String? = nil |
| var info: String! = nil |
| |
| let process = try bazelQueryProcess(query, |
| outputKind: outputKind, |
| additionalArguments: additionalArguments, |
| message: message, |
| loggingIdentifier: loggingIdentifier) { |
| (_: Process, returnedData: Data, stderrString: String?, debugInfo: String) in |
| data = returnedData |
| stderr = stderrString |
| info = debugInfo |
| semaphore.signal() |
| } |
| |
| process.currentDirectoryPath = workspaceRootURL.path |
| process.launch() |
| |
| _ = semaphore.wait(timeout: DispatchTime.distantFuture) |
| return (process, data, stderr, info) |
| } |
| |
| private func extractRuleInfosWithRuleInputsFromBazelXMLOutput(_ bazelOutput: Data) -> [RuleInfo: Set<BuildLabel>]? { |
| do { |
| var infos = [RuleInfo: Set<BuildLabel>]() |
| let doc = try XMLDocument(data: bazelOutput, options: XMLNode.Options(rawValue: 0)) |
| let rules = try doc.nodes(forXPath: "/query/rule") |
| for ruleNode in rules { |
| guard let ruleElement = ruleNode as? XMLElement else { |
| localizedMessageLogger.error("BazelResponseXMLNonElementType", |
| comment: "General error to show when the XML parser returns something other " + |
| "than an NSXMLElement. This should never happen in practice.") |
| continue |
| } |
| guard let ruleLabel = ruleElement.attribute(forName: "name")?.stringValue else { |
| localizedMessageLogger.error("BazelResponseMissingRequiredAttribute", |
| comment: "Bazel response XML element %1$@ was found but was missing an attribute named %2$@.", |
| values: ruleElement, "name") |
| continue |
| } |
| guard let ruleType = ruleElement.attribute(forName: "class")?.stringValue else { |
| localizedMessageLogger.error("BazelResponseMissingRequiredAttribute", |
| comment: "Bazel response XML element %1$@ was found but was missing an attribute named %2$@.", |
| values: ruleElement, "class") |
| continue |
| } |
| |
| func extractLabelsFromXpath(_ xpath: String) throws -> Set<BuildLabel> { |
| var labelSet = Set<BuildLabel>() |
| let nodes = try ruleElement.nodes(forXPath: xpath) |
| for node in nodes { |
| guard let label = node.stringValue else { |
| localizedMessageLogger.error("BazelResponseLabelAttributeInvalid", |
| comment: "Bazel response XML element %1$@ should have a valid string value but does not.", |
| values: node) |
| continue |
| } |
| labelSet.insert(BuildLabel(label)) |
| } |
| return labelSet |
| } |
| |
| // Retrieve the list of linked targets through the test_host attribute. This provides a |
| // link between the test target and the test host so they can be linked in Xcode. |
| var linkedTargetLabels = Set<BuildLabel>() |
| linkedTargetLabels.formUnion( |
| try extractLabelsFromXpath("./label[@name='test_host']/@value")) |
| |
| let entry = RuleInfo(label: BuildLabel(ruleLabel), |
| type: ruleType, |
| linkedTargetLabels: linkedTargetLabels) |
| |
| infos[entry] = try extractLabelsFromXpath("./rule-input/@name") |
| } |
| return infos |
| } catch let e as NSError { |
| localizedMessageLogger.error("BazelResponseXMLParsingFailed", |
| comment: "Extractor Bazel output failed to be parsed as XML with error %1$@. This may be a Bazel bug or a bad BUILD file.", |
| values: e.localizedDescription) |
| return nil |
| } |
| } |
| |
| private func extractRuleInfosFromBazelXMLOutput(_ bazelOutput: Data) -> [RuleInfo]? { |
| if let infoMap = extractRuleInfosWithRuleInputsFromBazelXMLOutput(bazelOutput) { |
| return [RuleInfo](infoMap.keys) |
| } |
| return nil |
| } |
| |
| private func extractSourceFileLabelsFromBazelXMLOutput(_ bazelOutput: Data) -> Set<BuildLabel>? { |
| do { |
| let doc = try XMLDocument(data: bazelOutput, options: XMLNode.Options(rawValue: 0)) |
| let fileLabels = try doc.nodes(forXPath: "/query/source-file/@name") |
| var extractedLabels = Set<BuildLabel>() |
| for labelNode in fileLabels { |
| guard let value = labelNode.stringValue else { |
| localizedMessageLogger.error("BazelResponseLabelAttributeInvalid", |
| comment: "Bazel response XML element %1$@ should have a valid string value but does not.", |
| values: labelNode) |
| continue |
| } |
| extractedLabels.insert(BuildLabel(value)) |
| } |
| return extractedLabels |
| } catch let e as NSError { |
| localizedMessageLogger.error("BazelResponseXMLParsingFailed", |
| comment: "Extractor Bazel output failed to be parsed as XML with error %1$@. This may be a Bazel bug or a bad BUILD file.", |
| values: e.localizedDescription) |
| return nil |
| } |
| } |
| |
| // MARK: - QueuedLogging |
| |
| func logQueuedInfoMessages() { |
| guard !self.queuedInfoMessages.isEmpty else { |
| return |
| } |
| localizedMessageLogger.debugMessage("Log of Bazel query output follows:") |
| for message in self.queuedInfoMessages { |
| localizedMessageLogger.debugMessage(message) |
| } |
| self.queuedInfoMessages.removeAll() |
| } |
| |
| var hasQueuedInfoMessages: Bool { |
| return !self.queuedInfoMessages.isEmpty |
| } |
| } |
