| // Copyright 2014 The Bazel Authors. All rights reserved. |
| // |
| // Licensed under the Apache License, Version 2.0 (the "License"); |
| // you may not use this file except in compliance with the License. |
| // You may obtain a copy of the License at |
| // |
| // http://www.apache.org/licenses/LICENSE-2.0 |
| // |
| // Unless required by applicable law or agreed to in writing, software |
| // distributed under the License is distributed on an "AS IS" BASIS, |
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| // See the License for the specific language governing permissions and |
| // limitations under the License. |
| |
| package com.google.devtools.build.lib.rules.repository; |
| |
| import com.google.common.annotations.VisibleForTesting; |
| import com.google.common.base.Optional; |
| import com.google.common.base.Preconditions; |
| import com.google.common.collect.ImmutableMap; |
| import com.google.common.collect.Maps; |
| import com.google.devtools.build.lib.analysis.BlazeDirectories; |
| import com.google.devtools.build.lib.cmdline.RepositoryName; |
| import com.google.devtools.build.lib.events.Event; |
| import com.google.devtools.build.lib.events.ExtendedEventHandler.FetchProgress; |
| import com.google.devtools.build.lib.packages.Rule; |
| import com.google.devtools.build.lib.packages.RuleFormatter; |
| import com.google.devtools.build.lib.repository.ExternalPackageException; |
| import com.google.devtools.build.lib.repository.ExternalPackageUtil; |
| import com.google.devtools.build.lib.repository.ExternalRuleNotFoundException; |
| import com.google.devtools.build.lib.repository.RepositoryFailedEvent; |
| import com.google.devtools.build.lib.rules.repository.RepositoryFunction.RepositoryFunctionException; |
| import com.google.devtools.build.lib.skyframe.PrecomputedValue; |
| import com.google.devtools.build.lib.skyframe.PrecomputedValue.Precomputed; |
| import com.google.devtools.build.lib.util.Fingerprint; |
| import com.google.devtools.build.lib.vfs.FileSystemUtils; |
| import com.google.devtools.build.lib.vfs.Path; |
| import com.google.devtools.build.lib.vfs.PathFragment; |
| import com.google.devtools.build.lib.vfs.RootedPath; |
| import com.google.devtools.build.skyframe.SkyFunction; |
| import com.google.devtools.build.skyframe.SkyFunctionException; |
| import com.google.devtools.build.skyframe.SkyFunctionException.Transience; |
| import com.google.devtools.build.skyframe.SkyKey; |
| import com.google.devtools.build.skyframe.SkyValue; |
| import java.io.IOException; |
| import java.nio.charset.StandardCharsets; |
| import java.util.Map; |
| import java.util.Set; |
| import java.util.TreeMap; |
| import java.util.concurrent.atomic.AtomicBoolean; |
| import java.util.function.Supplier; |
| import javax.annotation.Nullable; |
| |
| /** |
| * A {@link SkyFunction} that implements delegation to the correct repository fetcher. |
| * |
| * <p> |
| * Each repository in the WORKSPACE file is represented by a {@link SkyValue} that is computed by |
| * this function. |
| */ |
| public final class RepositoryDelegatorFunction implements SkyFunction { |
| public static final Precomputed<Map<RepositoryName, PathFragment>> REPOSITORY_OVERRIDES = |
| new Precomputed<>(PrecomputedValue.Key.create("repository_overrides")); |
| |
| public static final Precomputed<String> DEPENDENCY_FOR_UNCONDITIONAL_FETCHING = |
| new Precomputed<>( |
| PrecomputedValue.Key.create("dependency_for_unconditional_repository_fetching")); |
| |
| public static final Precomputed<Optional<RootedPath>> RESOLVED_FILE_FOR_VERIFICATION = |
| new Precomputed<>( |
| PrecomputedValue.Key.create("resolved_file_for_external_repository_verification")); |
| |
| public static final Precomputed<Set<String>> OUTPUT_VERIFICATION_REPOSITORY_RULES = |
| new Precomputed<>(PrecomputedValue.Key.create("output_verification_repository_rules")); |
| |
| public static final Precomputed<Optional<RootedPath>> RESOLVED_FILE_INSTEAD_OF_WORKSPACE = |
| new Precomputed<>(PrecomputedValue.Key.create("resolved_file_instead_of_workspace")); |
| |
| public static final String DONT_FETCH_UNCONDITIONALLY = ""; |
| |
| // The marker file version is inject in the rule key digest so the rule key is always different |
| // when we decide to update the format. |
| private static final int MARKER_FILE_VERSION = 3; |
| |
| // Mapping of rule class name to RepositoryFunction. |
| private final ImmutableMap<String, RepositoryFunction> handlers; |
| |
| // Delegate function to handle skylark remote repositories |
| private final RepositoryFunction skylarkHandler; |
| |
| // This is a reference to isFetch in BazelRepositoryModule, which tracks whether the current |
| // command is a fetch. Remote repository lookups are only allowed during fetches. |
| private final AtomicBoolean isFetch; |
| |
| private final BlazeDirectories directories; |
| |
| private final Supplier<Map<String, String>> clientEnvironmentSupplier; |
| |
| public RepositoryDelegatorFunction( |
| ImmutableMap<String, RepositoryFunction> handlers, |
| @Nullable RepositoryFunction skylarkHandler, |
| AtomicBoolean isFetch, |
| Supplier<Map<String, String>> clientEnvironmentSupplier, |
| BlazeDirectories directories) { |
| this.handlers = handlers; |
| this.skylarkHandler = skylarkHandler; |
| this.isFetch = isFetch; |
| this.clientEnvironmentSupplier = clientEnvironmentSupplier; |
| this.directories = directories; |
| } |
| |
| private void setupRepositoryRoot(Path repoRoot) throws RepositoryFunctionException { |
| try { |
| repoRoot.deleteTree(); |
| Preconditions.checkNotNull(repoRoot.getParentDirectory()).createDirectoryAndParents(); |
| } catch (IOException e) { |
| throw new RepositoryFunctionException(e, Transience.TRANSIENT); |
| } |
| } |
| |
| @Override |
| public SkyValue compute(SkyKey skyKey, Environment env) |
| throws SkyFunctionException, InterruptedException { |
| RepositoryName repositoryName = (RepositoryName) skyKey.argument(); |
| |
| Map<RepositoryName, PathFragment> overrides = REPOSITORY_OVERRIDES.get(env); |
| boolean doNotFetchUnconditionally = |
| DONT_FETCH_UNCONDITIONALLY.equals(DEPENDENCY_FOR_UNCONDITIONAL_FETCHING.get(env)); |
| |
| Path repoRoot = RepositoryFunction.getExternalRepositoryDirectory(directories) |
| .getRelative(repositoryName.strippedName()); |
| |
| if (Preconditions.checkNotNull(overrides).containsKey(repositoryName)) { |
| return setupOverride(overrides.get(repositoryName), env, repoRoot); |
| } |
| |
| Rule rule; |
| try { |
| rule = getRepository(repositoryName, env); |
| } catch (ExternalRuleNotFoundException e) { |
| return RepositoryDirectoryValue.NO_SUCH_REPOSITORY_VALUE; |
| } |
| if (rule == null) { |
| return null; |
| } |
| |
| RepositoryFunction handler = getHandler(rule); |
| if (handler == null) { |
| // If we refer to a non repository rule then the repository does not exist. |
| return RepositoryDirectoryValue.NO_SUCH_REPOSITORY_VALUE; |
| } |
| |
| byte[] ruleSpecificData = handler.getRuleSpecificMarkerData(rule, env); |
| if (env.valuesMissing()) { |
| return null; |
| } |
| DigestWriter digestWriter = |
| new DigestWriter( |
| directories, repositoryName, rule, Preconditions.checkNotNull(ruleSpecificData)); |
| |
| // Local repositories are fetched regardless of the marker file because the operation is |
| // generally fast and they do not depend on non-local data, so it does not make much sense to |
| // try to cache them from across server instances. |
| boolean fetchLocalRepositoryAlways = isFetch.get() && handler.isLocal(rule); |
| if (!fetchLocalRepositoryAlways) { |
| // For the non-local repositories, check if they are already up-to-date: |
| // 1) unconditional fetching is not enabled, AND |
| // 2) repository directory exists, AND |
| // 3) marker file correctly describes the current repository state |
| if (doNotFetchUnconditionally && repoRoot.exists()) { |
| byte[] markerHash = digestWriter.areRepositoryAndMarkerFileConsistent(handler, env); |
| if (env.valuesMissing()) { |
| return null; |
| } |
| if (markerHash != null) { |
| // Now that we know that it exists and that we should not fetch unconditionally, we can |
| // declare a Skyframe dependency on the repository root. |
| RepositoryFunction.getRepositoryDirectory(repoRoot, env); |
| if (env.valuesMissing()) { |
| return null; |
| } |
| return RepositoryDirectoryValue.builder().setPath(repoRoot).setDigest(markerHash).build(); |
| } |
| } |
| } |
| |
| if (isFetch.get()) { |
| // Fetching enabled, go ahead. |
| RepositoryDirectoryValue.Builder builder = |
| fetchRepository(skyKey, repoRoot, env, digestWriter.getMarkerData(), handler, rule); |
| if (builder == null) { |
| return null; |
| } |
| |
| // No new Skyframe dependencies must be added between calling the repository implementation |
| // and writing the marker file because if they aren't computed, it would cause a Skyframe |
| // restart thus calling the possibly very slow (networking, decompression...) fetch() |
| // operation again. So we write the marker file here immediately. |
| byte[] digest = digestWriter.writeMarkerFile(); |
| return builder.setDigest(digest).build(); |
| } |
| |
| if (!repoRoot.exists()) { |
| // The repository isn't on the file system, there is nothing we can do. |
| throw new RepositoryFunctionException( |
| new IOException("to fix, run\n\tbazel fetch //...\nExternal repository " + repositoryName |
| + " not found and fetching repositories is disabled."), |
| Transience.TRANSIENT); |
| } |
| |
| // Declare a Skyframe dependency so that this is re-evaluated when something happens to the |
| // directory. |
| RepositoryFunction.getRepositoryDirectory(repoRoot, env); |
| if (env.valuesMissing()) { |
| return null; |
| } |
| |
| // Try to build with whatever is on the file system and emit a warning. |
| env.getListener() |
| .handle(Event.warn(rule.getLocation(), |
| String.format( |
| "External repository '%s' is not up-to-date and fetching is disabled. To update, " |
| + "run the build without the '--nofetch' command line option.", |
| rule.getName()))); |
| |
| return RepositoryDirectoryValue.builder().setPath(repoRoot).setFetchingDelayed().build(); |
| } |
| |
| private RepositoryFunction getHandler(Rule rule) { |
| RepositoryFunction handler; |
| if (rule.getRuleClassObject().isSkylark()) { |
| handler = skylarkHandler; |
| } else { |
| handler = handlers.get(rule.getRuleClass()); |
| } |
| if (handler != null) { |
| handler.setClientEnvironment(clientEnvironmentSupplier.get()); |
| } |
| |
| return handler; |
| } |
| |
| private RepositoryDirectoryValue.Builder fetchRepository( |
| SkyKey skyKey, |
| Path repoRoot, |
| Environment env, |
| Map<String, String> markerData, |
| RepositoryFunction handler, |
| Rule rule) |
| throws SkyFunctionException, InterruptedException { |
| |
| setupRepositoryRoot(repoRoot); |
| |
| String repositoryName = ((RepositoryName) skyKey.argument()).getName(); |
| env.getListener().post(new RepositoryFetching(repositoryName, false)); |
| |
| RepositoryDirectoryValue.Builder repoBuilder; |
| try { |
| repoBuilder = handler.fetch(rule, repoRoot, directories, env, markerData, skyKey); |
| } catch (SkyFunctionException e) { |
| // Upon an exceptional exit, the fetching of that repository is over as well. |
| env.getListener().post(new RepositoryFetching(repositoryName, true)); |
| env.getListener().post(new RepositoryFailedEvent(repositoryName)); |
| throw e; |
| } |
| |
| if (env.valuesMissing()) { |
| env.getListener().post(new RepositoryFetching(repositoryName, false, "Restarting.")); |
| return null; |
| } |
| env.getListener().post(new RepositoryFetching(repositoryName, true)); |
| return Preconditions.checkNotNull(repoBuilder); |
| } |
| |
| /** |
| * Uses a remote repository name to fetch the corresponding Rule describing how to get it. This |
| * should be called from {@link SkyFunction#compute} functions, which should return null if this |
| * returns null. |
| */ |
| @Nullable |
| private static Rule getRepository( |
| RepositoryName repositoryName, Environment env) |
| throws ExternalPackageException, InterruptedException { |
| return ExternalPackageUtil.getRuleByName(repositoryName.strippedName(), env); |
| } |
| |
| @Override |
| public String extractTag(SkyKey skyKey) { |
| return null; |
| } |
| |
| private RepositoryDirectoryValue setupOverride( |
| PathFragment sourcePath, Environment env, Path repoRoot) |
| throws RepositoryFunctionException, InterruptedException { |
| setupRepositoryRoot(repoRoot); |
| RepositoryDirectoryValue.Builder directoryValue = |
| LocalRepositoryFunction.symlink(repoRoot, sourcePath, env); |
| if (directoryValue == null) { |
| return null; |
| } |
| byte[] digest = new byte[] {}; |
| return directoryValue.setDigest(digest).build(); |
| } |
| |
| // Escape a value for the marker file |
| @VisibleForTesting |
| static String escape(String str) { |
| return str == null ? "\\0" : str.replace("\\", "\\\\").replace("\n", "\\n").replace(" ", "\\s"); |
| } |
| |
| // Unescape a value from the marker file |
| @VisibleForTesting |
| static String unescape(String str) { |
| if (str.equals("\\0")) { |
| return null; // \0 == null string |
| } |
| StringBuilder result = new StringBuilder(); |
| boolean escaped = false; |
| for (int i = 0; i < str.length(); i++) { |
| char c = str.charAt(i); |
| if (escaped) { |
| if (c == 'n') { // n means new line |
| result.append("\n"); |
| } else if (c == 's') { // s means space |
| result.append(" "); |
| } else { // Any other escaped characters are just un-escaped |
| result.append(c); |
| } |
| escaped = false; |
| } else if (c == '\\') { |
| escaped = true; |
| } else { |
| result.append(c); |
| } |
| } |
| return result.toString(); |
| } |
| |
| private static class DigestWriter { |
| private final Path markerPath; |
| private final Rule rule; |
| private final Map<String, String> markerData; |
| private final String ruleKey; |
| |
| DigestWriter( |
| BlazeDirectories directories, |
| RepositoryName repositoryName, |
| Rule rule, |
| byte[] ruleSpecificData) { |
| ruleKey = computeRuleKey(rule, ruleSpecificData); |
| markerPath = getMarkerPath(directories, repositoryName.strippedName()); |
| this.rule = rule; |
| markerData = Maps.newHashMap(); |
| } |
| |
| byte[] writeMarkerFile() throws RepositoryFunctionException { |
| StringBuilder builder = new StringBuilder(); |
| builder.append(ruleKey).append("\n"); |
| for (Map.Entry<String, String> data : markerData.entrySet()) { |
| String key = data.getKey(); |
| String value = data.getValue(); |
| builder.append(escape(key)).append(" ").append(escape(value)).append("\n"); |
| } |
| String content = builder.toString(); |
| try { |
| FileSystemUtils.writeContent(markerPath, StandardCharsets.UTF_8, content); |
| } catch (IOException e) { |
| throw new RepositoryFunctionException(e, Transience.TRANSIENT); |
| } |
| return new Fingerprint().addString(content).digestAndReset(); |
| } |
| |
| /** |
| * Checks if the state of the repository in the file system is consistent with the rule in the |
| * WORKSPACE file. |
| * |
| * <p>Deletes the marker file if not so that no matter what happens after, the state of the file |
| * system stays consistent. |
| * |
| * <p>Returns null if the file system is not up to date and a hash of the marker file if the |
| * file system is up to date. |
| * |
| * <p>We check the repository root for existence here, but we can't depend on the FileValue, |
| * because it's possible that we eventually create that directory in which case the FileValue |
| * and the state of the file system would be inconsistent. |
| */ |
| byte[] areRepositoryAndMarkerFileConsistent(RepositoryFunction handler, Environment env) |
| throws RepositoryFunctionException, InterruptedException { |
| if (!markerPath.exists()) { |
| return null; |
| } |
| |
| Map<String, String> markerData = new TreeMap<>(); |
| String content; |
| try { |
| content = FileSystemUtils.readContent(markerPath, StandardCharsets.UTF_8); |
| String markerRuleKey = readMarkerFile(content, markerData); |
| boolean verified = false; |
| if (Preconditions.checkNotNull(ruleKey).equals(markerRuleKey)) { |
| verified = handler.verifyMarkerData(rule, markerData, env); |
| if (env.valuesMissing()) { |
| return null; |
| } |
| } |
| |
| if (verified) { |
| return new Fingerprint().addString(content).digestAndReset(); |
| } else { |
| // So that we are in a consistent state if something happens while fetching the repository |
| markerPath.delete(); |
| return null; |
| } |
| } catch (IOException e) { |
| throw new RepositoryFunctionException(e, Transience.TRANSIENT); |
| } |
| } |
| |
| Map<String, String> getMarkerData() { |
| return markerData; |
| } |
| |
| @Nullable |
| private String readMarkerFile(String content, Map<String, String> markerData) { |
| String markerRuleKey = null; |
| String[] lines = content.split("\n"); |
| |
| boolean firstLine = true; |
| for (String line : lines) { |
| if (firstLine) { |
| markerRuleKey = line; |
| firstLine = false; |
| } else { |
| int sChar = line.indexOf(' '); |
| String key = line; |
| String value = ""; |
| if (sChar > 0) { |
| key = unescape(line.substring(0, sChar)); |
| value = unescape(line.substring(sChar + 1)); |
| } |
| markerData.put(key, value); |
| } |
| } |
| return markerRuleKey; |
| } |
| |
| private String computeRuleKey(Rule rule, byte[] ruleSpecificData) { |
| return new Fingerprint() |
| .addBytes(RuleFormatter.serializeRule(rule).build().toByteArray()) |
| .addBytes(ruleSpecificData) |
| .addInt(MARKER_FILE_VERSION) |
| .hexDigestAndReset(); |
| } |
| |
| private static Path getMarkerPath(BlazeDirectories directories, String ruleName) { |
| return RepositoryFunction.getExternalRepositoryDirectory(directories) |
| .getChild("@" + ruleName + ".marker"); |
| } |
| } |
| |
| private class RepositoryFetching implements FetchProgress { |
| final String id; |
| final boolean finished; |
| final String message; |
| |
| RepositoryFetching(String name, boolean finished) { |
| this.id = name; |
| this.finished = finished; |
| this.message = finished ? "finished." : "fetching"; |
| } |
| |
| RepositoryFetching(String name, boolean finished, String message) { |
| this.id = name; |
| this.finished = finished; |
| this.message = message; |
| } |
| |
| @Override |
| public String getResourceIdentifier() { |
| return id; |
| } |
| |
| @Override |
| public String getProgress() { |
| return message; |
| } |
| |
| @Override |
| public boolean isFinished() { |
| return finished; |
| } |
| } |
| } |