Automated rollback of commit 7d7ceaec50c84480617e22707304c975d8dc2940.
*** Reason for rollback ***
Makes Exoblaze slower:
https://storage.cloud.google.com/blaze-performance-data/2021-01-19/nightly_report.html#yt-ios-local-blaze-bench-imacpro-2017-18c
*** Original change description ***
Simplify the decision on whether to extract include remotely. Instead of asking
the output service whether a given file is a remote file, just always extract
output files remotely.
RELNOTES: None.
PiperOrigin-RevId: 353812770
diff --git a/src/main/java/com/google/devtools/build/lib/includescanning/IncludeScanningModule.java b/src/main/java/com/google/devtools/build/lib/includescanning/IncludeScanningModule.java
index 97a3012..5d15e28 100644
--- a/src/main/java/com/google/devtools/build/lib/includescanning/IncludeScanningModule.java
+++ b/src/main/java/com/google/devtools/build/lib/includescanning/IncludeScanningModule.java
@@ -304,6 +304,7 @@
spawnScannerSupplier,
env.getExecRoot());
+ spawnScannerSupplier.get().setOutputService(env.getOutputService());
spawnScannerSupplier.get().setInMemoryOutput(options.inMemoryIncludesFiles);
}
}
diff --git a/src/main/java/com/google/devtools/build/lib/includescanning/SpawnIncludeScanner.java b/src/main/java/com/google/devtools/build/lib/includescanning/SpawnIncludeScanner.java
index bcde047..74c0c91 100644
--- a/src/main/java/com/google/devtools/build/lib/includescanning/SpawnIncludeScanner.java
+++ b/src/main/java/com/google/devtools/build/lib/includescanning/SpawnIncludeScanner.java
@@ -53,6 +53,7 @@
import com.google.devtools.build.lib.util.io.FileOutErr;
import com.google.devtools.build.lib.vfs.FileStatus;
import com.google.devtools.build.lib.vfs.IORuntimeException;
+import com.google.devtools.build.lib.vfs.OutputService;
import com.google.devtools.build.lib.vfs.Path;
import com.google.devtools.build.lib.vfs.PathFragment;
import com.google.devtools.build.lib.vfs.Symlinks;
@@ -75,6 +76,7 @@
ResourceSet.createWithRamCpu(/*memoryMb=*/ 10, /*cpuUsage=*/ 1);
private final Path execRoot;
+ private OutputService outputService;
private boolean inMemoryOutput;
private final int remoteExtractionThreshold;
private final AtomicReference<FilesystemCalls> syscallCache;
@@ -87,6 +89,11 @@
this.syscallCache = syscallCache;
}
+ public void setOutputService(OutputService outputService) {
+ Preconditions.checkState(this.outputService == null);
+ this.outputService = outputService;
+ }
+
public void setInMemoryOutput(boolean inMemoryOutput) {
this.inMemoryOutput = inMemoryOutput;
}
@@ -120,10 +127,11 @@
if (file.getRoot().getRoot().isAbsolute()) {
return false;
}
- // Output files are generally not locally available should be scanned remotely to avoid the
+ // Files written remotely that are not locally available should be scanned remotely to avoid the
// bandwidth and disk space penalty of bringing them across. Also, enable include scanning
- // remotely when the file size exceeds a certain size.
- if (remoteExtractionThreshold == 0 || !file.isSourceArtifact()) {
+ // remotely when explicitly directed to via a flag.
+ if (remoteExtractionThreshold == 0
+ || (outputService != null && outputService.isRemoteFile(file))) {
return true;
}
FileStatus status = syscallCache.get().statIfFound(file.getPath(), Symlinks.FOLLOW);
diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteOutputService.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteOutputService.java
index 6939e1e..957af4e 100644
--- a/src/main/java/com/google/devtools/build/lib/remote/RemoteOutputService.java
+++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteOutputService.java
@@ -28,6 +28,7 @@
import com.google.devtools.build.lib.vfs.FileSystem;
import com.google.devtools.build.lib.vfs.ModifiedFileSet;
import com.google.devtools.build.lib.vfs.OutputService;
+import com.google.devtools.build.lib.vfs.Path;
import com.google.devtools.build.lib.vfs.PathFragment;
import com.google.devtools.build.lib.vfs.Root;
import java.util.Map;
@@ -114,6 +115,13 @@
}
@Override
+ public boolean isRemoteFile(Artifact artifact) {
+ Path path = artifact.getPath();
+ return path.getFileSystem() instanceof RemoteActionFileSystem
+ && ((RemoteActionFileSystem) path.getFileSystem()).isRemote(path);
+ }
+
+ @Override
public boolean supportsPathResolverForArtifactValues() {
return actionFileSystemType() != ActionFileSystemType.DISABLED;
}
diff --git a/src/main/java/com/google/devtools/build/lib/vfs/OutputService.java b/src/main/java/com/google/devtools/build/lib/vfs/OutputService.java
index 21da650..88a4cff 100644
--- a/src/main/java/com/google/devtools/build/lib/vfs/OutputService.java
+++ b/src/main/java/com/google/devtools/build/lib/vfs/OutputService.java
@@ -141,6 +141,9 @@
*/
void clean() throws ExecException, InterruptedException;
+ /** @return true iff the file actually lives on a remote server */
+ boolean isRemoteFile(Artifact file);
+
default ActionFileSystemType actionFileSystemType() {
return ActionFileSystemType.DISABLED;
}
