src/main/java/com/google/devtools/build/lib/skyframe/PerBuildSyscallCache.java - bazel - Git at Google

 // Copyright 2014 The Bazel Authors. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
 //
 //    http://www.apache.org/licenses/LICENSE-2.0
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
 package com.google.devtools.build.lib.skyframe;

 import static com.google.common.base.MoreObjects.firstNonNull;

 import com.github.benmanes.caffeine.cache.Caffeine;
 import com.github.benmanes.caffeine.cache.LoadingCache;
 import com.google.devtools.build.lib.util.Pair;
 import com.google.devtools.build.lib.vfs.Dirent;
 import com.google.devtools.build.lib.vfs.FileStatus;
 import com.google.devtools.build.lib.vfs.Path;
 import com.google.devtools.build.lib.vfs.Symlinks;
 import com.google.devtools.build.lib.vfs.UnixGlob;
 import java.io.IOException;
 import java.util.Collection;

 /**
  * A per-build cache of filesystem operations.
  *
  * <p>Mostly used by non-Skyframe globbing and include parsing.
  */
 public final class PerBuildSyscallCache implements UnixGlob.FilesystemCalls {

   private final LoadingCache<Pair<Path, Symlinks>, Object> statCache;

   /* Caches the result of readdir(<path>, Symlinks.NOFOLLOW) calls. */
   private final LoadingCache<Path, Object> readdirCache;

   private static final FileStatus NO_STATUS = new FakeFileStatus();

   private PerBuildSyscallCache(
       LoadingCache<Pair<Path, Symlinks>, Object> statCache,
       LoadingCache<Path, Object> readdirCache) {
     this.statCache = statCache;
     this.readdirCache = readdirCache;
   }

   public static Builder newBuilder() {
     return new Builder();
   }

   /** Builder for a per-build filesystem cache. */
   public static final class Builder {
     private static final int UNSET = -1;
     private int maxStats = UNSET;
     private int maxReaddirs = UNSET;
     private int initialCapacity = UNSET;

     private Builder() {}

     /** Sets the upper bound of the 'stat' cache. This cache is unbounded by default. */
     public Builder setMaxStats(int maxStats) {
       this.maxStats = maxStats;
       return this;
     }

     /** Sets the upper bound of the 'readdir' cache. This cache is unbounded by default. */
     public Builder setMaxReaddirs(int maxReaddirs) {
       this.maxReaddirs = maxReaddirs;
       return this;
     }

     /** Sets the concurrency level of the caches. */
     public Builder setInitialCapacity(int initialCapacity) {
       this.initialCapacity = initialCapacity;
       return this;
     }

     public PerBuildSyscallCache build() {
       Caffeine<Object, Object> statCacheBuilder = Caffeine.newBuilder();
       if (maxStats != UNSET) {
         statCacheBuilder.maximumSize(maxStats);
       }
       Caffeine<Object, Object> readdirCacheBuilder = Caffeine.newBuilder();
       if (maxReaddirs != UNSET) {
         readdirCacheBuilder.maximumSize(maxReaddirs);
       }
       if (initialCapacity != UNSET) {
         statCacheBuilder.initialCapacity(initialCapacity);
         readdirCacheBuilder.initialCapacity(initialCapacity);
       }
       return new PerBuildSyscallCache(
           statCacheBuilder.build(PerBuildSyscallCache::statImpl),
           readdirCacheBuilder.build(PerBuildSyscallCache::readdirImpl));
     }
   }

   @Override
   @SuppressWarnings("unchecked")
   public Collection<Dirent> readdir(Path path) throws IOException {
     Object result = readdirCache.get(path);
     if (result instanceof IOException) {
       throw (IOException) result;
     }
     return (Collection<Dirent>) result; // unchecked cast
   }

   @Override
   public FileStatus statIfFound(Path path, Symlinks symlinks) throws IOException {
     // Try to load a Symlinks.NOFOLLOW result first. Symlinks are rare and this enables sharing the
     // cache for all non-symlink paths.
     Object result = statCache.get(Pair.of(path, Symlinks.NOFOLLOW));
     if (result instanceof IOException) {
       throw (IOException) result;
     }
     FileStatus status = (FileStatus) result;
     if (status != NO_STATUS && symlinks == Symlinks.FOLLOW && status.isSymbolicLink()) {
       result = statCache.get(Pair.of(path, Symlinks.FOLLOW));
       if (result instanceof IOException) {
         throw (IOException) result;
       }
       status = (FileStatus) result;
     }
     return (status == NO_STATUS) ? null : status;
   }

   @Override
   @SuppressWarnings("unchecked")
   public Dirent.Type getType(Path path, Symlinks symlinks) throws IOException {
     // Use a cached stat call if we have one. This is done first so that we don't need to iterate
     // over a list of directory entries as we do for cached readdir() entries. We don't ever expect
     // to get a cache hit if symlinks == Symlinks.NOFOLLOW and so we don't bother to check.
     if (symlinks == Symlinks.FOLLOW) {
       Pair<Path, Symlinks> key = Pair.of(path, symlinks);
       Object result = statCache.getIfPresent(key);
       if (result != null && !(result instanceof IOException)) {
         if (result == NO_STATUS) {
           return null;
         }
         return UnixGlob.statusToDirentType((FileStatus) result);
       }
     }

     // If this is a root directory, we must stat, there is no parent.
     Path parent = path.getParentDirectory();
     if (parent == null) {
       return UnixGlob.statusToDirentType(statIfFound(path, symlinks));
     }

     // Answer based on a cached readdir() call if possible. The cache might already be populated
     // from Skyframe directory lising (DirectoryListingFunction) or by globbing via
     // {@link UnixGlob}. We generally try to avoid following symlinks in readdir() calls as in a
     // directory with many symlinks, these would be resolved basically using a stat anyway and they
     // would be resolved sequentially which can be slow on high-latency file systems. If we request
     // the type of a file with FOLLOW, and find a symlink in the directory, we fall back to doing a
     // stat.
     Object result = readdirCache.getIfPresent(parent);
     if (result != null && !(result instanceof IOException)) {
       for (Dirent dirent : (Collection<Dirent>) result) { // unchecked cast
         // TODO(djasper): Dealing with filesystem case is a bit of a code smell. Figure out a better
         // way to store Dirents, e.g. with names normalized.
         if (path.getFileSystem().isFilePathCaseSensitive()
             && !dirent.getName().equals(path.getBaseName())) {
           continue;
         }
         if (!path.getFileSystem().isFilePathCaseSensitive()
             && !dirent.getName().equalsIgnoreCase(path.getBaseName())) {
           continue;
         }
         if (dirent.getType() == Dirent.Type.SYMLINK && symlinks == Symlinks.FOLLOW) {
           // See above: We don't want to follow symlinks with readdir(). Do a stat() instead.
           return UnixGlob.statusToDirentType(statIfFound(path, Symlinks.FOLLOW));
         }
         return dirent.getType();
       }
       return null;
     }

     return UnixGlob.statusToDirentType(statIfFound(path, symlinks));
   }

   public void clear() {
     statCache.invalidateAll();
     readdirCache.invalidateAll();
   }

   // This is used because the cache implementations don't allow null.
   private static final class FakeFileStatus implements FileStatus {
     @Override
     public long getLastChangeTime() {
       throw new UnsupportedOperationException();
     }

     @Override
     public long getNodeId() {
       throw new UnsupportedOperationException();
     }

     @Override
     public long getLastModifiedTime() {
       throw new UnsupportedOperationException();
     }

     @Override
     public long getSize() {
       throw new UnsupportedOperationException();
     }

     @Override
     public boolean isDirectory() {
       throw new UnsupportedOperationException();
     }

     @Override
     public boolean isFile() {
       throw new UnsupportedOperationException();
     }

     @Override
     public boolean isSpecialFile() {
       throw new UnsupportedOperationException();
     }

     @Override
     public boolean isSymbolicLink() {
       throw new UnsupportedOperationException();
     }
   }

   /** Returns {@link FileStatus} or {@link IOException}. */
   private static Object statImpl(Pair<Path, Symlinks> p) {
     try {
       FileStatus stat = p.first.statIfFound(p.second);
       return firstNonNull(stat, NO_STATUS);
     } catch (IOException e) {
       return e;
     }
   }

   /** Returns a collection of {@link Dirent} or {@link IOException}. */
   private static Object readdirImpl(Path p) {
     try {
       // TODO(bazel-team): Consider storing the Collection of Dirent values more compactly by
       // reusing DirectoryEntryListingStateValue#CompactSortedDirents.
       return p.readdir(Symlinks.NOFOLLOW);
     } catch (IOException e) {
       return e;
     }
   }
 }
	// Copyright 2014 The Bazel Authors. All rights reserved.
	//
	// Licensed under the Apache License, Version 2.0 (the "License");
	// you may not use this file except in compliance with the License.
	// You may obtain a copy of the License at
	//
	// http://www.apache.org/licenses/LICENSE-2.0
	//
	// Unless required by applicable law or agreed to in writing, software
	// distributed under the License is distributed on an "AS IS" BASIS,
	// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	// See the License for the specific language governing permissions and
	// limitations under the License.
	package com.google.devtools.build.lib.skyframe;

	import static com.google.common.base.MoreObjects.firstNonNull;

	import com.github.benmanes.caffeine.cache.Caffeine;
	import com.github.benmanes.caffeine.cache.LoadingCache;
	import com.google.devtools.build.lib.util.Pair;
	import com.google.devtools.build.lib.vfs.Dirent;
	import com.google.devtools.build.lib.vfs.FileStatus;
	import com.google.devtools.build.lib.vfs.Path;
	import com.google.devtools.build.lib.vfs.Symlinks;
	import com.google.devtools.build.lib.vfs.UnixGlob;
	import java.io.IOException;
	import java.util.Collection;

	/**
	* A per-build cache of filesystem operations.
	*
	* <p>Mostly used by non-Skyframe globbing and include parsing.
	*/
	public final class PerBuildSyscallCache implements UnixGlob.FilesystemCalls {

	private final LoadingCache<Pair<Path, Symlinks>, Object> statCache;

	/* Caches the result of readdir(<path>, Symlinks.NOFOLLOW) calls. */
	private final LoadingCache<Path, Object> readdirCache;

	private static final FileStatus NO_STATUS = new FakeFileStatus();

	private PerBuildSyscallCache(
	LoadingCache<Pair<Path, Symlinks>, Object> statCache,
	LoadingCache<Path, Object> readdirCache) {
	this.statCache = statCache;
	this.readdirCache = readdirCache;
	}

	public static Builder newBuilder() {
	return new Builder();
	}

	/** Builder for a per-build filesystem cache. */
	public static final class Builder {
	private static final int UNSET = -1;
	private int maxStats = UNSET;
	private int maxReaddirs = UNSET;
	private int initialCapacity = UNSET;

	private Builder() {}

	/** Sets the upper bound of the 'stat' cache. This cache is unbounded by default. */
	public Builder setMaxStats(int maxStats) {
	this.maxStats = maxStats;
	return this;
	}

	/** Sets the upper bound of the 'readdir' cache. This cache is unbounded by default. */
	public Builder setMaxReaddirs(int maxReaddirs) {
	this.maxReaddirs = maxReaddirs;
	return this;
	}

	/** Sets the concurrency level of the caches. */
	public Builder setInitialCapacity(int initialCapacity) {
	this.initialCapacity = initialCapacity;
	return this;
	}

	public PerBuildSyscallCache build() {
	Caffeine<Object, Object> statCacheBuilder = Caffeine.newBuilder();
	if (maxStats != UNSET) {
	statCacheBuilder.maximumSize(maxStats);
	}
	Caffeine<Object, Object> readdirCacheBuilder = Caffeine.newBuilder();
	if (maxReaddirs != UNSET) {
	readdirCacheBuilder.maximumSize(maxReaddirs);
	}
	if (initialCapacity != UNSET) {
	statCacheBuilder.initialCapacity(initialCapacity);
	readdirCacheBuilder.initialCapacity(initialCapacity);
	}
	return new PerBuildSyscallCache(
	statCacheBuilder.build(PerBuildSyscallCache::statImpl),
	readdirCacheBuilder.build(PerBuildSyscallCache::readdirImpl));
	}
	}

	@Override
	@SuppressWarnings("unchecked")
	public Collection<Dirent> readdir(Path path) throws IOException {
	Object result = readdirCache.get(path);
	if (result instanceof IOException) {
	throw (IOException) result;
	}
	return (Collection<Dirent>) result; // unchecked cast
	}

	@Override
	public FileStatus statIfFound(Path path, Symlinks symlinks) throws IOException {
	// Try to load a Symlinks.NOFOLLOW result first. Symlinks are rare and this enables sharing the
	// cache for all non-symlink paths.
	Object result = statCache.get(Pair.of(path, Symlinks.NOFOLLOW));
	if (result instanceof IOException) {
	throw (IOException) result;
	}
	FileStatus status = (FileStatus) result;
	if (status != NO_STATUS && symlinks == Symlinks.FOLLOW && status.isSymbolicLink()) {
	result = statCache.get(Pair.of(path, Symlinks.FOLLOW));
	if (result instanceof IOException) {
	throw (IOException) result;
	}
	status = (FileStatus) result;
	}
	return (status == NO_STATUS) ? null : status;
	}

	@Override
	@SuppressWarnings("unchecked")
	public Dirent.Type getType(Path path, Symlinks symlinks) throws IOException {
	// Use a cached stat call if we have one. This is done first so that we don't need to iterate
	// over a list of directory entries as we do for cached readdir() entries. We don't ever expect
	// to get a cache hit if symlinks == Symlinks.NOFOLLOW and so we don't bother to check.
	if (symlinks == Symlinks.FOLLOW) {
	Pair<Path, Symlinks> key = Pair.of(path, symlinks);
	Object result = statCache.getIfPresent(key);
	if (result != null && !(result instanceof IOException)) {
	if (result == NO_STATUS) {
	return null;
	}
	return UnixGlob.statusToDirentType((FileStatus) result);
	}
	}

	// If this is a root directory, we must stat, there is no parent.
	Path parent = path.getParentDirectory();
	if (parent == null) {
	return UnixGlob.statusToDirentType(statIfFound(path, symlinks));
	}

	// Answer based on a cached readdir() call if possible. The cache might already be populated
	// from Skyframe directory lising (DirectoryListingFunction) or by globbing via
	// {@link UnixGlob}. We generally try to avoid following symlinks in readdir() calls as in a
	// directory with many symlinks, these would be resolved basically using a stat anyway and they
	// would be resolved sequentially which can be slow on high-latency file systems. If we request
	// the type of a file with FOLLOW, and find a symlink in the directory, we fall back to doing a
	// stat.
	Object result = readdirCache.getIfPresent(parent);
	if (result != null && !(result instanceof IOException)) {
	for (Dirent dirent : (Collection<Dirent>) result) { // unchecked cast
	// TODO(djasper): Dealing with filesystem case is a bit of a code smell. Figure out a better
	// way to store Dirents, e.g. with names normalized.
	if (path.getFileSystem().isFilePathCaseSensitive()
	&& !dirent.getName().equals(path.getBaseName())) {
	continue;
	}
	if (!path.getFileSystem().isFilePathCaseSensitive()
	&& !dirent.getName().equalsIgnoreCase(path.getBaseName())) {
	continue;
	}
	if (dirent.getType() == Dirent.Type.SYMLINK && symlinks == Symlinks.FOLLOW) {
	// See above: We don't want to follow symlinks with readdir(). Do a stat() instead.
	return UnixGlob.statusToDirentType(statIfFound(path, Symlinks.FOLLOW));
	}
	return dirent.getType();
	}
	return null;
	}

	return UnixGlob.statusToDirentType(statIfFound(path, symlinks));
	}

	public void clear() {
	statCache.invalidateAll();
	readdirCache.invalidateAll();
	}

	// This is used because the cache implementations don't allow null.
	private static final class FakeFileStatus implements FileStatus {
	@Override
	public long getLastChangeTime() {
	throw new UnsupportedOperationException();
	}

	@Override
	public long getNodeId() {
	throw new UnsupportedOperationException();
	}

	@Override
	public long getLastModifiedTime() {
	throw new UnsupportedOperationException();
	}

	@Override
	public long getSize() {
	throw new UnsupportedOperationException();
	}

	@Override
	public boolean isDirectory() {
	throw new UnsupportedOperationException();
	}

	@Override
	public boolean isFile() {
	throw new UnsupportedOperationException();
	}

	@Override
	public boolean isSpecialFile() {
	throw new UnsupportedOperationException();
	}

	@Override
	public boolean isSymbolicLink() {
	throw new UnsupportedOperationException();
	}
	}

	/** Returns {@link FileStatus} or {@link IOException}. */
	private static Object statImpl(Pair<Path, Symlinks> p) {
	try {
	FileStatus stat = p.first.statIfFound(p.second);
	return firstNonNull(stat, NO_STATUS);
	} catch (IOException e) {
	return e;
	}
	}

	/** Returns a collection of {@link Dirent} or {@link IOException}. */
	private static Object readdirImpl(Path p) {
	try {
	// TODO(bazel-team): Consider storing the Collection of Dirent values more compactly by
	// reusing DirectoryEntryListingStateValue#CompactSortedDirents.
	return p.readdir(Symlinks.NOFOLLOW);
	} catch (IOException e) {
	return e;
	}
	}
	}