blob: 2e95d658c662c880997775658b42de1cf79f6f01 [file] [log] [blame]
// Copyright 2014 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.devtools.build.lib.unix;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe;
import com.google.devtools.build.lib.profiler.Profiler;
import com.google.devtools.build.lib.profiler.ProfilerTask;
import com.google.devtools.build.lib.unix.NativePosixFiles.Dirents;
import com.google.devtools.build.lib.unix.NativePosixFiles.ReadTypes;
import com.google.devtools.build.lib.util.Blocker;
import com.google.devtools.build.lib.vfs.AbstractFileSystemWithCustomStat;
import com.google.devtools.build.lib.vfs.DigestHashFunction;
import com.google.devtools.build.lib.vfs.Dirent;
import com.google.devtools.build.lib.vfs.FileStatus;
import com.google.devtools.build.lib.vfs.Path;
import com.google.devtools.build.lib.vfs.PathFragment;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import javax.annotation.Nullable;
/** This class implements the FileSystem interface using direct calls to the UNIX filesystem. */
@ThreadSafe
public class UnixFileSystem extends AbstractFileSystemWithCustomStat {
protected final String hashAttributeName;
public UnixFileSystem(DigestHashFunction hashFunction, String hashAttributeName) {
super(hashFunction);
this.hashAttributeName = hashAttributeName;
}
public static Dirent.Type getDirentFromMode(int mode) {
if (com.google.devtools.build.lib.unix.FileStatus.isSpecialFile(mode)) {
return Dirent.Type.UNKNOWN;
} else if (com.google.devtools.build.lib.unix.FileStatus.isFile(mode)) {
return Dirent.Type.FILE;
} else if (com.google.devtools.build.lib.unix.FileStatus.isDirectory(mode)) {
return Dirent.Type.DIRECTORY;
} else if (com.google.devtools.build.lib.unix.FileStatus.isSymbolicLink(mode)) {
return Dirent.Type.SYMLINK;
} else {
return Dirent.Type.UNKNOWN;
}
}
/**
* Eager implementation of FileStatus for file systems that have an atomic stat(2) syscall. A
* proxy for {@link com.google.devtools.build.lib.unix.FileStatus}. Note that isFile and
* getLastModifiedTime have slightly different meanings between UNIX and VFS.
*/
@VisibleForTesting
protected static class UnixFileStatus implements FileStatus {
private final com.google.devtools.build.lib.unix.FileStatus status;
UnixFileStatus(com.google.devtools.build.lib.unix.FileStatus status) {
this.status = status;
}
@Override
public boolean isFile() {
return !isDirectory() && !isSymbolicLink();
}
@Override
public boolean isDirectory() {
return status.isDirectory();
}
@Override
public boolean isSymbolicLink() {
return status.isSymbolicLink();
}
@Override
public boolean isSpecialFile() {
return isFile() && !status.isRegularFile();
}
@Override
public long getSize() {
return status.getSize();
}
@Override
public long getLastModifiedTime() {
return (status.getLastModifiedTime() * 1000)
+ (status.getFractionalLastModifiedTime() / 1000000);
}
@Override
public long getLastChangeTime() {
return (status.getLastChangeTime() * 1000) + (status.getFractionalLastChangeTime() / 1000000);
}
@Override
public long getNodeId() {
// Note that we may want to include more information in this id number going forward,
// especially the device number.
return status.getInodeNumber();
}
@Override
public int getPermissions() {
return status.getPermissions();
}
@Override
public String toString() {
return status.toString();
}
}
@Override
protected Collection<String> getDirectoryEntries(PathFragment path) throws IOException {
String name = path.getPathString();
String[] entries;
long startTime = Profiler.nanoTimeMaybe();
try {
entries = NativePosixFiles.readdir(name);
} finally {
profiler.logSimpleTask(startTime, ProfilerTask.VFS_DIR, name);
}
Collection<String> result = new ArrayList<>(entries.length);
for (String entry : entries) {
result.add(entry);
}
return result;
}
@Override
@Nullable
protected PathFragment resolveOneLink(PathFragment path) throws IOException {
// Beware, this seemingly simple code belies the complex specification of
// FileSystem.resolveOneLink().
return stat(path, false).isSymbolicLink() ? readSymbolicLink(path) : null;
}
/**
* Converts from {@link com.google.devtools.build.lib.unix.NativePosixFiles.Dirents.Type} to
* {@link com.google.devtools.build.lib.vfs.Dirent.Type}.
*/
private static Dirent.Type convertToDirentType(Dirents.Type type) {
switch (type) {
case FILE:
return Dirent.Type.FILE;
case DIRECTORY:
return Dirent.Type.DIRECTORY;
case SYMLINK:
return Dirent.Type.SYMLINK;
case UNKNOWN:
return Dirent.Type.UNKNOWN;
default:
throw new IllegalArgumentException("Unknown type " + type);
}
}
@Override
protected Collection<Dirent> readdir(PathFragment path, boolean followSymlinks)
throws IOException {
String name = path.getPathString();
long startTime = Profiler.nanoTimeMaybe();
try {
Dirents unixDirents =
NativePosixFiles.readdir(name, followSymlinks ? ReadTypes.FOLLOW : ReadTypes.NOFOLLOW);
Preconditions.checkState(unixDirents.hasTypes());
List<Dirent> dirents = Lists.newArrayListWithCapacity(unixDirents.size());
for (int i = 0; i < unixDirents.size(); i++) {
dirents.add(
new Dirent(unixDirents.getName(i), convertToDirentType(unixDirents.getType(i))));
}
return dirents;
} finally {
profiler.logSimpleTask(startTime, ProfilerTask.VFS_DIR, name);
}
}
@Override
protected FileStatus stat(PathFragment path, boolean followSymlinks) throws IOException {
return statInternal(path, followSymlinks);
}
@VisibleForTesting
protected UnixFileStatus statInternal(PathFragment path, boolean followSymlinks)
throws IOException {
String name = path.getPathString();
long startTime = Profiler.nanoTimeMaybe();
long comp = Blocker.begin();
try {
return new UnixFileStatus(
followSymlinks ? NativePosixFiles.stat(name) : NativePosixFiles.lstat(name));
} finally {
Blocker.end(comp);
profiler.logSimpleTask(startTime, ProfilerTask.VFS_STAT, name);
}
}
// Like stat(), but returns null instead of throwing.
// This is a performance optimization in the case where clients
// catch and don't re-throw.
@Override
@Nullable
protected FileStatus statNullable(PathFragment path, boolean followSymlinks) {
String name = path.getPathString();
long startTime = Profiler.nanoTimeMaybe();
long comp = Blocker.begin();
try {
ErrnoFileStatus stat =
followSymlinks ? NativePosixFiles.errnoStat(name) : NativePosixFiles.errnoLstat(name);
return stat.hasError() ? null : new UnixFileStatus(stat);
} finally {
Blocker.end(comp);
profiler.logSimpleTask(startTime, ProfilerTask.VFS_STAT, name);
}
}
@Override
protected boolean exists(PathFragment path, boolean followSymlinks) {
return statNullable(path, followSymlinks) != null;
}
/**
* Return true iff the {@code stat} of {@code path} resulted in an {@code ENOENT} or {@code
* ENOTDIR} error.
*/
@Override
@Nullable
protected FileStatus statIfFound(PathFragment path, boolean followSymlinks) throws IOException {
String name = path.getPathString();
long startTime = Profiler.nanoTimeMaybe();
long comp = Blocker.begin();
try {
ErrnoFileStatus stat =
followSymlinks ? NativePosixFiles.errnoStat(name) : NativePosixFiles.errnoLstat(name);
if (!stat.hasError()) {
return new UnixFileStatus(stat);
}
int errno = stat.getErrno();
if (errno == ErrnoFileStatus.ENOENT || errno == ErrnoFileStatus.ENOTDIR) {
return null;
}
// This should not return -- we are calling stat here just to throw the proper exception.
// However, since there may be transient IO errors, we cannot guarantee that an exception will
// be thrown.
// TODO(bazel-team): Extract the exception-construction code and make it visible separately in
// FilesystemUtils to avoid having to do a duplicate stat call.
return stat(path, followSymlinks);
} finally {
Blocker.end(comp);
profiler.logSimpleTask(startTime, ProfilerTask.VFS_STAT, name);
}
}
@Override
protected boolean isReadable(PathFragment path) throws IOException {
return (statInternal(path, true).getPermissions() & 0400) != 0;
}
@Override
protected boolean isWritable(PathFragment path) throws IOException {
return (statInternal(path, true).getPermissions() & 0200) != 0;
}
@Override
protected boolean isExecutable(PathFragment path) throws IOException {
return (statInternal(path, true).getPermissions() & 0100) != 0;
}
/**
* Adds or remove the bits specified in "permissionBits" to the permission mask of the file
* specified by {@code path}. If the argument {@code add} is true, the specified permissions are
* added, otherwise they are removed.
*
* @throws IOException if there was an error writing the file's metadata
*/
private void modifyPermissionBits(PathFragment path, int permissionBits, boolean add)
throws IOException {
int oldMode = statInternal(path, true).getPermissions();
int newMode = add ? (oldMode | permissionBits) : (oldMode & ~permissionBits);
long comp = Blocker.begin();
try {
NativePosixFiles.chmod(path.toString(), newMode);
} finally {
Blocker.end(comp);
}
}
@Override
protected void setReadable(PathFragment path, boolean readable) throws IOException {
modifyPermissionBits(path, 0400, readable);
}
@Override
public void setWritable(PathFragment path, boolean writable) throws IOException {
modifyPermissionBits(path, 0200, writable);
}
@Override
protected void setExecutable(PathFragment path, boolean executable) throws IOException {
modifyPermissionBits(path, 0111, executable);
}
@Override
protected void chmod(PathFragment path, int mode) throws IOException {
NativePosixFiles.chmod(path.toString(), mode);
}
@Override
public boolean supportsModifications(PathFragment path) {
return true;
}
@Override
public boolean supportsSymbolicLinksNatively(PathFragment path) {
return true;
}
@Override
public boolean supportsHardLinksNatively(PathFragment path) {
return true;
}
@Override
public boolean isFilePathCaseSensitive() {
return true;
}
@Override
public boolean createDirectory(PathFragment path) throws IOException {
long comp = Blocker.begin();
try {
// Note: UNIX mkdir(2), FilesystemUtils.mkdir() and createDirectory all
// have different ways of representing failure!
if (NativePosixFiles.mkdir(path.toString(), 0755)) {
return true; // successfully created
}
} finally {
Blocker.end(comp);
}
// false => EEXIST: something is already in the way (file/dir/symlink)
if (isDirectory(path, false)) {
return false; // directory already existed
} else {
throw new IOException(path + " (File exists)");
}
}
@Override
protected boolean createWritableDirectory(PathFragment path) throws IOException {
long comp = Blocker.begin();
try {
return NativePosixFiles.mkdirWritable(path.toString());
} finally {
Blocker.end(comp);
}
}
@Override
public void createDirectoryAndParents(PathFragment path) throws IOException {
long comp = Blocker.begin();
try {
NativePosixFiles.mkdirs(path.toString(), 0755);
} finally {
Blocker.end(comp);
}
}
@Override
protected void createSymbolicLink(PathFragment linkPath, PathFragment targetFragment)
throws IOException {
long comp = Blocker.begin();
try {
NativePosixFiles.symlink(targetFragment.getSafePathString(), linkPath.toString());
} finally {
Blocker.end(comp);
}
}
@Override
protected PathFragment readSymbolicLink(PathFragment path) throws IOException {
// Note that the default implementation of readSymbolicLinkUnchecked calls this method and thus
// is optimal since we only make one system call in here.
String name = path.toString();
long startTime = Profiler.nanoTimeMaybe();
long comp = Blocker.begin();
try {
return PathFragment.create(NativePosixFiles.readlink(name));
} catch (InvalidArgumentIOException e) {
throw new NotASymlinkException(path, e);
} finally {
Blocker.end(comp);
profiler.logSimpleTask(startTime, ProfilerTask.VFS_READLINK, name);
}
}
@Override
public void renameTo(PathFragment sourcePath, PathFragment targetPath) throws IOException {
long comp = Blocker.begin();
try {
NativePosixFiles.rename(sourcePath.toString(), targetPath.toString());
} finally {
Blocker.end(comp);
}
}
@Override
protected long getFileSize(PathFragment path, boolean followSymlinks) throws IOException {
return stat(path, followSymlinks).getSize();
}
@Override
protected boolean delete(PathFragment path) throws IOException {
String name = path.toString();
long startTime = Profiler.nanoTimeMaybe();
long comp = Blocker.begin();
try {
return NativePosixFiles.remove(name);
} finally {
Blocker.end(comp);
profiler.logSimpleTask(startTime, ProfilerTask.VFS_DELETE, name);
}
}
@Override
protected long getLastModifiedTime(PathFragment path, boolean followSymlinks) throws IOException {
return stat(path, followSymlinks).getLastModifiedTime();
}
@Override
public void setLastModifiedTime(PathFragment path, long newTime) throws IOException {
long comp = Blocker.begin();
try {
if (newTime == Path.NOW_SENTINEL_TIME) {
NativePosixFiles.utime(path.toString(), true, 0);
} else {
// newTime > MAX_INT => -ve unixTime
int unixTime = (int) (newTime / 1000);
NativePosixFiles.utime(path.toString(), false, unixTime);
}
} finally {
Blocker.end(comp);
}
}
@Override
@Nullable
public byte[] getxattr(PathFragment path, String name, boolean followSymlinks)
throws IOException {
String pathName = path.toString();
long startTime = Profiler.nanoTimeMaybe();
long comp = Blocker.begin();
try {
return followSymlinks
? NativePosixFiles.getxattr(pathName, name)
: NativePosixFiles.lgetxattr(pathName, name);
} catch (UnsupportedOperationException e) {
// getxattr() syscall is not supported by the underlying filesystem (it returned ENOTSUP).
// Per method contract, treat this as ENODATA.
return null;
} finally {
Blocker.end(comp);
profiler.logSimpleTask(startTime, ProfilerTask.VFS_XATTR, pathName);
}
}
@Override
@Nullable
protected byte[] getFastDigest(PathFragment path) throws IOException {
// Attempt to obtain the digest from an extended attribute attached to the file. This is much
// faster than reading and digesting the file's contents on the fly, especially for large files.
return hashAttributeName.isEmpty() ? null : getxattr(path, hashAttributeName, true);
}
@Override
protected byte[] getDigest(PathFragment path) throws IOException {
String name = path.toString();
long startTime = Profiler.nanoTimeMaybe();
try {
return super.getDigest(path);
} finally {
profiler.logSimpleTask(startTime, ProfilerTask.VFS_MD5, name);
}
}
@Override
protected void createFSDependentHardLink(PathFragment linkPath, PathFragment originalPath)
throws IOException {
long comp = Blocker.begin();
try {
NativePosixFiles.link(originalPath.toString(), linkPath.toString());
} finally {
Blocker.end(comp);
}
}
@Override
protected void deleteTreesBelow(PathFragment dir) throws IOException {
if (isDirectory(dir, /*followSymlinks=*/ false)) {
long startTime = Profiler.nanoTimeMaybe();
long comp = Blocker.begin();
try {
NativePosixFiles.deleteTreesBelow(dir.toString());
} finally {
Blocker.end(comp);
profiler.logSimpleTask(startTime, ProfilerTask.VFS_DELETE, dir.toString());
}
}
}
private static File createJavaIoFile(PathFragment path) {
final String pathStr = path.getPathString();
if (pathStr.chars().allMatch(c -> c < 128)) {
return new File(pathStr);
}
// Paths returned from NativePosixFiles are Strings containing raw bytes from the filesystem.
// Java's IO subsystem expects paths to be encoded per the `sun.jnu.encoding` setting. This
// is difficult to handle generically, but we can special-case the most common case (UTF-8).
if ("UTF-8".equals(System.getProperty("sun.jnu.encoding"))) {
final byte[] pathBytes = pathStr.getBytes(StandardCharsets.ISO_8859_1);
return new File(new String(pathBytes, StandardCharsets.UTF_8));
}
// This will probably fail but not much that can be done without migrating to `java.nio.Files`.
return new File(pathStr);
}
@Override
protected InputStream createFileInputStream(PathFragment path) throws IOException {
return new FileInputStream(createJavaIoFile(path));
}
protected OutputStream createFileOutputStream(PathFragment path, boolean append)
throws FileNotFoundException {
return createFileOutputStream(path, append, /* internal= */ false);
}
@Override
protected OutputStream createFileOutputStream(PathFragment path, boolean append, boolean internal)
throws FileNotFoundException {
final String name = path.toString();
if (!internal
&& profiler.isActive()
&& (profiler.isProfiling(ProfilerTask.VFS_WRITE)
|| profiler.isProfiling(ProfilerTask.VFS_OPEN))) {
long startTime = Profiler.nanoTimeMaybe();
long comp = Blocker.begin();
try {
return new ProfiledNativeFileOutputStream(NativePosixFiles.openWrite(name, append), name);
} finally {
Blocker.end(comp);
profiler.logSimpleTask(startTime, ProfilerTask.VFS_OPEN, name);
}
} else {
long comp = Blocker.begin();
try {
return new NativeFileOutputStream(NativePosixFiles.openWrite(name, append));
} finally {
Blocker.end(comp);
}
}
}
private static class NativeFileOutputStream extends OutputStream {
private final int fd;
private boolean closed = false;
NativeFileOutputStream(int fd) {
this.fd = fd;
}
@Override
protected void finalize() throws Throwable {
close();
super.finalize();
}
@Override
public synchronized void close() throws IOException {
if (!closed) {
long comp = Blocker.begin();
try {
NativePosixFiles.close(fd, this);
closed = true;
} finally {
Blocker.end(comp);
}
}
super.close();
}
@Override
public void write(int b) throws IOException {
write(new byte[] {(byte) (b & 0xFF)});
}
@Override
public void write(byte[] b) throws IOException {
write(b, 0, b.length);
}
@Override
@SuppressWarnings(
"UnsafeFinalization") // Finalizer invokes close; close and write are synchronized.
public synchronized void write(byte[] b, int off, int len) throws IOException {
if (closed) {
throw new IOException("attempt to write to a closed Outputstream backed by a native file");
}
long comp = Blocker.begin();
try {
NativePosixFiles.write(fd, b, off, len);
} finally {
Blocker.end(comp);
}
}
}
private static final class ProfiledNativeFileOutputStream extends NativeFileOutputStream {
private final String name;
public ProfiledNativeFileOutputStream(int fd, String name) throws FileNotFoundException {
super(fd);
this.name = name;
}
@Override
public synchronized void write(byte[] b, int off, int len) throws IOException {
long startTime = Profiler.nanoTimeMaybe();
try {
super.write(b, off, len);
} finally {
profiler.logSimpleTask(startTime, ProfilerTask.VFS_WRITE, name);
}
}
}
}