| # Lint as: python3 |
| # Copyright 2022 The Bazel Authors. All rights reserved. |
| # |
| # Licensed under the Apache License, Version 2.0 (the "License"); |
| # you may not use this file except in compliance with the License. |
| # You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| """This tool build tar files from a list of inputs.""" |
| |
| import argparse |
| import os |
| import tarfile |
| |
| # Use a deterministic mtime that doesn't confuse other programs. |
| # See: https://github.com/bazelbuild/bazel/issues/1299 |
| PORTABLE_MTIME = 946684800 # 2000-01-01 00:00:00.000 UTC |
| |
| |
| class TarFileWriter(object): |
| """A wrapper to write tar files.""" |
| |
| class Error(Exception): |
| pass |
| |
| def __init__(self, |
| name, |
| root_directory='', |
| default_uid=0, |
| default_gid=0, |
| default_mtime=None): |
| """TarFileWriter wraps tarfile.open(). |
| |
| Args: |
| name: the tar file name. |
| root_directory: virtual root to prepend to elements in the archive. |
| default_uid: uid to assign to files in the archive. |
| default_gid: gid to assign to files in the archive. |
| default_mtime: default mtime to use for elements in the archive. May be an |
| integer or the value 'portable' to use the date 2000-01-01, which is |
| compatible with non *nix OSes'. |
| """ |
| mode = 'w:' |
| self.name = name |
| self.root_directory = root_directory.strip('/') |
| self.default_gid = default_gid |
| self.default_uid = default_uid |
| if default_mtime is None: |
| self.default_mtime = 0 |
| elif default_mtime == 'portable': |
| self.default_mtime = PORTABLE_MTIME |
| else: |
| self.default_mtime = int(default_mtime) |
| self.tar = tarfile.open(name=name, mode=mode) |
| self.members = set() |
| self.directories = set(['.']) |
| |
| def __enter__(self): |
| return self |
| |
| def __exit__(self, t, v, traceback): |
| self.close() |
| |
| def close(self): |
| """Close the output tar file.""" |
| self.tar.close() |
| |
| def _addfile(self, info, fileobj=None): |
| """Add a file in the tar file if there is no conflict.""" |
| if info.type == tarfile.DIRTYPE: |
| # Enforce the ending / for directories so we correctly deduplicate. |
| if not info.name.endswith('/'): |
| info.name += '/' |
| if info.name not in self.members: |
| self.tar.addfile(info, fileobj) |
| self.members.add(info.name) |
| elif info.type != tarfile.DIRTYPE: |
| print(('Duplicate file in archive: %s, ' |
| 'picking first occurrence' % info.name)) |
| |
| def add_parents(self, path, mode=0o755): |
| """Add the parents of this path to the archive. |
| |
| Args: |
| path: destination path in archive. |
| mode: unix permission mode of the dir, default 0o755. |
| """ |
| |
| def add_dirs(path): |
| """Helper to add dirs.""" |
| path = path.strip('/') |
| if not path: |
| return |
| if path in self.directories: |
| return |
| components = path.rsplit('/', 1) |
| if len(components) > 1: |
| add_dirs(components[0]) |
| self.directories.add(path) |
| tarinfo = tarfile.TarInfo(path + '/') |
| tarinfo.mtime = self.default_mtime |
| tarinfo.uid = self.default_uid |
| tarinfo.gid = self.default_gid |
| tarinfo.type = tarfile.DIRTYPE |
| tarinfo.mode = mode or 0o755 |
| self.tar.addfile(tarinfo, fileobj=None) |
| |
| components = path.rsplit('/', 1) |
| if len(components) > 1: |
| add_dirs(components[0]) |
| |
| def add_tree(self, input_path, dest_path, mode=None): |
| """Recursively add a tree of files. |
| |
| Args: |
| input_path: the path of the directory to add. |
| dest_path: the destination path of the directory to add. |
| mode: unix permission mode of the file, default 0644 (0755). |
| """ |
| # Add the x bit to directories to prevent non-traversable directories. |
| # The x bit is set only to if the read bit is set. |
| dirmode = (mode | ((0o444 & mode) >> 2)) if mode else mode |
| self.add_parents(dest_path, mode=dirmode) |
| |
| if os.path.isdir(input_path): |
| dest_path = dest_path.rstrip('/') + '/' |
| # Iterate over the sorted list of file so we get a deterministic result. |
| filelist = os.listdir(input_path) |
| filelist.sort() |
| for f in filelist: |
| self.add_tree( |
| input_path=input_path + '/' + f, dest_path=dest_path + f, mode=mode) |
| else: |
| self.add_file_and_parents( |
| dest_path, tarfile.REGTYPE, file_content=input_path, mode=mode) |
| |
| def add_file_and_parents(self, |
| name, |
| kind=tarfile.REGTYPE, |
| link=None, |
| file_content=None, |
| mode=None): |
| """Add a file to the current tar. |
| |
| Creates parent directories if needed. |
| |
| Args: |
| name: the name of the file to add. |
| kind: the type of the file to add, see tarfile.*TYPE. |
| link: if the file is a link, the destination of the link. |
| file_content: file to read the content from. Provide either this one or |
| `content` to specifies a content for the file. |
| mode: unix permission mode of the file, default 0644 (0755). |
| """ |
| if self.root_directory and ( |
| not (name == self.root_directory or name.startswith('/') or |
| name.startswith(self.root_directory + '/'))): |
| name = self.root_directory + '/' + name |
| self.add_parents(name, mode=0o755) |
| |
| if kind == tarfile.DIRTYPE: |
| name = name.rstrip('/') |
| if name in self.directories: |
| return |
| |
| if file_content and os.path.isdir(file_content): |
| self.add_tree(input_path=file_content, dest_path=name, mode=mode) |
| return |
| |
| tarinfo = tarfile.TarInfo(name) |
| tarinfo.mtime = self.default_mtime |
| tarinfo.uid = self.default_uid |
| tarinfo.gid = self.default_gid |
| tarinfo.type = kind |
| if mode is None: |
| tarinfo.mode = 0o644 if kind == tarfile.REGTYPE else 0o755 |
| else: |
| tarinfo.mode = mode |
| if link: |
| tarinfo.linkname = link |
| if file_content: |
| with open(file_content, 'rb') as f: |
| tarinfo.size = os.fstat(f.fileno()).st_size |
| self._addfile(tarinfo, fileobj=f) |
| else: |
| self._addfile(tarinfo, fileobj=None) |
| |
| def add_file_at_dest(self, in_path, dest_path, mode=None): |
| """Add a file to the tar file. |
| |
| Args: |
| in_path: the path of the file to add to the artifact |
| dest_path: the name of the file in the artifact |
| mode: force to the specified mode. Default is mode from the file. |
| """ |
| # Make a clean, '/' deliminted destination path |
| dest = os.path.normpath(dest_path.strip('/')).replace(os.path.sep, '/') |
| # If mode is unspecified, derive the mode from the file's mode. |
| if mode is None: |
| mode = 0o755 if os.access(dest, os.X_OK) else 0o644 |
| self.add_file_and_parents(dest, file_content=in_path, mode=mode) |
| |
| |
| def unquote_and_split(arg, c): |
| """Split a string at the first unquoted occurrence of a character. |
| |
| Split the string arg at the first unquoted occurrence of the character c. |
| Here, in the first part of arg, the backslash is considered the |
| quoting character indicating that the next character is to be |
| added literally to the first part, even if it is the split character. |
| |
| Args: |
| arg: the string to be split |
| c: the character at which to split |
| |
| Returns: |
| The unquoted string before the separator and the string after the |
| separator. |
| """ |
| head = '' |
| i = 0 |
| while i < len(arg): |
| if arg[i] == c: |
| return (head, arg[i + 1:]) |
| elif arg[i] == '\\': |
| i += 1 |
| if i == len(arg): |
| # dangling quotation symbol |
| return (head, '') |
| else: |
| head += arg[i] |
| else: |
| head += arg[i] |
| i += 1 |
| # if we leave the loop, the character c was not found unquoted |
| return (head, '') |
| |
| |
| def main(): |
| parser = argparse.ArgumentParser( |
| description='Helper for building tar packages', fromfile_prefix_chars='@') |
| parser.add_argument( |
| '--output', required=True, help='The output file, mandatory.') |
| parser.add_argument( |
| '--mode', help='Force the mode on the added files (in octal).') |
| parser.add_argument( |
| '--directory', |
| help='Directory in which to store the file inside the layer') |
| parser.add_argument('--file', action='append', help='input_paty=dest_path') |
| parser.add_argument( |
| '--owner', |
| default='0.0', |
| help='Specify the numeric default owner of all files. E.g. 0.0') |
| options = parser.parse_args() |
| |
| # Parse modes arguments |
| default_mode = None |
| if options.mode: |
| # Convert from octal |
| default_mode = int(options.mode, 8) |
| |
| uid = gid = 0 |
| if options.owner: |
| ids = options.owner.split('.', 1) |
| uid = int(ids[0]) |
| gid = int(ids[1]) |
| |
| # Add objects to the tar file |
| with TarFileWriter( |
| name=options.output, |
| root_directory=options.directory or '', |
| default_uid=uid, |
| default_gid=gid, |
| default_mtime=PORTABLE_MTIME) as output: |
| for f in options.file: |
| (input_path, dest) = unquote_and_split(f, '=') |
| output.add_file_at_dest(input_path, dest, mode=default_mode) |
| |
| |
| if __name__ == '__main__': |
| main() |
