| #!/usr/bin/python2.7 | |
| # Copyright 2015 The Bazel Authors. All rights reserved. | |
| # | |
| # Licensed under the Apache License, Version 2.0 (the "License"); | |
| # you may not use this file except in compliance with the License. | |
| # You may obtain a copy of the License at | |
| # | |
| # http:#www.apache.org/licenses/LICENSE-2.0 | |
| # | |
| # Unless required by applicable law or agreed to in writing, software | |
| # distributed under the License is distributed on an "AS IS" BASIS, | |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| # See the License for the specific language governing permissions and | |
| # limitations under the License. | |
| """A script for J2ObjC dead code removal in Blaze. | |
| This script removes unused J2ObjC-translated classes from compilation and | |
| linking by: | |
| 1. Build a class dependency tree among translated source files. | |
| 2. Use user-provided Java class entry points to get a list of reachable | |
| classes. | |
| 3. Go through all translated source files and rewrite unreachable ones with | |
| dummy content. | |
| """ | |
| import argparse | |
| from collections import OrderedDict | |
| import multiprocessing | |
| import os | |
| import pipes # swap to shlex once on Python 3 | |
| import Queue | |
| import re | |
| import shutil | |
| import subprocess | |
| import threading | |
| PRUNED_SRC_CONTENT = 'static int DUMMY_unused __attribute__((unused,used)) = 0;' | |
| def BuildReachabilityTree(dependency_mapping_files, file_open=open): | |
| """Builds a reachability tree using entries from dependency mapping files. | |
| Args: | |
| dependency_mapping_files: A comma separated list of J2ObjC-generated | |
| dependency mapping files. | |
| file_open: Reference to the builtin open function so it may be | |
| overridden for testing. | |
| Returns: | |
| A dict mapping J2ObjC-generated source files to the corresponding direct | |
| dependent source files. | |
| """ | |
| tree = dict() | |
| for dependency_mapping_file in dependency_mapping_files.split(','): | |
| with file_open(dependency_mapping_file, 'r') as f: | |
| for line in f: | |
| entry = line.strip().split(':')[0] | |
| dep = line.strip().split(':')[1] | |
| if entry in tree: | |
| tree[entry].append(dep) | |
| else: | |
| tree[entry] = [dep] | |
| return tree | |
| def BuildHeaderMapping(header_mapping_files, file_open=open): | |
| """Builds a mapping between Java classes and J2ObjC-translated header files. | |
| Args: | |
| header_mapping_files: A comma separated list of J2ObjC-generated | |
| header mapping files. | |
| file_open: Reference to the builtin open function so it may be | |
| overridden for testing. | |
| Returns: | |
| An ordered dict mapping Java class names to corresponding J2ObjC-translated | |
| source files. | |
| """ | |
| header_mapping = OrderedDict() | |
| for header_mapping_file in header_mapping_files.split(','): | |
| with file_open(header_mapping_file, 'r') as f: | |
| for line in f: | |
| java_class_name = line.strip().split('=')[0] | |
| transpiled_file_name = os.path.splitext(line.strip().split('=')[1])[0] | |
| header_mapping[java_class_name] = transpiled_file_name | |
| return header_mapping | |
| def BuildReachableFileSet(entry_classes, reachability_tree, header_mapping, | |
| archive_source_file_mapping=None): | |
| """Builds a set of reachable translated files from entry Java classes. | |
| Args: | |
| entry_classes: A comma separated list of Java entry classes. | |
| reachability_tree: A dict mapping translated files to their direct | |
| dependencies. | |
| header_mapping: A dict mapping Java class names to translated source files. | |
| archive_source_file_mapping: A dict mapping source files to the associated | |
| archive file that contains them. | |
| Returns: | |
| A set of reachable translated files from the given list of entry classes. | |
| Raises: | |
| Exception: If there is an entry class that is not being transpiled in this | |
| j2objc_library. | |
| """ | |
| transpiled_entry_files = [] | |
| for entry_class in entry_classes.split(','): | |
| if entry_class not in header_mapping: | |
| raise Exception(entry_class + | |
| ' is not in the transitive Java deps of included ' + | |
| 'j2objc_library rules.') | |
| transpiled_entry_files.append(header_mapping[entry_class]) | |
| # Translated files going into the same static library archive with duplicated | |
| # base names also need to be added to the set of entry files. | |
| # | |
| # This edge case is ignored because we currently cannot correctly perform | |
| # dead code removal in this case. The object file entries in static library | |
| # archives are named by the base names of the original source files. If two | |
| # source files (e.g., foo/bar.m, bar/bar.m) go into the same archive and | |
| # share the same base name (bar.m), their object file entries inside the | |
| # archive will have the same name (bar.o). We cannot correctly handle this | |
| # case because current archive tools (ar, ranlib, etc.) do not handle this | |
| # case very well. | |
| if archive_source_file_mapping: | |
| transpiled_entry_files.extend(_DuplicatedFiles(archive_source_file_mapping)) | |
| # Translated files from package-info.java are also added to the entry files | |
| # because they are needed to resolve ObjC class names with prefixes and these | |
| # files may also have dependencies. | |
| for transpiled_file in reachability_tree: | |
| if transpiled_file.endswith('package-info'): | |
| transpiled_entry_files.append(transpiled_file) | |
| reachable_files = set() | |
| for transpiled_entry_file in transpiled_entry_files: | |
| reachable_files.add(transpiled_entry_file) | |
| current_level_deps = [] | |
| # We need to check if the transpiled file is in the reachability tree | |
| # because J2ObjC protos are not analyzed for dead code stripping and | |
| # therefore are not in the reachability tree at all. | |
| if transpiled_entry_file in reachability_tree: | |
| current_level_deps = reachability_tree[transpiled_entry_file] | |
| while current_level_deps: | |
| next_level_deps = [] | |
| for dep in current_level_deps: | |
| if dep not in reachable_files: | |
| reachable_files.add(dep) | |
| if dep in reachability_tree: | |
| next_level_deps.extend(reachability_tree[dep]) | |
| current_level_deps = next_level_deps | |
| return reachable_files | |
| def PruneFiles(input_files, output_files, objc_file_path, reachable_files, | |
| file_open=open, file_shutil=shutil): | |
| """Copies over translated files and remove the contents of unreachable files. | |
| Args: | |
| input_files: A comma separated list of input source files to prune. It has | |
| a one-on-one pair mapping with the output_file list. | |
| output_files: A comma separated list of output source files to write pruned | |
| source files to. It has a one-on-one pair mapping with the input_file | |
| list. | |
| objc_file_path: The file path which represents a directory where the | |
| generated ObjC files reside. | |
| reachable_files: A set of reachable source files. | |
| file_open: Reference to the builtin open function so it may be | |
| overridden for testing. | |
| file_shutil: Reference to the builtin shutil module so it may be | |
| overridden for testing. | |
| Returns: | |
| None. | |
| """ | |
| file_queue = Queue.Queue() | |
| for input_file, output_file in zip(input_files.split(','), | |
| output_files.split(',')): | |
| file_queue.put((input_file, output_file)) | |
| for _ in xrange(multiprocessing.cpu_count()): | |
| t = threading.Thread(target=_PruneFile, args=(file_queue, | |
| reachable_files, | |
| objc_file_path, | |
| file_open, | |
| file_shutil)) | |
| t.start() | |
| file_queue.join() | |
| def _PruneFile(file_queue, reachable_files, objc_file_path, file_open=open, | |
| file_shutil=shutil): | |
| while True: | |
| try: | |
| input_file, output_file = file_queue.get_nowait() | |
| except Queue.Empty: | |
| return | |
| file_name = os.path.relpath(os.path.splitext(input_file)[0], | |
| objc_file_path) | |
| if file_name in reachable_files: | |
| file_shutil.copy(input_file, output_file) | |
| else: | |
| with file_open(output_file, 'w') as f: | |
| # Use a static variable scoped to the source file to suppress | |
| # the "has no symbols" linker warning for empty object files. | |
| f.write(PRUNED_SRC_CONTENT) | |
| file_queue.task_done() | |
| def _DuplicatedFiles(archive_source_file_mapping): | |
| """Returns a list of file with duplicated base names in each archive file. | |
| Args: | |
| archive_source_file_mapping: A dict mapping source files to the associated | |
| archive file that contains them. | |
| Returns: | |
| A list containing files with duplicated base names. | |
| """ | |
| duplicated_files = [] | |
| dict_with_duplicates = dict() | |
| for source_files in archive_source_file_mapping.values(): | |
| for source_file in source_files: | |
| file_basename = os.path.basename(source_file) | |
| file_without_ext = os.path.splitext(source_file)[0] | |
| if file_basename in dict_with_duplicates: | |
| dict_with_duplicates[file_basename].append(file_without_ext) | |
| else: | |
| dict_with_duplicates[file_basename] = [file_without_ext] | |
| for basename in dict_with_duplicates: | |
| if len(dict_with_duplicates[basename]) > 1: | |
| duplicated_files.extend(dict_with_duplicates[basename]) | |
| dict_with_duplicates = dict() | |
| return duplicated_files | |
| def BuildArchiveSourceFileMapping(archive_source_mapping_files, file_open): | |
| """Builds a mapping between archive files and their associated source files. | |
| Args: | |
| archive_source_mapping_files: A comma separated list of J2ObjC-generated | |
| mapping between archive files and their associated source files. | |
| file_open: Reference to the builtin open function so it may be | |
| overridden for testing. | |
| Returns: | |
| A dict mapping between archive files and their associated source files. | |
| """ | |
| tree = dict() | |
| for archive_source_mapping_file in archive_source_mapping_files.split(','): | |
| with file_open(archive_source_mapping_file, 'r') as f: | |
| for line in f: | |
| entry = line.strip().split(':')[0] | |
| dep = line.strip().split(':')[1] | |
| if entry in tree: | |
| tree[entry].append(dep) | |
| else: | |
| tree[entry] = [dep] | |
| return tree | |
| def PruneSourceFiles(input_files, output_files, dependency_mapping_files, | |
| header_mapping_files, entry_classes, objc_file_path, | |
| file_open=open, file_shutil=shutil): | |
| """Copies over translated files and remove the contents of unreachable files. | |
| Args: | |
| input_files: A comma separated list of input source files to prune. It has | |
| a one-on-one pair mapping with the output_file list. | |
| output_files: A comma separated list of output source files to write pruned | |
| source files to. It has a one-on-one pair mapping with the input_file | |
| list. | |
| dependency_mapping_files: A comma separated list of J2ObjC-generated | |
| dependency mapping files. | |
| header_mapping_files: A comma separated list of J2ObjC-generated | |
| header mapping files. | |
| entry_classes: A comma separated list of Java entry classes. | |
| objc_file_path: The file path which represents a directory where the | |
| generated ObjC files reside. | |
| file_open: Reference to the builtin open function so it may be | |
| overridden for testing. | |
| file_shutil: Reference to the builtin shutil module so it may be | |
| overridden for testing. | |
| """ | |
| reachability_file_mapping = BuildReachabilityTree( | |
| dependency_mapping_files, file_open) | |
| header_map = BuildHeaderMapping(header_mapping_files, file_open) | |
| reachable_files_set = BuildReachableFileSet(entry_classes, | |
| reachability_file_mapping, | |
| header_map) | |
| PruneFiles(input_files, | |
| output_files, | |
| objc_file_path, | |
| reachable_files_set, | |
| file_open, | |
| file_shutil) | |
| def MatchObjectNamesInArchive(xcrunwrapper, archive, object_names): | |
| """Returns object names matching their identity in an archive file. | |
| The linker that blaze uses appends an md5 hash to object file | |
| names prior to inclusion in the archive file. Thus, object names | |
| such as 'foo.o' need to be matched to their appropriate name in | |
| the archive file, such as 'foo_<hash>.o'. | |
| Args: | |
| xcrunwrapper: A wrapper script over xcrun. | |
| archive: The location of the archive file. | |
| object_names: The expected basenames of object files to match, | |
| sans extension. For example 'foo' (not 'foo.o'). | |
| Returns: | |
| A list of basenames of matching members of the given archive | |
| """ | |
| ar_contents_cmd = [xcrunwrapper, 'ar', '-t', archive] | |
| real_object_names = subprocess.check_output(ar_contents_cmd) | |
| expected_object_name_regex = r'^(?:%s)(?:_[0-9a-f]{32}(?:-[0-9]+)?)?\.o$' % ( | |
| '|'.join([re.escape(name) for name in object_names])) | |
| return re.findall(expected_object_name_regex, real_object_names, | |
| flags=re.MULTILINE) | |
| def PruneArchiveFile(input_archive, output_archive, dummy_archive, | |
| dependency_mapping_files, header_mapping_files, | |
| archive_source_mapping_files, entry_classes, xcrunwrapper, | |
| file_open=open): | |
| """Remove unreachable objects from archive file. | |
| Args: | |
| input_archive: The source archive file to prune. | |
| output_archive: The location of the pruned archive file. | |
| dummy_archive: A dummy archive file that contains no object. | |
| dependency_mapping_files: A comma separated list of J2ObjC-generated | |
| dependency mapping files. | |
| header_mapping_files: A comma separated list of J2ObjC-generated | |
| header mapping files. | |
| archive_source_mapping_files: A comma separated list of J2ObjC-generated | |
| mapping between archive files and their associated source files. | |
| entry_classes: A comma separated list of Java entry classes. | |
| xcrunwrapper: A wrapper script over xcrun. | |
| file_open: Reference to the builtin open function so it may be | |
| overridden for testing. | |
| """ | |
| reachability_file_mapping = BuildReachabilityTree( | |
| dependency_mapping_files, file_open) | |
| header_map = BuildHeaderMapping(header_mapping_files, file_open) | |
| archive_source_file_mapping = BuildArchiveSourceFileMapping( | |
| archive_source_mapping_files, file_open) | |
| reachable_files_set = BuildReachableFileSet(entry_classes, | |
| reachability_file_mapping, | |
| header_map, | |
| archive_source_file_mapping) | |
| # Copy the current processes' environment, as xcrunwrapper depends on these | |
| # variables. | |
| cmd_env = dict(os.environ) | |
| j2objc_cmd = '' | |
| if input_archive in archive_source_file_mapping: | |
| source_files = archive_source_file_mapping[input_archive] | |
| unreachable_object_names = [] | |
| for source_file in source_files: | |
| if os.path.splitext(source_file)[0] not in reachable_files_set: | |
| unreachable_object_names.append( | |
| os.path.basename(os.path.splitext(source_file)[0])) | |
| # There are unreachable objects in the archive to prune | |
| if unreachable_object_names: | |
| # If all objects in the archive are unreachable, just copy over a dummy | |
| # archive that contains no object | |
| if len(unreachable_object_names) == len(source_files): | |
| j2objc_cmd = 'cp %s %s' % (pipes.quote(dummy_archive), | |
| pipes.quote(output_archive)) | |
| # Else we need to prune the archive of unreachable objects | |
| else: | |
| cmd_env['ZERO_AR_DATE'] = '1' | |
| # Copy the input archive to the output location | |
| j2objc_cmd += 'cp %s %s && ' % (pipes.quote(input_archive), | |
| pipes.quote(output_archive)) | |
| # Make the output archive editable | |
| j2objc_cmd += 'chmod +w %s && ' % (pipes.quote(output_archive)) | |
| # Remove the unreachable objects from the archive | |
| unreachable_object_names = MatchObjectNamesInArchive( | |
| xcrunwrapper, input_archive, unreachable_object_names) | |
| j2objc_cmd += '%s ar -d -s %s %s && ' % ( | |
| pipes.quote(xcrunwrapper), | |
| pipes.quote(output_archive), | |
| ' '.join(pipes.quote(uon) for uon in unreachable_object_names)) | |
| # Update the table of content of the archive file | |
| j2objc_cmd += '%s ranlib %s' % (pipes.quote(xcrunwrapper), | |
| pipes.quote(output_archive)) | |
| # There are no unreachable objects, we just copy over the original archive | |
| else: | |
| j2objc_cmd = 'cp %s %s' % (pipes.quote(input_archive), | |
| pipes.quote(output_archive)) | |
| # The archive cannot be pruned by J2ObjC dead code removal, just copy over | |
| # the original archive | |
| else: | |
| j2objc_cmd = 'cp %s %s' % (pipes.quote(input_archive), | |
| pipes.quote(output_archive)) | |
| try: | |
| subprocess.check_output( | |
| j2objc_cmd, stderr=subprocess.STDOUT, shell=True, env=cmd_env) | |
| except OSError as e: | |
| raise Exception( | |
| 'executing command failed: %s (%s)' % (j2objc_cmd, e.strerror)) | |
| # "Touch" the output file. | |
| # Prevents a pre-Xcode-8 bug in which passing zero-date archive files to ld | |
| # would cause ld to error. | |
| os.utime(output_archive, None) | |
| if __name__ == '__main__': | |
| parser = argparse.ArgumentParser(fromfile_prefix_chars='@') | |
| # TODO(rduan): Remove these three flags once J2ObjC compile actions are fully | |
| # moved to the edges. | |
| parser.add_argument( | |
| '--input_files', | |
| help=('The comma-separated file paths of translated source files to ' | |
| 'prune.')) | |
| parser.add_argument( | |
| '--output_files', | |
| help='The comma-separated file paths of pruned source files to write to.') | |
| parser.add_argument( | |
| '--objc_file_path', | |
| help='The file path which represents a directory where the generated ObjC' | |
| ' files reside') | |
| parser.add_argument( | |
| '--input_archive', | |
| help=('The path of the translated archive to prune.')) | |
| parser.add_argument( | |
| '--output_archive', | |
| help='The path of the pruned archive file to write to.') | |
| parser.add_argument( | |
| '--dummy_archive', | |
| help='The dummy archive file that contains no symbol.') | |
| parser.add_argument( | |
| '--dependency_mapping_files', | |
| help='The comma-separated file paths of dependency mapping files.') | |
| parser.add_argument( | |
| '--header_mapping_files', | |
| help='The comma-separated file paths of header mapping files.') | |
| parser.add_argument( | |
| '--archive_source_mapping_files', | |
| help='The comma-separated file paths of archive to source mapping files.' | |
| 'These mapping files should contain mappings between the ' | |
| 'translated source files and the archive file compiled from those ' | |
| 'source files.') | |
| parser.add_argument( | |
| '--entry_classes', | |
| help=('The comma-separated list of Java entry classes to be used as entry' | |
| ' point of the dead code anlysis.')) | |
| parser.add_argument( | |
| '--xcrunwrapper', | |
| help=('The xcrun wrapper script.')) | |
| args = parser.parse_args() | |
| if not args.entry_classes: | |
| raise Exception('J2objC dead code removal is on but no entry class is ', | |
| 'specified in any j2objc_library targets in the transitive', | |
| ' closure') | |
| if args.input_archive and args.output_archive: | |
| PruneArchiveFile( | |
| args.input_archive, | |
| args.output_archive, | |
| args.dummy_archive, | |
| args.dependency_mapping_files, | |
| args.header_mapping_files, | |
| args.archive_source_mapping_files, | |
| args.entry_classes, | |
| args.xcrunwrapper) | |
| else: | |
| # TODO(rduan): Remove once J2ObjC compile actions are fully moved to the | |
| # edges. | |
| PruneSourceFiles( | |
| args.input_files, | |
| args.output_files, | |
| args.dependency_mapping_files, | |
| args.header_mapping_files, | |
| args.entry_classes, | |
| args.objc_file_path) |