Internal only refactoring PiperOrigin-RevId: 221698301
diff --git a/examples/googmodule/a.ts b/examples/googmodule/a.ts index d8c9a7d..4f633f6 100644 --- a/examples/googmodule/a.ts +++ b/examples/googmodule/a.ts
@@ -1 +1 @@ -export const a: number = 1; +const a: number = 1;
diff --git a/internal/BUILD.bazel b/internal/BUILD.bazel index 908addd..3762b8c 100644 --- a/internal/BUILD.bazel +++ b/internal/BUILD.bazel
@@ -107,7 +107,10 @@ srcs = [], deps = [ ":test_lib", + "@npm//bytebuffer", "@npm//jasmine", + "@npm//protobufjs", + "@npm//source-map", "@npm//typescript", ], )
diff --git a/internal/tsc_wrapped/tsc_wrapped.ts b/internal/tsc_wrapped/tsc_wrapped.ts index 768127e..259f7d7 100644 --- a/internal/tsc_wrapped/tsc_wrapped.ts +++ b/internal/tsc_wrapped/tsc_wrapped.ts
@@ -1,3 +1,4 @@ +import * as fs from 'fs'; import * as path from 'path'; import * as tsickle from 'tsickle'; import * as ts from 'typescript'; @@ -7,6 +8,7 @@ import {CachedFileLoader, FileLoader, ProgramAndFileCache, UncachedFileLoader} from './cache'; import {CompilerHost} from './compiler_host'; import * as bazelDiagnostics from './diagnostics'; +import {constructManifest} from './manifest'; import * as perfTrace from './perf_trace'; import {PLUGIN as strictDepsPlugin} from './strict_deps'; import {BazelOptions, parseTsconfig, resolveNormalizedPath} from './tsconfig'; @@ -200,43 +202,21 @@ cache.putProgram(bazelOpts.target, program); - const compilationTargets = - program.getSourceFiles().filter(f => isCompilationTarget(bazelOpts, f)); - const emitResults: ts.EmitResult[] = []; - const diagnostics: ts.Diagnostic[] = []; - if (bazelOpts.tsickle) { - // The 'tsickle' import above is only used in type positions, so it won't - // result in a runtime dependency on tsickle. - // If the user requests the tsickle emit, then we dynamically require it - // here for use at runtime. - let optTsickle: typeof tsickle; - try { - // tslint:disable-next-line:no-require-imports dependency on tsickle only - // if requested - optTsickle = require('tsickle'); - } catch { - throw new Error( - 'When setting bazelOpts { tsickle: true }, ' + - 'you must also add a devDependency on the tsickle npm package'); - } - for (const sf of compilationTargets) { - emitResults.push(optTsickle.emitWithTsickle( - program, compilerHost, compilerHost, options, sf)); - } - diagnostics.push( - ...optTsickle.mergeEmitResults(emitResults as tsickle.EmitResult[]) - .diagnostics); - } else { - for (const sf of compilationTargets) { - emitResults.push(program.emit(sf)); - } + const compilationTargets = program.getSourceFiles().filter( + fileName => isCompilationTarget(bazelOpts, fileName)); - for (const d of emitResults) { - diagnostics.push(...d.diagnostics); - } + let diagnostics: ts.Diagnostic[] = []; + let useTsickleEmit = bazelOpts.tsickle; + if (useTsickleEmit) { + diagnostics = emitWithTsickle( + program, compilerHost, compilationTargets, options, bazelOpts); + } else { + diagnostics = emitWithTypescript(program, compilationTargets); } + if (diagnostics.length > 0) { console.error(bazelDiagnostics.format(bazelOpts.target, diagnostics)); + debug('compilation failed at', new Error().stack!); return false; } @@ -244,6 +224,113 @@ return true; } +function emitWithTypescript( + program: ts.Program, compilationTargets: ts.SourceFile[]): ts.Diagnostic[] { + const diagnostics: ts.Diagnostic[] = []; + for (const sf of compilationTargets) { + const result = program.emit(sf); + diagnostics.push(...result.diagnostics); + } + return diagnostics; +} + +function emitWithTsickle( + program: ts.Program, compilerHost: CompilerHost, + compilationTargets: ts.SourceFile[], options: ts.CompilerOptions, + bazelOpts: BazelOptions): ts.Diagnostic[] { + const emitResults: tsickle.EmitResult[] = []; + const diagnostics: ts.Diagnostic[] = []; + // The 'tsickle' import above is only used in type positions, so it won't + // result in a runtime dependency on tsickle. + // If the user requests the tsickle emit, then we dynamically require it + // here for use at runtime. + let optTsickle: typeof tsickle; + try { + // tslint:disable-next-line:no-require-imports + optTsickle = require('tsickle'); + } catch (e) { + if (e.code !== 'MODULE_NOT_FOUND') { + throw e; + } + throw new Error( + 'When setting bazelOpts { tsickle: true }, ' + + 'you must also add a devDependency on the tsickle npm package'); + } + perfTrace.wrap('emit', () => { + for (const sf of compilationTargets) { + perfTrace.wrap(`emit ${sf.fileName}`, () => { + emitResults.push(optTsickle.emitWithTsickle( + program, compilerHost, compilerHost, options, sf)); + }); + } + }); + const emitResult = optTsickle.mergeEmitResults(emitResults); + diagnostics.push(...emitResult.diagnostics); + + // If tsickle reported diagnostics, don't produce externs or manifest outputs. + if (diagnostics.length > 0) { + return diagnostics; + } + + let externs = '/** @externs */\n' + + '// generating externs was disabled using generate_externs=False\n'; + if (bazelOpts.tsickleGenerateExterns) { + externs = + optTsickle.getGeneratedExterns(emitResult.externs, options.rootDir!); + } + + if (bazelOpts.tsickleExternsPath) { + // Note: when tsickleExternsPath is provided, we always write a file as a + // marker that compilation succeeded, even if it's empty (just containing an + // @externs). + fs.writeFileSync(bazelOpts.tsickleExternsPath, externs); + + // When generating externs, generate an externs file for each of the input + // .d.ts files. + if (bazelOpts.tsickleGenerateExterns && + compilerHost.provideExternalModuleDtsNamespace) { + for (const extern of compilationTargets) { + if (!extern.isDeclarationFile) continue; + const outputBaseDir = options.outDir!; + const relativeOutputPath = + compilerHost.relativeOutputPath(extern.fileName); + mkdirp(outputBaseDir, path.dirname(relativeOutputPath)); + const outputPath = path.join(outputBaseDir, relativeOutputPath); + const moduleName = compilerHost.pathToModuleName('', extern.fileName); + fs.writeFileSync( + outputPath, + `goog.module('${moduleName}');\n` + + `// Export an empty object of unknown type to allow imports.\n` + + `// TODO: use typeof once available\n` + + `exports = /** @type {?} */ ({});\n`); + } + } + } + + if (bazelOpts.manifest) { + perfTrace.wrap('manifest', () => { + const manifest = + constructManifest(emitResult.modulesManifest, compilerHost); + fs.writeFileSync(bazelOpts.manifest, manifest); + }); + } + + return diagnostics; +} + +/** + * Creates directories subdir (a slash separated relative path) starting from + * base. + */ +function mkdirp(base: string, subdir: string) { + const steps = subdir.split(path.sep); + let current = base; + for (let i = 0; i < steps.length; i++) { + current = path.join(current, steps[i]); + if (!fs.existsSync(current)) fs.mkdirSync(current); + } +} + if (require.main === module) { // Do not call process.exit(), as that terminates the binary before
diff --git a/internal/tsc_wrapped/tsc_wrapped_test.ts b/internal/tsc_wrapped/tsc_wrapped_test.ts new file mode 100644 index 0000000..cc5af0a --- /dev/null +++ b/internal/tsc_wrapped/tsc_wrapped_test.ts
@@ -0,0 +1,287 @@ +import * as path from 'path'; +import {ModulesManifest} from 'tsickle'; +import * as ts from 'typescript'; + +import {UncachedFileLoader} from './cache'; +import {CompilerHost} from './compiler_host'; +import {constructManifest} from './manifest'; +import {writeTempFile} from './test_support'; +import {BazelOptions} from './tsconfig'; + +// tslint:disable-next-line:no-any mock for testing. +const throwingCompilerHostFake: ts.CompilerHost = null as any; + +const testFileLoader = new UncachedFileLoader(); + +const relativeOutputPath = (f: string) => f; + +type ModuleResolver = + (moduleName: string, containingFile: string, + compilerOptions: ts.CompilerOptions, host: ts.ModuleResolutionHost) => + ts.ResolvedModuleWithFailedLookupLocations; + +describe('tsc wrapped', () => { + function f( + res: ModulesManifest, fname: string, module: string, deps: string[]) { + res.addModule(fname, module); + for (let i = 0; i < deps.length; i++) { + res.addReferencedModule(fname, deps[i]); + } + return res; + } + + it('produces a topo-sorted manifest', () => { + const res = new ModulesManifest(); + f(res, 'src/f3.js', 'test$f3', ['test$f2', 'test$f1']); + f(res, 'src/f2.js', 'test$f2', + ['external$ts_source_not_included', 'test$f1']); + f(res, 'src/f1.js', 'test$f1', []); + expect(constructManifest(res, {relativeOutputPath})).toBe([ + 'src/f1.js\n', 'src/f2.js\n', 'src/f3.js\n' + ].join('')); + }); + + it('reports cyclical dependencies', () => { + const res = new ModulesManifest(); + f(res, 'src/f2', 'src$f2', ['src$f3']); + f(res, 'src/f3', 'src$f3', ['src$f1']); + f(res, 'src/f1', 'src$f1', ['src$f2']); + expect(() => constructManifest(res, {relativeOutputPath})) + .toThrowError(/src\/f2 -> src\/f3 -> src\/f1 -> src\/f2/g); + }); + + it('toposorts diamonds', () => { + // t + // l r + // b + const res = new ModulesManifest(); + f(res, 'bottom.js', 'bottom', ['left', 'right']); + f(res, 'right.js', 'right', ['top']); + f(res, 'left.js', 'left', ['top']); + f(res, 'top.js', 'top', []); + expect(constructManifest(res, {relativeOutputPath})).toBe([ + 'top.js\n', + 'left.js\n', + 'right.js\n', + 'bottom.js\n', + ].join('')); + }); + +}); + +// Create something that looks like CompilerOptions. +const COMPILER_OPTIONS: ts.CompilerOptions = { + rootDirs: [ + // Sorted by inverse length, as done by tsconfig.ts in production. + '/root/google3/blaze-out/k8-fastbuild/genfiles', + '/root/google3/blaze-out/k8-fastbuild/bin', + '/root/google3', + ], + outDir: '/root/google3/blaze-out/k8-fastbuild/bin', + rootDir: '/root/google3' +}; + + +const defaultBazelOpts = { + googmodule: true, + workspaceName: 'google3', + prelude: + `goog.require('google3.third_party.javascript.tslib.tslib_closure');`, +} as BazelOptions; + +describe('compiler host', () => { + const bazelOpts = { + ...defaultBazelOpts, + es5Mode: false, + } as BazelOptions; + + it('looks up files', () => { + const fn = writeTempFile('file_lookup', 'let x: number = 123;'); + const fn2 = writeTempFile('file_lookup2', 'let x: number = 124;'); + const host = new CompilerHost( + [fn /* but not fn2! */], COMPILER_OPTIONS, bazelOpts, + throwingCompilerHostFake, testFileLoader); + expect(host.fileExists(fn)).toBe(true); + expect(host.fileExists(fn2)).toBe(false); + }); + + describe('file writing', () => { + let writtenFiles: {[key: string]: string}; + beforeEach(() => writtenFiles = {}); + + const delegateHost = { + writeFile: (fn: string, d: string) => { + writtenFiles[fn.replace(/\\/g, '/')] = d; + } + // tslint:disable-next-line:no-any mock for testing. + } as any; + + function createFakeModuleResolver( + moduleRoots: {[moduleName: string]: string}): ModuleResolver { + return (moduleName: string, containingFile: string, + compilerOptions: ts.CompilerOptions, + host: ts.ModuleResolutionHost) => { + if (moduleName[0] === '.') { + moduleName = + path.posix.join(path.dirname(containingFile), moduleName); + } + for (const moduleRoot in moduleRoots) { + if (moduleName.indexOf(moduleRoot) === 0) { + const resolvedFileName = moduleRoots[moduleRoot] + + moduleName.substring(moduleRoot.length) + '.d.ts'; + return { + resolvedModule: {resolvedFileName, extension: ts.Extension.Dts}, + failedLookupLocations: [] + }; + } + } + + return { + resolvedModule: + {resolvedFileName: moduleName, extension: ts.Extension.Dts}, + failedLookupLocations: [] + }; + }; + } + + function createFakeGoogle3Host({ + es5 = false, + moduleRoots = {} as {[moduleName: string]: string}, + isJsTranspilation = false, + transpiledJsOutputFileName = undefined as string | undefined, + } = {}) { + const bazelOpts = { + ...defaultBazelOpts, + es5Mode: es5, + isJsTranspilation, + transpiledJsOutputFileName, + } as BazelOptions; + return new CompilerHost( + [], COMPILER_OPTIONS, bazelOpts, delegateHost, testFileLoader, + createFakeModuleResolver(moduleRoots)); + } + + describe('converts path to module names', () => { + let host: CompilerHost; + beforeEach(() => { + host = createFakeGoogle3Host({ + moduleRoots: { + 'module': 'path/to/module', + 'module2': 'path/to/module2', + 'path/to/module2': 'path/to/module2', + }, + }); + }); + + function expectPath(context: string, path: string) { + return expect(host.pathToModuleName(context, path)); + } + + it('mangles absolute paths', () => { + expectPath('whatever/context', 'some/absolute/module') + .toBe('google3.some.absolute.module'); + }); + + it('escapes special symbols', () => { + expectPath('', 'some|123').toBe('google3.some$7c123'); + expectPath('', '1some|').toBe('google3.1some$7c'); + expectPath('', 'bar/foo.bam.ts').toBe('google3.bar.foo$2ebam'); + // Underscore is unmodified, because it is common in google3 paths. + expectPath('', 'foo_bar').toBe('google3.foo_bar'); + }); + + it('resolves paths', () => { + const context = 'path/to/module'; + + expectPath(context, './module2').toBe('google3.path.to.module2'); + expectPath(context, '././module2').toBe('google3.path.to.module2'); + expectPath(context, '../to/module2').toBe('google3.path.to.module2'); + expectPath(context, '../to/.././to/module2') + .toBe('google3.path.to.module2'); + }); + + it('ignores extra google3 sections in paths', () => { + expectPath('', 'google3/foo/bar').toBe('google3.foo.bar'); + }); + + it('resolves absolute paths', () => { + const context = 'path/to/module/dir/file'; + + expectPath(context, '/root/google3/some/file.ts') + .toBe('google3.some.file'); + expectPath(context, '/root/google3/path/to/some/file') + .toBe('google3.path.to.some.file'); + expectPath( + context, '/root/google3/blaze-out/k8-fastbuild/bin/some/file') + .toBe('google3.some.file'); + expectPath( + context, '/root/google3/blaze-out/k8-fastbuild/genfiles/some/file') + .toBe('google3.some.file'); + }); + + describe('uses module name for resolved file paths', () => { + it('for goog.module module names', () => { + expectPath('', 'module/dir/file2') + .toBe('google3.path.to.module.dir.file2'); + expectPath('', 'module2/dir/file2') + .toBe('google3.path.to.module2.dir.file2'); + }); + + it('for imports of files from the same module', () => { + const context = 'path/to/module/dir/file'; + + expectPath(context, 'module/dir/file2') + .toBe('google3.path.to.module.dir.file2'); + expectPath(context, './foo/bar') + .toBe('google3.path.to.module.dir.foo.bar'); + expectPath(context, '../foo/bar') + .toBe('google3.path.to.module.foo.bar'); + expectPath(context, 'path/to/module/dir/file2') + .toBe('google3.path.to.module.dir.file2'); + }); + + it('for imports of files from a different module', () => { + const context = 'path/to/module/dir/file'; + + expectPath(context, 'module2/dir/file') + .toBe('google3.path.to.module2.dir.file'); + expectPath(context, '../../module2/dir/file') + .toBe('google3.path.to.module2.dir.file'); + expectPath(context, 'path/to/module2/dir/file') + .toBe('google3.path.to.module2.dir.file'); + }); + }); + }); + + describe('output files', () => { + it('writes to .closure.js in ES6 mode', () => { + createFakeGoogle3Host({ + es5: false, + }).writeFile('a.js', 'some.code();', false, undefined, []); + expect(Object.keys(writtenFiles)).toEqual([ + '/root/google3/blaze-out/k8-fastbuild/bin/a.closure.js' + ]); + }); + + it('writes to .js in ES5 mode', () => { + createFakeGoogle3Host({ + es5: true, + }).writeFile('a/b.js', 'some.code();', false, undefined, []); + expect(Object.keys(writtenFiles)).toEqual([ + '/root/google3/blaze-out/k8-fastbuild/bin/a/b.js' + ]); + }); + + it('writes to closureOptions.transpiledJsOutputFileName in JS transpilation mode', + () => { + createFakeGoogle3Host({ + isJsTranspilation: true, + transpiledJsOutputFileName: 'foo/bar/a/b.dev_es5.js', + }).writeFile('a/b.js', 'some.code();', false, undefined, []); + expect(Object.keys(writtenFiles)).toEqual([ + '/root/google3/blaze-out/k8-fastbuild/bin/foo/bar/a/b.dev_es5.js' + ]); + }); + }); + }); +});
diff --git a/package.json b/package.json index e4323aa..641b98f 100644 --- a/package.json +++ b/package.json
@@ -30,7 +30,7 @@ "http-server": "^0.11.1", "protractor": "^5.2.0", "shelljs": "^0.8.2", - "tsickle": "0.32.1", + "tsickle": "0.33.1", "typescript": "~2.9.1", "which": "~1.0.5" },
diff --git a/yarn.lock b/yarn.lock index b3206b4..8d09cb4 100644 --- a/yarn.lock +++ b/yarn.lock
@@ -1045,11 +1045,6 @@ resolved "https://registry.yarnpkg.com/di/-/di-0.0.1.tgz#806649326ceaa7caa3306d75d985ea2748ba913c" integrity sha1-gGZJMmzqp8qjMG112YXqJ0i6kTw= -diff@^3.2.0: - version "3.5.0" - resolved "https://registry.yarnpkg.com/diff/-/diff-3.5.0.tgz#800c0dd1e0a8bfbc95835c202ad220fe317e5a12" - integrity sha512-A46qtFgd+g7pDZinpnwiRJtxbC1hpgf0uzP3iG89scHk0AUC7A1TGxf5OiiOUv/JMZR8GOt8hL900hV0bOy5xA== - dom-serialize@^2.2.0: version "2.2.1" resolved "https://registry.yarnpkg.com/dom-serialize/-/dom-serialize-2.2.1.tgz#562ae8999f44be5ea3076f5419dcd59eb43ac95b" @@ -2153,13 +2148,6 @@ resolved "https://registry.yarnpkg.com/jasmine-core/-/jasmine-core-2.8.0.tgz#bcc979ae1f9fd05701e45e52e65d3a5d63f1a24e" integrity sha1-vMl5rh+f0FcB5F5S5l06XWPxok4= -jasmine-diff@^0.1.3: - version "0.1.3" - resolved "https://registry.yarnpkg.com/jasmine-diff/-/jasmine-diff-0.1.3.tgz#93ccc2dcc41028c5ddd4606558074839f2deeaa8" - integrity sha1-k8zC3MQQKMXd1GBlWAdIOfLe6qg= - dependencies: - diff "^3.2.0" - jasmine@^2.5.3: version "2.8.0" resolved "https://registry.yarnpkg.com/jasmine/-/jasmine-2.8.0.tgz#6b089c0a11576b1f16df11b80146d91d4e8b8a3e" @@ -3756,6 +3744,11 @@ resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.6.1.tgz#74722af32e9614e9c287a8d0bbde48b5e2f1a263" integrity sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g== +source-map@^0.7.3: + version "0.7.3" + resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.7.3.tgz#5302f8169031735226544092e64981f751750383" + integrity sha512-CkCj6giN3S+n9qrYiBTX5gystlENnRW5jZeNLHpe6aue+SrHcG5VYwujhW9s4dY31mEGsxBDrHR6oI69fTXsaQ== + spawn-command@^0.0.2-1: version "0.0.2" resolved "https://registry.yarnpkg.com/spawn-command/-/spawn-command-0.0.2.tgz#9544e1a43ca045f8531aac1a48cb29bdae62338e" @@ -4020,16 +4013,14 @@ source-map "^0.6.0" source-map-support "^0.5.0" -tsickle@0.32.1: - version "0.32.1" - resolved "https://registry.yarnpkg.com/tsickle/-/tsickle-0.32.1.tgz#f16e94ba80b32fc9ebe320dc94fbc2ca7f3521a5" - integrity sha512-JW9j+W0SaMSZGejIFZBk0AiPfnhljK3oLx5SaqxrJhjlvzFyPml5zqG1/PuScUj6yTe1muEqwk5CnDK0cOZmKw== +tsickle@0.33.1: + version "0.33.1" + resolved "https://registry.yarnpkg.com/tsickle/-/tsickle-0.33.1.tgz#eee4ebabeda3bcd8afc32cee34c822cbe3e839ec" + integrity sha512-SpW2G3PvDGs4a5sMXPlWnCWHWRviWjSlI3U0734e3fU3U39VAE0NPr8M3W1cuL/OU/YXheYipGeEwtIJ5k0NHQ== dependencies: - jasmine-diff "^0.1.3" minimist "^1.2.0" mkdirp "^0.5.1" - source-map "^0.6.0" - source-map-support "^0.5.0" + source-map "^0.7.3" tslib@^1.8.1: version "1.9.0"