Internal only refactoring
PiperOrigin-RevId: 221698301
diff --git a/examples/googmodule/a.ts b/examples/googmodule/a.ts
index d8c9a7d..4f633f6 100644
--- a/examples/googmodule/a.ts
+++ b/examples/googmodule/a.ts
@@ -1 +1 @@
-export const a: number = 1;
+const a: number = 1;
diff --git a/internal/BUILD.bazel b/internal/BUILD.bazel
index 908addd..3762b8c 100644
--- a/internal/BUILD.bazel
+++ b/internal/BUILD.bazel
@@ -107,7 +107,10 @@
srcs = [],
deps = [
":test_lib",
+ "@npm//bytebuffer",
"@npm//jasmine",
+ "@npm//protobufjs",
+ "@npm//source-map",
"@npm//typescript",
],
)
diff --git a/internal/tsc_wrapped/tsc_wrapped.ts b/internal/tsc_wrapped/tsc_wrapped.ts
index 768127e..259f7d7 100644
--- a/internal/tsc_wrapped/tsc_wrapped.ts
+++ b/internal/tsc_wrapped/tsc_wrapped.ts
@@ -1,3 +1,4 @@
+import * as fs from 'fs';
import * as path from 'path';
import * as tsickle from 'tsickle';
import * as ts from 'typescript';
@@ -7,6 +8,7 @@
import {CachedFileLoader, FileLoader, ProgramAndFileCache, UncachedFileLoader} from './cache';
import {CompilerHost} from './compiler_host';
import * as bazelDiagnostics from './diagnostics';
+import {constructManifest} from './manifest';
import * as perfTrace from './perf_trace';
import {PLUGIN as strictDepsPlugin} from './strict_deps';
import {BazelOptions, parseTsconfig, resolveNormalizedPath} from './tsconfig';
@@ -200,43 +202,21 @@
cache.putProgram(bazelOpts.target, program);
- const compilationTargets =
- program.getSourceFiles().filter(f => isCompilationTarget(bazelOpts, f));
- const emitResults: ts.EmitResult[] = [];
- const diagnostics: ts.Diagnostic[] = [];
- if (bazelOpts.tsickle) {
- // The 'tsickle' import above is only used in type positions, so it won't
- // result in a runtime dependency on tsickle.
- // If the user requests the tsickle emit, then we dynamically require it
- // here for use at runtime.
- let optTsickle: typeof tsickle;
- try {
- // tslint:disable-next-line:no-require-imports dependency on tsickle only
- // if requested
- optTsickle = require('tsickle');
- } catch {
- throw new Error(
- 'When setting bazelOpts { tsickle: true }, ' +
- 'you must also add a devDependency on the tsickle npm package');
- }
- for (const sf of compilationTargets) {
- emitResults.push(optTsickle.emitWithTsickle(
- program, compilerHost, compilerHost, options, sf));
- }
- diagnostics.push(
- ...optTsickle.mergeEmitResults(emitResults as tsickle.EmitResult[])
- .diagnostics);
- } else {
- for (const sf of compilationTargets) {
- emitResults.push(program.emit(sf));
- }
+ const compilationTargets = program.getSourceFiles().filter(
+ fileName => isCompilationTarget(bazelOpts, fileName));
- for (const d of emitResults) {
- diagnostics.push(...d.diagnostics);
- }
+ let diagnostics: ts.Diagnostic[] = [];
+ let useTsickleEmit = bazelOpts.tsickle;
+ if (useTsickleEmit) {
+ diagnostics = emitWithTsickle(
+ program, compilerHost, compilationTargets, options, bazelOpts);
+ } else {
+ diagnostics = emitWithTypescript(program, compilationTargets);
}
+
if (diagnostics.length > 0) {
console.error(bazelDiagnostics.format(bazelOpts.target, diagnostics));
+ debug('compilation failed at', new Error().stack!);
return false;
}
@@ -244,6 +224,113 @@
return true;
}
+function emitWithTypescript(
+ program: ts.Program, compilationTargets: ts.SourceFile[]): ts.Diagnostic[] {
+ const diagnostics: ts.Diagnostic[] = [];
+ for (const sf of compilationTargets) {
+ const result = program.emit(sf);
+ diagnostics.push(...result.diagnostics);
+ }
+ return diagnostics;
+}
+
+function emitWithTsickle(
+ program: ts.Program, compilerHost: CompilerHost,
+ compilationTargets: ts.SourceFile[], options: ts.CompilerOptions,
+ bazelOpts: BazelOptions): ts.Diagnostic[] {
+ const emitResults: tsickle.EmitResult[] = [];
+ const diagnostics: ts.Diagnostic[] = [];
+ // The 'tsickle' import above is only used in type positions, so it won't
+ // result in a runtime dependency on tsickle.
+ // If the user requests the tsickle emit, then we dynamically require it
+ // here for use at runtime.
+ let optTsickle: typeof tsickle;
+ try {
+ // tslint:disable-next-line:no-require-imports
+ optTsickle = require('tsickle');
+ } catch (e) {
+ if (e.code !== 'MODULE_NOT_FOUND') {
+ throw e;
+ }
+ throw new Error(
+ 'When setting bazelOpts { tsickle: true }, ' +
+ 'you must also add a devDependency on the tsickle npm package');
+ }
+ perfTrace.wrap('emit', () => {
+ for (const sf of compilationTargets) {
+ perfTrace.wrap(`emit ${sf.fileName}`, () => {
+ emitResults.push(optTsickle.emitWithTsickle(
+ program, compilerHost, compilerHost, options, sf));
+ });
+ }
+ });
+ const emitResult = optTsickle.mergeEmitResults(emitResults);
+ diagnostics.push(...emitResult.diagnostics);
+
+ // If tsickle reported diagnostics, don't produce externs or manifest outputs.
+ if (diagnostics.length > 0) {
+ return diagnostics;
+ }
+
+ let externs = '/** @externs */\n' +
+ '// generating externs was disabled using generate_externs=False\n';
+ if (bazelOpts.tsickleGenerateExterns) {
+ externs =
+ optTsickle.getGeneratedExterns(emitResult.externs, options.rootDir!);
+ }
+
+ if (bazelOpts.tsickleExternsPath) {
+ // Note: when tsickleExternsPath is provided, we always write a file as a
+ // marker that compilation succeeded, even if it's empty (just containing an
+ // @externs).
+ fs.writeFileSync(bazelOpts.tsickleExternsPath, externs);
+
+ // When generating externs, generate an externs file for each of the input
+ // .d.ts files.
+ if (bazelOpts.tsickleGenerateExterns &&
+ compilerHost.provideExternalModuleDtsNamespace) {
+ for (const extern of compilationTargets) {
+ if (!extern.isDeclarationFile) continue;
+ const outputBaseDir = options.outDir!;
+ const relativeOutputPath =
+ compilerHost.relativeOutputPath(extern.fileName);
+ mkdirp(outputBaseDir, path.dirname(relativeOutputPath));
+ const outputPath = path.join(outputBaseDir, relativeOutputPath);
+ const moduleName = compilerHost.pathToModuleName('', extern.fileName);
+ fs.writeFileSync(
+ outputPath,
+ `goog.module('${moduleName}');\n` +
+ `// Export an empty object of unknown type to allow imports.\n` +
+ `// TODO: use typeof once available\n` +
+ `exports = /** @type {?} */ ({});\n`);
+ }
+ }
+ }
+
+ if (bazelOpts.manifest) {
+ perfTrace.wrap('manifest', () => {
+ const manifest =
+ constructManifest(emitResult.modulesManifest, compilerHost);
+ fs.writeFileSync(bazelOpts.manifest, manifest);
+ });
+ }
+
+ return diagnostics;
+}
+
+/**
+ * Creates directories subdir (a slash separated relative path) starting from
+ * base.
+ */
+function mkdirp(base: string, subdir: string) {
+ const steps = subdir.split(path.sep);
+ let current = base;
+ for (let i = 0; i < steps.length; i++) {
+ current = path.join(current, steps[i]);
+ if (!fs.existsSync(current)) fs.mkdirSync(current);
+ }
+}
+
if (require.main === module) {
// Do not call process.exit(), as that terminates the binary before
diff --git a/internal/tsc_wrapped/tsc_wrapped_test.ts b/internal/tsc_wrapped/tsc_wrapped_test.ts
new file mode 100644
index 0000000..cc5af0a
--- /dev/null
+++ b/internal/tsc_wrapped/tsc_wrapped_test.ts
@@ -0,0 +1,287 @@
+import * as path from 'path';
+import {ModulesManifest} from 'tsickle';
+import * as ts from 'typescript';
+
+import {UncachedFileLoader} from './cache';
+import {CompilerHost} from './compiler_host';
+import {constructManifest} from './manifest';
+import {writeTempFile} from './test_support';
+import {BazelOptions} from './tsconfig';
+
+// tslint:disable-next-line:no-any mock for testing.
+const throwingCompilerHostFake: ts.CompilerHost = null as any;
+
+const testFileLoader = new UncachedFileLoader();
+
+const relativeOutputPath = (f: string) => f;
+
+type ModuleResolver =
+ (moduleName: string, containingFile: string,
+ compilerOptions: ts.CompilerOptions, host: ts.ModuleResolutionHost) =>
+ ts.ResolvedModuleWithFailedLookupLocations;
+
+describe('tsc wrapped', () => {
+ function f(
+ res: ModulesManifest, fname: string, module: string, deps: string[]) {
+ res.addModule(fname, module);
+ for (let i = 0; i < deps.length; i++) {
+ res.addReferencedModule(fname, deps[i]);
+ }
+ return res;
+ }
+
+ it('produces a topo-sorted manifest', () => {
+ const res = new ModulesManifest();
+ f(res, 'src/f3.js', 'test$f3', ['test$f2', 'test$f1']);
+ f(res, 'src/f2.js', 'test$f2',
+ ['external$ts_source_not_included', 'test$f1']);
+ f(res, 'src/f1.js', 'test$f1', []);
+ expect(constructManifest(res, {relativeOutputPath})).toBe([
+ 'src/f1.js\n', 'src/f2.js\n', 'src/f3.js\n'
+ ].join(''));
+ });
+
+ it('reports cyclical dependencies', () => {
+ const res = new ModulesManifest();
+ f(res, 'src/f2', 'src$f2', ['src$f3']);
+ f(res, 'src/f3', 'src$f3', ['src$f1']);
+ f(res, 'src/f1', 'src$f1', ['src$f2']);
+ expect(() => constructManifest(res, {relativeOutputPath}))
+ .toThrowError(/src\/f2 -> src\/f3 -> src\/f1 -> src\/f2/g);
+ });
+
+ it('toposorts diamonds', () => {
+ // t
+ // l r
+ // b
+ const res = new ModulesManifest();
+ f(res, 'bottom.js', 'bottom', ['left', 'right']);
+ f(res, 'right.js', 'right', ['top']);
+ f(res, 'left.js', 'left', ['top']);
+ f(res, 'top.js', 'top', []);
+ expect(constructManifest(res, {relativeOutputPath})).toBe([
+ 'top.js\n',
+ 'left.js\n',
+ 'right.js\n',
+ 'bottom.js\n',
+ ].join(''));
+ });
+
+});
+
+// Create something that looks like CompilerOptions.
+const COMPILER_OPTIONS: ts.CompilerOptions = {
+ rootDirs: [
+ // Sorted by inverse length, as done by tsconfig.ts in production.
+ '/root/google3/blaze-out/k8-fastbuild/genfiles',
+ '/root/google3/blaze-out/k8-fastbuild/bin',
+ '/root/google3',
+ ],
+ outDir: '/root/google3/blaze-out/k8-fastbuild/bin',
+ rootDir: '/root/google3'
+};
+
+
+const defaultBazelOpts = {
+ googmodule: true,
+ workspaceName: 'google3',
+ prelude:
+ `goog.require('google3.third_party.javascript.tslib.tslib_closure');`,
+} as BazelOptions;
+
+describe('compiler host', () => {
+ const bazelOpts = {
+ ...defaultBazelOpts,
+ es5Mode: false,
+ } as BazelOptions;
+
+ it('looks up files', () => {
+ const fn = writeTempFile('file_lookup', 'let x: number = 123;');
+ const fn2 = writeTempFile('file_lookup2', 'let x: number = 124;');
+ const host = new CompilerHost(
+ [fn /* but not fn2! */], COMPILER_OPTIONS, bazelOpts,
+ throwingCompilerHostFake, testFileLoader);
+ expect(host.fileExists(fn)).toBe(true);
+ expect(host.fileExists(fn2)).toBe(false);
+ });
+
+ describe('file writing', () => {
+ let writtenFiles: {[key: string]: string};
+ beforeEach(() => writtenFiles = {});
+
+ const delegateHost = {
+ writeFile: (fn: string, d: string) => {
+ writtenFiles[fn.replace(/\\/g, '/')] = d;
+ }
+ // tslint:disable-next-line:no-any mock for testing.
+ } as any;
+
+ function createFakeModuleResolver(
+ moduleRoots: {[moduleName: string]: string}): ModuleResolver {
+ return (moduleName: string, containingFile: string,
+ compilerOptions: ts.CompilerOptions,
+ host: ts.ModuleResolutionHost) => {
+ if (moduleName[0] === '.') {
+ moduleName =
+ path.posix.join(path.dirname(containingFile), moduleName);
+ }
+ for (const moduleRoot in moduleRoots) {
+ if (moduleName.indexOf(moduleRoot) === 0) {
+ const resolvedFileName = moduleRoots[moduleRoot] +
+ moduleName.substring(moduleRoot.length) + '.d.ts';
+ return {
+ resolvedModule: {resolvedFileName, extension: ts.Extension.Dts},
+ failedLookupLocations: []
+ };
+ }
+ }
+
+ return {
+ resolvedModule:
+ {resolvedFileName: moduleName, extension: ts.Extension.Dts},
+ failedLookupLocations: []
+ };
+ };
+ }
+
+ function createFakeGoogle3Host({
+ es5 = false,
+ moduleRoots = {} as {[moduleName: string]: string},
+ isJsTranspilation = false,
+ transpiledJsOutputFileName = undefined as string | undefined,
+ } = {}) {
+ const bazelOpts = {
+ ...defaultBazelOpts,
+ es5Mode: es5,
+ isJsTranspilation,
+ transpiledJsOutputFileName,
+ } as BazelOptions;
+ return new CompilerHost(
+ [], COMPILER_OPTIONS, bazelOpts, delegateHost, testFileLoader,
+ createFakeModuleResolver(moduleRoots));
+ }
+
+ describe('converts path to module names', () => {
+ let host: CompilerHost;
+ beforeEach(() => {
+ host = createFakeGoogle3Host({
+ moduleRoots: {
+ 'module': 'path/to/module',
+ 'module2': 'path/to/module2',
+ 'path/to/module2': 'path/to/module2',
+ },
+ });
+ });
+
+ function expectPath(context: string, path: string) {
+ return expect(host.pathToModuleName(context, path));
+ }
+
+ it('mangles absolute paths', () => {
+ expectPath('whatever/context', 'some/absolute/module')
+ .toBe('google3.some.absolute.module');
+ });
+
+ it('escapes special symbols', () => {
+ expectPath('', 'some|123').toBe('google3.some$7c123');
+ expectPath('', '1some|').toBe('google3.1some$7c');
+ expectPath('', 'bar/foo.bam.ts').toBe('google3.bar.foo$2ebam');
+ // Underscore is unmodified, because it is common in google3 paths.
+ expectPath('', 'foo_bar').toBe('google3.foo_bar');
+ });
+
+ it('resolves paths', () => {
+ const context = 'path/to/module';
+
+ expectPath(context, './module2').toBe('google3.path.to.module2');
+ expectPath(context, '././module2').toBe('google3.path.to.module2');
+ expectPath(context, '../to/module2').toBe('google3.path.to.module2');
+ expectPath(context, '../to/.././to/module2')
+ .toBe('google3.path.to.module2');
+ });
+
+ it('ignores extra google3 sections in paths', () => {
+ expectPath('', 'google3/foo/bar').toBe('google3.foo.bar');
+ });
+
+ it('resolves absolute paths', () => {
+ const context = 'path/to/module/dir/file';
+
+ expectPath(context, '/root/google3/some/file.ts')
+ .toBe('google3.some.file');
+ expectPath(context, '/root/google3/path/to/some/file')
+ .toBe('google3.path.to.some.file');
+ expectPath(
+ context, '/root/google3/blaze-out/k8-fastbuild/bin/some/file')
+ .toBe('google3.some.file');
+ expectPath(
+ context, '/root/google3/blaze-out/k8-fastbuild/genfiles/some/file')
+ .toBe('google3.some.file');
+ });
+
+ describe('uses module name for resolved file paths', () => {
+ it('for goog.module module names', () => {
+ expectPath('', 'module/dir/file2')
+ .toBe('google3.path.to.module.dir.file2');
+ expectPath('', 'module2/dir/file2')
+ .toBe('google3.path.to.module2.dir.file2');
+ });
+
+ it('for imports of files from the same module', () => {
+ const context = 'path/to/module/dir/file';
+
+ expectPath(context, 'module/dir/file2')
+ .toBe('google3.path.to.module.dir.file2');
+ expectPath(context, './foo/bar')
+ .toBe('google3.path.to.module.dir.foo.bar');
+ expectPath(context, '../foo/bar')
+ .toBe('google3.path.to.module.foo.bar');
+ expectPath(context, 'path/to/module/dir/file2')
+ .toBe('google3.path.to.module.dir.file2');
+ });
+
+ it('for imports of files from a different module', () => {
+ const context = 'path/to/module/dir/file';
+
+ expectPath(context, 'module2/dir/file')
+ .toBe('google3.path.to.module2.dir.file');
+ expectPath(context, '../../module2/dir/file')
+ .toBe('google3.path.to.module2.dir.file');
+ expectPath(context, 'path/to/module2/dir/file')
+ .toBe('google3.path.to.module2.dir.file');
+ });
+ });
+ });
+
+ describe('output files', () => {
+ it('writes to .closure.js in ES6 mode', () => {
+ createFakeGoogle3Host({
+ es5: false,
+ }).writeFile('a.js', 'some.code();', false, undefined, []);
+ expect(Object.keys(writtenFiles)).toEqual([
+ '/root/google3/blaze-out/k8-fastbuild/bin/a.closure.js'
+ ]);
+ });
+
+ it('writes to .js in ES5 mode', () => {
+ createFakeGoogle3Host({
+ es5: true,
+ }).writeFile('a/b.js', 'some.code();', false, undefined, []);
+ expect(Object.keys(writtenFiles)).toEqual([
+ '/root/google3/blaze-out/k8-fastbuild/bin/a/b.js'
+ ]);
+ });
+
+ it('writes to closureOptions.transpiledJsOutputFileName in JS transpilation mode',
+ () => {
+ createFakeGoogle3Host({
+ isJsTranspilation: true,
+ transpiledJsOutputFileName: 'foo/bar/a/b.dev_es5.js',
+ }).writeFile('a/b.js', 'some.code();', false, undefined, []);
+ expect(Object.keys(writtenFiles)).toEqual([
+ '/root/google3/blaze-out/k8-fastbuild/bin/foo/bar/a/b.dev_es5.js'
+ ]);
+ });
+ });
+ });
+});
diff --git a/package.json b/package.json
index e4323aa..641b98f 100644
--- a/package.json
+++ b/package.json
@@ -30,7 +30,7 @@
"http-server": "^0.11.1",
"protractor": "^5.2.0",
"shelljs": "^0.8.2",
- "tsickle": "0.32.1",
+ "tsickle": "0.33.1",
"typescript": "~2.9.1",
"which": "~1.0.5"
},
diff --git a/yarn.lock b/yarn.lock
index b3206b4..8d09cb4 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -1045,11 +1045,6 @@
resolved "https://registry.yarnpkg.com/di/-/di-0.0.1.tgz#806649326ceaa7caa3306d75d985ea2748ba913c"
integrity sha1-gGZJMmzqp8qjMG112YXqJ0i6kTw=
-diff@^3.2.0:
- version "3.5.0"
- resolved "https://registry.yarnpkg.com/diff/-/diff-3.5.0.tgz#800c0dd1e0a8bfbc95835c202ad220fe317e5a12"
- integrity sha512-A46qtFgd+g7pDZinpnwiRJtxbC1hpgf0uzP3iG89scHk0AUC7A1TGxf5OiiOUv/JMZR8GOt8hL900hV0bOy5xA==
-
dom-serialize@^2.2.0:
version "2.2.1"
resolved "https://registry.yarnpkg.com/dom-serialize/-/dom-serialize-2.2.1.tgz#562ae8999f44be5ea3076f5419dcd59eb43ac95b"
@@ -2153,13 +2148,6 @@
resolved "https://registry.yarnpkg.com/jasmine-core/-/jasmine-core-2.8.0.tgz#bcc979ae1f9fd05701e45e52e65d3a5d63f1a24e"
integrity sha1-vMl5rh+f0FcB5F5S5l06XWPxok4=
-jasmine-diff@^0.1.3:
- version "0.1.3"
- resolved "https://registry.yarnpkg.com/jasmine-diff/-/jasmine-diff-0.1.3.tgz#93ccc2dcc41028c5ddd4606558074839f2deeaa8"
- integrity sha1-k8zC3MQQKMXd1GBlWAdIOfLe6qg=
- dependencies:
- diff "^3.2.0"
-
jasmine@^2.5.3:
version "2.8.0"
resolved "https://registry.yarnpkg.com/jasmine/-/jasmine-2.8.0.tgz#6b089c0a11576b1f16df11b80146d91d4e8b8a3e"
@@ -3756,6 +3744,11 @@
resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.6.1.tgz#74722af32e9614e9c287a8d0bbde48b5e2f1a263"
integrity sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==
+source-map@^0.7.3:
+ version "0.7.3"
+ resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.7.3.tgz#5302f8169031735226544092e64981f751750383"
+ integrity sha512-CkCj6giN3S+n9qrYiBTX5gystlENnRW5jZeNLHpe6aue+SrHcG5VYwujhW9s4dY31mEGsxBDrHR6oI69fTXsaQ==
+
spawn-command@^0.0.2-1:
version "0.0.2"
resolved "https://registry.yarnpkg.com/spawn-command/-/spawn-command-0.0.2.tgz#9544e1a43ca045f8531aac1a48cb29bdae62338e"
@@ -4020,16 +4013,14 @@
source-map "^0.6.0"
source-map-support "^0.5.0"
-tsickle@0.32.1:
- version "0.32.1"
- resolved "https://registry.yarnpkg.com/tsickle/-/tsickle-0.32.1.tgz#f16e94ba80b32fc9ebe320dc94fbc2ca7f3521a5"
- integrity sha512-JW9j+W0SaMSZGejIFZBk0AiPfnhljK3oLx5SaqxrJhjlvzFyPml5zqG1/PuScUj6yTe1muEqwk5CnDK0cOZmKw==
+tsickle@0.33.1:
+ version "0.33.1"
+ resolved "https://registry.yarnpkg.com/tsickle/-/tsickle-0.33.1.tgz#eee4ebabeda3bcd8afc32cee34c822cbe3e839ec"
+ integrity sha512-SpW2G3PvDGs4a5sMXPlWnCWHWRviWjSlI3U0734e3fU3U39VAE0NPr8M3W1cuL/OU/YXheYipGeEwtIJ5k0NHQ==
dependencies:
- jasmine-diff "^0.1.3"
minimist "^1.2.0"
mkdirp "^0.5.1"
- source-map "^0.6.0"
- source-map-support "^0.5.0"
+ source-map "^0.7.3"
tslib@^1.8.1:
version "1.9.0"
