Move rules_typescript to protobufjs 6.8. protobufjs 6 is a full, API incompatible rewrite. This change updates any proto use, and also rewrite the partial message reading logic. PiperOrigin-RevId: 240776103
diff --git a/internal/BUILD.bazel b/internal/BUILD.bazel index 8b42aa8..375a5ce 100644 --- a/internal/BUILD.bazel +++ b/internal/BUILD.bazel
@@ -67,6 +67,7 @@ # Workaround for https://github.com/Microsoft/TypeScript/issues/22208 deps = [ "@npm//@types/node", + "@npm//protobufjs", "@npm//tsickle", "@npm//tsutils", "@npm//typescript",
diff --git a/internal/tsc_wrapped/worker.ts b/internal/tsc_wrapped/worker.ts index 7e52ba8..2998318 100644 --- a/internal/tsc_wrapped/worker.ts +++ b/internal/tsc_wrapped/worker.ts
@@ -1,20 +1,24 @@ import * as path from 'path'; -/* tslint:disable:no-require-imports */ -const protobufjs = require('protobufjs'); -// tslint:disable-next-line:variable-name: ByteBuffer is instantiatable. -const ByteBuffer = require('bytebuffer'); +import * as protobufjs from 'protobufjs'; // Equivalent of running node with --expose-gc // but easier to write tooling since we don't need to inject that arg to // nodejs_binary if (typeof global.gc !== 'function') { + // tslint:disable-next-line:no-require-imports require('v8').setFlagsFromString('--expose_gc'); + // tslint:disable-next-line:no-require-imports global.gc = require('vm').runInNewContext('gc'); } +/** + * Whether to print debug messages (to console.error) from the debug function + * below. + */ export const DEBUG = false; -export function debug(...args: Array<{}>) { +/** Maybe print a debug message (depending on a flag defaulting to false). */ +export function debug(...args: Array<unknown>) { if (DEBUG) console.error.apply(console, args); } @@ -26,14 +30,52 @@ console.error.apply(console, args); } +/** + * runAsWorker returns true if the given arguments indicate the process should + * run as a persistent worker. + */ export function runAsWorker(args: string[]) { return args.indexOf('--persistent_worker') !== -1; } -const workerpb = (function loadWorkerPb() { - const protoPath = '../worker_protocol.proto'; +/** + * workerProto declares the static type of the object constructed at runtime by + * protobufjs, based on reading the protocol buffer definition. + */ +declare namespace workerProto { + /** Input represents the blaze.worker.Input message. */ + interface Input extends protobufjs.Message<Input> { + path: string; + /** + * In Node, digest is a Buffer. In the browser, it's a replacement + * implementation. We only care about its toString(encoding) method. + */ + digest: {toString(encoding: string): string}; + } - // Use node module resolution so we can find the .proto file in any of the root dirs + /** WorkRequest repesents the blaze.worker.WorkRequest message. */ + interface WorkRequest extends protobufjs.Message<WorkRequest> { + arguments: string[]; + inputs: Input[]; + } + + // tslint:disable:variable-name reflected, constructable types. + const WorkRequest: protobufjs.Type; + const WorkResponse: protobufjs.Type; + // tslint:enable:variable-name +} + +/** + * loadWorkerPb finds and loads the protocol buffer definition for bazel's + * worker protocol using protobufjs. In protobufjs, this means it's a reflection + * object that also contains properties for the individual messages. + */ +function loadWorkerPb() { + const protoPath = + '../worker_protocol.proto'; + + // Use node module resolution so we can find the .proto file in any of the + // root dirs let protofile; try { // Look for the .proto file relative in its @bazel/typescript npm package @@ -50,35 +92,32 @@ '../../third_party/github.com/bazelbuild/bazel/src/main/protobuf/worker_protocol.proto'); } - // Under Bazel, we use the version of TypeScript installed in the user's - // workspace This means we also use their version of protobuf.js. Handle both. - // v5 and v6 by checking which one is present. - if (protobufjs.loadProtoFile) { - // Protobuf.js v5 - const protoNamespace = protobufjs.loadProtoFile(protofile); - if (!protoNamespace) { - throw new Error('Cannot find ' + path.resolve(protoPath)); - } - return protoNamespace.build('blaze.worker'); - } else { - // Protobuf.js v6 - const protoNamespace = protobufjs.loadSync(protofile); - if (!protoNamespace) { - throw new Error('Cannot find ' + path.resolve(protoPath)); - } - return protoNamespace.lookup('blaze.worker'); + const protoNamespace = protobufjs.loadSync(protofile); + if (!protoNamespace) { + throw new Error('Cannot find ' + path.resolve(protoPath)); } -})(); - -interface Input { - getPath(): string; - getDigest(): {toString(encoding: string): string}; // npm:ByteBuffer -} -interface WorkRequest { - getArguments(): string[]; - getInputs(): Input[]; + const workerpb = protoNamespace.lookup('blaze.worker'); + if (!workerpb) { + throw new Error(`Cannot find namespace blaze.worker`); + } + return workerpb as protobufjs.ReflectionObject & typeof workerProto; } +/** + * workerpb contains the runtime representation of the worker protocol buffer, + * including accessor for the defined messages. + */ +const workerpb = loadWorkerPb(); + +/** + * runWorkerLoop handles the interacton between bazel workers and the + * TypeScript compiler. It reads compilation requests from stdin, unmarshals the + * data, and dispatches into `runOneBuild` for the actual compilation to happen. + * + * The compilation handler is parameterized so that this code can be used by + * different compiler entry points (currently TypeScript compilation and Angular + * compilation). + */ export function runWorkerLoop( runOneBuild: (args: string[], inputs?: {[path: string]: string}) => boolean) { @@ -88,39 +127,54 @@ // user as expected. let consoleOutput = ''; process.stderr.write = - (chunk: string | Buffer, ...otherArgs: any[]): boolean => { + (chunk: string|Buffer, ...otherArgs: Array<unknown>): boolean => { consoleOutput += chunk.toString(); return true; }; // Accumulator for asynchronously read input. - // tslint:disable-next-line:no-any protobufjs is untyped - let buf: any; + // protobufjs uses node's Buffer, but has its own reader abstraction on top of + // it (for browser compatiblity). It ignores Buffer's builtin start and + // offset, which means the handling code below cannot use Buffer in a + // meaningful way (such as cycling data through it). The handler below reads + // any data available on stdin, concatenating it into this buffer. It then + // attempts to read a delimited Message from it. If a message is incomplete, + // it exits and waits for more input. If a message has been read, it strips + // its data of this buffer. + let buf: Buffer = Buffer.alloc(0); process.stdin.on('readable', () => { - const chunk = process.stdin.read(); + const chunk = process.stdin.read() as Buffer; if (!chunk) return; - - const wrapped = ByteBuffer.wrap(chunk); - buf = buf ? ByteBuffer.concat([buf, wrapped]) : wrapped; + buf = Buffer.concat([buf, chunk]); try { - let req: WorkRequest; + const reader = new protobufjs.Reader(buf); // Read all requests that have accumulated in the buffer. - while ((req = workerpb.WorkRequest.decodeDelimited(buf)) != null) { + while (reader.len - reader.pos > 0) { + const messageStart = reader.len; + const msgLength: number = reader.uint32(); + // chunk might be an incomplete read from stdin. If there are not enough + // bytes for the next full message, wait for more input. + if ((reader.len - reader.pos) < msgLength) return; + + const req = workerpb.WorkRequest.decode(reader, msgLength) as + workerProto.WorkRequest; + // Once a message has been read, remove it from buf so that if we pause + // to read more input, this message will not be processed again. + buf = buf.slice(messageStart); debug('=== Handling new build request'); // Reset accumulated log output. consoleOutput = ''; - const args = req.getArguments(); + const args = req.arguments; const inputs: {[path: string]: string} = {}; - for (const input of req.getInputs()) { - inputs[input.getPath()] = input.getDigest().toString('hex'); + for (const input of req.inputs) { + inputs[input.path] = input.digest.toString('hex'); } debug('Compiling with:\n\t' + args.join('\n\t')); const exitCode = runOneBuild(args, inputs) ? 0 : 1; - process.stdout.write(new workerpb.WorkResponse() - .setExitCode(exitCode) - .setOutput(consoleOutput) - .encodeDelimited() - .toBuffer()); + process.stdout.write((workerpb.WorkResponse.encodeDelimited({ + exitCode, + output: consoleOutput, + })).finish() as Buffer); // Force a garbage collection pass. This keeps our memory usage // consistent across multiple compilations, and allows the file // cache to use the current memory usage as a guideline for expiring @@ -128,17 +182,19 @@ // we want to gc only after all its locals have gone out of scope. global.gc(); } - // Avoid growing the buffer indefinitely. - buf.compact(); + // All messages have been handled, make sure the invariant holds and + // Buffer is empty once all messages have been read. + if (buf.length > 0) { + throw new Error('buffer not empty after reading all messages'); + } } catch (e) { log('Compilation failed', e.stack); - process.stdout.write(new workerpb.WorkResponse() - .setExitCode(1) - .setOutput(consoleOutput) - .encodeDelimited() - .toBuffer()); + process.stdout.write( + workerpb.WorkResponse + .encodeDelimited({exitCode: 1, output: consoleOutput}) + .finish() as Buffer); // Clear buffer so the next build won't read an incomplete request. - buf = null; + buf = Buffer.alloc(0); } }); }
diff --git a/package.json b/package.json index dd1a029..b0579be 100644 --- a/package.json +++ b/package.json
@@ -3,7 +3,7 @@ "homepage": "https://github.com/bazelbuild/rules_typescript", "license": "Apache-2.0", "devDependencies": { - "protobufjs": "5.0.3", + "protobufjs": "6.8.8", "semver": "5.6.0", "source-map-support": "0.5.9", "tsutils": "2.27.2",