From 90e6bfc2038e053ee8409d1942df29d7105f5e2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Ole=C5=9B?= Date: Sun, 28 Feb 2021 19:08:41 +0100 Subject: [PATCH 1/2] feat: improve API of the sandbox --- README.md | 25 ++++----- src/package.ts | 30 ++--------- src/sandbox.ts | 141 +++++++++++++++++++++++++++++++++++-------------- 3 files changed, 118 insertions(+), 78 deletions(-) diff --git a/README.md b/README.md index f5b38f8..3f62f8c 100644 --- a/README.md +++ b/README.md @@ -27,21 +27,22 @@ Example: import { createSandbox, Sandbox, - packLocalPackage, - externalPackage, - Package + packLocalPackage } from "karton"; import path from 'path'; describe('my-package', () => { let sandbox: Sandbox; - let myPackage: Package; beforeAll(async () => { - myPackage = await packLocalPackage( - path.resolve(__dirname, '../../') - ); - sandbox = await createSandbox(); + sandbox = await createSandbox({ + lockDirectory: path.resolve(__dirname, '__locks__'), + fixedDependencies: { + 'my-package': `file:${await packLocalPackage( + path.resolve(__dirname, '../../') + )}` + } + }); }); afterEach(async () => { await sandbox.reset(); @@ -51,11 +52,11 @@ describe('my-package', () => { }) it.each([ - externalPackage('webpack', '^4.0.0'), - externalPackage('webpack', '^5.0.0') - ])('works with webpack %p', async (webpack) => { + [{ 'webpack': '^4.0.0' }], + [{ 'webpack': '^5.0.0' }] + ])('works with %p', async (dependencies) => { await sandbox.load(path.join(__dirname, 'fixtures/basic')); - await sandbox.install('yarn', [myPackage, webpack]); + await sandbox.install('yarn', dependencies); const result = await sandbox.exec('node src/test.js'); expect(result).toEqual('my-package awesome output'); diff --git a/src/package.ts b/src/package.ts index 7d1cfff..3a16a03 100644 --- a/src/package.ts +++ b/src/package.ts @@ -4,21 +4,7 @@ import path from "path"; import fs from "fs-extra"; import os from "os"; -interface Package { - name: string; - version: string; - immutable?: boolean; -} - -function externalPackage(name: string, version: string): Package { - const pkg = { name, version, immutable: true }; - Object.assign(pkg, { - toString: () => `${pkg.name}@${pkg.version}`, - }); - return pkg; -} - -async function packLocalPackage(directory: string): Promise { +async function packLocalPackage(directory: string): Promise { const packageJSONPath = path.resolve(directory, "package.json"); if (!(await fs.pathExists(packageJSONPath))) { throw new Error( @@ -30,7 +16,7 @@ async function packLocalPackage(directory: string): Promise { ); const npmPacked = path.resolve(directory, `${name}-${version}.tgz`); - return new Promise((resolve, reject) => { + return new Promise((resolve, reject) => { const childProcess = exec( "npm pack", { @@ -52,15 +38,7 @@ async function packLocalPackage(directory: string): Promise { await fs.move(npmPacked, packagePath); - const pkg = { - name, - version: `file:${packagePath}`, - immutable: false, - }; - Object.assign(pkg, { - toString: () => `${name}@${version}`, - }); - resolve(pkg); + resolve(packagePath); } else { reject( new Error(`Cannot find 'npm pack' output file: ${npmPacked}`) @@ -79,4 +57,4 @@ async function packLocalPackage(directory: string): Promise { }); } -export { externalPackage, packLocalPackage, Package }; +export { packLocalPackage }; diff --git a/src/sandbox.ts b/src/sandbox.ts index 77bb850..93d5c72 100644 --- a/src/sandbox.ts +++ b/src/sandbox.ts @@ -7,18 +7,37 @@ import spawn from "cross-spawn"; import stripAnsi from "strip-ansi"; import treeKill from "tree-kill"; import { defaultLogger, Logger } from "./logger"; -import { Package } from "./package"; import { retry, RetryOptions, wait } from "./async"; -interface CommandOptions { +interface SandboxOptions { + logger?: Logger; + lockDirectory?: string; + fixedDependencies?: Record; +} + +interface ExecOptions extends RetryOptions { cwd?: string; env?: Record; + fail?: boolean; } -interface InstallOptions { - lockDirectory?: string; +interface SpawnOptions { + cwd?: string; + env?: Record; } +type BufferEncoding = + | "ascii" + | "utf8" + | "utf-8" + | "utf16le" + | "ucs2" + | "ucs-2" + | "base64" + | "latin1" + | "binary" + | "hex"; + interface Sandbox { context: string; reset(options?: RetryOptions): Promise; @@ -26,11 +45,20 @@ interface Sandbox { load(directory: string, options?: RetryOptions): Promise; install( manager: "yarn" | "npm", - overwrites: Package[], - options?: InstallOptions & RetryOptions + dependencies: Record, + options?: RetryOptions + ): Promise; + write( + path: string, + content: string | Buffer, + options?: RetryOptions ): Promise; - write(path: string, content: string, options?: RetryOptions): Promise; - read(path: string, options?: RetryOptions): Promise; + read( + path: string, + encoding: BufferEncoding, + options?: RetryOptions + ): Promise; + read(path: string, options?: RetryOptions): Promise; exists(path: string, options?: RetryOptions): Promise; remove(path: string, options?: RetryOptions): Promise; patch( @@ -39,11 +67,9 @@ interface Sandbox { replacement: string, options?: RetryOptions ): Promise; - exec( - command: string, - options?: CommandOptions & RetryOptions - ): Promise; - spawn(command: string, options?: CommandOptions): ChildProcess; + list(path: string, options?: RetryOptions): Promise; + exec(command: string, options?: ExecOptions): Promise; + spawn(command: string, options?: SpawnOptions): ChildProcess; kill(childProcess: ChildProcess, options?: RetryOptions): Promise; } @@ -51,7 +77,13 @@ function normalizeEol(content: string): string { return content.split(/\r\n?|\n/).join("\n"); } -async function createSandbox(logger: Logger = defaultLogger): Promise { +async function createSandbox(options: SandboxOptions = {}): Promise { + const { + logger = defaultLogger, + fixedDependencies = {}, + lockDirectory, + } = options; + const context = fs.realpathSync.native( await fs.mkdtemp(resolve(os.tmpdir(), "karton-sandbox-")) ); @@ -101,8 +133,8 @@ async function createSandbox(logger: Logger = defaultLogger): Promise { }, install: async ( manager: "yarn" | "npm", - overwrites: Package[], - options?: InstallOptions & RetryOptions + dependencies: Record, + options?: RetryOptions ) => { logger.log("Installing dependencies..."); @@ -113,17 +145,16 @@ async function createSandbox(logger: Logger = defaultLogger): Promise { } const originalPackageJSON = JSON.parse( - await sandbox.read("package.json", options) + await sandbox.read("package.json", "utf8", options) ); const packageJSON = { ...originalPackageJSON, - dependencies: originalPackageJSON.dependencies - ? { ...originalPackageJSON.dependencies } - : {}, + dependencies: { + ...(originalPackageJSON.dependencies || {}), + ...fixedDependencies, + ...dependencies, + }, }; - for (const { name, version } of overwrites) { - packageJSON.dependencies[name] = version; - } await sandbox.write( "package.json", JSON.stringify(packageJSON, undefined, " "), @@ -131,15 +162,15 @@ async function createSandbox(logger: Logger = defaultLogger): Promise { ); let lockFile: string | undefined; - if (options?.lockDirectory) { + if (lockDirectory) { lockFile = resolve( - options.lockDirectory, + lockDirectory, `${crypto .createHash("md5") .update( JSON.stringify([ manager, - overwrites.filter((pkg) => pkg.immutable), + dependencies, originalPackageJSON.dependencies, originalPackageJSON.devDependencies, ]) @@ -192,7 +223,11 @@ async function createSandbox(logger: Logger = defaultLogger): Promise { break; } }, - write: async (path: string, content: string, options?: RetryOptions) => { + write: async ( + path: string, + content: string | Buffer, + options?: RetryOptions + ) => { logger.log(`Writing file ${path}...`); const realPath = resolve(context, path); const dirPath = dirname(realPath); @@ -211,19 +246,38 @@ async function createSandbox(logger: Logger = defaultLogger): Promise { await wait(); return retry( - () => fs.writeFile(realPath, normalizeEol(content)), + () => + fs.writeFile( + realPath, + typeof content === "string" ? normalizeEol(content) : content + ), logger, options ); }, - read: (path: string, options?: RetryOptions) => { + read: ( + path: string, + encodingOrOptions?: BufferEncoding | RetryOptions, + options?: RetryOptions + ): Promise | Promise => { logger.log(`Reading file ${path}...`); - return retry( - () => fs.readFile(resolve(context, path), "utf-8").then(normalizeEol), - logger, - options - ); + if (typeof encodingOrOptions === "string") { + return retry( + () => + fs + .readFile(resolve(context, path), encodingOrOptions) + .then(normalizeEol), + logger, + options + ); + } else { + return retry( + () => fs.readFile(resolve(context, path)), + logger, + options + ); + } }, exists: (path: string) => fs.pathExists(resolve(context, path)), remove: (path: string, options?: RetryOptions) => { @@ -261,7 +315,13 @@ async function createSandbox(logger: Logger = defaultLogger): Promise { options ); }, - exec: (command: string, options: CommandOptions & RetryOptions = {}) => + list: (path: string, options?: RetryOptions) => + retry( + () => fs.readdir(resolve(context, path), { withFileTypes: true }), + logger, + options + ), + exec: (command: string, options: ExecOptions = {}) => retry( () => new Promise((resolve, reject) => { @@ -280,10 +340,11 @@ async function createSandbox(logger: Logger = defaultLogger): Promise { }, }, (error, stdout, stderr) => { - if (error) { - reject(stdout + stderr); + const results = stripAnsi(stdout + stderr); + if ((error && options.fail) || !error) { + resolve(results); } else { - resolve(stdout + stderr); + reject(results); } childProcesses.delete(childProcess); } @@ -301,7 +362,7 @@ async function createSandbox(logger: Logger = defaultLogger): Promise { logger, options ), - spawn: (command: string, options: CommandOptions = {}) => { + spawn: (command: string, options: SpawnOptions = {}) => { logger.log(`Spawning "${command}" command...`); const env = options.env || {}; @@ -352,7 +413,7 @@ async function createSandbox(logger: Logger = defaultLogger): Promise { } childProcesses.delete(childProcess); }, - }; + } as Sandbox; return sandbox; } From 08b91ffb359e55429f70b5815f3ee7fa49e1e55c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Ole=C5=9B?= Date: Sun, 28 Feb 2021 19:09:15 +0100 Subject: [PATCH 2/2] feat: add process driver for spawn processes --- src/index.ts | 1 + src/listener.ts | 133 ++++++++++++++++++++++++++++++++++++++++++ src/process-driver.ts | 104 +++++++++++++++++++++++++++++++++ 3 files changed, 238 insertions(+) create mode 100644 src/listener.ts create mode 100644 src/process-driver.ts diff --git a/src/index.ts b/src/index.ts index 2563c0d..2df5c3d 100644 --- a/src/index.ts +++ b/src/index.ts @@ -2,3 +2,4 @@ export * from "./sandbox"; export * from "./logger"; export * from "./async"; export * from "./package"; +export * from "./process-driver"; diff --git a/src/listener.ts b/src/listener.ts new file mode 100644 index 0000000..573de4d --- /dev/null +++ b/src/listener.ts @@ -0,0 +1,133 @@ +interface Listener { + resolve(value: TValue): void; + reject(error: unknown): void; +} + +type ListenerStatus = "pending" | "resolved" | "rejected"; + +interface QueuedListener extends Listener { + apply>(listener: TListener): void; + resolved: TValue | undefined; + rejected: unknown | undefined; + status: ListenerStatus; +} + +interface BufferedListener + extends QueuedListener { + apply>( + listener: TListener, + ingest?: (chunk: TChunk, listener: Listener) => TChunk | undefined + ): void; + ingest(chunk: TChunk): void; + buffer: TChunk[]; +} + +function createBufferedListener( + buffer: TChunk[] = [] +): BufferedListener { + let resolve: (value: TValue) => void | undefined; + let reject: (error: unknown) => void | undefined; + + const bufferedListener: BufferedListener = { + resolve(value) { + if (bufferedListener.status === "pending") { + bufferedListener.resolved = value; + bufferedListener.status = "resolved"; + + if (resolve) { + resolve(value); + } + } + }, + reject(error) { + if (bufferedListener.status === "pending") { + bufferedListener.rejected = error; + bufferedListener.status = "rejected"; + + if (reject) { + reject(error); + } + } + }, + ingest(chunk: TChunk) { + bufferedListener.buffer.push(chunk); + }, + apply(listener, ingest) { + switch (bufferedListener.status) { + case "resolved": + listener.resolve(bufferedListener.resolved!); + break; + case "rejected": + listener.reject(bufferedListener.rejected); + break; + case "pending": + resolve = listener.resolve; + reject = listener.reject; + break; + } + + if (ingest) { + const ingestListener: Listener = { + resolve(value) { + if (bufferedListener.status === "pending") { + bufferedListener.resolved = value; + bufferedListener.status = "resolved"; + } + }, + reject(error) { + if (bufferedListener.status === "pending") { + bufferedListener.rejected = error; + bufferedListener.status = "rejected"; + } + }, + }; + + const applyStatusChange = () => { + switch (bufferedListener.status) { + case "resolved": + listener.resolve(bufferedListener.resolved!); + break; + case "rejected": + listener.reject(bufferedListener.rejected); + break; + } + }; + + bufferedListener.ingest = (chunk: TChunk) => { + if (bufferedListener.status === "pending") { + const ingestedChunk = ingest(chunk, ingestListener); + + if (ingestedChunk) { + bufferedListener.buffer.push(ingestedChunk); + } + + applyStatusChange(); + } else { + bufferedListener.buffer.push(chunk); + } + }; + + // process buffer + bufferedListener.buffer = bufferedListener.buffer + .map((chunk) => { + if (bufferedListener.status === "pending") { + return ingest(chunk, ingestListener); + } else { + return chunk; + } + }) + .filter((chunk) => chunk !== undefined) as TChunk[]; + + applyStatusChange(); + } + }, + resolved: undefined, + rejected: undefined, + status: "pending", + buffer: [...buffer], + }; + + return bufferedListener; +} + +export { Listener, QueuedListener, BufferedListener, createBufferedListener }; diff --git a/src/process-driver.ts b/src/process-driver.ts new file mode 100644 index 0000000..fb08224 --- /dev/null +++ b/src/process-driver.ts @@ -0,0 +1,104 @@ +import { ChildProcess } from "child_process"; +import { BufferedListener, createBufferedListener } from "./listener"; +import stripAnsi from "strip-ansi"; + +interface ProcessDriver { + process: ChildProcess; + waitForStdoutIncludes( + pattern: string | string[], + timeout?: number + ): Promise; + waitForStderrIncludes( + pattern: string | string[], + timeout?: number + ): Promise; +} + +type ProcessOutput = "stdout" | "stderr"; + +function createProcessDriver( + process: ChildProcess, + defaultTimeout = 30000 +): ProcessDriver { + const listeners: Record> = { + stdout: createBufferedListener(), + stderr: createBufferedListener(), + }; + + if (process.stdout) { + process.stdout.on("data", (data) => { + const content = stripAnsi(data.toString()); + listeners.stdout.ingest(content); + }); + } + + if (process.stderr) { + process.stderr.on("data", (data) => { + const content = stripAnsi(data.toString()); + listeners.stderr.ingest(content); + }); + } + + const waitForOutputIncludes = ( + output: ProcessOutput, + pattern: string | string[], + timeout: number + ) => + new Promise((resolve, reject) => { + const timeoutId = setTimeout(() => { + reject( + new Error( + `Exceeded time on waiting for "${pattern}" to appear in the ${output}.` + ) + ); + }, timeout); + + listeners[output].apply( + { + resolve: () => { + clearTimeout(timeoutId); + listeners[output] = createBufferedListener( + listeners[output].buffer + ); + resolve(); + }, + reject: (error) => { + clearTimeout(timeoutId); + listeners[output] = createBufferedListener( + listeners[output].buffer + ); + reject(error); + }, + string: pattern, + }, + (chunk, listener) => { + let index = -1; + const strings = Array.isArray(pattern) ? pattern : [pattern]; + + strings.forEach((string) => { + const stringIndex = chunk.indexOf(string); + + index = Math.max( + index, + stringIndex === -1 ? -1 : stringIndex + string.length + ); + }); + + if (index !== -1) { + listener.resolve(); + return chunk.slice(index); + } + } + ); + }); + + return { + process, + waitForStdoutIncludes: (string, timeout = defaultTimeout) => + waitForOutputIncludes("stdout", string, timeout), + waitForStderrIncludes: (string, timeout = defaultTimeout) => + waitForOutputIncludes("stderr", string, timeout), + }; +} + +export { createProcessDriver, ProcessDriver };