diff --git a/package.json b/package.json index 7399b4ba6c1e..9a9403eb9df0 100644 --- a/package.json +++ b/package.json @@ -89,5 +89,8 @@ "loupe": "^2.3.6", "vite": "^5.3.4", "testcontainers/**/nan": "^2.19.0" + }, + "dependencies": { + "progress-stream": "^2.0.0" } } diff --git a/packages/beacon-node/package.json b/packages/beacon-node/package.json index 87d80d27039c..295bc14f2c37 100644 --- a/packages/beacon-node/package.json +++ b/packages/beacon-node/package.json @@ -133,6 +133,7 @@ "@lodestar/utils": "^1.23.0", "@lodestar/validator": "^1.23.0", "@multiformats/multiaddr": "^12.1.3", + "axios": "^1.7.7", "c-kzg": "^2.1.2", "datastore-core": "^9.1.1", "datastore-level": "^10.1.1", @@ -144,16 +145,19 @@ "jwt-simple": "0.5.6", "libp2p": "1.4.3", "multiformats": "^11.0.1", + "progress-stream": "^2.0.0", "prom-client": "^15.1.0", "qs": "^6.11.1", "snappyjs": "^0.7.0", "strict-event-emitter-types": "^2.0.0", "systeminformation": "^5.22.9", + "tar": "^7.4.3", "uint8arraylist": "^2.4.7", "xxhash-wasm": "1.0.2" }, "devDependencies": { "@types/datastore-level": "^3.0.0", + "@types/progress-stream": "^2.0.5", "@types/qs": "^6.9.7", "@types/tmp": "^0.2.3", "it-drain": "^3.0.3", diff --git a/packages/spec-test-util/src/downloadTests.ts b/packages/spec-test-util/src/downloadTests.ts index 53113e716eea..5629a05274b4 100644 --- a/packages/spec-test-util/src/downloadTests.ts +++ b/packages/spec-test-util/src/downloadTests.ts @@ -4,6 +4,7 @@ import path from "node:path"; import stream from "node:stream"; import {promisify} from "node:util"; import {retry} from "@lodestar/utils"; +import progressStream from "progress-stream"; import axios from "axios"; import {rimraf} from "rimraf"; import {x as extractTar} from "tar"; @@ -15,9 +16,9 @@ const logEmpty = (): void => {}; export type DownloadTestsOptions = { specVersion: string; outputDir: string; - /** Root Github URL `https://github.com/ethereum/consensus-spec-tests` */ + /** Root Github URL `https://github.com/ethereum/consensus-spec-tests` */ specTestsRepoUrl: string; - /** Release files names to download without prefix `["general", "mainnet", "minimal"]` */ + /** Release files names to download without prefix `["general", "mainnet", "minimal"]` */ testsToDownload: string[]; }; @@ -74,13 +75,29 @@ export async function downloadGenericSpecTests( timeout: 30 * 60 * 1000, }); - const totalSize = headers["content-length"] as string; + const totalSize = parseInt(headers["content-length"], 10); log(`Downloading ${url} - ${totalSize} bytes`); - // extract tar into output directory - await promisify(stream.pipeline)(data, extractTar({cwd: outputDir})); + const progress = progressStream({ length: totalSize, time: 100 }); + const barLength = 40; // Length of the progress bar + let lastPercentage = 0; + + progress.on('progress', (progressData) => { + const currentPercentage = Math.floor(progressData.percentage); + + if (currentPercentage > lastPercentage) { + lastPercentage = currentPercentage; + const filledLength = Math.round(barLength * currentPercentage / 100); + const bar = '#'.repeat(filledLength) + '-'.repeat(barLength - filledLength); + // Move the cursor to the beginning of the line and overwrite it + process.stdout.write(`\r[${bar}] ${currentPercentage}% (${progressData.transferred} bytes)`); + } + }); - log(`Downloaded ${url}`); + // Use pipeline to handle the stream and extract the tar + await promisify(stream.pipeline)(data.pipe(progress), extractTar({ cwd: outputDir })); + console.log(); // Move to the next line after the download is complete + log(`Downloaded ${url}`); }, { retries: 3, diff --git a/packages/types/src/progress-stream.d.ts b/packages/types/src/progress-stream.d.ts new file mode 100644 index 000000000000..c1c7c76a096a --- /dev/null +++ b/packages/types/src/progress-stream.d.ts @@ -0,0 +1,13 @@ +declare module 'progress-stream' { + import { Transform } from 'stream'; + + interface ProgressStreamOptions { + length: number; // Total length of the data being processed + time?: number; // Time interval to emit progress events + } + + function progressStream(options?: ProgressStreamOptions): Transform; + + export = progressStream; + } + \ No newline at end of file diff --git a/tsconfig.json b/tsconfig.json index c2cf3e5f258a..0dbd3ac9baed 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -2,6 +2,8 @@ "extends": "./tsconfig.build.json", "compilerOptions": { "emitDeclarationOnly": false, + "esModuleInterop": true, + "allowSyntheticDefaultImports": true, "incremental": false, // Required to run benchmark command from root directory "typeRoots": ["node_modules/@types", "./types"], diff --git a/yarn.lock b/yarn.lock index f59b3d082456..e9215ffcd6ac 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2979,6 +2979,13 @@ resolved "https://registry.yarnpkg.com/@swc/core-linux-x64-musl/-/core-linux-x64-musl-1.7.0.tgz#0868518771004753acb07d9871cf5b143b51dc30" integrity sha512-cEJ2ebtV1v/5Ilb55E05J6F5SrHKQWzUttIhR5Mkayyo+yvPslcpByuFC3D+J7X1ebziTOBpWuMpUdjLfh3SMQ== +"@types/progress-stream@^2.0.5": + version "2.0.5" + resolved "https://registry.npmjs.org/@types/progress-stream/-/progress-stream-2.0.5.tgz" + integrity sha512-5YNriuEZkHlFHHepLIaxzq3atGeav1qCTGzB74HKWpo66qjfostF+rHc785YYYHeBytve8ZG3ejg42jEIfXNiQ== + dependencies: + "@types/node" "*" + "@swc/core-win32-arm64-msvc@1.7.0": version "1.7.0" resolved "https://registry.yarnpkg.com/@swc/core-win32-arm64-msvc/-/core-win32-arm64-msvc-1.7.0.tgz#43048d67f871e798566c3883c991bed521ef3b9e" @@ -10185,6 +10192,19 @@ pathval@^2.0.0: resolved "https://registry.yarnpkg.com/pathval/-/pathval-2.0.0.tgz#7e2550b422601d4f6b8e26f1301bc8f15a741a25" integrity sha512-vE7JKRyES09KiunauX7nd2Q9/L7lhok4smP9RZTDeD4MVs72Dp2qNFVz39Nz5a0FVEW0BJR6C0DYrq6unoziZA== +progress-stream@^2.0.0: + version "2.0.0" + resolved "https://registry.npmjs.org/progress-stream/-/progress-stream-2.0.0.tgz" + integrity sha512-xJwOWR46jcXUq6EH9yYyqp+I52skPySOeHfkxOZ2IY1AiBi/sFJhbhAKHoV3OTw/omQ45KTio9215dRJ2Yxd3Q== + dependencies: + speedometer "~1.0.0" + through2 "~2.0.3" + +progress@^2.0.3, progress@2.0.3: + version "2.0.3" + resolved "https://registry.npmjs.org/progress/-/progress-2.0.3.tgz" + integrity sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA== + pbkdf2@^3.0.3: version "3.1.2" resolved "https://registry.npmjs.org/pbkdf2/-/pbkdf2-3.1.2.tgz"