From 541b24206f4c3b52c061b9543b1d898a8c8b1c40 Mon Sep 17 00:00:00 2001 From: busma13 Date: Mon, 2 Dec 2024 10:41:17 -0700 Subject: [PATCH 01/21] Add --verbose to yarn build command to see CI errors --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 4d2d49d0088..13082ac6bdd 100644 --- a/package.json +++ b/package.json @@ -14,7 +14,7 @@ ], "scripts": { "prebuild": "./packages/xlucene-parser/scripts/generate-engine.js", - "build": "tsc --build && yarn workspace teraslice-cli build", + "build": "tsc --build --verbose && yarn workspace teraslice-cli build --verbose", "build:cleanup": "./scripts/build-cleanup.sh", "build:doctor": "./scripts/build-doctor.sh", "build:fix": "echo '[DEPRECATED], use yarn run build:doctor instead'", From 0e0f5ce7fb3f5fe6bfc17660a68bb4d6ca2ceb9b Mon Sep 17 00:00:00 2001 From: busma13 Date: Thu, 5 Dec 2024 09:09:53 -0700 Subject: [PATCH 02/21] assign port range to all tests using findPort() --- packages/teraslice/src/lib/utils/port_utils.ts | 2 ++ packages/teraslice/test/services/api-spec.ts | 2 +- packages/teraslice/test/services/assets-spec.ts | 13 ++++++++++--- .../execution-controller-spec.ts | 4 ++-- .../execution-special-test-cases-spec.ts | 2 +- .../execution-test-cases-spec.ts | 2 +- .../teraslice/test/workers/helpers/test-context.ts | 2 +- .../teraslice/test/workers/worker/worker-spec.ts | 2 +- 8 files changed, 19 insertions(+), 10 deletions(-) diff --git a/packages/teraslice/src/lib/utils/port_utils.ts b/packages/teraslice/src/lib/utils/port_utils.ts index 7772e3f50e5..3b7a6d80de7 100644 --- a/packages/teraslice/src/lib/utils/port_utils.ts +++ b/packages/teraslice/src/lib/utils/port_utils.ts @@ -26,6 +26,8 @@ export interface PortOptions { } export async function findPort(options: PortOptions = {}) { + // If using this function within a test file, avoid port collisions by + // setting PortOptions start and end to a range not used by other tests. const { start = 8002, end = 40000, diff --git a/packages/teraslice/test/services/api-spec.ts b/packages/teraslice/test/services/api-spec.ts index 3e18087c63a..9d0b2f68f39 100644 --- a/packages/teraslice/test/services/api-spec.ts +++ b/packages/teraslice/test/services/api-spec.ts @@ -31,7 +31,7 @@ describe('HTTP API', () => { let server: http.Server; beforeAll(async () => { - port = await findPort(); + port = await findPort({ assetsPort: 10000, start: 10001, end: 12000 }); baseUrl = `http://localhost:${port}`; diff --git a/packages/teraslice/test/services/assets-spec.ts b/packages/teraslice/test/services/assets-spec.ts index 43fb2de7865..3d1ab72a107 100644 --- a/packages/teraslice/test/services/assets-spec.ts +++ b/packages/teraslice/test/services/assets-spec.ts @@ -6,6 +6,7 @@ import { createClient } from 'elasticsearch-store'; import { createS3Client } from '@terascope/file-asset-apis'; import { AssetsService } from '../../src/lib/cluster/services/assets'; import { TEST_INDEX_PREFIX } from '../test.config'; +import { findPort } from '../../src/lib/utils/port_utils.js'; describe('Assets Service', () => { const contextOptions: TestContextOptions = { @@ -59,11 +60,17 @@ describe('Assets Service', () => { context.sysconfig.teraslice.asset_storage_connection = 'default'; context.sysconfig.teraslice.asset_storage_bucket = 'assets-spec-test-bucket'; context.sysconfig.teraslice.api_response_timeout = 30000; - /// Setting port for the asset service - process.env.port = '55678'; - const service = new AssetsService(context); + + let service: AssetsService; beforeAll(async () => { + /// Setting port for the asset service + process.env.port = (await findPort({ + assetsPort: 12000, + start: 12001, + end: 14000 + })).toString(); + service = new AssetsService(context); await service.initialize(); }); diff --git a/packages/teraslice/test/workers/execution-controller/execution-controller-spec.ts b/packages/teraslice/test/workers/execution-controller/execution-controller-spec.ts index 967837fd088..8e61c5832a8 100644 --- a/packages/teraslice/test/workers/execution-controller/execution-controller-spec.ts +++ b/packages/teraslice/test/workers/execution-controller/execution-controller-spec.ts @@ -12,7 +12,7 @@ describe('ExecutionController', () => { let executionStorage!: ExecutionStorage; beforeEach(async () => { - const port = await findPort(); + const port = await findPort({ assetsPort: 14000, start: 14001, end: 16000 }); testContext = new TestContext({ assignment: 'execution_controller', @@ -92,7 +92,7 @@ describe('ExecutionController', () => { const probationWindow = 500; beforeEach(async () => { - const port = await findPort(); + const port = await findPort({ assetsPort: 16000, start: 16001, end: 18000 }); testContext = new TestContext({ assignment: 'execution_controller', diff --git a/packages/teraslice/test/workers/execution-controller/execution-special-test-cases-spec.ts b/packages/teraslice/test/workers/execution-controller/execution-special-test-cases-spec.ts index 18b87d1a46c..5088741f885 100644 --- a/packages/teraslice/test/workers/execution-controller/execution-special-test-cases-spec.ts +++ b/packages/teraslice/test/workers/execution-controller/execution-special-test-cases-spec.ts @@ -352,7 +352,7 @@ describe('ExecutionController Special Tests', () => { beforeAll(async () => { slices = []; - const port = await findPort(); + const port = await findPort({ assetsPort: 16000, start: 16001, end: 18000 }); testContext = new TestContext({ assignment: 'execution_controller', diff --git a/packages/teraslice/test/workers/execution-controller/execution-test-cases-spec.ts b/packages/teraslice/test/workers/execution-controller/execution-test-cases-spec.ts index f7e776da65e..ae41240cd43 100644 --- a/packages/teraslice/test/workers/execution-controller/execution-test-cases-spec.ts +++ b/packages/teraslice/test/workers/execution-controller/execution-test-cases-spec.ts @@ -158,7 +158,7 @@ describe('ExecutionController Test Cases', () => { beforeAll(async () => { slices = []; - const port = await findPort(); + const port = await findPort({ assetsPort: 18000, start: 18001, end: 20000 }); testContext = new TestContext({ assignment: 'execution_controller', diff --git a/packages/teraslice/test/workers/helpers/test-context.ts b/packages/teraslice/test/workers/helpers/test-context.ts index f5e77b09a64..ac1169ff0da 100644 --- a/packages/teraslice/test/workers/helpers/test-context.ts +++ b/packages/teraslice/test/workers/helpers/test-context.ts @@ -152,7 +152,7 @@ export class TestContext { async addClusterMaster() { if (this.clusterMaster) return this.clusterMaster; - const port = await findPort(); + const port = await findPort({ assetsPort: 20000, start: 20001, end: 22000 }); const networkLatencyBuffer = get( this.context, 'sysconfig.teraslice.network_latency_buffer' diff --git a/packages/teraslice/test/workers/worker/worker-spec.ts b/packages/teraslice/test/workers/worker/worker-spec.ts index 3cc815a8b38..b6daaf17508 100644 --- a/packages/teraslice/test/workers/worker/worker-spec.ts +++ b/packages/teraslice/test/workers/worker/worker-spec.ts @@ -15,7 +15,7 @@ describe('Worker', () => { } async function setupTest(options: any = {}): Promise { - const slicerPort = await findPort(); + const slicerPort = await findPort({ assetsPort: 22000, start: 22001, end: 24000 }); options.slicerPort = slicerPort; const testContext = new TestContext(options); From 7fd5867149fb79a59e5fe25a44f91296eeadd1c4 Mon Sep 17 00:00:00 2001 From: busma13 Date: Thu, 5 Dec 2024 12:45:39 -0700 Subject: [PATCH 03/21] log ports in TIME_WAIT state before starting a service with docker --- packages/scripts/src/helpers/scripts.ts | 12 ++++++++++++ packages/scripts/src/helpers/test-runner/services.ts | 4 +++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/packages/scripts/src/helpers/scripts.ts b/packages/scripts/src/helpers/scripts.ts index 56dc123de73..5ddedde835f 100644 --- a/packages/scripts/src/helpers/scripts.ts +++ b/packages/scripts/src/helpers/scripts.ts @@ -767,3 +767,15 @@ async function showAssets(tsPort: string) { return err; } } + +export async function logTimeWaitPorts() { + try { + const netstat = await execa('netstat', ['-an', '|', 'grep', 'TIME_WAIT'], { shell: true, reject: false }); + signale.info('Ports in TIME_WAIT:\n', netstat.stdout); + } catch (err) { + signale.error('Netstat command failed trying to log ports in TIME_WAIT: ', err); + signale.error('stderr: ', err.stderr); + signale.error('stdout: ', err.stdout); + signale.error('message: ', err.message); + } +} diff --git a/packages/scripts/src/helpers/test-runner/services.ts b/packages/scripts/src/helpers/test-runner/services.ts index a42812b0703..34aeeb43b4f 100644 --- a/packages/scripts/src/helpers/test-runner/services.ts +++ b/packages/scripts/src/helpers/test-runner/services.ts @@ -13,7 +13,7 @@ import { getServicesForSuite, getRootDir } from '../misc.js'; import { dockerRun, DockerRunOptions, getContainerInfo, dockerStop, k8sStartService, k8sStopService, - loadThenDeleteImageFromCache, dockerPull + loadThenDeleteImageFromCache, dockerPull, logTimeWaitPorts } from '../scripts.js'; import { Kind } from '../kind.js'; import { TestOptions } from './interfaces.js'; @@ -850,6 +850,8 @@ async function startService(options: TestOptions, service: Service): Promise<() await stopService(service); + await logTimeWaitPorts(); + const fn = await dockerRun( services[service], version, From 51825262530295f98b032acd55b18f3f314506f4 Mon Sep 17 00:00:00 2001 From: busma13 Date: Thu, 5 Dec 2024 13:23:42 -0700 Subject: [PATCH 04/21] update netstat command --- packages/scripts/src/helpers/scripts.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/scripts/src/helpers/scripts.ts b/packages/scripts/src/helpers/scripts.ts index 5ddedde835f..daa26b0c99e 100644 --- a/packages/scripts/src/helpers/scripts.ts +++ b/packages/scripts/src/helpers/scripts.ts @@ -770,7 +770,7 @@ async function showAssets(tsPort: string) { export async function logTimeWaitPorts() { try { - const netstat = await execa('netstat', ['-an', '|', 'grep', 'TIME_WAIT'], { shell: true, reject: false }); + const netstat = await execa('netstat', ['-an', '-f', 'inet', '-p', 'tcp'], { shell: true, reject: false }); signale.info('Ports in TIME_WAIT:\n', netstat.stdout); } catch (err) { signale.error('Netstat command failed trying to log ports in TIME_WAIT: ', err); From 2f7160902ddd9ee2a9e23e31e90734fb0cd81bc3 Mon Sep 17 00:00:00 2001 From: busma13 Date: Thu, 5 Dec 2024 13:33:33 -0700 Subject: [PATCH 05/21] fix message --- packages/scripts/src/helpers/scripts.ts | 6 +++--- packages/scripts/src/helpers/test-runner/services.ts | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/scripts/src/helpers/scripts.ts b/packages/scripts/src/helpers/scripts.ts index daa26b0c99e..7820e89ccf8 100644 --- a/packages/scripts/src/helpers/scripts.ts +++ b/packages/scripts/src/helpers/scripts.ts @@ -768,12 +768,12 @@ async function showAssets(tsPort: string) { } } -export async function logTimeWaitPorts() { +export async function logTCPPorts() { try { const netstat = await execa('netstat', ['-an', '-f', 'inet', '-p', 'tcp'], { shell: true, reject: false }); - signale.info('Ports in TIME_WAIT:\n', netstat.stdout); + signale.info('TCP Ports:\n', netstat.stdout); } catch (err) { - signale.error('Netstat command failed trying to log ports in TIME_WAIT: ', err); + signale.error('Netstat command failed trying to log ports: ', err); signale.error('stderr: ', err.stderr); signale.error('stdout: ', err.stdout); signale.error('message: ', err.message); diff --git a/packages/scripts/src/helpers/test-runner/services.ts b/packages/scripts/src/helpers/test-runner/services.ts index 34aeeb43b4f..39921de7d7d 100644 --- a/packages/scripts/src/helpers/test-runner/services.ts +++ b/packages/scripts/src/helpers/test-runner/services.ts @@ -13,7 +13,7 @@ import { getServicesForSuite, getRootDir } from '../misc.js'; import { dockerRun, DockerRunOptions, getContainerInfo, dockerStop, k8sStartService, k8sStopService, - loadThenDeleteImageFromCache, dockerPull, logTimeWaitPorts + loadThenDeleteImageFromCache, dockerPull, logTCPPorts } from '../scripts.js'; import { Kind } from '../kind.js'; import { TestOptions } from './interfaces.js'; @@ -850,7 +850,7 @@ async function startService(options: TestOptions, service: Service): Promise<() await stopService(service); - await logTimeWaitPorts(); + await logTCPPorts(); const fn = await dockerRun( services[service], From 9f8840fc00eb3b369b9dcb699fefb6128816f611 Mon Sep 17 00:00:00 2001 From: busma13 Date: Thu, 5 Dec 2024 14:10:25 -0700 Subject: [PATCH 06/21] rework command. inet not supported in linux --- packages/scripts/src/helpers/scripts.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/scripts/src/helpers/scripts.ts b/packages/scripts/src/helpers/scripts.ts index 7820e89ccf8..218cbe47a8a 100644 --- a/packages/scripts/src/helpers/scripts.ts +++ b/packages/scripts/src/helpers/scripts.ts @@ -770,7 +770,7 @@ async function showAssets(tsPort: string) { export async function logTCPPorts() { try { - const netstat = await execa('netstat', ['-an', '-f', 'inet', '-p', 'tcp'], { shell: true, reject: false }); + const netstat = await execa('netstat', ['-an', '|', 'grep', 'tcp4'], { shell: true, reject: false }); signale.info('TCP Ports:\n', netstat.stdout); } catch (err) { signale.error('Netstat command failed trying to log ports: ', err); From 2425ebe17aa78763d1a6e668fe4c405a88050aa1 Mon Sep 17 00:00:00 2001 From: busma13 Date: Thu, 5 Dec 2024 14:30:05 -0700 Subject: [PATCH 07/21] try again to get the command right --- packages/scripts/src/helpers/scripts.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/scripts/src/helpers/scripts.ts b/packages/scripts/src/helpers/scripts.ts index 218cbe47a8a..476b35b5d69 100644 --- a/packages/scripts/src/helpers/scripts.ts +++ b/packages/scripts/src/helpers/scripts.ts @@ -770,7 +770,7 @@ async function showAssets(tsPort: string) { export async function logTCPPorts() { try { - const netstat = await execa('netstat', ['-an', '|', 'grep', 'tcp4'], { shell: true, reject: false }); + const netstat = await execa('netstat', ['-an', '-t', 'tcp'], { shell: true, reject: false }); signale.info('TCP Ports:\n', netstat.stdout); } catch (err) { signale.error('Netstat command failed trying to log ports: ', err); From 085c16f420f784aeb030211466665c525ed7abec Mon Sep 17 00:00:00 2001 From: busma13 Date: Thu, 5 Dec 2024 14:40:58 -0700 Subject: [PATCH 08/21] log ports during k8s service setup --- packages/scripts/src/helpers/test-runner/services.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/scripts/src/helpers/test-runner/services.ts b/packages/scripts/src/helpers/test-runner/services.ts index 39921de7d7d..2b49091b0ae 100644 --- a/packages/scripts/src/helpers/test-runner/services.ts +++ b/packages/scripts/src/helpers/test-runner/services.ts @@ -844,6 +844,7 @@ async function startService(options: TestOptions, service: Service): Promise<() options.skipImageDeletion ); await k8sStopService(service); + await logTCPPorts(); await k8sStartService(service, services[service].image, version, kind); return () => { }; } From 9f63c9c2ea0a42c1492672779602268b217859a2 Mon Sep 17 00:00:00 2001 From: busma13 Date: Fri, 6 Dec 2024 09:37:58 -0700 Subject: [PATCH 09/21] Add comment about netstat command --- packages/scripts/src/helpers/scripts.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/packages/scripts/src/helpers/scripts.ts b/packages/scripts/src/helpers/scripts.ts index 476b35b5d69..b53c86d5d33 100644 --- a/packages/scripts/src/helpers/scripts.ts +++ b/packages/scripts/src/helpers/scripts.ts @@ -770,12 +770,11 @@ async function showAssets(tsPort: string) { export async function logTCPPorts() { try { + // These netstat flags were chosen because they work in + // both the mac and linux versions of the command const netstat = await execa('netstat', ['-an', '-t', 'tcp'], { shell: true, reject: false }); signale.info('TCP Ports:\n', netstat.stdout); } catch (err) { signale.error('Netstat command failed trying to log ports: ', err); - signale.error('stderr: ', err.stderr); - signale.error('stdout: ', err.stdout); - signale.error('message: ', err.message); } } From 427984db090b72fe49af89dd8af55ff330ab79da Mon Sep 17 00:00:00 2001 From: busma13 Date: Fri, 6 Dec 2024 10:03:11 -0700 Subject: [PATCH 10/21] fix port ranges --- .../execution-controller/execution-special-test-cases-spec.ts | 2 +- .../workers/execution-controller/execution-test-cases-spec.ts | 2 +- packages/teraslice/test/workers/helpers/test-context.ts | 2 +- packages/teraslice/test/workers/worker/worker-spec.ts | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/teraslice/test/workers/execution-controller/execution-special-test-cases-spec.ts b/packages/teraslice/test/workers/execution-controller/execution-special-test-cases-spec.ts index 5088741f885..32033bab735 100644 --- a/packages/teraslice/test/workers/execution-controller/execution-special-test-cases-spec.ts +++ b/packages/teraslice/test/workers/execution-controller/execution-special-test-cases-spec.ts @@ -352,7 +352,7 @@ describe('ExecutionController Special Tests', () => { beforeAll(async () => { slices = []; - const port = await findPort({ assetsPort: 16000, start: 16001, end: 18000 }); + const port = await findPort({ assetsPort: 18000, start: 18001, end: 20000 }); testContext = new TestContext({ assignment: 'execution_controller', diff --git a/packages/teraslice/test/workers/execution-controller/execution-test-cases-spec.ts b/packages/teraslice/test/workers/execution-controller/execution-test-cases-spec.ts index ae41240cd43..6b7cdb3bfff 100644 --- a/packages/teraslice/test/workers/execution-controller/execution-test-cases-spec.ts +++ b/packages/teraslice/test/workers/execution-controller/execution-test-cases-spec.ts @@ -158,7 +158,7 @@ describe('ExecutionController Test Cases', () => { beforeAll(async () => { slices = []; - const port = await findPort({ assetsPort: 18000, start: 18001, end: 20000 }); + const port = await findPort({ assetsPort: 20000, start: 20001, end: 22000 }); testContext = new TestContext({ assignment: 'execution_controller', diff --git a/packages/teraslice/test/workers/helpers/test-context.ts b/packages/teraslice/test/workers/helpers/test-context.ts index ac1169ff0da..7a715b16f4c 100644 --- a/packages/teraslice/test/workers/helpers/test-context.ts +++ b/packages/teraslice/test/workers/helpers/test-context.ts @@ -152,7 +152,7 @@ export class TestContext { async addClusterMaster() { if (this.clusterMaster) return this.clusterMaster; - const port = await findPort({ assetsPort: 20000, start: 20001, end: 22000 }); + const port = await findPort({ assetsPort: 22000, start: 22001, end: 24000 }); const networkLatencyBuffer = get( this.context, 'sysconfig.teraslice.network_latency_buffer' diff --git a/packages/teraslice/test/workers/worker/worker-spec.ts b/packages/teraslice/test/workers/worker/worker-spec.ts index b6daaf17508..1cdc412f3d4 100644 --- a/packages/teraslice/test/workers/worker/worker-spec.ts +++ b/packages/teraslice/test/workers/worker/worker-spec.ts @@ -15,7 +15,7 @@ describe('Worker', () => { } async function setupTest(options: any = {}): Promise { - const slicerPort = await findPort({ assetsPort: 22000, start: 22001, end: 24000 }); + const slicerPort = await findPort({ assetsPort: 24000, start: 24001, end: 26000 }); options.slicerPort = slicerPort; const testContext = new TestContext(options); From bcf8fddc78e8bc573e7155cd24c85094619b52d4 Mon Sep 17 00:00:00 2001 From: busma13 Date: Mon, 9 Dec 2024 09:42:37 -0700 Subject: [PATCH 11/21] fix yarn build:watch in root package.json --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 13082ac6bdd..cdb7ad5c353 100644 --- a/package.json +++ b/package.json @@ -19,7 +19,7 @@ "build:doctor": "./scripts/build-doctor.sh", "build:fix": "echo '[DEPRECATED], use yarn run build:doctor instead'", "build:pkg": "./scripts/build-pkg.sh", - "build:watch": "yarn run build --watch", + "build:watch": "tsc --build --watch", "bump": "ts-scripts bump", "docker:limit": "./scripts/docker-limit-check.sh", "docker:listImages": "ts-scripts images list", From 25378f8c8159162e28e740d5546f00eb2c5dec49 Mon Sep 17 00:00:00 2001 From: busma13 Date: Mon, 9 Dec 2024 15:50:32 -0700 Subject: [PATCH 12/21] switch from porty to get-port, rework findPort() --- packages/scripts/src/helpers/scripts.ts | 2 +- packages/teraslice-messaging/package.json | 2 +- .../src/messenger/server.ts | 26 +++++++- .../test/helpers/find-port.ts | 14 +---- packages/teraslice/package.json | 2 +- .../teraslice/src/lib/utils/port_utils.ts | 60 +++++++------------ packages/teraslice/test/services/api-spec.ts | 2 +- .../teraslice/test/services/assets-spec.ts | 6 +- .../execution-controller-spec.ts | 4 +- .../execution-special-test-cases-spec.ts | 2 +- .../execution-test-cases-spec.ts | 2 +- .../test/workers/helpers/test-context.ts | 2 +- .../test/workers/worker/worker-spec.ts | 2 +- types/porty/index.d.ts | 13 ---- yarn.lock | 10 ++-- 15 files changed, 64 insertions(+), 85 deletions(-) delete mode 100644 types/porty/index.d.ts diff --git a/packages/scripts/src/helpers/scripts.ts b/packages/scripts/src/helpers/scripts.ts index b53c86d5d33..16bff63985e 100644 --- a/packages/scripts/src/helpers/scripts.ts +++ b/packages/scripts/src/helpers/scripts.ts @@ -772,7 +772,7 @@ export async function logTCPPorts() { try { // These netstat flags were chosen because they work in // both the mac and linux versions of the command - const netstat = await execa('netstat', ['-an', '-t', 'tcp'], { shell: true, reject: false }); + const netstat = await execa('netstat', ['-an', '-p', 'tcp'], { shell: true, reject: false }); signale.info('TCP Ports:\n', netstat.stdout); } catch (err) { signale.error('Netstat command failed trying to log ports: ', err); diff --git a/packages/teraslice-messaging/package.json b/packages/teraslice-messaging/package.json index 686edd3a6d3..c1e4cd64378 100644 --- a/packages/teraslice-messaging/package.json +++ b/packages/teraslice-messaging/package.json @@ -40,7 +40,7 @@ "ms": "~2.1.3", "nanoid": "~5.0.9", "p-event": "~6.0.1", - "porty": "~3.1.1", + "get-port": "~7.1.0", "socket.io": "~1.7.4", "socket.io-client": "~1.7.4" }, diff --git a/packages/teraslice-messaging/src/messenger/server.ts b/packages/teraslice-messaging/src/messenger/server.ts index 2a5c41a3aec..27b663bb64d 100644 --- a/packages/teraslice-messaging/src/messenger/server.ts +++ b/packages/teraslice-messaging/src/messenger/server.ts @@ -1,5 +1,4 @@ import http from 'node:http'; -import porty from 'porty'; import SocketIOServer from 'socket.io'; import { get, @@ -94,8 +93,31 @@ export class Server extends Core { } async listen(): Promise { + // Check if port is available before using it + const testPort = async function (port: number) { + return new Promise((resolve) => { + const portTestServer = http.createServer(); + + portTestServer.unref(); + + portTestServer.once('error', () => { + portTestServer.close(() => { + return resolve(false); + }); + }); + + portTestServer.once('listening', () => { + portTestServer.close(() => { + return resolve(true); + }); + }); + + portTestServer.listen(port); + }); + }; + await pRetry(async () => { - const portAvailable = await porty.test(this.port); + const portAvailable = await testPort(this.port); if (!portAvailable) { throw new Error(`Port ${this.port} is already in-use`); } diff --git a/packages/teraslice-messaging/test/helpers/find-port.ts b/packages/teraslice-messaging/test/helpers/find-port.ts index 00056843db7..99d2e45c8a9 100644 --- a/packages/teraslice-messaging/test/helpers/find-port.ts +++ b/packages/teraslice-messaging/test/helpers/find-port.ts @@ -1,19 +1,11 @@ -import porty from 'porty'; -import { random } from '@terascope/utils'; +import getPort from 'get-port'; const usedPorts: number[] = []; export default async function findPort(): Promise { - const min = random(8000, 40000); - const max = min + 100; - - const port = await porty.find({ - min, - max, - avoids: usedPorts, - }); + // getPort will return an open port between 1024 and 65535, excluding usedPorts + const port = await getPort({ exclude: usedPorts }); usedPorts.push(port); - return port; } diff --git a/packages/teraslice/package.json b/packages/teraslice/package.json index 327d63b604c..f6bdd649bb3 100644 --- a/packages/teraslice/package.json +++ b/packages/teraslice/package.json @@ -58,7 +58,7 @@ "kubernetes-client": "~9.0.0", "ms": "~2.1.3", "nanoid": "~5.0.9", - "porty": "~3.1.1", + "get-port": "~7.1.0", "semver": "~7.6.3", "socket.io": "~1.7.4", "socket.io-client": "~1.7.4", diff --git a/packages/teraslice/src/lib/utils/port_utils.ts b/packages/teraslice/src/lib/utils/port_utils.ts index 3b7a6d80de7..177cd922c52 100644 --- a/packages/teraslice/src/lib/utils/port_utils.ts +++ b/packages/teraslice/src/lib/utils/port_utils.ts @@ -1,23 +1,11 @@ import { get, times, pDelay, - toIntegerOrThrow, shuffle + toIntegerOrThrow, shuffle, range } from '@terascope/utils'; -import porty from 'porty'; -import { Context } from '@terascope/job-components'; - -const _portLists = new Map(); -function listPorts(start: number, end: number): number[] { - const key = `${start}:${end}`; - if (_portLists.has(key)) { - return _portLists.get(key); - } - - // this should only be done once - const ports = shuffle(times((end - start) + 1, (n) => n + start)); - _portLists.set(key, ports); - return ports; -} +import getPort from 'get-port'; +import { Context } from '@terascope/job-components'; +import range from 'lodash/range.js'; export interface PortOptions { start?: number; @@ -25,36 +13,30 @@ export interface PortOptions { assetsPort?: number; } +const usedPorts: number[] = []; + +/** + * Return a random open port between start and end, excluding usedPorts. + * If all ports in that range are being used a random port between 1024 and 65535 + * will be returned instead. + * @param {PortOptions} options Start, end, and assetsPort to exclude can be specified + * @returns {number} + */ export async function findPort(options: PortOptions = {}) { - // If using this function within a test file, avoid port collisions by - // setting PortOptions start and end to a range not used by other tests. const { - start = 8002, - end = 40000, + start = 1024, + end = 65536, // exclusive assetsPort = 8003 } = options; - const ports = listPorts(start, end); - - const tested: number[] = []; - let port: number | undefined; - - while (ports.length) { - port = ports.shift() as number; - if (port === assetsPort) continue; - // these will be enqueue - tested.push(port); + usedPorts.push(assetsPort); - const available = await porty.test(port); - if (available) { - break; - } else { - await pDelay(100); - port = undefined; - } - } + const port = await getPort({ + port: shuffle(range(start, end)), + exclude: usedPorts + }); - _portLists[`${start}:${end}`] = ports.concat(tested); + usedPorts.push(port); if (port) return port; throw new Error(`No available port between ${start}-${end}`); diff --git a/packages/teraslice/test/services/api-spec.ts b/packages/teraslice/test/services/api-spec.ts index 9d0b2f68f39..3e18087c63a 100644 --- a/packages/teraslice/test/services/api-spec.ts +++ b/packages/teraslice/test/services/api-spec.ts @@ -31,7 +31,7 @@ describe('HTTP API', () => { let server: http.Server; beforeAll(async () => { - port = await findPort({ assetsPort: 10000, start: 10001, end: 12000 }); + port = await findPort(); baseUrl = `http://localhost:${port}`; diff --git a/packages/teraslice/test/services/assets-spec.ts b/packages/teraslice/test/services/assets-spec.ts index 3d1ab72a107..4bddc2b51a8 100644 --- a/packages/teraslice/test/services/assets-spec.ts +++ b/packages/teraslice/test/services/assets-spec.ts @@ -65,11 +65,7 @@ describe('Assets Service', () => { beforeAll(async () => { /// Setting port for the asset service - process.env.port = (await findPort({ - assetsPort: 12000, - start: 12001, - end: 14000 - })).toString(); + process.env.port = (await findPort()).toString(); service = new AssetsService(context); await service.initialize(); }); diff --git a/packages/teraslice/test/workers/execution-controller/execution-controller-spec.ts b/packages/teraslice/test/workers/execution-controller/execution-controller-spec.ts index 8e61c5832a8..967837fd088 100644 --- a/packages/teraslice/test/workers/execution-controller/execution-controller-spec.ts +++ b/packages/teraslice/test/workers/execution-controller/execution-controller-spec.ts @@ -12,7 +12,7 @@ describe('ExecutionController', () => { let executionStorage!: ExecutionStorage; beforeEach(async () => { - const port = await findPort({ assetsPort: 14000, start: 14001, end: 16000 }); + const port = await findPort(); testContext = new TestContext({ assignment: 'execution_controller', @@ -92,7 +92,7 @@ describe('ExecutionController', () => { const probationWindow = 500; beforeEach(async () => { - const port = await findPort({ assetsPort: 16000, start: 16001, end: 18000 }); + const port = await findPort(); testContext = new TestContext({ assignment: 'execution_controller', diff --git a/packages/teraslice/test/workers/execution-controller/execution-special-test-cases-spec.ts b/packages/teraslice/test/workers/execution-controller/execution-special-test-cases-spec.ts index 32033bab735..18b87d1a46c 100644 --- a/packages/teraslice/test/workers/execution-controller/execution-special-test-cases-spec.ts +++ b/packages/teraslice/test/workers/execution-controller/execution-special-test-cases-spec.ts @@ -352,7 +352,7 @@ describe('ExecutionController Special Tests', () => { beforeAll(async () => { slices = []; - const port = await findPort({ assetsPort: 18000, start: 18001, end: 20000 }); + const port = await findPort(); testContext = new TestContext({ assignment: 'execution_controller', diff --git a/packages/teraslice/test/workers/execution-controller/execution-test-cases-spec.ts b/packages/teraslice/test/workers/execution-controller/execution-test-cases-spec.ts index 6b7cdb3bfff..f7e776da65e 100644 --- a/packages/teraslice/test/workers/execution-controller/execution-test-cases-spec.ts +++ b/packages/teraslice/test/workers/execution-controller/execution-test-cases-spec.ts @@ -158,7 +158,7 @@ describe('ExecutionController Test Cases', () => { beforeAll(async () => { slices = []; - const port = await findPort({ assetsPort: 20000, start: 20001, end: 22000 }); + const port = await findPort(); testContext = new TestContext({ assignment: 'execution_controller', diff --git a/packages/teraslice/test/workers/helpers/test-context.ts b/packages/teraslice/test/workers/helpers/test-context.ts index 7a715b16f4c..f5e77b09a64 100644 --- a/packages/teraslice/test/workers/helpers/test-context.ts +++ b/packages/teraslice/test/workers/helpers/test-context.ts @@ -152,7 +152,7 @@ export class TestContext { async addClusterMaster() { if (this.clusterMaster) return this.clusterMaster; - const port = await findPort({ assetsPort: 22000, start: 22001, end: 24000 }); + const port = await findPort(); const networkLatencyBuffer = get( this.context, 'sysconfig.teraslice.network_latency_buffer' diff --git a/packages/teraslice/test/workers/worker/worker-spec.ts b/packages/teraslice/test/workers/worker/worker-spec.ts index 1cdc412f3d4..3cc815a8b38 100644 --- a/packages/teraslice/test/workers/worker/worker-spec.ts +++ b/packages/teraslice/test/workers/worker/worker-spec.ts @@ -15,7 +15,7 @@ describe('Worker', () => { } async function setupTest(options: any = {}): Promise { - const slicerPort = await findPort({ assetsPort: 24000, start: 24001, end: 26000 }); + const slicerPort = await findPort(); options.slicerPort = slicerPort; const testContext = new TestContext(options); diff --git a/types/porty/index.d.ts b/types/porty/index.d.ts deleted file mode 100644 index 96f8e70a9c4..00000000000 --- a/types/porty/index.d.ts +++ /dev/null @@ -1,13 +0,0 @@ -/** Declaration file generated by dts-gen */ - -export const HOST: string; - -export const MAX: number; - -export const MIN: number; - -export function find(...args: any[]): any; - -export function get(...args: any[]): any; - -export function test(port: any): any; diff --git a/yarn.lock b/yarn.lock index 6882e2ac0af..1660b76db9b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5594,6 +5594,11 @@ get-package-type@^0.1.0: resolved "https://registry.yarnpkg.com/get-package-type/-/get-package-type-0.1.0.tgz#8de2d803cff44df3bc6c456e6668b36c3926e11a" integrity sha512-pjzuKtY64GYfWizNAJ0fr9VqttZkNiK2iS430LtIHzjBEr6bX8Am2zm4sW4Ro5wjWW5cAlRL1qAMTcXbjNAO2Q== +get-port@~7.1.0: + version "7.1.0" + resolved "https://registry.yarnpkg.com/get-port/-/get-port-7.1.0.tgz#d5a500ebfc7aa705294ec2b83cc38c5d0e364fec" + integrity sha512-QB9NKEeDg3xxVwCCwJQ9+xycaz6pBB6iQ76wiWMl1927n0Kir6alPiP+yuiICLLU4jpMe08dXfpebuQppFA2zw== + get-stdin@^9.0.0: version "9.0.0" resolved "https://registry.yarnpkg.com/get-stdin/-/get-stdin-9.0.0.tgz#3983ff82e03d56f1b2ea0d3e60325f39d703a575" @@ -8288,11 +8293,6 @@ point-in-polygon-hao@^1.1.0, point-in-polygon-hao@~1.1.0: resolved "https://registry.yarnpkg.com/point-in-polygon-hao/-/point-in-polygon-hao-1.1.0.tgz#37f5f4fbe14e89fa8a3bb7f67c9158079d2ede7c" integrity sha512-3hTIM2j/v9Lio+wOyur3kckD4NxruZhpowUbEgmyikW+a2Kppjtu1eN+AhnMQtoHW46zld88JiYWv6fxpsDrTQ== -porty@~3.1.1: - version "3.1.1" - resolved "https://registry.yarnpkg.com/porty/-/porty-3.1.1.tgz#3477f31c6b15bcda6a27c4292057181798e5c3c1" - integrity sha512-aNg4WE90L7vE/AFYeSOghnbcDMIfIYEDI4LvWeAP4iwSHB6sljSQGeGw5h04RQPN+ArhhE+NY+75WOndwMNFIA== - possible-typed-array-names@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/possible-typed-array-names/-/possible-typed-array-names-1.0.0.tgz#89bb63c6fada2c3e90adc4a647beeeb39cc7bf8f" From bac25aff31c8db779da23b18d44c635c699edaa1 Mon Sep 17 00:00:00 2001 From: busma13 Date: Mon, 9 Dec 2024 16:08:35 -0700 Subject: [PATCH 13/21] try lsof instead of netstat --- packages/scripts/src/helpers/scripts.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/scripts/src/helpers/scripts.ts b/packages/scripts/src/helpers/scripts.ts index 16bff63985e..42440681c65 100644 --- a/packages/scripts/src/helpers/scripts.ts +++ b/packages/scripts/src/helpers/scripts.ts @@ -772,7 +772,7 @@ export async function logTCPPorts() { try { // These netstat flags were chosen because they work in // both the mac and linux versions of the command - const netstat = await execa('netstat', ['-an', '-p', 'tcp'], { shell: true, reject: false }); + const netstat = await execa('lsof', ['-iTCP', '-n'], { shell: true, reject: false }); signale.info('TCP Ports:\n', netstat.stdout); } catch (err) { signale.error('Netstat command failed trying to log ports: ', err); From 56ff503d781a07a92b5a550ffdbdedb1a0c2e87d Mon Sep 17 00:00:00 2001 From: busma13 Date: Mon, 9 Dec 2024 16:09:58 -0700 Subject: [PATCH 14/21] yarn sync --- packages/teraslice-messaging/package.json | 2 +- packages/teraslice/package.json | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/teraslice-messaging/package.json b/packages/teraslice-messaging/package.json index c1e4cd64378..db56fc6c1f1 100644 --- a/packages/teraslice-messaging/package.json +++ b/packages/teraslice-messaging/package.json @@ -37,10 +37,10 @@ "dependencies": { "@terascope/types": "~1.3.1", "@terascope/utils": "~1.5.0", + "get-port": "~7.1.0", "ms": "~2.1.3", "nanoid": "~5.0.9", "p-event": "~6.0.1", - "get-port": "~7.1.0", "socket.io": "~1.7.4", "socket.io-client": "~1.7.4" }, diff --git a/packages/teraslice/package.json b/packages/teraslice/package.json index f6bdd649bb3..2517fc7609f 100644 --- a/packages/teraslice/package.json +++ b/packages/teraslice/package.json @@ -53,6 +53,7 @@ "express": "~4.21.2", "fs-extra": "~11.2.0", "gc-stats": "~1.4.0", + "get-port": "~7.1.0", "got": "~13.0.0", "ip": "~2.0.1", "kubernetes-client": "~9.0.0", From 4a8d21ac5448f68fac79b4fa44aa4d46624bc337 Mon Sep 17 00:00:00 2001 From: busma13 Date: Mon, 9 Dec 2024 16:25:38 -0700 Subject: [PATCH 15/21] remove --silent from YARN_SETUP_ARGS --- .github/workflows/test.yml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d70f4e40b2d..e338a182cb5 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -43,7 +43,7 @@ jobs: - name: Install and build packages run: yarn setup env: - YARN_SETUP_ARGS: "--prod=false --silent" + YARN_SETUP_ARGS: "--prod=false" check-docker-limit-before: uses: terascope/workflows/.github/workflows/check-docker-limit.yml@f5e098fa861fe7744fa61842e82124f806364be9 @@ -73,7 +73,7 @@ jobs: - name: Install and build packages run: yarn setup env: - YARN_SETUP_ARGS: "--prod=false --silent" + YARN_SETUP_ARGS: "--prod=false" - name: Test run: yarn --silent test -- --suite unit -- @@ -107,7 +107,7 @@ jobs: - name: Install and build packages run: yarn setup env: - YARN_SETUP_ARGS: "--prod=false --silent" + YARN_SETUP_ARGS: "--prod=false" - name: Create Docker Image List run: | @@ -154,7 +154,7 @@ jobs: - name: Install and build packages run: yarn setup env: - YARN_SETUP_ARGS: "--prod=false --silent" + YARN_SETUP_ARGS: "--prod=false" - name: Create Docker Image List run: | @@ -190,7 +190,7 @@ jobs: - name: Install and build packages run: yarn setup env: - YARN_SETUP_ARGS: "--prod=false --silent" + YARN_SETUP_ARGS: "--prod=false" - name: Lint codebase run: yarn lint @@ -227,7 +227,7 @@ jobs: - name: Install and build packages run: yarn setup env: - YARN_SETUP_ARGS: "--prod=false --silent" + YARN_SETUP_ARGS: "--prod=false" - name: Create Docker Image List run: | @@ -274,7 +274,7 @@ jobs: - name: Install and build packages run: yarn setup env: - YARN_SETUP_ARGS: "--prod=false --silent" + YARN_SETUP_ARGS: "--prod=false" - name: Create Docker Image List run: | @@ -324,7 +324,7 @@ jobs: - name: Install and build packages run: yarn setup env: - YARN_SETUP_ARGS: "--prod=false --silent" + YARN_SETUP_ARGS: "--prod=false" - name: Create Docker Image List run: | @@ -379,7 +379,7 @@ jobs: - name: Install and build packages run: yarn setup env: - YARN_SETUP_ARGS: "--prod=false --silent" + YARN_SETUP_ARGS: "--prod=false" - name: Create Docker Image List run: | @@ -434,7 +434,7 @@ jobs: - name: Install and build packages run: yarn setup env: - YARN_SETUP_ARGS: "--prod=false --silent" + YARN_SETUP_ARGS: "--prod=false" - name: Create Docker Image List run: | @@ -484,7 +484,7 @@ jobs: - name: Install and build packages run: yarn setup env: - YARN_SETUP_ARGS: "--prod=false --silent" + YARN_SETUP_ARGS: "--prod=false" - name: Create Docker Image List run: | From 1353c4da839cdaff9f834e2ccca7f0d46a9d4f61 Mon Sep 17 00:00:00 2001 From: busma13 Date: Tue, 10 Dec 2024 08:49:06 -0700 Subject: [PATCH 16/21] Change port logging command based on platform --- packages/scripts/src/helpers/scripts.ts | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/packages/scripts/src/helpers/scripts.ts b/packages/scripts/src/helpers/scripts.ts index 42440681c65..636f1caefb7 100644 --- a/packages/scripts/src/helpers/scripts.ts +++ b/packages/scripts/src/helpers/scripts.ts @@ -770,11 +770,15 @@ async function showAssets(tsPort: string) { export async function logTCPPorts() { try { - // These netstat flags were chosen because they work in - // both the mac and linux versions of the command - const netstat = await execa('lsof', ['-iTCP', '-n'], { shell: true, reject: false }); - signale.info('TCP Ports:\n', netstat.stdout); + // Determine the appropriate command based on the OS + const command = process.platform === 'darwin' ? 'netstat' : 'ss'; + const args = process.platform === 'darwin' + ? ['-an', '-f', 'inet', '-p', 'tcp'] + : ['-tan4']; + + const { stdout } = await execa(command, args, { shell: true, reject: false }); + signale.info('TCP Ports:\n', stdout); } catch (err) { - signale.error('Netstat command failed trying to log ports: ', err); + signale.error('Execa command failed trying to log ports: ', err); } } From 7f3a1676889f543bbde92af3614bb772a0639ed5 Mon Sep 17 00:00:00 2001 From: busma13 Date: Wed, 11 Dec 2024 07:28:14 -0700 Subject: [PATCH 17/21] replace ternaries with if/else --- packages/scripts/src/helpers/scripts.ts | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/packages/scripts/src/helpers/scripts.ts b/packages/scripts/src/helpers/scripts.ts index 636f1caefb7..1c8f8123287 100644 --- a/packages/scripts/src/helpers/scripts.ts +++ b/packages/scripts/src/helpers/scripts.ts @@ -770,11 +770,16 @@ async function showAssets(tsPort: string) { export async function logTCPPorts() { try { - // Determine the appropriate command based on the OS - const command = process.platform === 'darwin' ? 'netstat' : 'ss'; - const args = process.platform === 'darwin' - ? ['-an', '-f', 'inet', '-p', 'tcp'] - : ['-tan4']; + let command: string; + let args: string[]; + + if (process.platform === 'darwin') { + command = 'netstat'; + args = ['-an', '-f', 'inet', '-p', 'tcp']; + } else { + command = 'ss'; + args = ['-tan4']; + } const { stdout } = await execa(command, args, { shell: true, reject: false }); signale.info('TCP Ports:\n', stdout); From 12ead0961b1904473925f1efe3f0d9f35ea9a2f6 Mon Sep 17 00:00:00 2001 From: busma13 Date: Wed, 11 Dec 2024 08:41:51 -0700 Subject: [PATCH 18/21] fix duplicate package import --- packages/teraslice/package.json | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/teraslice/package.json b/packages/teraslice/package.json index 2517fc7609f..3f964798a20 100644 --- a/packages/teraslice/package.json +++ b/packages/teraslice/package.json @@ -59,7 +59,6 @@ "kubernetes-client": "~9.0.0", "ms": "~2.1.3", "nanoid": "~5.0.9", - "get-port": "~7.1.0", "semver": "~7.6.3", "socket.io": "~1.7.4", "socket.io-client": "~1.7.4", From 0992646eae843b0a2517dec128197d85b61a2f18 Mon Sep 17 00:00:00 2001 From: busma13 Date: Fri, 13 Dec 2024 09:17:32 -0700 Subject: [PATCH 19/21] comments, clean up --- package.json | 2 +- packages/teraslice/src/lib/utils/port_utils.ts | 9 ++++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/package.json b/package.json index cdb7ad5c353..13082ac6bdd 100644 --- a/package.json +++ b/package.json @@ -19,7 +19,7 @@ "build:doctor": "./scripts/build-doctor.sh", "build:fix": "echo '[DEPRECATED], use yarn run build:doctor instead'", "build:pkg": "./scripts/build-pkg.sh", - "build:watch": "tsc --build --watch", + "build:watch": "yarn run build --watch", "bump": "ts-scripts bump", "docker:limit": "./scripts/docker-limit-check.sh", "docker:listImages": "ts-scripts images list", diff --git a/packages/teraslice/src/lib/utils/port_utils.ts b/packages/teraslice/src/lib/utils/port_utils.ts index 177cd922c52..00f39c2dc4e 100644 --- a/packages/teraslice/src/lib/utils/port_utils.ts +++ b/packages/teraslice/src/lib/utils/port_utils.ts @@ -6,6 +6,9 @@ import { import getPort from 'get-port'; import { Context } from '@terascope/job-components'; import range from 'lodash/range.js'; +import getPort from 'get-port'; +import { get, toIntegerOrThrow } from '@terascope/utils'; +import { Context } from '@terascope/job-components'; export interface PortOptions { start?: number; @@ -16,16 +19,16 @@ export interface PortOptions { const usedPorts: number[] = []; /** - * Return a random open port between start and end, excluding usedPorts. + * Return a random open port between start(inclusive) and end(exclusive), excluding usedPorts. * If all ports in that range are being used a random port between 1024 and 65535 * will be returned instead. - * @param {PortOptions} options Start, end, and assetsPort to exclude can be specified + * @param {PortOptions} options Start, end, and assetsPort to exclude * @returns {number} */ export async function findPort(options: PortOptions = {}) { const { start = 1024, - end = 65536, // exclusive + end = 65536, assetsPort = 8003 } = options; From 0a1e3ed79f5fd3864d593fe3018a09122450bc54 Mon Sep 17 00:00:00 2001 From: busma13 Date: Mon, 16 Dec 2024 11:22:44 -0700 Subject: [PATCH 20/21] add range to utils lodash imports --- packages/teraslice/src/lib/utils/port_utils.ts | 7 +------ packages/utils/src/deps.ts | 6 ++++-- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/packages/teraslice/src/lib/utils/port_utils.ts b/packages/teraslice/src/lib/utils/port_utils.ts index 00f39c2dc4e..ff99ee9369b 100644 --- a/packages/teraslice/src/lib/utils/port_utils.ts +++ b/packages/teraslice/src/lib/utils/port_utils.ts @@ -1,14 +1,9 @@ import { - get, times, pDelay, - toIntegerOrThrow, shuffle, range + get, toIntegerOrThrow, shuffle, range } from '@terascope/utils'; import getPort from 'get-port'; import { Context } from '@terascope/job-components'; -import range from 'lodash/range.js'; -import getPort from 'get-port'; -import { get, toIntegerOrThrow } from '@terascope/utils'; -import { Context } from '@terascope/job-components'; export interface PortOptions { start?: number; diff --git a/packages/utils/src/deps.ts b/packages/utils/src/deps.ts index 6c395c43378..e7ed3db39b2 100644 --- a/packages/utils/src/deps.ts +++ b/packages/utils/src/deps.ts @@ -6,7 +6,8 @@ import { has, set, get, unset, merge, debounce, padEnd, difference, throttle, chain, - orderBy, shuffle, defaultsDeep, sortBy + orderBy, shuffle, defaultsDeep, + sortBy, range } from 'lodash-es'; import { isPlainObject as _isPlainObject } from 'is-plain-object'; import _clone from 'shallow-clone'; @@ -131,5 +132,6 @@ export { orderBy, shuffle, defaultsDeep, - multiFieldSort + multiFieldSort, + range }; From 83be72cf7e819b8ba9eb382fc73b19e65a9bfb4f Mon Sep 17 00:00:00 2001 From: busma13 Date: Mon, 16 Dec 2024 13:35:32 -0700 Subject: [PATCH 21/21] release: (minor) teraslice@2.10.0 bump: (minor) @terascope/teraslice-messaging@1.9.0, @terascope/utils@1.6.0 bump: (minor) @terascope/data-types@1.6.0, @terascope/data-mate@1.6.0 bump: (minor) elasticsearch-store@1.6.0, terafoundation@1.8.0 bump: (minor) ts-transforms@1.6.0, xlucene-parser@1.6.0 bump: (minor) xlucene-translator@1.6.0, @terascope/elasticsearch-api@4.6.0 bump: (minor) @terascope/teraslice-state-storage@1.6.0, @terascope/job-components@1.8.0 --- e2e/package.json | 2 +- packages/data-mate/package.json | 8 ++++---- packages/data-types/package.json | 4 ++-- packages/elasticsearch-api/package.json | 6 +++--- packages/elasticsearch-store/package.json | 10 +++++----- packages/job-components/package.json | 4 ++-- packages/scripts/package.json | 4 ++-- packages/terafoundation/package.json | 6 +++--- packages/teraslice-cli/package.json | 4 ++-- packages/teraslice-client-js/package.json | 4 ++-- packages/teraslice-messaging/package.json | 4 ++-- packages/teraslice-state-storage/package.json | 6 +++--- packages/teraslice-test-harness/package.json | 4 ++-- packages/teraslice/package.json | 10 +++++----- packages/ts-transforms/package.json | 6 +++--- packages/utils/package.json | 2 +- packages/xlucene-parser/package.json | 4 ++-- packages/xlucene-translator/package.json | 6 +++--- packages/xpressions/package.json | 4 ++-- 19 files changed, 49 insertions(+), 49 deletions(-) diff --git a/e2e/package.json b/e2e/package.json index 08670936c19..2bed25a0419 100644 --- a/e2e/package.json +++ b/e2e/package.json @@ -45,7 +45,7 @@ "devDependencies": { "@terascope/types": "~1.3.1", "bunyan": "~1.8.15", - "elasticsearch-store": "~1.5.0", + "elasticsearch-store": "~1.6.0", "fs-extra": "~11.2.0", "ms": "~2.1.3", "nanoid": "~5.0.9", diff --git a/packages/data-mate/package.json b/packages/data-mate/package.json index 3e76359a129..dad220334fa 100644 --- a/packages/data-mate/package.json +++ b/packages/data-mate/package.json @@ -1,7 +1,7 @@ { "name": "@terascope/data-mate", "displayName": "Data-Mate", - "version": "1.5.0", + "version": "1.6.0", "description": "Library of data validations/transformations", "homepage": "https://github.com/terascope/teraslice/tree/master/packages/data-mate#readme", "repository": { @@ -30,9 +30,9 @@ "test:watch": "ts-scripts test --watch . --" }, "dependencies": { - "@terascope/data-types": "~1.5.0", + "@terascope/data-types": "~1.6.0", "@terascope/types": "~1.3.1", - "@terascope/utils": "~1.5.0", + "@terascope/utils": "~1.6.0", "@types/validator": "~13.12.2", "awesome-phonenumber": "~7.2.0", "date-fns": "~4.1.0", @@ -45,7 +45,7 @@ "uuid": "~11.0.3", "valid-url": "~1.0.9", "validator": "~13.12.0", - "xlucene-parser": "~1.5.0" + "xlucene-parser": "~1.6.0" }, "devDependencies": { "@types/ip6addr": "~0.2.6", diff --git a/packages/data-types/package.json b/packages/data-types/package.json index 21b935d279e..4f0f8b6a1db 100644 --- a/packages/data-types/package.json +++ b/packages/data-types/package.json @@ -1,7 +1,7 @@ { "name": "@terascope/data-types", "displayName": "Data Types", - "version": "1.5.0", + "version": "1.6.0", "description": "A library for defining the data structures and mapping", "homepage": "https://github.com/terascope/teraslice/tree/master/packages/data-types#readme", "bugs": { @@ -28,7 +28,7 @@ }, "dependencies": { "@terascope/types": "~1.3.1", - "@terascope/utils": "~1.5.0", + "@terascope/utils": "~1.6.0", "graphql": "~16.9.0", "yargs": "~17.7.2" }, diff --git a/packages/elasticsearch-api/package.json b/packages/elasticsearch-api/package.json index b98e3dfe621..d3da7b80e27 100644 --- a/packages/elasticsearch-api/package.json +++ b/packages/elasticsearch-api/package.json @@ -1,7 +1,7 @@ { "name": "@terascope/elasticsearch-api", "displayName": "Elasticsearch API", - "version": "4.5.0", + "version": "4.6.0", "description": "Elasticsearch client api used across multiple services, handles retries and exponential backoff", "homepage": "https://github.com/terascope/teraslice/tree/master/packages/elasticsearch-api#readme", "bugs": { @@ -25,7 +25,7 @@ }, "dependencies": { "@terascope/types": "~1.3.1", - "@terascope/utils": "~1.5.0", + "@terascope/utils": "~1.6.0", "bluebird": "~3.7.2", "setimmediate": "~1.0.5" }, @@ -33,7 +33,7 @@ "@opensearch-project/opensearch": "~1.2.0", "@types/elasticsearch": "~5.0.43", "elasticsearch": "~15.4.1", - "elasticsearch-store": "~1.5.0", + "elasticsearch-store": "~1.6.0", "elasticsearch6": "npm:@elastic/elasticsearch@~6.8.0", "elasticsearch7": "npm:@elastic/elasticsearch@~7.17.0", "elasticsearch8": "npm:@elastic/elasticsearch@~8.15.0" diff --git a/packages/elasticsearch-store/package.json b/packages/elasticsearch-store/package.json index 470532d8edc..93db96a2a2f 100644 --- a/packages/elasticsearch-store/package.json +++ b/packages/elasticsearch-store/package.json @@ -1,7 +1,7 @@ { "name": "elasticsearch-store", "displayName": "Elasticsearch Store", - "version": "1.5.0", + "version": "1.6.0", "description": "An API for managing an elasticsearch index, with versioning and migration support.", "homepage": "https://github.com/terascope/teraslice/tree/master/packages/elasticsearch-store#readme", "bugs": { @@ -30,10 +30,10 @@ "test:watch": "ts-scripts test --watch . --" }, "dependencies": { - "@terascope/data-mate": "~1.5.0", - "@terascope/data-types": "~1.5.0", + "@terascope/data-mate": "~1.6.0", + "@terascope/data-types": "~1.6.0", "@terascope/types": "~1.3.1", - "@terascope/utils": "~1.5.0", + "@terascope/utils": "~1.6.0", "ajv": "~8.17.1", "ajv-formats": "~3.0.1", "elasticsearch6": "npm:@elastic/elasticsearch@~6.8.0", @@ -43,7 +43,7 @@ "opensearch2": "npm:@opensearch-project/opensearch@~2.12.0", "setimmediate": "~1.0.5", "uuid": "~11.0.3", - "xlucene-translator": "~1.5.0" + "xlucene-translator": "~1.6.0" }, "devDependencies": { "@types/uuid": "~10.0.0" diff --git a/packages/job-components/package.json b/packages/job-components/package.json index 242b3713ab1..1eb38f683aa 100644 --- a/packages/job-components/package.json +++ b/packages/job-components/package.json @@ -1,7 +1,7 @@ { "name": "@terascope/job-components", "displayName": "Job Components", - "version": "1.7.0", + "version": "1.8.0", "description": "A teraslice library for validating jobs schemas, registering apis, and defining and running new Job APIs", "homepage": "https://github.com/terascope/teraslice/tree/master/packages/job-components#readme", "bugs": { @@ -33,7 +33,7 @@ }, "dependencies": { "@terascope/types": "~1.3.1", - "@terascope/utils": "~1.5.0", + "@terascope/utils": "~1.6.0", "convict": "~6.2.4", "convict-format-with-moment": "~6.2.0", "convict-format-with-validator": "~6.2.0", diff --git a/packages/scripts/package.json b/packages/scripts/package.json index 90d40d2ae35..63f89d559fa 100644 --- a/packages/scripts/package.json +++ b/packages/scripts/package.json @@ -1,7 +1,7 @@ { "name": "@terascope/scripts", "displayName": "Scripts", - "version": "1.6.0", + "version": "1.7.0", "description": "A collection of terascope monorepo scripts", "homepage": "https://github.com/terascope/teraslice/tree/master/packages/scripts#readme", "bugs": { @@ -33,7 +33,7 @@ }, "dependencies": { "@kubernetes/client-node": "~0.22.3", - "@terascope/utils": "~1.5.0", + "@terascope/utils": "~1.6.0", "codecov": "~3.8.3", "execa": "~9.5.2", "fs-extra": "~11.2.0", diff --git a/packages/terafoundation/package.json b/packages/terafoundation/package.json index 3971a9eabcd..9b4ef1a8f7c 100644 --- a/packages/terafoundation/package.json +++ b/packages/terafoundation/package.json @@ -1,7 +1,7 @@ { "name": "terafoundation", "displayName": "Terafoundation", - "version": "1.7.0", + "version": "1.8.0", "description": "A Clustering and Foundation tool for Terascope Tools", "homepage": "https://github.com/terascope/teraslice/tree/master/packages/terafoundation#readme", "bugs": { @@ -30,14 +30,14 @@ "dependencies": { "@terascope/file-asset-apis": "~1.0.3", "@terascope/types": "~1.3.1", - "@terascope/utils": "~1.5.0", + "@terascope/utils": "~1.6.0", "bluebird": "~3.7.2", "bunyan": "~1.8.15", "convict": "~6.2.4", "convict-format-with-moment": "~6.2.0", "convict-format-with-validator": "~6.2.0", "elasticsearch": "~15.4.1", - "elasticsearch-store": "~1.5.0", + "elasticsearch-store": "~1.6.0", "express": "~4.21.2", "js-yaml": "~4.1.0", "nanoid": "~5.0.9", diff --git a/packages/teraslice-cli/package.json b/packages/teraslice-cli/package.json index 911619ed672..b760bdb04ca 100644 --- a/packages/teraslice-cli/package.json +++ b/packages/teraslice-cli/package.json @@ -43,7 +43,7 @@ "devDependencies": { "@terascope/fetch-github-release": "~1.0.0", "@terascope/types": "~1.3.1", - "@terascope/utils": "~1.5.0", + "@terascope/utils": "~1.6.0", "@types/decompress": "~4.2.7", "@types/diff": "~6.0.0", "@types/ejs": "~3.1.5", @@ -67,7 +67,7 @@ "pretty-bytes": "~6.1.1", "prompts": "~2.4.2", "signale": "~1.4.0", - "teraslice-client-js": "~1.5.0", + "teraslice-client-js": "~1.6.0", "tmp": "~0.2.0", "tty-table": "~4.2.3", "yargs": "~17.7.2" diff --git a/packages/teraslice-client-js/package.json b/packages/teraslice-client-js/package.json index 759281d68ec..a6b33022cdb 100644 --- a/packages/teraslice-client-js/package.json +++ b/packages/teraslice-client-js/package.json @@ -1,7 +1,7 @@ { "name": "teraslice-client-js", "displayName": "Teraslice Client (JavaScript)", - "version": "1.5.0", + "version": "1.6.0", "description": "A Node.js client for teraslice jobs, assets, and cluster references.", "keywords": [ "elasticsearch", @@ -33,7 +33,7 @@ }, "dependencies": { "@terascope/types": "~1.3.1", - "@terascope/utils": "~1.5.0", + "@terascope/utils": "~1.6.0", "auto-bind": "~5.0.1", "got": "~13.0.0" }, diff --git a/packages/teraslice-messaging/package.json b/packages/teraslice-messaging/package.json index db56fc6c1f1..4e457713b4f 100644 --- a/packages/teraslice-messaging/package.json +++ b/packages/teraslice-messaging/package.json @@ -1,7 +1,7 @@ { "name": "@terascope/teraslice-messaging", "displayName": "Teraslice Messaging", - "version": "1.8.0", + "version": "1.9.0", "description": "An internal teraslice messaging library using socket.io", "homepage": "https://github.com/terascope/teraslice/tree/master/packages/teraslice-messaging#readme", "bugs": { @@ -36,7 +36,7 @@ }, "dependencies": { "@terascope/types": "~1.3.1", - "@terascope/utils": "~1.5.0", + "@terascope/utils": "~1.6.0", "get-port": "~7.1.0", "ms": "~2.1.3", "nanoid": "~5.0.9", diff --git a/packages/teraslice-state-storage/package.json b/packages/teraslice-state-storage/package.json index 7ad80dc26e0..94642aa24c8 100644 --- a/packages/teraslice-state-storage/package.json +++ b/packages/teraslice-state-storage/package.json @@ -1,7 +1,7 @@ { "name": "@terascope/teraslice-state-storage", "displayName": "Teraslice State Storage", - "version": "1.5.0", + "version": "1.6.0", "description": "State storage operation api for teraslice", "homepage": "https://github.com/terascope/teraslice/tree/master/packages/teraslice-state-storage#readme", "bugs": { @@ -24,8 +24,8 @@ "test:watch": "ts-scripts test --watch . --" }, "dependencies": { - "@terascope/elasticsearch-api": "~4.5.0", - "@terascope/utils": "~1.5.0" + "@terascope/elasticsearch-api": "~4.6.0", + "@terascope/utils": "~1.6.0" }, "engines": { "node": ">=18.18.0", diff --git a/packages/teraslice-test-harness/package.json b/packages/teraslice-test-harness/package.json index 3a80967d65e..2eefcd6c756 100644 --- a/packages/teraslice-test-harness/package.json +++ b/packages/teraslice-test-harness/package.json @@ -36,10 +36,10 @@ "fs-extra": "~11.2.0" }, "devDependencies": { - "@terascope/job-components": "~1.7.0" + "@terascope/job-components": "~1.8.0" }, "peerDependencies": { - "@terascope/job-components": ">=1.7.0" + "@terascope/job-components": ">=1.8.0" }, "engines": { "node": ">=18.18.0", diff --git a/packages/teraslice/package.json b/packages/teraslice/package.json index 3f964798a20..68c88e355d0 100644 --- a/packages/teraslice/package.json +++ b/packages/teraslice/package.json @@ -39,11 +39,11 @@ }, "dependencies": { "@kubernetes/client-node": "~0.22.3", - "@terascope/elasticsearch-api": "~4.5.0", - "@terascope/job-components": "~1.7.0", - "@terascope/teraslice-messaging": "~1.8.0", + "@terascope/elasticsearch-api": "~4.6.0", + "@terascope/job-components": "~1.8.0", + "@terascope/teraslice-messaging": "~1.9.0", "@terascope/types": "~1.3.1", - "@terascope/utils": "~1.5.0", + "@terascope/utils": "~1.6.0", "async-mutex": "~0.5.0", "barbe": "~3.0.16", "body-parser": "~1.20.2", @@ -62,7 +62,7 @@ "semver": "~7.6.3", "socket.io": "~1.7.4", "socket.io-client": "~1.7.4", - "terafoundation": "~1.7.0", + "terafoundation": "~1.8.0", "uuid": "~11.0.3" }, "devDependencies": { diff --git a/packages/ts-transforms/package.json b/packages/ts-transforms/package.json index 5c151ccbc99..de8fef0fee9 100644 --- a/packages/ts-transforms/package.json +++ b/packages/ts-transforms/package.json @@ -1,7 +1,7 @@ { "name": "ts-transforms", "displayName": "TS Transforms", - "version": "1.5.0", + "version": "1.6.0", "description": "An ETL framework built upon xlucene-evaluator", "homepage": "https://github.com/terascope/teraslice/tree/master/packages/ts-transforms#readme", "bugs": { @@ -36,9 +36,9 @@ "test:watch": "ts-scripts test --watch . --" }, "dependencies": { - "@terascope/data-mate": "~1.5.0", + "@terascope/data-mate": "~1.6.0", "@terascope/types": "~1.3.1", - "@terascope/utils": "~1.5.0", + "@terascope/utils": "~1.6.0", "awesome-phonenumber": "~7.2.0", "graphlib": "~2.1.8", "jexl": "~2.3.0", diff --git a/packages/utils/package.json b/packages/utils/package.json index e44faf2b0f7..5dd2328de25 100644 --- a/packages/utils/package.json +++ b/packages/utils/package.json @@ -1,7 +1,7 @@ { "name": "@terascope/utils", "displayName": "Utils", - "version": "1.5.0", + "version": "1.6.0", "description": "A collection of Teraslice Utilities", "homepage": "https://github.com/terascope/teraslice/tree/master/packages/utils#readme", "bugs": { diff --git a/packages/xlucene-parser/package.json b/packages/xlucene-parser/package.json index 494e072536d..4d0cf9c0869 100644 --- a/packages/xlucene-parser/package.json +++ b/packages/xlucene-parser/package.json @@ -1,7 +1,7 @@ { "name": "xlucene-parser", "displayName": "xLucene Parser", - "version": "1.5.0", + "version": "1.6.0", "description": "Flexible Lucene-like evaluator and language parser", "homepage": "https://github.com/terascope/teraslice/tree/master/packages/xlucene-parser#readme", "repository": { @@ -34,7 +34,7 @@ }, "dependencies": { "@terascope/types": "~1.3.1", - "@terascope/utils": "~1.5.0", + "@terascope/utils": "~1.6.0", "peggy": "~4.2.0", "ts-pegjs": "~4.2.1" }, diff --git a/packages/xlucene-translator/package.json b/packages/xlucene-translator/package.json index cb8eed7185f..0e01abb7808 100644 --- a/packages/xlucene-translator/package.json +++ b/packages/xlucene-translator/package.json @@ -1,7 +1,7 @@ { "name": "xlucene-translator", "displayName": "xLucene Translator", - "version": "1.5.0", + "version": "1.6.0", "description": "Translate xlucene query to database queries", "homepage": "https://github.com/terascope/teraslice/tree/master/packages/xlucene-translator#readme", "repository": { @@ -30,9 +30,9 @@ }, "dependencies": { "@terascope/types": "~1.3.1", - "@terascope/utils": "~1.5.0", + "@terascope/utils": "~1.6.0", "@types/elasticsearch": "~5.0.43", - "xlucene-parser": "~1.5.0" + "xlucene-parser": "~1.6.0" }, "devDependencies": { "elasticsearch": "~15.4.1" diff --git a/packages/xpressions/package.json b/packages/xpressions/package.json index 44207e7fe49..3a998607a06 100644 --- a/packages/xpressions/package.json +++ b/packages/xpressions/package.json @@ -1,7 +1,7 @@ { "name": "xpressions", "displayName": "Xpressions", - "version": "1.5.0", + "version": "1.6.0", "description": "Variable expressions with date-math support", "homepage": "https://github.com/terascope/teraslice/tree/master/packages/xpressions#readme", "bugs": { @@ -24,7 +24,7 @@ "test:watch": "ts-scripts test --watch . --" }, "dependencies": { - "@terascope/utils": "~1.5.0" + "@terascope/utils": "~1.6.0" }, "devDependencies": { "@terascope/types": "~1.3.1"