[ts-scripts] Add support for kafka and minio tools/brokers in k8s and e2e

e2e/k8s/kafkaDeployment.yaml

@@ -28,15 +28,17 @@ spec:
           - name: KAFKA_ZOOKEEPER_CONNECT
             value: zookeeper:2181
           - name: KAFKA_ADVERTISED_LISTENERS
-            value: INTERNAL://cpkafka.services-dev1:9092
+            value: INTERNAL://cpkafka.services-dev1:9092,EXTERNAL://localhost:9094
           - name: KAFKA_LISTENER_SECURITY_PROTOCOL_MAP
-            value: INTERNAL:PLAINTEXT
+            value: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT
           - name: KAFKA_INTER_BROKER_LISTENER_NAME
             value: INTERNAL
           - name: KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR
             value: "1"
         ports:
         - containerPort: 9092
+        - containerPort: 9094
+
 ---
 kind: Service
 apiVersion: v1
@@ -54,3 +56,7 @@ spec:
     name: cpkafka
     targetPort: 9092
     nodePort: 30092
+  - port: 9094
+    name: cpkafkalocal
+    targetPort: 9094
+    nodePort: 30094

e2e/k8s/kindConfigDefaultPorts.yaml

@@ -10,8 +10,12 @@ nodes:
     hostPort: 5678
   - containerPort: 30092 # Map internal kafka service to host port
     hostPort: 9092
+  - containerPort: 30094 # Map external kafka service to host port
+    hostPort: 9094
   - containerPort: 30900 # Map internal minio service to host port
     hostPort: 9000
+  - containerPort: 30901 # Map internal minio-ui service to host port
+    hostPort: 9001
   extraMounts:
   - hostPath: ./e2e/autoload
     containerPath: /autoload

e2e/k8s/kindConfigDefaultPortsDev.yaml

@@ -14,8 +14,12 @@ nodes:
     hostPort: 5678
   - containerPort: 30092 # Map internal kafka service to host port
     hostPort: 9092
+  - containerPort: 30094 # Map external kafka service to host port
+    hostPort: 9094
   - containerPort: 30900 # Map internal minio service to host port
     hostPort: 9000
+  - containerPort: 30901 # Map internal minio-ui service to host port
+    hostPort: 9001
   extraMounts:
   - hostPath: ./e2e/autoload
     containerPath: /autoload

e2e/k8s/kindConfigTestPorts.yaml

@@ -10,8 +10,12 @@ nodes:
     hostPort: 45678
   - containerPort: 30092 # Map internal kafka service to host port
     hostPort: 49092
+  - containerPort: 30094 # Map external kafka service to host port
+    hostPort: 49094
   - containerPort: 30900 # Map internal minio service to host port
     hostPort: 49000
+  - containerPort: 30901 # Map internal minio-ui service to host port
+    hostPort: 49001
   extraMounts:
   - hostPath: ./e2e/autoload
     containerPath: /autoload

e2e/k8s/minioDeployment.yaml

@@ -20,11 +20,14 @@ spec:
       containers:
       - name: minio
         image: minio/minio:RELEASE.2020-02-07T23-28-16Z
+        command:
+        - /bin/bash
+        - -c
         args:
-        - server
-        - /data
+        - minio server /data --console-address :9001
         ports:
         - containerPort: 9000
+        - containerPort: 9001
         env:
           - name: MINIO_ROOT_USER
             value: minioadmin
@@ -47,3 +50,7 @@ spec:
     name: minio
     targetPort: 9000
     nodePort: 30900
+  - port: 9001
+    name: minio-ui
+    targetPort: 9001
+    nodePort: 30901

packages/scripts/src/helpers/config.ts

@@ -66,6 +66,7 @@ export const KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR = process.env.KAFKA_OFFSETS_
 export const MINIO_NAME = process.env.MINIO_NAME || 'minio';
 export const MINIO_HOSTNAME = process.env.MINIO_HOSTNAME || HOST_IP;
 export const MINIO_PORT = process.env.MINIO_PORT || '49000';
+export const MINIO_UI_PORT = process.env.MINIO_UI_PORT || '49001';
 export const ENCRYPT_MINIO = toBoolean(process.env.ENCRYPT_MINIO ?? false);
 export const MINIO_HOST = `http${ENCRYPT_MINIO ? 's' : ''}://${MINIO_HOSTNAME}:${MINIO_PORT}`;
 export const MINIO_VERSION = process.env.MINIO_VERSION || 'RELEASE.2022-09-22T18-57-27Z';

packages/scripts/src/helpers/test-runner/services.ts

@@ -132,7 +132,7 @@ const services: Readonly<Record<Service, Readonly<DockerRunOptions>>> = {
         tmpfs: config.SERVICES_USE_TMPFS
             ? ['/data']
             : undefined,
-        ports: [`${config.MINIO_PORT}:${config.MINIO_PORT}`],
+        ports: [`${config.MINIO_PORT}:${config.MINIO_PORT}`, `${config.MINIO_UI_PORT}:${config.MINIO_UI_PORT}`],
         mount: config.ENCRYPT_MINIO
             ? [`type=bind,source=${path.join(getRootDir(), '/e2e/test/certs')},target=/opt/certs`]
             : [],
@@ -142,8 +142,8 @@ const services: Readonly<Record<Service, Readonly<DockerRunOptions>>> = {
         },
         network: config.DOCKER_NETWORK_NAME,
         args: config.ENCRYPT_MINIO
-            ? ['server', '-S', '/opt/certs', '--address', `0.0.0.0:${config.MINIO_PORT}`, '/data']
-            : ['server', '--address', `0.0.0.0:${config.MINIO_PORT}`, '/data']
+            ? ['server', '-S', '/opt/certs', '--address', `0.0.0.0:${config.MINIO_PORT}`, '--console-address', `:${config.MINIO_UI_PORT}`, '/data']
+            : ['server', '--address', `0.0.0.0:${config.MINIO_PORT}`, '--console-address', `:${config.MINIO_UI_PORT}`, '/data']
     },
     [Service.RabbitMQ]: {
         image: config.RABBITMQ_DOCKER_IMAGE,