diff --git a/.gitignore b/.gitignore index 5c06b0162..cd0856e5c 100644 --- a/.gitignore +++ b/.gitignore @@ -95,3 +95,6 @@ gentx-*.json # Binaries VSCode builds for debugging cmd/poktrolld/*debug_bin* + +# The temporary file created by e2e scripts +job.yaml \ No newline at end of file diff --git a/Makefile b/Makefile index 28cb34a04..1a46832ec 100644 --- a/Makefile +++ b/Makefile @@ -16,6 +16,7 @@ TESTNET_RPC ?= https://testnet-validated-validator-rpc.poktroll.com/ # TestNet R APPGATE_SERVER ?= http://localhost:42069 GATEWAY_URL ?= http://localhost:42079 POCKET_ADDR_PREFIX = pokt +LOAD_TEST_CUSTOM_MANIFEST ?= loadtest_manifest_example.yaml # The domain ending in ".town" is staging, ".city" is production GROVE_GATEWAY_STAGING_ETH_MAINNET = https://eth-mainnet.rpc.grove.town @@ -402,11 +403,11 @@ test_e2e_settlement: test_e2e_env ## Run only the E2E suite that exercises the s test_e2e_params: test_e2e_env ## Run only the E2E suite that exercises parameter updates for all modules go test -v ./e2e/tests/... -tags=e2e,test --features-path=update_params.feature -.PHONY: test_load_relays_stress_example -test_load_relays_stress_example: ## Run the stress test for E2E relays on a persistent (non-ephemeral) remote (devnet) chain. Note that this is just an example. +.PHONY: test_load_relays_stress_custom +test_load_relays_stress_custom: ## Run the stress test for E2E relays using custom manifest. "loadtest_manifest_example.yaml" manifest is used by default. Set `LOAD_TEST_CUSTOM_MANIFEST` environment variable to use the different manifest. go test -v -count=1 ./load-testing/tests/... \ -tags=load,test -run LoadRelays --log-level=debug --timeout=30m \ - --manifest ./load-testing/loadtest_manifest_example.yaml + --manifest ./load-testing/$(LOAD_TEST_CUSTOM_MANIFEST) .PHONY: test_load_relays_stress_localnet test_load_relays_stress_localnet: test_e2e_env warn_message_local_stress_test ## Run the stress test for E2E relays on LocalNet. diff --git a/docusaurus/docs/operate/infrastructure/devnet.md b/docusaurus/docs/operate/infrastructure/devnet.md index 228789f79..05ce3b679 100644 --- a/docusaurus/docs/operate/infrastructure/devnet.md +++ b/docusaurus/docs/operate/infrastructure/devnet.md @@ -1,9 +1,9 @@ --- sidebar_position: 3 -title: DevNet +title: DevNets --- -# DevNet +# DevNets :::note @@ -20,6 +20,15 @@ to set up your access to GCP. - [Grafana logs](#grafana-logs) - [Infrastructure Provisioning](#infrastructure-provisioning) - [Configuration](#configuration) +- [Interaction with DevNet](#interaction-with-devnet) +- [DevNet types](#devnet-types) +- [Manually provisioned DevNets](#manually-provisioned-devnets) + - [How to create](#how-to-create) + - [How to delete](#how-to-delete) + - [Configuring and updating DevNets](#configuring-and-updating-devnets) + - [Scaling actors up and down](#scaling-actors-up-and-down) + - [Run e2e tests on a manually provisioned DevNet](#run-e2e-tests-on-a-manually-provisioned-devnet) + - [Staking actors](#staking-actors) ## GCP Console @@ -54,3 +63,127 @@ Each DevNet ArgoCD App (following the App of Apps pattern) provisions a Helm cha Each `full-network` includes other ArgoCD applications that deploy Validators and off-chain actors. Each Helm chart receives a list of configuration files. For example, see the [relayminer configuration](https://github.com/pokt-network/protocol-infra/blob/main/charts/full-network/templates/Application-Relayminer.yaml#L37). All possible values can be found in the `values.yaml` of the Helm chart, such as the [relayminer Helm chart](https://github.com/pokt-network/helm-charts/blob/main/charts/relayminer/values.yaml). + +## Interaction with DevNet + +:::note +Devnets are provisioned with the same mnemonc phrases as LocalNet, so it is possible to reuse the same keys +from the keybase - the user just needs to change the `--node=` flag to point to the DevNet RPC endpoint when using the `poktrolld` CLI. +::: + +The DevNet RPC endpoint is exposed on `https://devnet-**NETWORK_NAME**-validator-rpc.poktroll.com`. + +It is possible to reuse the LocalNet make targets with DevNet endpoints. For example + +```bash +POCKET_NODE=https://devnet-issue-420-validator-rpc.poktroll.com make supplier_list +``` + +## DevNet types + +We have two types of DevNets: +1. Automatically provisioned by `devnet-test-e2e` GitHub label. + - Their main purpose is to automatically run e2e tests for each PR this label has been assigned to. + - They are not easily modified and configured. E.g. we automatically provision single instance of each actor only. +2. Manually provisioned by creating a file in [protocol-infra repo under `devnet-configs` directory](https://github.com/pokt-network/protocol-infra/tree/main/devnets-configs). + - Their main purpose is for testing new features or configurations that require more complex setups than the first type can provide. + - They are easily modified and configured, allowing you to add multiple instances of each actor. + +## Manually provisioned DevNets + +### How to create + +Commit a new YAML file in the [devnets-configs](https://github.com/pokt-network/protocol-infra/tree/main/devnets-configs) +directory on the `main` branch. Use [this template](https://github.com/pokt-network/protocol-infra/blob/main/devnets-configs/_TEMPLATE_YAML_) +as a reference. If you are not a member of the protocol team, open a pull request (PR) to add the new YAML file to the +`main` branch. Protocol team members can bypass the `main` branch protection to commit directly without going through +the PR flow. + +### How to delete + +Remove the devnet config file by committing the deletion to the `main` branch. If you are not a member of the protocol +team, open a pull request (PR) to delete the YAML file from the `main` branch. Protocol team members can bypass the +`main` branch protection to commit the deletion directly without going through the PR flow. + +### Configuring and updating DevNets + +New container images are automatically built on each merge to the `main` branch and each push to non-main branches for which there exists a PR with the `push-image` label. +Images named according to the following format: + +``` +ghcr.io/pokt-network/poktrolld:sha-7042be3 +ghcr.io/pokt-network/poktrolld:sha-7042be3922245fb4313ee90e1f28d0f402b700a0 +``` + +You can update the version of DevNet by changing this parameter in the devnet config file: + +```yaml +image: + tag: sha-7042be3 +``` + +All parameters in this config are passed to the [downstream helm chart](https://github.com/pokt-network/protocol-infra/tree/main/charts/full-network) via an [ArgoCD Application](https://github.com/pokt-network/protocol-infra/blob/main/clusters/protocol-us-central1/devnets-persistent.yaml). +Here is a [list of all options](https://github.com/pokt-network/protocol-infra/blob/main/charts/full-network/values.yaml) which can be modified, in addition to the image tag. + +### Scaling actors up and down + +You can modify the number of each actor by changing the devnet config file: + +:::info +We use the same ignite `config.yaml` to provision genesis in devnet as is used in localnet. Because localnet supports a max of 3 of each actor type, any devnet deployment with more actors would also require additional corresponding genesis state to be included in `config.yaml`, or on-demand account funding and staking. General rule of thumb: don't go over `3`. +::: + + +```yaml +appgateservers: + count: 1 +gateways: + count: 1 +relayminers: + count: 1 +``` + +### Run e2e tests on a manually provisioned DevNet + +To run e2e tests on a manually provisioned DevNet, use the following command from the root of the poktroll repo: + +```bash +IMAGE_TAG=**IMAGE TAG NAME FROM DEVNET CONFIG** NAMESPACE=devnet-**NETWORK NAME** JOB_NAME=e2e-test-**GITSHA FROM IMAGE TAG** POCKET_NODE=tcp://devnet-**NETWORK NAME**-validator-poktrolld:26657 bash .github/workflows-helpers/run-e2e-test.sh +``` + +**Environment Variables**: + +- **IMAGE_TAG**: The tag of the image from the devnet config YAML file. This tag must match the tag of the image used in the DevNet configuration. +- **NAMESPACE**: The name of the devnet. This should be specified in the environment variables and follow the format devnet-**NETWORK NAME**. +- **JOB_NAME**: A unique identifier for the e2e test job. It follows the format e2e-test-**GITSHA FROM IMAGE TAG**. +- **POCKET_NODE**: The address of the pocket node, following the format tcp://devnet-**NETWORK NAME**-validator-poktrolld:26657. + + +**Example** + +```bash +IMAGE_TAG=sha-7042be3 NAMESPACE=devnet-sophon JOB_NAME=e2e-test-7042be3 POCKET_NODE=tcp://devnet-sophon-validator-poktrolld:26657 bash .github/workflows-helpers/run-e2e-test.sh +``` + +:::info +- The `IMAGE_TAG` must match the tag of the image from the devnet config YAML file. +- The name of the devnet in the environment variables must be specified. +- The Kubernetes context must be pointed to the protocol cluster. This command can be used to change context: + ```bash + kubectl config set-context gke_protocol-us-central1-d505_us-central1_protocol-us-central1 + ``` +::: + +### Staking actors + +Since the genesis actor keys are the same as LocalNet, we can use the same commands (and keyring) to stake actors on +DevNet. Additionally, DevNets match the hostnames with LocalNet, making it possible for the same stake configs to work on different networks. +For example, this command stakes supplier2 on DevNet `devnet-sophon`: + +```bash +POCKET_NODE=https://devnet-sophon-validator-rpc.poktroll.com make supplier2_stake +``` + +:::note +Only manually provisioned DevNets have more than one actor to stake. +::: \ No newline at end of file diff --git a/docusaurus/docs/operate/testing/load_testing.md b/docusaurus/docs/operate/testing/load_testing.md index c30604278..f1b802c8c 100644 --- a/docusaurus/docs/operate/testing/load_testing.md +++ b/docusaurus/docs/operate/testing/load_testing.md @@ -9,129 +9,111 @@ Poktroll load-testing suite. - [Overview](#overview) - [Dependencies](#dependencies) -- [How to run tests](#how-to-run-tests) - - [Common k6 CLI flags](#common-k6-cli-flags) -- [Understanding Output](#understanding-output) - - [CLI Output](#cli-output) - - [HTML Report](#html-report) - - [Adding a new load test](#adding-a-new-load-test) - - [Sophisticated Reporting](#sophisticated-reporting) -- [Troubleshooting](#troubleshooting) -- [File structure](#file-structure) +- [Load Test Manifests](#load-test-manifests) +- [Test Features](#test-features) +- [Executing Tests](#executing-tests) + - [LocalNet Environment](#localnet-environment) + - [Interpreting Results](#interpreting-results) + - [Non-Ephemeral Networks (TestNets, MainNet, etc)](#non-ephemeral-networks-testnets-mainnet-etc) + - [Prerequisites](#prerequisites) + - [Manifest Modification](#manifest-modification) + - [Test Execution](#test-execution) + - [Result Analysis](#result-analysis) +- [Developing Custom Tests](#developing-custom-tests) ## Overview -We use [k6](https://k6.io/) for load testing. For detailed information about k6 internals and best practices, please refer to the [k6 documentation](https://grafana.com/docs/k6/latest/). +The load-testing suite is built on [Gherkin](https://cucumber.io/docs/gherkin/), enabling the creation of simple and human-readable tests. ## Dependencies -- [k6](https://grafana.com/docs/k6/latest/get-started/installation/) -- (For local suite execution) [LocalNet](../infrastructure/localnet.md) +- [LocalNet](../infrastructure/localnet.md) (for local suite execution) +- [Golang](https://go.dev/dl/) -## How to run tests +## Load Test Manifests -Tests are stored in the [load-testing/tests](https://github.com/pokt-network/poktroll/tree/main/load-testing/tests) folder, covering various use cases. +Load test manifests are YAML files that define the network environment for test execution. These files specify properties such as blockchain account addresses for funding and staking applications, and for utilization by suppliers and gateways. The [LocalNet manifest](https://github.com/pokt-network/poktroll/blob/main/load-testing/loadtest_manifest_localnet.yaml) serves as a comprehensive example, including detailed comments for each manifest property. -For instance, here's a basic load test sending requests through `AppGateServer`, which proxies them to `RelayMiner`: +## Test Features -```bash -k6 run load-testing/tests/appGateServerEtherium.js -``` +Test features are located in the [load-testing/tests](https://github.com/pokt-network/poktroll/tree/main/load-testing/tests) directory, encompassing various scenarios. -### Common k6 CLI flags +Feature files are composed of one or more [`Scenario`](https://cucumber.io/docs/gherkin/reference/?sbsearch=Scenarios)s (or [`Scenario Outline`](https://cucumber.io/docs/gherkin/reference/?sbsearch=Scenarios#scenario-outline)s), each of which is composed of one or more [steps](https://cucumber.io/docs/gherkin/reference/#steps). For example: -The `k6` load testing tool provides various command-line flags to customize and control your load tests. Below are some of the key flags that you can use: +```gherkin +Feature: Loading anvil node only + Scenario Outline: Anvil can handle the maximum number of concurrent users + Given anvil is running + And load of concurrent requests for the "eth_blockNumber" JSON-RPC method + Then load is handled within seconds -- `--vus`: Specifies the number of virtual users (VUs) to simulate. This flag allows you to set the concurrency level of your load test. + Examples: + | num_requests | timeout | + | 10 | 1 | + | 100 | 1 | + | 1000 | 5 | + | 10000 | 10 | +``` - - Example: `k6 run script.js --vus=50` runs the test with 50 virtual users. +This natural language syntax is parsed and used to match and execute the corresponding [step definitions](https://cucumber.io/docs/cucumber/step-definitions/?lang=javascript) using [gocuke](https://github.com/regen-network/gocuke). -- `--duration`: Defines the duration for which the test should run. You can specify the time in seconds (s), minutes (m), or hours (h). +## Executing Tests - - Example: `k6 run script.js --duration=1m` runs the test for 1 minute. +### LocalNet Environment -- `--http-debug`: Enables HTTP debugging. This flag can be set to `full` for detailed logging of all HTTP requests and responses. Useful for troubleshooting and debugging your tests. - - Use `--http-debug` for a summary of HTTP requests. - - Use `--http-debug=full` for detailed request and response logging. - - Example: `k6 run script.js --http-debug=full` provides a detailed log of HTTP transactions. +To execute tests on LocalNet: -The default configurations for VUs and duration are in the [config file](https://github.com/pokt-network/poktroll/tree/main/load-testing/config/index.js). Override these defaults by passing the appropriate flags to the `k6` command. For example: +1. Ensure [LocalNet](../infrastructure/localnet.md) is operational. +2. In the `localnet_config.yaml` file, set `gateways.count` and `relayminers.count` to `3`. +3. Run `make acc_initialize_pubkeys` to initialize blockchain state public keys. +4. Run `make test_load_relays_stress_localnet` to run the LocalNet stress-test. -```bash -k6 run load-testing/tests/appGateServerEtherium.js --vus=300 --duration=30s -``` +#### Interpreting Results -For a comprehensive list of available flags and their usage, refer to the [k6 documentation](https://grafana.com/docs/k6/latest/). +- The CLI output displays standard Go test results. Successful tests are indicated by `PASS`, while failures are denoted by `FAIL` with accompanying error messages. +- During test execution, the observability stack continuously collects metric data from off-chain actors. On LocalNet, [Grafana is accessible on port 3003](http://localhost:3003/?orgId=1). The + [Stress test](http://localhost:3003/d/ddkakqetrti4gb/protocol-stress-test?orgId=1&refresh=5s) + and [Load Testing](http://localhost:3003/d/fdjwb9u9t9ts0e/protocol-load-testing?orgId=1) dashboards provide valuable + insights into system status. -## Understanding Output +### Non-Ephemeral Networks (TestNets, MainNet, etc) -### CLI Output +These networks are generated with random addresses, necessitating modifications to the load test manifest to reflect network-specific accounts. -Post-test, basic performance metrics are available: +:::info +Note: Such networks typically involve other participants, allowing load testing against off-chain actors deployed by third parties. Consequently, metrics and logs may not be available when testing against uncontrolled software. For comprehensive observability, consider creating a new service with custom gateways and suppliers, and conduct tests against this controlled environment. +::: -``` - scenarios: (100.00%) 1 scenario, 100 max VUs, 1m30s max duration (incl. graceful stop): - * default: 100 looping VUs for 1m0s (gracefulStop: 30s) - -INFO[0061] [k6-reporter v2.3.0] Generating HTML summary report source=console - ✓ is status 200 - ✓ is successful JSON-RPC response - - checks.........................: 100.00% ✓ 12000 ✗ 0 - data_received..................: 1.6 MB 27 kB/s - data_sent......................: 1.2 MB 19 kB/s - http_req_blocked...............: avg=52.52µs min=0s med=2µs max=7.08ms p(90)=4µs p(95)=6µs - http_req_connecting............: avg=37.73µs min=0s med=0s max=5.48ms p(90)=0s p(95)=0s - http_req_duration..............: avg=8.51ms min=1.58ms med=5.98ms max=81.98ms p(90)=15.45ms p(95)=19.97ms - { expected_response:true }...: avg=8.51ms min=1.58ms med=5.98ms max=81.98ms p(90)=15.45ms p(95)=19.97ms - http_req_failed................: 0.00% ✓ 0 ✗ 6000 - http_req_receiving.............: avg=42.18µs min=5µs med=25µs max=5.71ms p(90)=65µs p(95)=125µs - http_req_sending...............: avg=26.5µs min=2µs med=9µs max=7.2ms p(90)=21µs p(95)=42µs - http_req_tls_handshaking.......: avg=0s min=0s med=0s max=0s p(90)=0s p(95)=0s - http_req_waiting...............: avg=8.44ms min=1.55ms med=5.93ms max=81.28ms p(90)=15.38ms p(95)=19.82ms - http_reqs......................: 6000 98.915624/s - iteration_duration.............: avg=1s min=1s med=1s max=1.08s p(90)=1.01s p(95)=1.02s - iterations.....................: 6000 98.915624/s - vus............................: 100 min=100 max=100 - -running (1m00.7s), 000/100 VUs, 6000 complete and 0 interrupted iterations -default ✓ [======================================] 100 VUs 1m0s -``` +#### Prerequisites -### HTML Report +- A account with sufficient tokens for application funding and staking, accessible on the local keychain (e.g., `poktrolld keys list`). +- A list of target gateways to which relay requests will be sent throughout the course of the test. +- For custom service testing, ensure supplier(s') corresponding relayminer(s) and custom service process are properly configured, running, and ready to process requests. -An HTML report is generated in the execution directory. Open it in your default browser: +#### Manifest Modification -```bash -open summary.html -``` +Using [loadtest_manifest_example.yaml](https://github.com/pokt-network/poktroll/blob/main/load-testing/loadtest_manifest_example.yaml) as a template, modify the values to align with the test requirements. + +#### Test Execution -### Adding a new load test +Utilize the provided makefile target to run the `relays_stress.feature` with the modified manifest. By default, the example manifest file is used. You can specify a different manifest file by setting the `LOAD_TEST_CUSTOM_MANIFEST` environment variable. -TODO_DOCUMENT(@okdas): Add link to PR next time a new type of load test is added. +To run the stress test using the default manifest: -### Sophisticated Reporting +```bash +make test_load_relays_stress_custom +``` -We're developing advanced reporting that integrates additional tags set in the code. This will require time-series databases and is planned for DevNets. +To run the stress test with a custom manifest: -## Troubleshooting +```bash +LOAD_TEST_CUSTOM_MANIFEST=your_new_manifest.yaml make test_load_relays_stress_custom +``` -To debug, activate the logging feature: +#### Result Analysis -`--http-debug` or `--http-debug=full` +For tests conducted against community-hosted suppliers and gateways, analysis is limited to blockchain transactions and test output. When testing against a custom-deployed service, comprehensive observability is available, including metrics, logs, and system behavior under load, accessible through Grafana or other configured monitoring tools. -## File structure +## Developing Custom Tests -``` -load-testing -├── README.md -├── config -│ └── index.js -├── modules # reusable code for scenarios and tests -│ └── etheriumRequests.js -├── scenarios # different scenarios for tests -│ └── requestBlockNumberEtherium.js -└── tests # test scripts - ├── anvilDirectEtherium.js - └── appGateServerEtherium.js -``` +For custom test development, refer to the [gocuke documentation](https://github.com/regen-network/gocuke?tab=readme-ov-file#quick-start). The [anvil test](https://github.com/pokt-network/poktroll/blob/main/load-testing/tests/anvil_test.go) provides a small but practical reference implementation. \ No newline at end of file diff --git a/docusaurus/docs/operate/testing/load_testing_devnet.md b/docusaurus/docs/operate/testing/load_testing_devnet.md new file mode 100644 index 000000000..faf2429a0 --- /dev/null +++ b/docusaurus/docs/operate/testing/load_testing_devnet.md @@ -0,0 +1,115 @@ +--- +sidebar_position: 2 +title: Load Testing - DevNet +--- + +# Load Testing on DevNets + +A guide on how to perform load testing on DevNets. + +- [Overview](#overview) +- [Prerequisites](#prerequisites) + - [1. Create and configure the DevNet](#1-create-and-configure-the-devnet) + - [2. Stake the necessary actors](#2-stake-the-necessary-actors) + - [3. Configure the load testing manifest](#3-configure-the-load-testing-manifest) +- [Full example](#full-example) + +## Overview + +We can create DevNets that are suitable for running load tests. + +:::warning +DevNets created with GitHub PRs using `devnet-test-e2e` tags are not suitable for load testing, as they only provision a +single instance of each off-chain actor. We can create custom DevNets with multiple instances of each off-chain actor for load testing purposes. +::: + +## Prerequisites + +### 1. Create and configure the DevNet + +Please refer to the DevNet creation guide [here](../infrastructure/devnet.md#how-to-create). + +### 2. Stake the necessary actors + +- Depending on your load testing requirements, you may need to stake one or more `gateways` and `suppliers`. +- [DevNet documentation](../infrastructure/devnet.md#stake-actors) provides more details about staking actors in DevNets. + +### 3. Configure the load test manifest + +[Load Testing documentation](./load_testing.md#manifest-modification) provides information on how the load test manifest +can be modified to run against DevNets. DevNets are not much different from TestNets, but they **do not** have randomly +generated accounts. Instead, the accounts are derived from the ignite `config.yaml` (as in LocalNet) for convenience. + +## Full example + +1. DevNet `sophon` can be created with the following devnet-config file: + +```yaml +networkName: "sophon" + +image: + tag: sha-7042be3 + +appgateservers: + count: 1 +gateways: + count: 3 +relayminers: + count: 3 +``` + +2. Gateways and suppliers can be staked using the following commands: + +:::info +`supplier1` and `gateway1` are pre-staked as part of the genesis generation process. +::: + +```bash +POCKET_NODE=https://devnet-sophon-validator-rpc.poktroll.com make gateway2_stake +POCKET_NODE=https://devnet-sophon-validator-rpc.poktroll.com make gateway3_stake +POCKET_NODE=https://devnet-sophon-validator-rpc.poktroll.com make supplier2_stake +POCKET_NODE=https://devnet-sophon-validator-rpc.poktroll.com make supplier3_stake +``` + +3. Update the manifest. The content of +[loadtest_manifest_example.yaml](https://github.com/pokt-network/poktroll/blob/main/load-testing/loadtest_manifest_example.yaml) +can be modified as follows: + +```yaml +# This file is used to configure the load test for non-ephemeral chains. +# It is intended to target a remote environment, such as a devnet or testnet. +is_ephemeral_chain: false + +# testnet_node is the URL of the node that the load test will use to query the +# chain and submit transactions. +testnet_node: https://devnet-sophon-validator-rpc.poktroll.com + +# The service ID to request relays from. +service_id: "anvil" + +# The address of the account that will be used to fund the application accounts +# so that they can stake on the network. +# TODO_TECHDEBT(@bryanchriswhite, #512): Replace with faucet address. +funding_account_address: pokt1eeeksh2tvkh7wzmfrljnhw4wrhs55lcuvmekkw # address for pnf account + +# In non-ephemeral chains, the gateways are identified by their addresses. +gateways: + - address: pokt15vzxjqklzjtlz7lahe8z2dfe9nm5vxwwmscne4 + exposed_url: https://devnet-sophon-gateway-1.poktroll.com + - address: pokt15w3fhfyc0lttv7r585e2ncpf6t2kl9uh8rsnyz + exposed_url: https://devnet-sophon-gateway-2.poktroll.com + - address: pokt1zhmkkd0rh788mc9prfq0m2h88t9ge0j83gnxya + exposed_url: https://devnet-sophon-gateway-3.poktroll.com +``` + +4. Run the test: + +```bash +make test_load_relays_stress_example +``` + +5. Observe the results: + +You can see the performance of the requests on [Grafana dashboards](https://grafana.poktroll.com/d/nginx/nginx-ingress-controller). +The DevNets have LoadBalancers which allow for more metrics about network load and latency. When looking +at the `NGINX Ingress controller` Dashboard, make sure to change the namespace to match the DevNet name. \ No newline at end of file diff --git a/docusaurus/docs/operate/testing/load_testing_plan_1.md b/docusaurus/docs/operate/testing/load_testing_plan_1.md index 601cb77c0..33f717bea 100644 --- a/docusaurus/docs/operate/testing/load_testing_plan_1.md +++ b/docusaurus/docs/operate/testing/load_testing_plan_1.md @@ -1,5 +1,5 @@ --- -sidebar_position: 2 +sidebar_position: 5 title: Load Testing - Plan #1 --- @@ -126,10 +126,10 @@ pie showData | | RAM | CPU | Network | Disk | Time | | ----------------------- | --- | --- | ------- | ---- | ---- | -| Proof Validation | ❓ | ❓ | | | ❓ | -| Block Generation | ❓ | | | | | -| Block Publishing | | | ❓ | | | -| Data Availability State | | | | ❓ | | +| Proof Validation | ❓ | ❓ | | | ❓ | +| Block Generation | ❓ | | | | | +| Block Publishing | | | ❓ | | | +| Data Availability State | | | | ❓ | | #### 3. AppGate Server (Application, Gateway, etc…) @@ -144,10 +144,10 @@ pie showData | | RAM | CPU | Network | Disk | Time | | ------------------ | --- | --- | ------- | ---- | ---- | -| Relay Proxies | | | ❓ | | ❓ | -| Caches & State | ❓ | ❓ | | | | -| Request Processing | | | | | ❓ | -| ??? | | | | ❓ | | +| Relay Proxies | | | ❓ | | ❓ | +| Caches & State | ❓ | ❓ | | | | +| Request Processing | | | | | ❓ | +| ??? | | | | ❓ | | #### 4. RelayMiner (Supplier, SMT, etc..) @@ -162,10 +162,10 @@ pie showData | | RAM | CPU | Network | Disk | Time | | ------------------- | --- | --- | ------- | ---- | ---- | -| SMT | ❓ | ❓ | | ❓ | ❓ | -| Caches & State | ❓ | | | | | -| Request Processing | | | | | ❓ | -| Response Generation | | ❓ | ❓ | | ❓ | +| SMT | ❓ | ❓ | | ❓ | ❓ | +| Caches & State | ❓ | | | | | +| Request Processing | | | | | ❓ | +| Response Generation | | ❓ | ❓ | | ❓ | ### Out-of-scope diff --git a/docusaurus/docs/operate/testing/load_testing_testnet.md b/docusaurus/docs/operate/testing/load_testing_testnet.md new file mode 100644 index 000000000..60878eb41 --- /dev/null +++ b/docusaurus/docs/operate/testing/load_testing_testnet.md @@ -0,0 +1,35 @@ +--- +sidebar_position: 2 +title: Load Testing - TestNet +--- + +# Load Testing on TestNets + +A guide on how to perform load testing on TestNets. + +- [Overview](#overview) +- [Load Testing Steps](#load-testing-steps) +- [Examples](#examples) + +## Overview + +Load Testing on TestNets is very similar to testing on DevNets. The main difference is that the test might run against software you don't control. Gateways and Suppliers can be hosted by other teams or organizations, so you need to ensure that your tests do not adversely affect their operations. As another side effect, you won't be able to collect metrics and logs on the software you don't run. + +## Load Testing Steps + +Please refer to the generic [load testing documentation](./load_testing.md#non-ephemeral-networks-testnets-mainnet-etc) for non-ephemeral networks (TestNets, MainNet, etc.). The steps are very similar to those for DevNets, which are thoroughly documented on the dedicated [Load Testing on DevNets](./load_testing_devnet.md) page. + +Key points to consider: +1. Identify the TestNet endpoints and actors (gateways, suppliers) you'll be testing against. +2. Ensure you have the necessary permissions and have communicated your intent to run load tests. +3. Configure your load test manifest to target the TestNet resources. +4. Start with lower load and gradually increase to avoid overwhelming the network. +5. Monitor the TestNet's performance metrics during the test. + +## Examples + +:::info +TODO_DOCUMENT(@okdas): add a few examples on how to perform load testing on TestNets: +- How to run the test against gateways and suppliers that are not under your control. +- How to run the test against gateways and suppliers deployed in a separate service for isolation. +::: \ No newline at end of file diff --git a/load-testing/README.md b/load-testing/README.md deleted file mode 100644 index e69de29bb..000000000 diff --git a/load-testing/loadtest_manifest_example.yaml b/load-testing/loadtest_manifest_example.yaml index 33a2fc6e0..2e3612388 100644 --- a/load-testing/loadtest_manifest_example.yaml +++ b/load-testing/loadtest_manifest_example.yaml @@ -1,24 +1,23 @@ -# The load test uses the provided gateways and the staked suppliers to send relays. - # This file is used to configure the load test for non-ephemeral chains. # It is intended to target a remote environment, such as a devnet or testnet. is_ephemeral_chain: false # testnet_node is the URL of the node that the load test will use to query the # chain and submit transactions. -testnet_node: https://testnet-validated-validator-rpc.poktroll.com +testnet_node: https://devnet-sophon-validator-rpc.poktroll.com # The service ID to request relays from. -service_id: "0021" +service_id: "anvil" # The address of the account that will be used to fund the the application accounts # so that they can stake on the network. -funding_account_address: pokt14eh973xt99s7edugnyvd5d5u50d6j0hysw2vsm # address for pnf account +funding_account_address: pokt1eeeksh2tvkh7wzmfrljnhw4wrhs55lcuvmekkw # address for pnf account # In non-ephemeral chains, the gateways are identified by their address. gateways: - # address is the bech32 pokt address of the gateway. - - address: pokt16sty9mjdh4u2fwgj8ptufg42cysvh6gsyx6wfp - # The url used to send relays to the gateway on. - # TODO_INVESTIGATE(@okdas): Why is this port number so low? What should it be? - exposed_url: http://localhost:84 + - address: pokt15vzxjqklzjtlz7lahe8z2dfe9nm5vxwwmscne4 + exposed_url: https://devnet-sophon-gateway-1.poktroll.com + - address: pokt15w3fhfyc0lttv7r585e2ncpf6t2kl9uh8rsnyz + exposed_url: https://devnet-sophon-gateway-2.poktroll.com + - address: pokt1zhmkkd0rh788mc9prfq0m2h88t9ge0j83gnxya + exposed_url: https://devnet-sophon-gateway-3.poktroll.com diff --git a/load-testing/loadtest_manifest_localnet.yaml b/load-testing/loadtest_manifest_localnet.yaml index ff952d390..e75e55bf0 100644 --- a/load-testing/loadtest_manifest_localnet.yaml +++ b/load-testing/loadtest_manifest_localnet.yaml @@ -1,6 +1,7 @@ # NB: The number of pre-provisioned **LocalNet** actors are managed in # 'localnet_config.yaml' by the respective actors `count` property. -is_ephemeral_chain: true + +is_ephemeral_chain: true # This should be `true` for LocalNet as it is an ephemeral network # The service ID to use for the load test. service_id: anvil @@ -12,6 +13,9 @@ funding_account_address: pokt1eeeksh2tvkh7wzmfrljnhw4wrhs55lcuvmekkw # address f # List of pre-provisioned suppliers used for load testing. # These suppliers will be progressively staked during the load test, according # to the test plan that is being executed. +# +# When running on **LocalNet**, the number of the suppliers defined below should match +# the `relayminers.count` in your `localnet_config.yaml`. suppliers: # The supplier address that is available in the load test's environment keyring, # used to identify the supplier and sign relays and transactions with. @@ -33,6 +37,9 @@ suppliers: # List of pre-provisioned gateways used for load testing. # These gateways will be progressively staked and delegated to during the load test. +# +# When running on LocalNet, the number of the gateways defined below should match +# the `gateways.count` in your `localnet_config.yaml`. gateways: # The gateway address that is available in the load test's environment keyring, # used to identify the gateway and sign relays and transactions with.