Merge remote-tracking branch 'origin/main' into yevhenii/cache

.env

@@ -17,30 +17,36 @@ POSTGRES_HOST_PORT=5432
 REDIS_CONTAINER_PORT=6379
 REDIS_HOST_PORT=6379
 
+KAVA_CONTAINER_TAG=v0.24.0
 KAVA_CONTAINER_EVM_RPC_PORT=8545
-KAVA_HOST_EVM_RPC_PORT=8545
 KAVA_CONTAINER_COSMOS_RPC_PORT=26657
+
+KAVA_HOST_EVM_RPC_PORT=8545
 KAVA_HOST_COSMOS_RPC_PORT=26657
+KAVA_PRUNING_HOST_EVM_RPC_PORT=8555
+KAVA_PRUNING_HOST_COSMOS_RPC_PORT=26667
 
 PROXY_CONTAINER_PORT=7777
-PROXY_CONTAINER_EVM_RPC_DATA_PORT=7778
+PROXY_CONTAINER_EVM_RPC_PRUNING_PORT=7778
 PROXY_HOST_PORT=7777
 PROXY_CONTAINER_DEBUG_PORT=2345
 PROXY_HOST_DEBUG_PORT=2345
 
 ##### E2E Testing Config
 TEST_PROXY_SERVICE_EVM_RPC_URL=http://localhost:7777
 TEST_PROXY_SERVICE_EVM_RPC_HOSTNAME=localhost:7777
-TEST_PROXY_SERVICE_EVM_RPC_DATA_URL=http://localhost:7778
+TEST_PROXY_SERVICE_EVM_RPC_PRUNING_URL=http://localhost:7778
 TEST_PROXY_BACKEND_EVM_RPC_HOST_URL=http://localhost:8545
 TEST_DATABASE_ENDPOINT_URL=localhost:5432
-TEST_PROXY_BACKEND_HOST_URL_MAP=localhost:7777>http://kava:8545,localhost:7778>http://kava:8545
+TEST_PROXY_BACKEND_HOST_URL_MAP=localhost:7777>http://kava-validator:8545,localhost:7778>http://kava-pruning:8545
+TEST_PROXY_HEIGHT_BASED_ROUTING_ENABLED=true
+TEST_PROXY_PRUNING_BACKEND_HOST_URL_MAP=localhost:7777>http://kava-pruning:8545,localhost:7778>http://kava-pruning:8545
 # What level of logging to use for service objects constructed during
 # unit tests
 TEST_SERVICE_LOG_LEVEL=ERROR
 # endpoint the proxy service should use for querying
 # evm blockchain information related to proxied requests
-TEST_EVM_QUERY_SERVICE_URL=http://kava:8545
+TEST_EVM_QUERY_SERVICE_URL=http://kava-validator:8545
 # TEST_REDIS_ENDPOINT_URL is an url of redis
 TEST_REDIS_ENDPOINT_URL=localhost:6379
 
@@ -54,7 +60,12 @@ LOG_LEVEL=TRACE
 HTTP_READ_TIMEOUT_SECONDS=30
 HTTP_WRITE_TIMEOUT_SECONDS=60
 # Address of the origin server to proxy all requests to
-PROXY_BACKEND_HOST_URL_MAP=localhost:7777>http://kava:8545,localhost:7778>http://kava:8545
+PROXY_BACKEND_HOST_URL_MAP=localhost:7777>http://kava-validator:8545,localhost:7778>http://kava-pruning:8545
+# height-based routing will look at the height of an incoming EVM request
+# iff. the height is "latest", it routes to the corresponding PROXY_PRUNING_BACKEND_HOST_URL_MAP value
+# otherwise, it falls back to the value in PROXY_BACKEND_HOST_URL_MAP
+PROXY_HEIGHT_BASED_ROUTING_ENABLED=true
+PROXY_PRUNING_BACKEND_HOST_URL_MAP=localhost:7777>http://kava-pruning:8545,localhost:7778>http://kava-pruning:8545
 # Configuration for the servcie to connect to it's database
 DATABASE_NAME=postgres
 DATABASE_ENDPOINT_URL=postgres:5432
@@ -74,7 +85,7 @@ DATABASE_QUERY_LOGGING_ENABLED=true
 METRIC_COMPACTION_ROUTINE_INTERVAL_SECONDS=5
 # endpoint the proxy service should use for querying
 # evm blockchain information related to proxied requests
-EVM_QUERY_SERVICE_URL=http://kava:8545
+EVM_QUERY_SERVICE_URL=http://kava-validator:8545
 # Whether the proxy service should attempt to track and store metrics
 # related to proxied requests
 METRIC_COLLECTION_ENABLED=true

.gitignore

@@ -21,4 +21,8 @@ vendor/
 
 # ignore editor files
 .vscode/
-.idea/
+.idea/
+
+# ignore e2e test validator files
+docker/shared/genesis.json
+docker/shared/VALIDATOR_NODE_ID

DEVELOPMENT.md

@@ -96,6 +96,19 @@ The e2e tests won't pass if the proxy service and it's dependencies aren't fully
 make ready e2e-test
 ```
 
+For details on the local E2E setup, see [the `docker` directory](./docker/README.md).
+
+#### Against testnet
+
+The Continuous Integration (CI) for this repo is setup to run a local proxy service with database & redis, but configures the service to use public testnet urls for the kava requests. This allows for testing the proxy service against a production-like environment (requests are routed to public testnet).
+
+You can emulate the CI configuration in your local environment:
+```bash
+make ci-setup
+```
+
+At that point, running `make e2e-test` will run the end-to-end tests with requests routing to public testnet.
+
 ## Test Coverage Report
 
 The test commands `make test`, `make unit-test`, and `make e2e-test` generate a `cover.out` raw test coverage report. The coverage can be converted into a user-friendly webpage:

Makefile

@@ -44,6 +44,14 @@ unit-test:
 e2e-test:
 	go test -count=1 -v -cover -coverprofile cover.out --race ./... -run "^TestE2ETest*"
 
+.PHONY: ci-setup
+# set up your local environment such that running `make e2e-test` runs against testnet (like in CI)
+ci-setup:
+	docker compose -f ci.docker-compose.yml pull
+	docker compose -f ci.docker-compose.yml up -d --build --force-recreate
+	PROXY_CONTAINER_PORT=7777 bash scripts/wait-for-proxy-service-running.sh
+	PROXY_CONTAINER_PORT=7777 MINIMUM_REQUIRED_PARTITIONS=30 bash scripts/wait-for-proxy-service-running.sh
+
 .PHONY: it
 # run any test matching the provided pattern, can pass a regex or a string
 # of the exact test to run
@@ -68,6 +76,8 @@ up:
 .PHONY: down
 # stop the service and it's dependencies
 down:
+	rm docker/shared/genesis.json
+	rm docker/shared/VALIDATOR_NODE_ID
 	docker compose down
 
 .PHONY: restart
@@ -78,6 +88,8 @@ restart:
 .PHONY: reset
 # wipe state and restart the service and all it's dependencies
 reset: lint
+	rm docker/shared/genesis.json || exit 0
+	rm docker/shared/VALIDATOR_NODE_ID || exit 0
 	docker compose up -d --build --remove-orphans --renew-anon-volumes --force-recreate
 
 .PHONY: refresh

architecture/MIDDLEWARE.MD

@@ -61,6 +61,8 @@ server := &http.Server{
 
 1. Times the roundtrip latency for the response from the backend origin server and stores the latency in the context key `X-KAVA-PROXY-ORIGIN-ROUNDTRIP-LATENCY-MILLISECONDS` for use by other middleware.
 
+See [Proxy Routing](./PROXY_ROUTING.md) for details on configuration and how requests are routed.
+
 ### After Proxy Middleware
 
 1. Parses the request body and latency from context key values and creates a request metric for the proxied request.

architecture/MIGRATIONS.MD

@@ -24,3 +24,54 @@ $ date '+%Y%m%d%H%M%S'
 ### Transactional Migrations
 
 Each migration file will run in a single transaction, which means that you can not have for example multiple `CREATE INDEX CONCURRENTLY` statements in a given migration file. Instead break each statement out into it's own migration file.
+
+## Troubleshooting
+
+### Migration Table Locks
+
+Bun (the db client) uses a simple locking mechanism while running migrations that locks an entire table by attempting to write a row `(id, table_name)` to a table called `bun_migration_locks`. The table uses `table_name` as a unique key so that only one process can lock a table at a time. After the process finishes modifying the table, it deletes its lock.
+
+However, a problem occurs if the process dies while still holding a lock, as the lock will never be deleted & no other process will be allowed to lock the table.
+
+If you find the migrations aren't running, check the following:
+
+Enable logging by setting the following:
+```sh
+# first, make sure you're actually trying to run the migrations... :)
+RUN_DATABASE_MIGRATIONS=true
+
+# debug logging for the service logger
+LOG_LEVEL=DEBUG
+# enable database query logging
+DATABASE_QUERY_LOGGING_ENABLED=true
+```
+
+With `LOG_LEVEL=DEBUG`, check that the service is attempting to run migrations. It will produce the following log when migrations are enabled:
+```json
+{
+    "level": "debug",
+    "time": "2023-10-17T20:46:17Z",
+    "caller": "/app/service/service.go:155",
+    "message": "running migrations on database"
+}
+```
+
+Repeated database query logs that look like this indicate the problem is with the table locks:
+```log
+[bun]  20:46:58.822   INSERT                2.529ms  INSERT INTO bun_migration_locks ("id", "table_name") VALUES (DEFAULT, 'bun_migrations') RETURNING "id" 	  pgdriver.Error: ERROR: duplicate key value violates unique constraint "bun_migration_locks_table_name_key" (SQLSTATE=23505)
+```
+
+To resolve, manually connect to the database and delete the lock:
+```sql
+SELECT * FROM bun_migration_locks;
+-- verify and locate the bad lock
+| --  | id    | table_name     |
+| --- | ----- | -------------- |
+| --  | 12345 | bun_migrations |
+
+-- DANGEROUS DELETE OPERATION!
+-- THINK BEFORE YOU PASTE!
+DELETE FROM bun_migration_locks WHERE id = 12345;
+```
+
+The service, which is retrying the lock acquisition, should be able to acquire the lock and complete the migrations (successfully removing its lock after it finishes).

architecture/PROXY_ROUTING.md

@@ -0,0 +1,152 @@
+# Proxy Routing
+
+The proxy chooses where to route a request primarily by the incoming Host URL to which the client
+originally made their request. The routing is configured by maps of the Host to a backend url in
+the environment variables.
+
+All possible configurations use the `PROXY_BACKEND_HOST_URL_MAP` environment variable. This encodes
+the default backend to route all requests from a given host. Additional functionality is available
+via the `PROXY_HEIGHT_BASED_ROUTING_ENABLED` env variable (see [Rudimentary Sharding](#rudimentary-sharding)).
+
+Consider the simplest case: a host-based-only routing proxy configured for one host
+```
+PROXY_HEIGHT_BASED_ROUTING_ENABLED=false
+PROXY_BACKEND_HOST_URL_MAP=localhost:7777>http://kava:8545
+```
+This value is parsed into a map that looks like the following:
+```
+{
+    "localhost:7777" => "http://kava:8545",
+}
+```
+Any request to the service will be routed according to this map.
+ie. all requests to local port 7777 get forwarded to `http://kava:8545`
+
+Implementations of the [`Proxies` interface](../service/proxy.go#L13) contain logic for deciding
+the backend host url to which a request is routed. This is used in the ProxyRequestMiddleware to
+route requests.
+
+Any request made to a host not in the map responds 502 Bad Gateway.
+
+## More Examples of Host-only routing
+
+Here is a diagram of the above network topology:
+
+![Proxy Service configured for one host](images/proxy_service_simple_one_host.jpg)
+
+In this simple configuration of only having default hosts, you can do many things:
+
+**Many hosts -> One backend**
+
+```
+PROXY_HEIGHT_BASED_ROUTING_ENABLED=false
+PROXY_BACKEND_HOST_URL_MAP=localhost:7777>http://kava:8545,localhost:7778>http://kava:8545
+```
+This value is parsed into a map that looks like the following:
+```
+{
+    "localhost:7777" => "http://kava:8545",
+    "localhost:7778" => "http://kava:8545",
+}
+```
+All requests to local ports 7777 & 7778 route to the same cluster at kava:8545
+
+![Proxy Service configured for many hosts for one backend](images/proxy_service_many_hosts_one_backend.jpg)
+
+**Many hosts -> Many backends**
+
+```
+PROXY_HEIGHT_BASED_ROUTING_ENABLED=false
+PROXY_BACKEND_HOST_URL_MAP=evm.kava.io>http://kava-pruning:8545,evm.data.kava.io>http://kava-archive:8545
+```
+This value is parsed into a map that looks like the following:
+```
+{
+    "evm.kava.io"      => "http://kava-pruning:8545",
+    "evm.data.kava.io" => "http://kava-archive:8545",
+}
+```
+Requests made to evm.kava.io route to a pruning cluster.
+Those made to evm.data.kava.io route to an archive cluster.
+
+![Proxy Service configured for many hosts with many backends](images/proxy_service_many_hosts_many_backends.jpg)
+
+## Rudimentary Sharding
+
+Now suppose you want multiple backends for the same host.
+
+The proxy service supports height-based routing to direct requests that only require the most recent
+block to a different cluster.
+This support is handled via the [`HeightShardingProxies` implementation](../service/shard.go#L16).
+
+This is configured via the `PROXY_HEIGHT_BASED_ROUTING_ENABLED` and `PROXY_PRUNING_BACKEND_HOST_URL_MAP`
+environment variables.
+* `PROXY_HEIGHT_BASED_ROUTING_ENABLED` - flag to toggle this functionality
+* `PROXY_PRUNING_BACKEND_HOST_URL_MAP` - like `PROXY_BACKEND_HOST_URL_MAP`, but only used for JSON-RPC
+  requests that target the latest block (or are stateless, like `eth_chainId`, `eth_coinbase`, etc).
+
+For example, to lighten the load for your resource-intensive (& expensive) archive cluster, you can
+route all requests for the "latest" block to a less resource-intensive (& cheaper) pruning cluster:
+```
+PROXY_HEIGHT_BASED_ROUTING_ENABLED=true
+PROXY_BACKEND_HOST_URL_MAP=evm.data.kava.io>http://kava-archive:8545
+PROXY_PRUNING_BACKEND_HOST_URL_MAP=evm.data.kava.io>http://kava-pruning:8545
+```
+This value is parsed into a map that looks like the following:
+```
+{
+  "default": {
+    "evm.data.kava.io" => "http://kava-archive:8545",
+  },
+  "pruning": {
+    "evm.data.kava.io" => "http://kava-pruning:8545",
+  }
+}
+```
+All traffic to evm.data.kava.io that targets the latest block (or requires no history) routes to the pruning cluster.
+Otherwise, all traffic is sent to the archive cluster.
+
+![Proxy Service configured with rudimentary sharding](images/proxy_service_rudimentary_sharding.jpg)
+
+### Default vs Pruning Backend Routing
+
+When `PROXY_HEIGHT_BASED_ROUTING_ENABLED` is `true`, the following cases will cause requests to route
+to the the backend url defined in `PROXY_PRUNING_BACKEND_HOST_URL_MAP` (if present):
+* requests that include any of the following block tags:
+  * `"latest"`
+  * `"finalized"`
+  * `"pending"`
+  * `"safe"`
+  * empty/missing block tag (interpreted as `"latest"`)
+* requests for methods that require no historic state, including transaction broadcasting
+  * for a full list of methods, see [`NoHistoryMethods`](../decode/evm_rpc.go#L89)
+
+All other requests fallback to the default backend url defined in `PROXY_BACKEND_HOST_URL_MAP`.
+This includes
+* requests for hosts not included in `PROXY_PRUNING_BACKEND_HOST_URL_MAP`
+* requests targeting any specific height by number
+  * NOTE: the service does not track the current height of the chain. if the tip of the chain is at
+    block 1000, a query for block 1000 will still route to the default (not pruning) backend
+* requests for methods that use block hash, like `eth_getBlockByHash`
+* requests with unparsable (invalid) block numbers
+* requests for block tag `"earliest"`
+
+The service will panic on startup if a host in `PROXY_PRUNING_BACKEND_HOST_URL_MAP` is not present
+in `PROXY_BACKEND_HOST_URL_MAP`.
+
+Any request made to a host not in the `PROXY_BACKEND_HOST_URL_MAP` map responds 502 Bad Gateway.
+
+## Metrics
+
+When metrics are enabled, the `proxied_request_metrics` table tracks the backend to which requests
+are routed in the `response_backend` column.
+
+When height-based sharding is disabled (`PROXY_HEIGHT_BASED_ROUTING_ENABLED=false`), the value is
+always `DEFAULT`.
+
+When enabled, the column will have one of the following values:
+* `DEFAULT` - the request was routed to the backend defined in `PROXY_BACKEND_HOST_URL_MAP`
+* `PRUNING` - the request was routed to the backend defined in `PROXY_PRUNING_BACKEND_HOST_URL_MAP`
+
+Additionally, the actual URL to which the request is routed to is tracked in the
+`response_backend_route` column.

ci.docker-compose.yml

@@ -22,12 +22,14 @@ services:
       dockerfile: ci.Dockerfile
     env_file: .env
     environment:
+      PROXY_HEIGHT_BASED_ROUTING_ENABLED: true
       # use public testnet as backend origin server to avoid having
       # to self-host a beefy Github Action runner
       # to build and run a kava node each execution
-      PROXY_BACKEND_HOST_URL_MAP: localhost:7777>https://evmrpc.internal.testnet.proxy.kava.io,localhost:7778>https://evmrpcdata.internal.testnet.proxy.kava.io
+      PROXY_BACKEND_HOST_URL_MAP: localhost:7777>https://evmrpcdata.internal.testnet.proxy.kava.io,localhost:7778>https://evmrpc.internal.testnet.proxy.kava.io
+      PROXY_PRUNING_BACKEND_HOST_URL_MAP: localhost:7777>https://evmrpc.internal.testnet.proxy.kava.io
       EVM_QUERY_SERVICE_URL: https://evmrpc.internal.testnet.proxy.kava.io
     ports:
       - "${PROXY_HOST_PORT}:${PROXY_CONTAINER_PORT}"
-      - "${PROXY_CONTAINER_EVM_RPC_DATA_PORT}:${PROXY_CONTAINER_PORT}"
+      - "${PROXY_CONTAINER_EVM_RPC_PRUNING_PORT}:${PROXY_CONTAINER_PORT}"
       - "${PROXY_HOST_DEBUG_PORT}:${PROXY_CONTAINER_DEBUG_PORT}"

clients/database/migrations/20231013122742_add_response_backend.up.sql

@@ -0,0 +1,9 @@
+-- add response backend column, backfilling with "DEFAULT" (the only value to exist up until now)
+-- new metrics that omit its value are assumed to have been routed to "DEFAULT" backend
+-- also add response backend route. this is the backend url the request was routed to.
+ALTER TABLE
+  IF EXISTS proxied_request_metrics
+ADD
+  response_backend character varying DEFAULT 'DEFAULT',
+ADD
+  response_backend_route character varying;

clients/database/request_metric.go

@@ -27,6 +27,8 @@ type ProxiedRequestMetric struct {
 	UserAgent                   *string
 	Referer                     *string
 	Origin                      *string
+	ResponseBackend             string
+	ResponseBackendRoute        string
 }
 
 // Save saves the current ProxiedRequestMetric to