diff --git a/.env.dev b/.env.dev index d40b85238..390e34774 100644 --- a/.env.dev +++ b/.env.dev @@ -7,13 +7,12 @@ POKTROLLD_HOME=./localnet/poktrolld POCKET_NODE=tcp://127.0.0.1:26657 # TestNet RPC endpoint for validator maintained by Grove. Needs to be updated if there's another "primary" testnet. TESTNET_RPC=https://testnet-validated-validator-rpc.poktroll.com/ -APPGATE_SERVER=http://localhost:42069 -GATEWAY_URL=http://localhost:42079 +PATH_URL=http://localhost:3000 POCKET_ADDR_PREFIX=pokt CHAIN_ID=poktroll # The domain ending in ".town" is staging, ".city" is production -GROVE_GATEWAY_STAGING_ETH_MAINNET=https://eth-mainnet.rpc.grove.town +GROVE_PORTAL_STAGING_ETH_MAINNET=https://eth-mainnet.rpc.grove.town # The "protocol" field here instructs the Grove gateway which network to use JSON_RPC_DATA_ETH_BLOCK_HEIGHT='{"protocol": "shannon-testnet","jsonrpc":"2.0","id":"0","method":"eth_blockNumber", "params": []}' diff --git a/.github/workflows-helpers/run-e2e-test-job-template.yaml b/.github/workflows-helpers/run-e2e-test-job-template.yaml index 1efdfe5fb..9db41ed01 100644 --- a/.github/workflows-helpers/run-e2e-test-job-template.yaml +++ b/.github/workflows-helpers/run-e2e-test-job-template.yaml @@ -11,8 +11,8 @@ spec: pokt.network/purpose: e2e-tests spec: initContainers: - # We need to make sure the services like validator, relayminer, appgate server, etc. work before we execute - # an e2e test. + # We need to ensure services like Validator, RelayMiner, PATH, etc. work + # before executing an e2e test. - name: check-services image: ruby:3.0 command: ["ruby"] @@ -49,10 +49,14 @@ spec: value: "false" # Flip to true to see the command and result of the execution - name: POKTROLLD_HOME value: /root/.poktroll - - name: APPGATE_SERVER_URL - value: http://${NAMESPACE}-appgate-server:80 - - name: GATEWAY_URL - value: http://${NAMESPACE}-gateway:80 + - name: PATH_URL + value: http://${NAMESPACE}-path:3000/v1 + # PATH relies on subdomains to get the requested service but our DevNet infra is not + # built to expose arbitrary subdomains and supporting it would be a significant effort. + # As a workaround, PATH_HOST_OVERRIDE is used as the host:port to connect to PATH while + # the subdomain is passed as a Host header in the request. + - name: PATH_HOST_OVERRIDE + value: ${NAMESPACE}-path:3000 volumeMounts: - mountPath: /root/.poktroll/keyring-test/ name: writable-keys-volume diff --git a/Makefile b/Makefile index 6d46f90be..200d37f06 100644 --- a/Makefile +++ b/Makefile @@ -5,13 +5,12 @@ SHELL = /bin/sh POKTROLLD_HOME ?= ./localnet/poktrolld POCKET_NODE ?= tcp://127.0.0.1:26657 # The pocket node (validator in the localnet context) TESTNET_RPC ?= https://testnet-validated-validator-rpc.poktroll.com/ # TestNet RPC endpoint for validator maintained by Grove. Needs to be update if there's another "primary" testnet. -APPGATE_SERVER ?= http://localhost:42069 -GATEWAY_URL ?= http://localhost:42079 +PATH_URL ?= http://localhost:3000 POCKET_ADDR_PREFIX = pokt LOAD_TEST_CUSTOM_MANIFEST ?= loadtest_manifest_example.yaml # The domain ending in ".town" is staging, ".city" is production -GROVE_GATEWAY_STAGING_ETH_MAINNET = https://eth-mainnet.rpc.grove.town +GROVE_PORTAL_STAGING_ETH_MAINNET = https://eth-mainnet.rpc.grove.town # JSON RPC data for a test relay request JSON_RPC_DATA_ETH_BLOCK_HEIGHT = '{"jsonrpc":"2.0","id":"0","method":"eth_blockNumber", "params": []}' @@ -349,14 +348,19 @@ docusaurus_start: check_npm check_node ## Start the Docusaurus server docs_update_gov_params_page: ## Update the page in Docusaurus documenting all the governance parameters go run tools/scripts/docusaurus/generate_docs_params.go -###################### -### Ignite Helpers ### -###################### +####################### +### Keyring Helpers ### +####################### + .PHONY: poktrolld_addr poktrolld_addr: ## Retrieve the address for an account by ACC_NAME @echo $(shell poktrolld --home=$(POKTROLLD_HOME) keys show -a $(ACC_NAME)) +.PHONY: poktrolld_key +poktrolld_key: ## Retrieve the private key for an account by ACC_NAME + @echo $(shell poktrolld --home=$(POKTROLLD_HOME) keys export --unsafe --unarmored-hex $(ACC_NAME)) + ################### ### Act Helpers ### ################### @@ -415,13 +419,13 @@ release_tag_minor_release: ## Tag a new minor release (e.g. v1.0.0 -> v1.1.0) @echo " git push origin $(NEW_TAG)" @echo "And draft a new release at https://github.com/pokt-network/poktroll/releases/new" -############################# -### Grove Gateway Helpers ### -############################# +############################ +### Grove Portal Helpers ### +############################ .PHONY: grove_staging_eth_block_height grove_staging_eth_block_height: ## Sends a relay through the staging grove gateway to the eth-mainnet chain. Must have GROVE_STAGING_PORTAL_APP_ID environment variable set. - curl $(GROVE_GATEWAY_STAGING_ETH_MAINNET)/v1/$(GROVE_STAGING_PORTAL_APP_ID) \ + curl $(GROVE_PORTAL_STAGING_ETH_MAINNET)/v1/$(GROVE_STAGING_PORTAL_APP_ID) \ -H 'Content-Type: application/json' \ -H 'Protocol: shannon-testnet' \ --data $(JSON_RPC_DATA_ETH_BLOCK_HEIGHT) @@ -437,6 +441,7 @@ grove_staging_eth_block_height: ## Sends a relay through the staging grove gatew ############### ### Imports ### ############### + include ./makefiles/warnings.mk include ./makefiles/todos.mk include ./makefiles/checks.mk @@ -450,3 +455,4 @@ include ./makefiles/suppliers.mk include ./makefiles/gateways.mk include ./makefiles/session.mk include ./makefiles/claims.mk +include ./makefiles/relay.mk diff --git a/Tiltfile b/Tiltfile index 03cf76bb4..4c1d74b1c 100644 --- a/Tiltfile +++ b/Tiltfile @@ -46,20 +46,6 @@ localnet_config_defaults = { "level": "debug", }, }, - "gateways": { - "count": 1, - "delve": {"enabled": False}, - "logs": { - "level": "debug", - }, - }, - "appgateservers": { - "count": 1, - "delve": {"enabled": False}, - "logs": { - "level": "debug", - }, - }, "ollama": { "enabled": False, "model": "qwen:0.5b", @@ -67,8 +53,14 @@ localnet_config_defaults = { "rest": { "enabled": True, }, + "path_gateways": { + "count": 1, + }, - # NOTE: git submodule usage was explicitly avoided to reduce environment complexity. + ############# + # NOTE: git submodule usage was explicitly avoided for the repositories below + # to reduce environment complexity. + ############# # By default, we use the `helm_repo` function below to point to the remote repository # but can update it to the locally cloned repo for testing & development @@ -76,11 +68,19 @@ localnet_config_defaults = { "enabled": False, "path": os.path.join("..", "helm-charts") }, + + # By default, we use a pre-built PATH image, but can update it to use a local + # repo instead. + "path_local_repo": { + "enabled": False, + "path": "../path" + }, + "indexer": { "repo_path": os.path.join("..", "pocketdex"), "enabled": True, "clone_if_not_present": False, - } + }, } localnet_config_file = read_yaml(localnet_config_path, default=localnet_config_defaults) # Initial empty config @@ -96,9 +96,12 @@ if (localnet_config_file != localnet_config) or (not os.path.exists(localnet_con print("Updating " + localnet_config_path + " with defaults") local("cat - > " + localnet_config_path, stdin=encode_yaml(localnet_config)) -# Configure helm chart reference. If using a local repo, set the path to the local repo; otherwise, use our own helm repo. +# Configure helm chart reference. +# If using a local repo, set the path to the local repo; otherwise, use our own helm repo. helm_repo("pokt-network", "https://pokt-network.github.io/helm-charts/") -# TODO_IMPROVE: Use os.path.join to make this more OS-agnostic. +helm_repo("buildwithgrove", "https://buildwithgrove.github.io/helm-charts/") + +# Configure POKT chart references chart_prefix = "pokt-network/" if localnet_config["helm_chart_local_repo"]["enabled"]: helm_chart_local_repo = localnet_config["helm_chart_local_repo"]["path"] @@ -107,6 +110,15 @@ if localnet_config["helm_chart_local_repo"]["enabled"]: # TODO_IMPROVE: Use os.path.join to make this more OS-agnostic. chart_prefix = helm_chart_local_repo + "/charts/" +# Configure PATH references +grove_chart_prefix = "buildwithgrove/" +# If using a local repo, set the path to the local repo; otherwise, use our own helm repo. +path_local_repo = "" +if localnet_config["path_local_repo"]["enabled"]: + path_local_repo = localnet_config["path_local_repo"]["path"] + hot_reload_dirs.append(path_local_repo) + print("Using local PATH repo " + path_local_repo) + # Observability print("Observability enabled: " + str(localnet_config["observability"]["enabled"])) if localnet_config["observability"]["enabled"]: @@ -290,91 +302,70 @@ for x in range(localnet_config["relayminers"]["count"]): ], ) -# Provision AppGate Servers +if localnet_config["path_local_repo"]["enabled"]: + docker_build("path-local", path_local_repo) + +# TODO_MAINNET(@okdas): Find and replace all `appgateserver` in ./localnet/grafana-dashboards` +# with PATH metrics (see the .json files) +# Ref: https://github.com/buildwithgrove/path/pull/72 + +# Provision PATH Gateway(s) actor_number = 0 -for x in range(localnet_config["appgateservers"]["count"]): - actor_number = actor_number + 1 - helm_resource( - "appgateserver" + str(actor_number), - chart_prefix + "appgate-server", - flags=[ - "--values=./localnet/kubernetes/values-common.yaml", - "--values=./localnet/kubernetes/values-appgateserver.yaml", - "--set=config.signing_key=app" + str(actor_number), - "--set=metrics.serviceMonitor.enabled=" + str(localnet_config["observability"]["enabled"]), - "--set=development.delve.enabled=" + str(localnet_config["appgateservers"]["delve"]["enabled"]), - "--set=logLevel=" + str(localnet_config["appgateservers"]["logs"]["level"]), - "--set=image.repository=poktrolld", - ], - image_deps=["poktrolld"], - image_keys=[("image.repository", "image.tag")], - ) - k8s_resource( - "appgateserver" + str(actor_number), - labels=["gateways"], - resource_deps=["validator"], - links=[ - link( - "http://localhost:3003/d/appgateserver/protocol-appgate-server?orgId=1&refresh=5s&var-appgateserver=appgateserver" - + str(actor_number), - "Grafana dashboard", - ), - ], - port_forwards=[ - str(42068 + actor_number) + ":42069", # appgateserver1 - exposes 42069, appgateserver2 exposes 42070, etc. - str(40054 + actor_number) - + ":40004", # DLV port. appgateserver1 - exposes 40055, appgateserver2 exposes 40056, etc. - # Run `curl localhost:PORT` to see the current snapshot of appgateserver metrics. - str(9079 + actor_number) - + ":9090", # appgateserver metrics port. appgateserver1 - exposes 9080, appgateserver2 exposes 9081, etc. - # Use with pprof like this: `go tool pprof -http=:3333 http://localhost:6080/debug/pprof/goroutine` - str(6079 + actor_number) - + ":6090", # appgateserver metrics port. appgateserver1 - exposes 6080, appgateserver2 exposes 6081, etc. - ], +# Loop to configure and apply multiple PATH gateway deployments +for x in range(localnet_config["path_gateways"]["count"]): + actor_number += 1 + + resource_flags = [ + "--values=./localnet/kubernetes/values-common.yaml", + "--set=metrics.serviceMonitor.enabled=" + str(localnet_config["observability"]["enabled"]), + "--set=path.mountConfigMaps[0].name=path-config-" + str(actor_number), + "--set=path.mountConfigMaps[0].mountPath=/app/config/", + ] + + if localnet_config["path_local_repo"]["enabled"]: + path_image_deps = ["path-local"] + path_image_keys = [("image.repository", "image.tag")] + path_deps=["path-local"] + resource_flags.append("--set=global.imagePullPolicy=Never") + else: + path_image_deps = [] + path_image_keys = [] + path_deps=[] + + configmap_create( + "path-config-" + str(actor_number), + from_file=".config.yaml=./localnet/kubernetes/config-path-" + str(actor_number) + ".yaml" ) -# Provision Gateways -actor_number = 0 -for x in range(localnet_config["gateways"]["count"]): - actor_number = actor_number + 1 helm_resource( - "gateway" + str(actor_number), - chart_prefix + "appgate-server", - flags=[ - "--values=./localnet/kubernetes/values-common.yaml", - "--values=./localnet/kubernetes/values-gateway.yaml", - "--set=config.signing_key=gateway" + str(actor_number), - "--set=metrics.serviceMonitor.enabled=" + str(localnet_config["observability"]["enabled"]), - "--set=development.delve.enabled=" + str(localnet_config["gateways"]["delve"]["enabled"]), - "--set=logLevel=" + str(localnet_config["gateways"]["logs"]["level"]), - "--set=image.repository=poktrolld", - ], - image_deps=["poktrolld"], - image_keys=[("image.repository", "image.tag")], + "path" + str(actor_number), + grove_chart_prefix + "path", + flags=resource_flags, + image_deps=path_image_deps, + image_keys=path_image_keys, ) + + # Apply the deployment to Kubernetes using Tilt k8s_resource( - "gateway" + str(actor_number), + "path" + str(actor_number), labels=["gateways"], - resource_deps=["validator"], - links=[ - link( - "http://localhost:3003/d/appgateserver/protocol-appgate-server?orgId=1&refresh=5s&var-appgateserver=gateway" - + str(actor_number), - "Grafana dashboard", - ), - ], + resource_deps=path_deps, + # TODO_IMPROVE(@okdas): Update this once PATH has grafana dashboards + # links=[ + # link( + # "http://localhost:3003/d/path/protocol-path?orgId=1&refresh=5s&var-path=gateway" + # + str(actor_number), + # "Grafana dashboard", + # ), + # ], + # TODO_IMPROVE(@okdas): Add port forwards to grafana, pprof, like the other resources port_forwards=[ - str(42078 + actor_number) + ":42069", # gateway1 - exposes 42079, gateway2 exposes 42080, etc. - str(40064 + actor_number) + ":40004", # DLV port. gateway1 - exposes 40065, gateway2 exposes 40066, etc. - # Run `curl localhost:PORT` to see the current snapshot of gateway metrics. - str(9059 + actor_number) - + ":9090", # gateway metrics port. gateway1 - exposes 9060, gateway2 exposes 9061, etc. - # Use with pprof like this: `go tool pprof -http=:3333 http://localhost:6090/debug/pprof/goroutine` - str(6059 + actor_number) - + ":6060", # gateway metrics port. gateway1 - exposes 6060, gateway2 exposes 6061, etc. + str(2999 + actor_number) + ":3000" ], ) + +# Provision Validators k8s_resource( "validator", labels=["pocket_network"], @@ -393,8 +384,10 @@ k8s_resource( ], ) +# Provision anvil (test Ethereum) service nodes k8s_resource("anvil", labels=["data_nodes"], port_forwards=["8547"]) +# Provision ollama (LLM) service nodes if localnet_config["ollama"]["enabled"]: print("Ollama enabled: " + str(localnet_config["ollama"]["enabled"])) @@ -413,6 +406,7 @@ if localnet_config["ollama"]["enabled"]: resource_deps=["ollama"], ) +# Provision RESTful (not JSON-RPC) test service nodes if localnet_config["rest"]["enabled"]: print("REST enabled: " + str(localnet_config["rest"]["enabled"])) deployment_create("rest", image="davarski/go-rest-api-demo") diff --git a/cmd/poktrolld/cmd/root.go b/cmd/poktrolld/cmd/root.go index 86fd01f81..768542be9 100644 --- a/cmd/poktrolld/cmd/root.go +++ b/cmd/poktrolld/cmd/root.go @@ -26,7 +26,6 @@ import ( "github.com/spf13/pflag" "github.com/pokt-network/poktroll/app" - appgateservercmd "github.com/pokt-network/poktroll/pkg/appgateserver/cmd" relayercmd "github.com/pokt-network/poktroll/pkg/relayer/cmd" ) @@ -135,10 +134,9 @@ func NewRootCmd() *cobra.Command { relayercmd.RelayerCmd(), ) - // add the appgate server command - rootCmd.AddCommand( - appgateservercmd.AppGateServerCmd(), - ) + // TODO_MAINNET(@commoddity): Consider adding an entrypoint to deploy a PATH + // gateway to the localnet to streamline it for users. + // For reference, see how we removed appgateserver in #879. return rootCmd } diff --git a/e2e/tests/init_test.go b/e2e/tests/init_test.go index 2c152c232..680806c87 100644 --- a/e2e/tests/init_test.go +++ b/e2e/tests/init_test.go @@ -61,10 +61,8 @@ var ( flagFeaturesPath string keyRingFlag = "--keyring-backend=test" chainIdFlag = "--chain-id=poktroll" - // Keeping localhost by default because that is how we run the tests on our machines locally - // gatewayUrl is pointing to a non-sovereign app gate server so multiple - // apps could relay through it. - gatewayUrl = "http://localhost:42079" + // pathUrl points to a local gateway using the PATH framework in centralized mode. + pathUrl = "http://localhost:3000/v1" // localhost is kept as the default to streamline local development & testing. ) func init() { @@ -74,9 +72,9 @@ func init() { flag.StringVar(&flagFeaturesPath, "features-path", "*.feature", "Specifies glob paths for the runner to look up .feature files") - // If "GATEWAY_URL" envar is present, use it for appGateServerUrl - if url := os.Getenv("GATEWAY_URL"); url != "" { - gatewayUrl = url + // If "PATH_URL" ENV variable is present, use it for pathUrl + if url := os.Getenv("PATH_URL"); url != "" { + pathUrl = url } } @@ -464,12 +462,12 @@ func (s *suite) TheApplicationSendsTheSupplierASuccessfulRequestForServiceWithPa appAddr := accNameToAddrMap[appName] - res, err := s.pocketd.RunCurlWithRetry(gatewayUrl, serviceId, method, path, appAddr, requestData, 5) + res, err := s.pocketd.RunCurlWithRetry(pathUrl, serviceId, method, path, appAddr, requestData, 5) require.NoError(s, err, "error sending relay request from app %q to supplier %q for service %q", appName, supplierOperatorName, serviceId) var jsonContent json.RawMessage err = json.Unmarshal([]byte(res.Stdout), &jsonContent) - require.NoError(s, err, `Expected valid JSON, got: %s`) + require.NoErrorf(s, err, `Expected valid JSON, got: %s`, res.Stdout) jsonMap, err := jsonToMap(jsonContent) require.NoError(s, err, "error converting JSON to map") diff --git a/e2e/tests/node.go b/e2e/tests/node.go index ada189be0..4dc1726de 100644 --- a/e2e/tests/node.go +++ b/e2e/tests/node.go @@ -5,6 +5,7 @@ package e2e import ( "bytes" "fmt" + "net" "net/url" "os" "os/exec" @@ -23,8 +24,13 @@ var ( defaultRPCHost = "127.0.0.1" // defaultHome is the default home directory for pocketd defaultHome = os.Getenv("POKTROLLD_HOME") - // defaultAppGateServerURL used by curl commands to send relay requests - defaultAppGateServerURL = os.Getenv("APPGATE_SERVER") + // defaultPathURL used by curl commands to send relay requests + defaultPathURL = os.Getenv("PATH_URL") + // defaultPathHostOverride overrides the host in the URL used to send requests + // Since the current DevNet infrastructure does not support arbitrary subdomains, + // this is used to specify the host to connect to and the full host (with the service as a subdomain) + // will be sent in the "Host" request header. + defaultPathHostOverride = os.Getenv("PATH_HOST_OVERRIDE") // defaultDebugOutput provides verbose output on manipulations with binaries (cli command, stdout, stderr) defaultDebugOutput = os.Getenv("E2E_DEBUG_OUTPUT") ) @@ -101,7 +107,7 @@ func (p *pocketdBin) RunCommandOnHostWithRetry(rpcUrl string, numRetries uint8, // RunCurl runs a curl command on the local machine func (p *pocketdBin) RunCurl(rpcUrl, service, method, path, appAddr, data string, args ...string) (*commandResult, error) { if rpcUrl == "" { - rpcUrl = defaultAppGateServerURL + rpcUrl = defaultPathURL } return p.runCurlCmd(rpcUrl, service, method, path, appAddr, data, args...) } @@ -109,6 +115,11 @@ func (p *pocketdBin) RunCurl(rpcUrl, service, method, path, appAddr, data string // RunCurlWithRetry runs a curl command on the local machine with multiple retries. // It also accounts for an ephemeral error that may occur due to DNS resolution such as "no such host". func (p *pocketdBin) RunCurlWithRetry(rpcUrl, service, method, path, appAddr, data string, numRetries uint8, args ...string) (*commandResult, error) { + if service == "" { + err := fmt.Errorf("Missing service name for curl request with url: %s", rpcUrl) + return nil, err + } + // No more retries left if numRetries <= 0 { return p.RunCurl(rpcUrl, service, method, path, appAddr, data, args...) @@ -178,8 +189,20 @@ func (p *pocketdBin) runCurlCmd(rpcBaseURL, service, method, path, appAddr, data return nil, err } - if len(service) > 0 { - rpcUrl.Path = rpcUrl.Path + service + // Get the virtual host that will be sent in the "Host" request header + virtualHost := getVirtualHostFromUrlForService(rpcUrl, service) + + // TODO_HACK: As of PR #879, the DevNet infrastructure does not support routing + // requests to arbitrary subdomains due to TLS certificate-related complexities. + // In such environment, defaultPathHostOverride (which contains no subdomain) + // is used as: + // 1. The gateway's 'host:port' to connect to + // 2. A base to which the service is added as a subdomain then set as the "Host" request header. + // (i.e. Host: .) + // + // Override the actual connection address if the environment requires it. + if defaultPathHostOverride != "" { + rpcUrl.Host = defaultPathHostOverride } // Ensure that if a path is provided, it starts with a "/". @@ -188,30 +211,21 @@ func (p *pocketdBin) runCurlCmd(rpcBaseURL, service, method, path, appAddr, data if len(path) > 0 && path[0] != '/' { path = "/" + path } - rpcUrl.Path = rpcUrl.Path + path - // When sending a relay request, through a gateway (i.e. non-sovereign application) - // then, the application address must be provided. - if len(appAddr) > 0 { - queryValues := rpcUrl.Query() - queryValues.Set("applicationAddr", appAddr) - rpcUrl.RawQuery = queryValues.Encode() - } - base := []string{ - "-v", // verbose output - "-sS", // silent with error - "-X", method, // HTTP method - "-H", "Content-Type: application/json", // HTTP headers + "-v", // verbose output + "-sS", // silent with error + "-H", `Content-Type: application/json`, // HTTP headers + "-H", fmt.Sprintf("Host: %s", virtualHost), // Add virtual host header + "-H", fmt.Sprintf("X-App-Address: %s", appAddr), rpcUrl.String(), } if method == "POST" { base = append(base, "--data", data) } else if len(data) > 0 { - fmt.Println(fmt.Sprintf("WARN: data provided but not being included in the %s request", method)) - + fmt.Printf("WARN: data provided but not being included in the %s request because it is not of type POST", method) } args = append(base, args...) commandStr := "curl " + strings.Join(args, " ") // Create a string representation of the command @@ -240,3 +254,40 @@ func (p *pocketdBin) runCurlCmd(rpcBaseURL, service, method, path, appAddr, data return r, err } + +// formatURLString returns RESTful or JSON-RPC API endpoint URL depending +// on the parameters provided. +func formatURLString(serviceAlias, rpcUrl, path string) string { + // For JSON-RPC APIs, the path should be empty + if len(path) == 0 { + return fmt.Sprintf("http://%s.%s/v1", serviceAlias, rpcUrl) + } + + // For RESTful APIs, the path should not be empty. + // We remove the leading / to make the format string below easier to read. + if path[0] == '/' { + path = path[1:] + } + return fmt.Sprintf("http://%s.%s/v1/%s", serviceAlias, rpcUrl, path) +} + +// getVirtualHostFromUrlForService returns a virtual host taking into consideration +// the URL's host and the service if it's non-empty. +// Specifically, it: +// 1. Extract's the host from the rpcURL +// 2. Prefixes the service as a subdomain to (1) the given rpcUrl host stripped of the port +// +// TODO_HACK: This is needed as of PR #879 because the DevNet infrastructure does +// not support arbitrary subdomains due to TLS certificate-related complexities. +func getVirtualHostFromUrlForService(rpcUrl *url.URL, service string) string { + // Strip port if it exists and add service prefix + host, _, err := net.SplitHostPort(rpcUrl.Host) + if err != nil { + // err is non-nil if rpcUrl.Host does not have a port. + // Use the entire host as is + host = rpcUrl.Host + } + virtualHost := fmt.Sprintf("%s.%s", service, host) + + return virtualHost +} diff --git a/e2e/tests/relay.feature b/e2e/tests/relay.feature index 023eedf06..d01ec6119 100644 --- a/e2e/tests/relay.feature +++ b/e2e/tests/relay.feature @@ -8,17 +8,19 @@ Feature: Relay Namespace And the session for application "app1" and service "anvil" contains the supplier "supplier1" Then the application "app1" sends the supplier "supplier1" a successful request for service "anvil" with path "" and data '{"jsonrpc":"2.0","method":"eth_blockNumber","params":[],"id":1}' - Scenario: App can send a REST relay to Supplier - Given the user has the pocketd binary installed - # Account "app2" is staked for service "rest" - And the application "app2" is staked for service "rest" - And the supplier "supplier1" is staked for service "rest" - And the session for application "app2" and service "rest" contains the supplier "supplier1" - When the application "app2" sends the supplier "supplier1" a successful request for service "rest" with path "/quote" - Then a "tokenomics" module "ClaimSettled" end block event is broadcast + # TODO_MAINNET(@red-0ne): Enable this test once PATH Gateway supports REST. + # See https://github.com/buildwithgrove/path/issues/87 + # Scenario: App can send a REST relay to Supplier + # Given the user has the pocketd binary installed + # # Account "app2" is staked for service "rest" + # And the application "app2" is staked for service "rest" + # And the supplier "supplier1" is staked for service "rest" + # And the session for application "app2" and service "rest" contains the supplier "supplier1" + # When the application "app2" sends the supplier "supplier1" a successful request for service "rest" with path "/quote" + # Then a "tokenomics" module "ClaimSettled" end block event is broadcast # TODO_TEST(@Olshansk): - # - Successful relay through applicat's sovereign appgate server + # - Successful relay through PATH's Trusted mode # - Successful relay through gateway app is delegation to # - Successful relay through gateway when app is delegating to multiple gateways # - Failed relay through gateway app is not delegation to diff --git a/go.mod b/go.mod index 024364e1b..70303d360 100644 --- a/go.mod +++ b/go.mod @@ -61,7 +61,6 @@ require ( github.com/prometheus/client_golang v1.19.0 github.com/regen-network/gocuke v1.1.0 github.com/rs/zerolog v1.32.0 - github.com/slok/go-http-metrics v0.11.0 github.com/spf13/cobra v1.8.1 github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.18.2 @@ -167,11 +166,13 @@ require ( github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/getsentry/sentry-go v0.27.0 // indirect github.com/ghodss/yaml v1.0.0 // indirect + github.com/gin-gonic/gin v1.9.1 // indirect github.com/go-chi/chi/v5 v5.1.0 // indirect github.com/go-kit/log v0.2.1 // indirect github.com/go-logfmt/logfmt v0.6.0 // indirect github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-playground/validator/v10 v10.15.5 // indirect github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 // indirect github.com/gofrs/uuid v4.4.0+incompatible // indirect github.com/gofrs/uuid/v5 v5.2.0 // indirect diff --git a/go.sum b/go.sum index 62dc61bb0..eddbf91bd 100644 --- a/go.sum +++ b/go.sum @@ -1077,8 +1077,6 @@ github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrf github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= -github.com/slok/go-http-metrics v0.11.0 h1:ABJUpekCZSkQT1wQrFvS4kGbhea/w6ndFJaWJeh3zL0= -github.com/slok/go-http-metrics v0.11.0/go.mod h1:ZGKeYG1ET6TEJpQx18BqAJAvxw9jBAZXCHU7bWQqqAc= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= diff --git a/load-testing/loadtest_manifest_localnet.yaml b/load-testing/loadtest_manifest_localnet.yaml index 97a914995..763771576 100644 --- a/load-testing/loadtest_manifest_localnet.yaml +++ b/load-testing/loadtest_manifest_localnet.yaml @@ -43,17 +43,17 @@ suppliers: gateways: # The gateway address that is available in the load test's environment keyring, # used to identify the gateway and sign relays and transactions with. - # It must be the address corresponding to the provided signing_key_name in the - # `appgate_server_config.yaml` file. + # It must be the address provided in the PATH configurations in + # `./localnet/kubernetes/config-path-*.yaml` files. # Gateway 1; http://localhost:10350/r/gateway1/overview - address: pokt15vzxjqklzjtlz7lahe8z2dfe9nm5vxwwmscne4 - exposed_url: http://localhost:42079 # The url used to send relays to the gateway on. + exposed_url: http://anvil.localhost/v1:3000 # The gateway url that the user sends relays to (e.g. curl) # Gateway 2; http://localhost:10350/r/gateway2/overview - address: pokt15w3fhfyc0lttv7r585e2ncpf6t2kl9uh8rsnyz - exposed_url: http://localhost:42080 + exposed_url: http://anvil.localhost/v1:3001 # Gateway 3; http://localhost:10350/r/gateway3/overview - address: pokt1zhmkkd0rh788mc9prfq0m2h88t9ge0j83gnxya - exposed_url: http://localhost:42081 + exposed_url: http://anvil.localhost/v1:3002 diff --git a/load-testing/loadtest_manifest_localnet_single_supplier.yaml b/load-testing/loadtest_manifest_localnet_single_supplier.yaml index f7e257077..c455eaa8f 100644 --- a/load-testing/loadtest_manifest_localnet_single_supplier.yaml +++ b/load-testing/loadtest_manifest_localnet_single_supplier.yaml @@ -35,17 +35,17 @@ suppliers: gateways: # The gateway address that is available in the load test's environment keyring, # used to identify the gateway and sign relays and transactions with. - # It must be the address corresponding to the provided signing_key_name in the - # `appgate_server_config.yaml` file. + # It must be the address provided in the PATH configurations in + # `./localnet/kubernetes/config-path-*.yaml` files. # Gateway 1; http://localhost:10350/r/gateway1/overview - address: pokt15vzxjqklzjtlz7lahe8z2dfe9nm5vxwwmscne4 - exposed_url: http://localhost:42079 # The gateway url that the user sends relays to (e.g. curl) + exposed_url: http://anvil.localhost/v1:3000 # The gateway url that the user sends relays to (e.g. curl) # Gateway 2; http://localhost:10350/r/gateway2/overview - address: pokt15w3fhfyc0lttv7r585e2ncpf6t2kl9uh8rsnyz - exposed_url: http://localhost:42080 + exposed_url: http://anvil.localhost/v1:3001 # Gateway 3; http://localhost:10350/r/gateway3/overview - address: pokt1zhmkkd0rh788mc9prfq0m2h88t9ge0j83gnxya - exposed_url: http://localhost:42081 + exposed_url: http://anvil.localhost/v1:3002 diff --git a/load-testing/tests/relays_stress_helpers_test.go b/load-testing/tests/relays_stress_helpers_test.go index 97d5b1339..aa55dcf49 100644 --- a/load-testing/tests/relays_stress_helpers_test.go +++ b/load-testing/tests/relays_stress_helpers_test.go @@ -420,7 +420,7 @@ func (s *relaysSuite) mapSessionInfoWhenStakingNewSuppliersAndGatewaysFn() chann // method since they are activated at the end of the session so they are // available for the beginning of the next one. // This is because the suppliers involvement is out of control of the test - // suite and is driven by the AppGateServer's supplier endpoint selection. + // suite and is driven by the Gateway's endpoint selection. if suppliersPlan.shouldIncrementSupplierCount(s.sharedParams, notif, activeSuppliers, s.testStartHeight) { newSuppliers = s.sendStakeSuppliersTxs(notif, &suppliersPlan) } diff --git a/load-testing/tests/relays_stress_test.go b/load-testing/tests/relays_stress_test.go index 5aa7fbb65..ae7f07a12 100644 --- a/load-testing/tests/relays_stress_test.go +++ b/load-testing/tests/relays_stress_test.go @@ -170,7 +170,7 @@ type relaysSuite struct { // gatewayUrls is a map of gatewayAddress->URL representing the provisioned gateways. // These gateways are not staked yet but have their off-chain instance running // and ready to be staked and used in the test. - // Since AppGateServers are pre-provisioned, and already assigned a signingAddress + // Since Gateways are pre-provisioned, and already assigned a signingAddress // and an URL to send relays to, the test suite does not create new ones but picks // from this list. // The max gateways used in the test must be less than or equal to the number of diff --git a/localnet/grafana-dashboards/appgateserver.json b/localnet/grafana-dashboards/appgateserver.json deleted file mode 100644 index 81e6c9813..000000000 --- a/localnet/grafana-dashboards/appgateserver.json +++ /dev/null @@ -1,828 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "links": [], - "panels": [ - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 8, - "panels": [], - "title": "Relay info", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 1 - }, - "id": 9, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "sum (appgateserver_requests_total{pod=~\"^$appgateserver.*\"}) by (request_type, service_id)", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Relay requests", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 1 - }, - "id": 10, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "sum (appgateserver_errors_total{pod=~\"^$appgateserver.*\"}) by (request_type, service_id)", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Relay errors", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 9 - }, - "id": 2, - "panels": [], - "title": "Golang metrics", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 10 - }, - "id": 1, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "sum (go_goroutines{job=~\"$appgateserver-.*\"}) by (pod)", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Goroutines", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 18 - }, - "id": 5, - "panels": [], - "title": "OS metrics", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 100, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 0, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "requests" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F2495C", - "mode": "fixed" - } - }, - { - "id": "custom.fillOpacity", - "value": 0 - }, - { - "id": "custom.lineWidth", - "value": 2 - }, - { - "id": "custom.stacking", - "value": { - "group": "A", - "mode": "none" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "limits" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FF9830", - "mode": "fixed" - } - }, - { - "id": "custom.fillOpacity", - "value": 0 - }, - { - "id": "custom.lineWidth", - "value": 2 - }, - { - "id": "custom.stacking", - "value": { - "group": "A", - "mode": "none" - } - } - ] - } - ] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 19 - }, - "id": 6, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "10.4.0", - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "editorMode": "code", - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{pod=~\"^$appgateserver.*\"}) by (container)", - "format": "time_series", - "legendFormat": "{{container}}", - "range": true, - "refId": "A" - }, - { - "datasource": { - "uid": "$datasource" - }, - "editorMode": "code", - "expr": "sum(\n kube_pod_container_resource_requests{pod=~\"^$appgateserver.*\", resource=\"cpu\"}\n)\n", - "format": "time_series", - "legendFormat": "requests", - "range": true, - "refId": "B" - }, - { - "datasource": { - "uid": "$datasource" - }, - "editorMode": "code", - "expr": "sum(\n kube_pod_container_resource_limits{pod=~\"^$appgateserver.*\", resource=\"cpu\"}\n)\n", - "format": "time_series", - "legendFormat": "limits", - "range": true, - "refId": "C" - } - ], - "title": "CPU Usage", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 100, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 0, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "requests" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F2495C", - "mode": "fixed" - } - }, - { - "id": "custom.fillOpacity", - "value": 0 - }, - { - "id": "custom.lineWidth", - "value": 2 - }, - { - "id": "custom.stacking", - "value": { - "group": "A", - "mode": "none" - } - }, - { - "id": "custom.lineStyle", - "value": { - "dash": [10, 10], - "fill": "dash" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "limits" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FF9830", - "mode": "fixed" - } - }, - { - "id": "custom.fillOpacity", - "value": 0 - }, - { - "id": "custom.lineWidth", - "value": 2 - }, - { - "id": "custom.stacking", - "value": { - "group": "A", - "mode": "none" - } - }, - { - "id": "custom.lineStyle", - "value": { - "dash": [10, 10], - "fill": "dash" - } - } - ] - } - ] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 19 - }, - "id": 7, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "10.4.0", - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "editorMode": "code", - "expr": "sum(container_memory_working_set_bytes{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", pod=~\"^$appgateserver.*\", container!=\"\", image!=\"\"}) by (container)", - "format": "time_series", - "legendFormat": "{{container}}", - "range": true, - "refId": "A" - }, - { - "datasource": { - "uid": "$datasource" - }, - "editorMode": "code", - "expr": "sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", pod=~\"^$appgateserver.*\", resource=\"memory\"}\n)\n", - "format": "time_series", - "legendFormat": "requests", - "range": true, - "refId": "B" - }, - { - "datasource": { - "uid": "$datasource" - }, - "editorMode": "code", - "expr": "sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", pod=~\"^$appgateserver.*\", resource=\"memory\"}\n)\n", - "format": "time_series", - "legendFormat": "limits", - "range": true, - "refId": "C" - } - ], - "title": "Memory Usage (WSS)", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 25 - }, - "id": 3, - "panels": [], - "title": "Logs", - "type": "row" - }, - { - "datasource": { - "type": "loki", - "uid": "P8E80F9AEF21F6940" - }, - "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 26 - }, - "id": 4, - "options": { - "dedupStrategy": "none", - "enableLogDetails": true, - "prettifyLogMessage": false, - "showCommonLabels": false, - "showLabels": false, - "showTime": true, - "sortOrder": "Descending", - "wrapLogMessage": true - }, - "targets": [ - { - "datasource": { - "type": "loki", - "uid": "P8E80F9AEF21F6940" - }, - "editorMode": "builder", - "expr": "{instance=\"$appgateserver\"} |= `` | json", - "queryType": "range", - "refId": "A" - } - ], - "title": "Logs", - "type": "logs" - } - ], - "refresh": "5s", - "schemaVersion": 39, - "tags": ["protocol"], - "templating": { - "list": [ - { - "current": { - "selected": false, - "text": "appgateserver1", - "value": "appgateserver1" - }, - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "definition": "label_values(go_info{container=\"appgate-server\"},job)", - "hide": 0, - "includeAll": false, - "multi": false, - "name": "appgateserver", - "options": [], - "query": { - "qryType": 1, - "query": "label_values(go_info{container=\"appgate-server\"},job)", - "refId": "PrometheusVariableQueryEditor-VariableQuery" - }, - "refresh": 1, - "regex": "/^(?.*)-appgate/", - "skipUrlSync": false, - "sort": 0, - "type": "query" - } - ] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timepicker": {}, - "timezone": "browser", - "title": "Protocol / AppGate Server", - "uid": "appgateserver", - "version": 1, - "weekStart": "" -} diff --git a/localnet/kubernetes/config-path-1.yaml b/localnet/kubernetes/config-path-1.yaml new file mode 100644 index 000000000..7a5fb58c7 --- /dev/null +++ b/localnet/kubernetes/config-path-1.yaml @@ -0,0 +1,15 @@ +shannon_config: + full_node_config: + rpc_url: tcp://validator-poktroll-validator:26657 + grpc_config: + host_port: validator-poktroll-validator:9090 + insecure: true + lazy_mode: true + gateway_config: + gateway_mode: delegated + gateway_address: pokt15vzxjqklzjtlz7lahe8z2dfe9nm5vxwwmscne4 # gateway1 + gateway_private_key_hex: cf09805c952fa999e9a63a9f434147b0a5abfd10f268879694c6b5a70e1ae177 + +services: + anvil: + alias: anvil diff --git a/localnet/kubernetes/config-path-2.yaml b/localnet/kubernetes/config-path-2.yaml new file mode 100644 index 000000000..9dcb6fedc --- /dev/null +++ b/localnet/kubernetes/config-path-2.yaml @@ -0,0 +1,15 @@ +shannon_config: + full_node_config: + rpc_url: tcp://validator-poktroll-validator:26657 + grpc_config: + host_port: validator-poktroll-validator:9090 + insecure: true + lazy_mode: true + gateway_config: + gateway_mode: delegated + gateway_address: pokt15w3fhfyc0lttv7r585e2ncpf6t2kl9uh8rsnyz # gateway2 + gateway_private_key_hex: 177ba43cec962ea407f71da9c3994ba685708e82d5d7a6d7da3268e74119bf88 + +services: + anvil: + alias: anvil diff --git a/localnet/kubernetes/config-path-3.yaml b/localnet/kubernetes/config-path-3.yaml new file mode 100644 index 000000000..933a36643 --- /dev/null +++ b/localnet/kubernetes/config-path-3.yaml @@ -0,0 +1,15 @@ +shannon_config: + full_node_config: + rpc_url: tcp://validator-poktroll-validator:26657 + grpc_config: + host_port: validator-poktroll-validator:9090 + insecure: true + lazy_mode: true + gateway_config: + gateway_mode: delegated + gateway_address: pokt1zhmkkd0rh788mc9prfq0m2h88t9ge0j83gnxya # gateway3 + gateway_private_key_hex: f73b6f7f0b9c99603c7eeddbf1c419c6f6bbc241f3798e3e4c8da9769ca81c26 + +services: + anvil: + alias: anvil diff --git a/localnet/kubernetes/values-appgateserver.yaml b/localnet/kubernetes/values-appgateserver.yaml deleted file mode 100644 index 1375175b4..000000000 --- a/localnet/kubernetes/values-appgateserver.yaml +++ /dev/null @@ -1,9 +0,0 @@ -config: - query_node_rpc_url: tcp://validator-poktroll-validator:26657 - query_node_grpc_url: tcp://validator-poktroll-validator:9090 - metrics: - enabled: true - addr: :9090 - pprof: - enabled: true - addr: localhost:6060 diff --git a/localnet/kubernetes/values-gateway.yaml b/localnet/kubernetes/values-gateway.yaml deleted file mode 100644 index fe4d46d79..000000000 --- a/localnet/kubernetes/values-gateway.yaml +++ /dev/null @@ -1,10 +0,0 @@ -config: - query_node_rpc_url: tcp://validator-poktroll-validator:26657 - query_node_grpc_url: tcp://validator-poktroll-validator:9090 - self_signing: false - metrics: - enabled: true - addr: :9090 - pprof: - enabled: true - addr: localhost:6060 diff --git a/localnet/poktrolld/config/appgate_server_config.yaml b/localnet/poktrolld/config/appgate_server_config.yaml deleted file mode 100644 index 16c48b79c..000000000 --- a/localnet/poktrolld/config/appgate_server_config.yaml +++ /dev/null @@ -1,11 +0,0 @@ -query_node_rpc_url: tcp://poktroll-validator:26657 -query_node_grpc_url: tcp://poktroll-validator:9090 -signing_key: app1 -self_signing: true -listening_endpoint: http://localhost:42069 -metrics: - enabled: true - addr: :9090 -pprof: - enabled: true - addr: localhost:6060 diff --git a/localnet/poktrolld/config/appgate_server_config_example.yaml b/localnet/poktrolld/config/appgate_server_config_example.yaml deleted file mode 100644 index 8c93c006c..000000000 --- a/localnet/poktrolld/config/appgate_server_config_example.yaml +++ /dev/null @@ -1,20 +0,0 @@ -# Pocket node URL that exposes CometBFT JSON-RPC API. -# This can be used by the Cosmos client SDK, event subscriptions, etc... -query_node_rpc_url: tcp://poktroll-validator:26657 -# Pocket node URL that exposes the Cosmos gRPC service, dedicated to querying purposes. -query_node_grpc_url: tcp://poktroll-validator:9090 -# The name of the key (in the keyring) that will be used to sign relays -signing_key: app1 -# Whether the server should sign all incoming requests with its own ring (for applications) -self_signing: true -# The host and port that the appgate server will listen on -listening_endpoint: http://localhost:42069 -# Prometheus exporter configuration -metrics: - # Turn on/off the metrics exporter - enabled: true - # The address that the metrics exporter will listen on. Can be just a port, or host:port - addr: :9090 -pprof: - enabled: true - addr: localhost:6060 diff --git a/localnet/poktrolld/config/appgate_server_config_localnet_vscode.yaml b/localnet/poktrolld/config/appgate_server_config_localnet_vscode.yaml deleted file mode 100644 index 2dab42371..000000000 --- a/localnet/poktrolld/config/appgate_server_config_localnet_vscode.yaml +++ /dev/null @@ -1,11 +0,0 @@ -query_node_rpc_url: tcp://localhost:26657 -query_node_grpc_url: tcp://localhost:36658 -signing_key: app1 -self_signing: true -listening_endpoint: http://0.0.0.0:42069 -metrics: - enabled: true - addr: :9090 -pprof: - enabled: true - addr: localhost:6060 diff --git a/makefiles/localnet.mk b/makefiles/localnet.mk index 1df6113cf..d43cdccb6 100644 --- a/makefiles/localnet.mk +++ b/makefiles/localnet.mk @@ -27,5 +27,5 @@ localnet_regenesis: check_yq warn_message_acc_initialize_pubkeys ## Regenerate t @cp -r ${HOME}/.poktroll/config $(POKTROLLD_HOME)/ .PHONY: cosmovisor_start_node -cosmovisor_start_node: # Starts the node using cosmovisor that waits for an upgrade plan +cosmovisor_start_node: ## Starts the node using cosmovisor that waits for an upgrade plan bash tools/scripts/upgrades/cosmovisor-start-node.sh diff --git a/makefiles/query.mk b/makefiles/query.mk index c55b7558b..943e5e295 100644 --- a/makefiles/query.mk +++ b/makefiles/query.mk @@ -1,6 +1,7 @@ ##################### ### Query Helpers ### ##################### + .PHONY: query_tx query_tx: ## Query for a transaction by hash and output as YAML (default). poktrolld --home=$(POKTROLLD_HOME) query tx $(HASH) --node $(POCKET_NODE) diff --git a/makefiles/relay.mk b/makefiles/relay.mk index 2270bf012..38c41bfcd 100644 --- a/makefiles/relay.mk +++ b/makefiles/relay.mk @@ -1,21 +1,23 @@ ##################### ### Relay Helpers ### ##################### -.PHONY: send_relay_sovereign_app_JSONRPC -send_relay_sovereign_app_JSONRPC: # Send a JSONRPC relay through the AppGateServer as a sovereign application - curl -X POST -H "Content-Type: application/json" \ - --data '{"jsonrpc":"2.0","method":"eth_blockNumber","params":[],"id":1}' \ - $(APPGATE_SERVER)/anvil -.PHONY: send_relay_delegating_app_JSONRPC -send_relay_delegating_app_JSONRPC: # Send a relay through the gateway as an application that's delegating to this gateway - @appAddr=$$(poktrolld keys show app1 -a) && \ +.PHONY: send_relay_path_JSONRPC +send_relay_path_JSONRPC: test_e2e_env ## Send a JSONRPC relay through PATH to a local anvil (test ETH) node curl -X POST -H "Content-Type: application/json" \ - --data '{"jsonrpc":"2.0","method":"eth_blockNumber","params":[],"id":1}' \ - $(GATEWAY_URL)/anvil?applicationAddr=$$appAddr + -H "X-App-Address: pokt1mrqt5f7qh8uxs27cjm9t7v9e74a9vvdnq5jva4" \ + --data '{"jsonrpc":"2.0","method":"eth_blockNumber","params":[],"id":1}' \ + http://anvil.localhost:3000/v1 +# $(subst http://,http://anvil.,$(PATH_URL))/v1 -.PHONY: send_relay_sovereign_app_REST -send_relay_sovereign_app_REST: # Send a REST relay through the AppGateServer as a sovereign application - curl -X POST -H "Content-Type: application/json" \ - --data '{"model": "qwen:0.5b", "stream": false, "messages": [{"role": "user", "content":"count from 1 to 10"}]}' \ - $(APPGATE_SERVER)/ollama/api/chat +# TODO_MAINNET(@red-0ne): Re-enable this once PATH Gateway supports REST. +# See https://github.com/buildwithgrove/path/issues/87 +.PHONY: send_relay_path_REST +send_relay_path_REST: acc_initialize_pubkeys ## Send a REST relay through PATH to a local ollama (LLM) service + @echo "Not implemented yet. Check if PATH supports REST relays yet: https://github.com/buildwithgrove/path/issues/87" +# curl -X POST -H "Content-Type: application/json" -H "X-App-Address: pokt1mrqt5f7qh8uxs27cjm9t7v9e74a9vvdnq5jva4" \ +# --data '{"model": "qwen:0.5b", "stream": false, "messages": [{"role": "user", "content":"count from 1 to 10"}]}' \ +# $(subst http://,http://ollama.,$(PATH_URL))/api/chat + +# TODO_MAINNET(@olshansk): Add all the permissionless/delegated/centralized variations once +# the following documentation is ready: https://www.notion.so/buildwithgrove/Different-Modes-of-Operation-PATH-LocalNet-Discussions-122a36edfff6805e9090c9a14f72f3b5?pvs=4#151a36edfff680d681a2dd7f4e5fee55 \ No newline at end of file diff --git a/makefiles/tests.mk b/makefiles/tests.mk index 9f721ead8..7bb292e3d 100644 --- a/makefiles/tests.mk +++ b/makefiles/tests.mk @@ -5,7 +5,7 @@ .PHONY: test_e2e_env test_e2e_env: warn_message_acc_initialize_pubkeys ## Setup the default env vars for E2E tests export POCKET_NODE=$(POCKET_NODE) && \ - export APPGATE_SERVER=$(APPGATE_SERVER) && \ + export PATH_URL=$(PATH_URL) && \ export POKTROLLD_HOME=../../$(POKTROLLD_HOME) .PHONY: test_e2e diff --git a/pkg/appgateserver/cmd/cmd.go b/pkg/appgateserver/cmd/cmd.go deleted file mode 100644 index 124d8ceb5..000000000 --- a/pkg/appgateserver/cmd/cmd.go +++ /dev/null @@ -1,209 +0,0 @@ -package cmd - -import ( - "context" - "errors" - "fmt" - "net/http" - "net/url" - "os" - - "cosmossdk.io/depinject" - cosmosflags "github.com/cosmos/cosmos-sdk/client/flags" - "github.com/spf13/cobra" - - "github.com/pokt-network/poktroll/cmd/signals" - "github.com/pokt-network/poktroll/pkg/appgateserver" - appgateconfig "github.com/pokt-network/poktroll/pkg/appgateserver/config" - "github.com/pokt-network/poktroll/pkg/deps/config" - "github.com/pokt-network/poktroll/pkg/polylog" - "github.com/pokt-network/poktroll/pkg/polylog/polyzero" -) - -// We're `explicitly omitting default` so that the appgateserver crashes if these aren't specified. -const omittedDefaultFlagValue = "explicitly omitting default" - -var ( - // flagAppGateConfig is the variable containing the AppGate config filepath - // sourced from the `--config` flag. - flagAppGateConfig string - // flagNodeRPCURL is the variable containing the Cosmos node RPC URL flag value. - flagNodeRPCURL string - // flagNodeGRPCURL is the variable containing the Cosmos node GRPC URL flag value. - flagNodeGRPCURL string - // flagLogLevel is the variable to set a log level (used by cosmos and polylog). - flagLogLevel string -) - -// AppGateServerCmd returns the Cobra command for running the AppGate server. -func AppGateServerCmd() *cobra.Command { - cmd := &cobra.Command{ - Use: "appgate-server", - Short: "Starts the AppGate server", - Long: `Starts the AppGate server that listens for incoming relay requests and handles -the necessary on-chain interactions (sessions, suppliers, etc) to receive the -respective relay response. - --- App Mode -- -If the server is started with a defined 'self-signing' configuration directive, -it will behave as an Application. Any incoming requests will be signed by using -the private key and ring associated with the 'signing_key' configuration directive. - --- Gateway Mode -- -If the 'self_signing' configuration directive is not provided, the server will -behave as a Gateway. -It will sign relays on behalf of any Application sending it relays, provided -that the address associated with 'signing_key' has been delegated to. This is -necessary for the application<->gateway ring signature to function. - --- App Mode (HTTP) -- -If an application doesn't provide the 'self_signing' configuration directive, -it can still send relays to the AppGate server and function as an Application, -provided that: -1. Each request contains the '?applicationAddr=[address]' query parameter -2. The key associated with the 'signing_key' configuration directive belongs - to the address provided in the request, otherwise the ring signature will not be valid.`, - Args: cobra.NoArgs, - RunE: runAppGateServer, - } - - // Custom flags - cmd.Flags().StringVar(&flagAppGateConfig, "config", "", "The path to the appgate config file") - - // Cosmos flags - // TODO_TECHDEBT(#256): Remove unneeded cosmos flags. - cmd.Flags().String(cosmosflags.FlagKeyringBackend, "", "Select keyring's backend (os|file|kwallet|pass|test)") - cmd.Flags().StringVar(&flagNodeRPCURL, cosmosflags.FlagNode, omittedDefaultFlagValue, "Register the default Cosmos node flag, which is needed to initialize the Cosmos query context correctly. It can be used to override the `QueryNodeUrl` field in the config file if specified.") - cmd.Flags().StringVar(&flagNodeGRPCURL, cosmosflags.FlagGRPC, omittedDefaultFlagValue, "Register the default Cosmos node grpc flag, which is needed to initialize the Cosmos query context with grpc correctly. It can be used to override the `QueryNodeGRPCUrl` field in the config file if specified.") - cmd.Flags().Bool(cosmosflags.FlagGRPCInsecure, true, "Used to initialize the Cosmos query context with grpc security options. It can be used to override the `QueryNodeGRPCInsecure` field in the config file if specified.") - cmd.Flags().StringVar(&flagLogLevel, cosmosflags.FlagLogLevel, "debug", "The logging level (debug|info|warn|error)") - - return cmd -} - -func runAppGateServer(cmd *cobra.Command, _ []string) error { - // Create a context that is canceled when the command is interrupted - ctx, cancelCtx := context.WithCancel(cmd.Context()) - defer cancelCtx() - - // Handle interrupt and kill signals asynchronously. - signals.GoOnExitSignal(cancelCtx) - - configContent, err := os.ReadFile(flagAppGateConfig) - if err != nil { - return err - } - - // TODO_TECHDEBT: add logger level and output options to the config. - appGateConfigs, err := appgateconfig.ParseAppGateServerConfigs(configContent) - if err != nil { - return err - } - - // TODO_TECHDEBT: populate logger from the config (ideally, from viper). - loggerOpts := []polylog.LoggerOption{ - polyzero.WithLevel(polyzero.ParseLevel(flagLogLevel)), - polyzero.WithOutput(os.Stderr), - } - - // Construct a logger and associate it with the command context. - logger := polyzero.NewLogger(loggerOpts...) - ctx = logger.WithContext(ctx) - cmd.SetContext(ctx) - - // Setup the AppGate server dependencies. - appGateServerDeps, err := setupAppGateServerDependencies(ctx, cmd, appGateConfigs) - if err != nil { - return fmt.Errorf("failed to setup AppGate server dependencies: %w", err) - } - - logger.Info().Msg("Creating AppGate server...") - - // Create the AppGate server. - appGateServer, err := appgateserver.NewAppGateServer( - appGateServerDeps, - appgateserver.WithSigningInformation(&appgateserver.SigningInformation{ - // provide the name of the key to use for signing all incoming requests - SigningKeyName: appGateConfigs.SigningKey, - // provide whether the appgate server should sign all incoming requests - // with its own ring (for applications) or not (for gateways) - SelfSigning: appGateConfigs.SelfSigning, - }), - appgateserver.WithListeningUrl(appGateConfigs.ListeningEndpoint), - ) - if err != nil { - return fmt.Errorf("failed to create AppGate server: %w", err) - } - - logger.Info(). - Str("listening_endpoint", appGateConfigs.ListeningEndpoint.String()). - Msg("Starting AppGate server...") - - if appGateConfigs.Metrics.Enabled { - err = appGateServer.ServeMetrics(appGateConfigs.Metrics.Addr) - if err != nil { - return fmt.Errorf("failed to start metrics endpoint: %w", err) - } - } - - if appGateConfigs.Pprof.Enabled { - err = appGateServer.ServePprof(ctx, appGateConfigs.Pprof.Addr) - if err != nil { - return fmt.Errorf("failed to start pprof endpoint: %w", err) - } - } - - // Start the AppGate server. - if err := appGateServer.Start(ctx); err != nil && !errors.Is(err, http.ErrServerClosed) { - return fmt.Errorf("failed to start app gate server: %w", err) - } else if errors.Is(err, http.ErrServerClosed) { - logger.Info().Msg("AppGate server stopped") - } - - return nil -} - -func setupAppGateServerDependencies( - ctx context.Context, - cmd *cobra.Command, - appGateConfig *appgateconfig.AppGateServerConfig, -) (_ depinject.Config, err error) { - queryNodeRPCURL := appGateConfig.QueryNodeRPCUrl - queryNodeGRPCURL := appGateConfig.QueryNodeGRPCUrl - - // Override the config file's `QueryNodeGRPCUrl` field - // with the `--grpc-addr` flag if it was specified. - // TODO_TECHDEBT(#223) Remove this check once viper is used as SoT for overridable config values. - if flagNodeGRPCURL != omittedDefaultFlagValue { - queryNodeGRPCURL, err = url.Parse(flagNodeGRPCURL) - if err != nil { - return nil, fmt.Errorf("failed to parse grpc query URL: %w", err) - } - } - - // Override the config file's `QueryNodeRPCURL` field - // with the `--node` flag if it was specified. - // TODO_TECHDEBT(#223) Remove this check once viper is used as SoT for overridable config values. - if flagNodeRPCURL != omittedDefaultFlagValue { - queryNodeRPCURL, err = url.Parse(flagNodeRPCURL) - if err != nil { - return nil, fmt.Errorf("failed to parse rpc query URL: %w", err) - } - } - - supplierFuncs := []config.SupplierFn{ - config.NewSupplyLoggerFromCtx(ctx), - config.NewSupplyEventsQueryClientFn(queryNodeRPCURL), // leaf - config.NewSupplyBlockQueryClientFn(queryNodeRPCURL), // leaf - config.NewSupplyBlockClientFn(queryNodeRPCURL), // leaf - config.NewSupplyQueryClientContextFn(queryNodeGRPCURL), // leaf - config.NewSupplyAccountQuerierFn(), // leaf - config.NewSupplyApplicationQuerierFn(), // leaf - config.NewSupplySessionQuerierFn(), // leaf - config.NewSupplySharedQueryClientFn(), // leaf - - config.NewSupplyShannonSDKFn(appGateConfig.SigningKey), - } - - return config.SupplyConfig(ctx, cmd, supplierFuncs) -} diff --git a/pkg/appgateserver/config/appgate_configs_reader.go b/pkg/appgateserver/config/appgate_configs_reader.go deleted file mode 100644 index 18c38a745..000000000 --- a/pkg/appgateserver/config/appgate_configs_reader.go +++ /dev/null @@ -1,128 +0,0 @@ -package config - -import ( - "net/url" - - "gopkg.in/yaml.v2" -) - -// YAMLAppGateServerConfig is the structure used to unmarshal the AppGateServer config file -// TODO_MAINNET(@red-0ne): Rename self_signing parameter to `sovereign` in code, configs -// and documentation -type YAMLAppGateServerConfig struct { - ListeningEndpoint string `yaml:"listening_endpoint"` - Metrics YAMLAppGateServerMetricsConfig `yaml:"metrics"` - QueryNodeGRPCUrl string `yaml:"query_node_grpc_url"` - QueryNodeRPCUrl string `yaml:"query_node_rpc_url"` - SelfSigning bool `yaml:"self_signing"` - SigningKey string `yaml:"signing_key"` - Pprof YAMLAppGateServerPprofConfig `yaml:"pprof"` -} - -// YAMLAppGateServerMetricsConfig is the structure used to unmarshal the metrics -// section of the AppGateServer config file -type YAMLAppGateServerMetricsConfig struct { - Enabled bool `yaml:"enabled"` - Addr string `yaml:"addr"` -} - -// YAMLAppGateServerPprofConfig is the structure used to unmarshal the config -// for `pprof`. -type YAMLAppGateServerPprofConfig struct { - Enabled bool `yaml:"enabled,omitempty"` - Addr string `yaml:"addr,omitempty"` -} - -// AppGateServerConfig is the structure describing the AppGateServer config -type AppGateServerConfig struct { - ListeningEndpoint *url.URL - Metrics *AppGateServerMetricsConfig - QueryNodeGRPCUrl *url.URL - QueryNodeRPCUrl *url.URL - SelfSigning bool - SigningKey string - Pprof *AppGateServerPprofConfig -} - -// AppGateServerMetricsConfig is the structure resulting from parsing the metrics -// section of the AppGateServer config file. -type AppGateServerMetricsConfig struct { - Enabled bool - Addr string -} - -// AppGateServerPprofConfig is the structure resulting from parsing the pprof -// section of the AppGateServer config file. -type AppGateServerPprofConfig struct { - Enabled bool - Addr string -} - -// ParseAppGateServerConfigs parses the stake config file into a AppGateConfig -// NOTE: If SelfSigning is not defined in the config file, it will default to false -func ParseAppGateServerConfigs(configContent []byte) (*AppGateServerConfig, error) { - var yamlAppGateServerConfig YAMLAppGateServerConfig - - if len(configContent) == 0 { - return nil, ErrAppGateConfigEmpty - } - - // Unmarshal the stake config file into a yamlAppGateConfig - if err := yaml.Unmarshal(configContent, &yamlAppGateServerConfig); err != nil { - return nil, ErrAppGateConfigUnmarshalYAML.Wrap(err.Error()) - } - - if len(yamlAppGateServerConfig.SigningKey) == 0 { - return nil, ErrAppGateConfigEmptySigningKey - } - - if len(yamlAppGateServerConfig.ListeningEndpoint) == 0 { - return nil, ErrAppGateConfigInvalidListeningEndpoint - } - - listeningEndpoint, err := url.Parse(yamlAppGateServerConfig.ListeningEndpoint) - if err != nil { - return nil, ErrAppGateConfigInvalidListeningEndpoint.Wrap(err.Error()) - } - - if len(yamlAppGateServerConfig.QueryNodeGRPCUrl) == 0 { - return nil, ErrAppGateConfigInvalidQueryNodeGRPCUrl - } - - queryNodeGRPCUrl, err := url.Parse(yamlAppGateServerConfig.QueryNodeGRPCUrl) - if err != nil { - return nil, ErrAppGateConfigInvalidQueryNodeGRPCUrl.Wrap(err.Error()) - } - - if len(yamlAppGateServerConfig.QueryNodeRPCUrl) == 0 { - return nil, ErrAppGateConfigInvalidQueryNodeRPCUrl - } - - queryNodeRPCUrl, err := url.Parse(yamlAppGateServerConfig.QueryNodeRPCUrl) - if err != nil { - return nil, ErrAppGateConfigInvalidQueryNodeRPCUrl.Wrap(err.Error()) - } - - // Populate the appGateServerConfig with the values from the yamlAppGateServerConfig - appGateServerConfig := &AppGateServerConfig{ - QueryNodeRPCUrl: queryNodeRPCUrl, - QueryNodeGRPCUrl: queryNodeGRPCUrl, - SigningKey: yamlAppGateServerConfig.SigningKey, - SelfSigning: yamlAppGateServerConfig.SelfSigning, - ListeningEndpoint: listeningEndpoint, - } - - // Not doing additinal validation on metrics, as the server would not start if the value is invalid, - // providing the user with a descriptive error message. - appGateServerConfig.Metrics = &AppGateServerMetricsConfig{ - Enabled: yamlAppGateServerConfig.Metrics.Enabled, - Addr: yamlAppGateServerConfig.Metrics.Addr, - } - - appGateServerConfig.Pprof = &AppGateServerPprofConfig{ - Enabled: yamlAppGateServerConfig.Pprof.Enabled, - Addr: yamlAppGateServerConfig.Pprof.Addr, - } - - return appGateServerConfig, nil -} diff --git a/pkg/appgateserver/config/appgate_configs_reader_test.go b/pkg/appgateserver/config/appgate_configs_reader_test.go deleted file mode 100644 index f1faafa12..000000000 --- a/pkg/appgateserver/config/appgate_configs_reader_test.go +++ /dev/null @@ -1,176 +0,0 @@ -package config_test - -import ( - "net/url" - "testing" - - sdkerrors "cosmossdk.io/errors" - "github.com/gogo/status" - "github.com/stretchr/testify/require" - - "github.com/pokt-network/poktroll/pkg/appgateserver/config" - "github.com/pokt-network/poktroll/testutil/yaml" -) - -func Test_ParseAppGateConfigs(t *testing.T) { - tests := []struct { - desc string - - inputConfigYAML string - - expectedErr *sdkerrors.Error - expectedConfig *config.AppGateServerConfig - }{ - // Valid Configs - { - desc: "valid: AppGateServer config", - - inputConfigYAML: ` - query_node_rpc_url: tcp://127.0.0.1:26657 - query_node_grpc_url: tcp://127.0.0.1:9090 - signing_key: app1 - self_signing: true - listening_endpoint: http://localhost:42069 - `, - - expectedErr: nil, - expectedConfig: &config.AppGateServerConfig{ - QueryNodeRPCUrl: &url.URL{Scheme: "tcp", Host: "127.0.0.1:26657"}, - QueryNodeGRPCUrl: &url.URL{Scheme: "tcp", Host: "127.0.0.1:9090"}, - SigningKey: "app1", - SelfSigning: true, - ListeningEndpoint: &url.URL{Scheme: "http", Host: "localhost:42069"}, - }, - }, - { - desc: "valid: AppGateServer config with undefined self signing", - - inputConfigYAML: ` - query_node_rpc_url: tcp://127.0.0.1:26657 - query_node_grpc_url: tcp://127.0.0.1:9090 - signing_key: app1 - listening_endpoint: http://localhost:42069 - `, - - expectedErr: nil, - expectedConfig: &config.AppGateServerConfig{ - QueryNodeRPCUrl: &url.URL{Scheme: "tcp", Host: "127.0.0.1:26657"}, - QueryNodeGRPCUrl: &url.URL{Scheme: "tcp", Host: "127.0.0.1:9090"}, - SigningKey: "app1", - SelfSigning: false, - ListeningEndpoint: &url.URL{Scheme: "http", Host: "localhost:42069"}, - }, - }, - // Invalid Configs - { - desc: "invalid: empty AppGateServer config", - - inputConfigYAML: ``, - - expectedErr: config.ErrAppGateConfigEmpty, - }, - { - desc: "invalid: no signing key", - - inputConfigYAML: ` - query_node_rpc_url: tcp://127.0.0.1:26657 - query_node_grpc_url: tcp://127.0.0.1:9090 - # NB: explicitly missing signing key - self_signing: true - listening_endpoint: http://localhost:42069 - `, - - expectedErr: config.ErrAppGateConfigEmptySigningKey, - }, - { - desc: "invalid: invalid listening endpoint", - - inputConfigYAML: ` - query_node_rpc_url: tcp://127.0.0.1:26657 - query_node_grpc_url: tcp://127.0.0.1:9090 - signing_key: app1 - self_signing: true - listening_endpoint: l&ocalhost:42069 - `, - - expectedErr: config.ErrAppGateConfigInvalidListeningEndpoint, - }, - { - desc: "invalid: invalid query node grpc url", - - inputConfigYAML: ` - query_node_rpc_url: tcp://127.0.0.1:26657 - query_node_grpc_url: 1&27.0.0.1:9090 - signing_key: app1 - self_signing: true - listening_endpoint: http://localhost:42069 - `, - - expectedErr: config.ErrAppGateConfigInvalidQueryNodeGRPCUrl, - }, - { - desc: "invalid: missing query node grpc url", - - inputConfigYAML: ` - query_node_rpc_url: tcp://127.0.0.1:26657 - # NB: explicitly missing query_node_grpc_url - signing_key: app1 - self_signing: true - listening_endpoint: http://localhost:42069 - `, - - expectedErr: config.ErrAppGateConfigInvalidQueryNodeGRPCUrl, - }, - { - desc: "invalid: invalid query node rpc url", - - inputConfigYAML: ` - query_node_rpc_url: 1&27.0.0.1:26657 - query_node_grpc_url: tcp://127.0.0.1:9090 - signing_key: app1 - self_signing: true - listening_endpoint: http://localhost:42069 - `, - - expectedErr: config.ErrAppGateConfigInvalidQueryNodeRPCUrl, - }, - { - desc: "invalid: missing query node rpc url", - - inputConfigYAML: ` - # NB: explicitly missing query_node_rpc_url - query_node_grpc_url: tcp://127.0.0.1:9090 - signing_key: app1 - self_signing: true - listening_endpoint: http://localhost:42069 - `, - - expectedErr: config.ErrAppGateConfigInvalidQueryNodeRPCUrl, - }, - } - - for _, test := range tests { - t.Run(test.desc, func(t *testing.T) { - normalizedConfig := yaml.NormalizeYAMLIndentation(test.inputConfigYAML) - config, err := config.ParseAppGateServerConfigs([]byte(normalizedConfig)) - - if test.expectedErr != nil { - require.ErrorIs(t, err, test.expectedErr) - require.Nil(t, config) - stat, ok := status.FromError(test.expectedErr) - require.True(t, ok) - require.Contains(t, stat.Message(), test.expectedErr.Error()) - require.Nil(t, config) - return - } - - require.NoError(t, err) - - require.Equal(t, test.expectedConfig.SelfSigning, config.SelfSigning) - require.Equal(t, test.expectedConfig.SigningKey, config.SigningKey) - require.Equal(t, test.expectedConfig.ListeningEndpoint.String(), config.ListeningEndpoint.String()) - require.Equal(t, test.expectedConfig.QueryNodeGRPCUrl.String(), config.QueryNodeGRPCUrl.String()) - require.Equal(t, test.expectedConfig.QueryNodeGRPCUrl.String(), config.QueryNodeGRPCUrl.String()) - }) - } -} diff --git a/pkg/appgateserver/config/errors.go b/pkg/appgateserver/config/errors.go deleted file mode 100644 index 882c70a18..000000000 --- a/pkg/appgateserver/config/errors.go +++ /dev/null @@ -1,13 +0,0 @@ -package config - -import sdkerrors "cosmossdk.io/errors" - -var ( - codespace = "appgate_config" - ErrAppGateConfigUnmarshalYAML = sdkerrors.Register(codespace, 2100, "config reader cannot unmarshal yaml content") - ErrAppGateConfigEmptySigningKey = sdkerrors.Register(codespace, 2101, "empty signing key in AppGateServer config") - ErrAppGateConfigInvalidListeningEndpoint = sdkerrors.Register(codespace, 2102, "invalid listening endpoint in AppGateServer config") - ErrAppGateConfigInvalidQueryNodeGRPCUrl = sdkerrors.Register(codespace, 2103, "invalid query node grpc url in AppGateServer config") - ErrAppGateConfigInvalidQueryNodeRPCUrl = sdkerrors.Register(codespace, 2104, "invalid query node rpc url in AppGateServer config") - ErrAppGateConfigEmpty = sdkerrors.Register(codespace, 2105, "empty AppGateServer config") -) diff --git a/pkg/appgateserver/endpoint_selector.go b/pkg/appgateserver/endpoint_selector.go deleted file mode 100644 index e5d406cea..000000000 --- a/pkg/appgateserver/endpoint_selector.go +++ /dev/null @@ -1,84 +0,0 @@ -package appgateserver - -import ( - "net/url" - "strconv" - - shannonsdk "github.com/pokt-network/shannon-sdk" - - sharedtypes "github.com/pokt-network/poktroll/x/shared/types" -) - -// getRelayerUrl returns the next relayer endpoint to use for the given serviceId and rpcType. -// NB: This is a naive implementation of the endpoint selection strategy. -// It is intentionally kept simple for the sake of a clear example, and future -// optimizations (i.e. quality of service implementations) are left as an exercise -// to gateways. -func (app *appGateServer) getRelayerUrl( - rpcType sharedtypes.RPCType, - sessionFilter shannonsdk.SessionFilter, - requestUrlStr string, -) (supplierEndpoint shannonsdk.Endpoint, err error) { - // AppGateServer uses the custom getRelayerUrl instead of leveraging the SDK's - // filter to select the next endpoint to use. - // This is because it needs to maintain the state of the last selected endpoint - // and have a view on the original request URL to determine the next endpoint. - // This behavior is specific to the AppGateServer and needed by clients that - // need to instrument the endpoint selection strategy, such as the Load testing tool. - endpoints, err := sessionFilter.AllEndpoints() - if err != nil { - return nil, err - } - - // Filter out the supplier endpoints that match the requested serviceId. - matchingRPCTypeEndpoints := []shannonsdk.Endpoint{} - - for _, supplierEndpoints := range endpoints { - for _, supplierEndpoint := range supplierEndpoints { - // Collect the endpoints that match the request's RpcType. - if supplierEndpoint.Endpoint().RpcType == rpcType { - matchingRPCTypeEndpoints = append(matchingRPCTypeEndpoints, supplierEndpoint) - } - } - } - - // Return an error if no relayer endpoints were found. - if len(matchingRPCTypeEndpoints) == 0 { - return nil, ErrAppGateNoRelayEndpoints - } - - // Protect the endpointSelectionIndex update from concurrent relay requests. - app.endpointSelectionIndexMu.Lock() - defer app.endpointSelectionIndexMu.Unlock() - - // Select the next endpoint in the list by rotating the index. - // This does not necessarily start from the first endpoint of a new session - // but will cycle through all valid endpoints of the same session if enough - // requests are made. - // This is a naive strategy that is used to ensure all endpoints are leveraged - // throughout the lifetime of the session. It is primarily used as a foundation - // for testing or development purposes but a more enhanced strategy is expected - // to be adopted by prod gateways. - - requestUrl, err := url.Parse(requestUrlStr) - if err != nil { - return nil, err - } - - // If a `relayCount` query parameter is provided, use it to determine the next endpoint; - // otherwise, continue the rotation based off the last selected endpoint index. - relayCount := requestUrl.Query().Get("relayCount") - nextEndpointIdx := 0 - if relayCount != "" { - relayCountNum, err := strconv.Atoi(relayCount) - if err != nil { - relayCountNum = 0 - } - nextEndpointIdx = relayCountNum % len(matchingRPCTypeEndpoints) - } else { - app.endpointSelectionIndex = (app.endpointSelectionIndex + 1) % len(matchingRPCTypeEndpoints) - nextEndpointIdx = app.endpointSelectionIndex - } - - return matchingRPCTypeEndpoints[nextEndpointIdx], nil -} diff --git a/pkg/appgateserver/error_reply.go b/pkg/appgateserver/error_reply.go deleted file mode 100644 index dd03663e4..000000000 --- a/pkg/appgateserver/error_reply.go +++ /dev/null @@ -1,37 +0,0 @@ -package appgateserver - -import ( - "net/http" - - sdktypes "github.com/pokt-network/shannon-sdk/types" - - sharedtypes "github.com/pokt-network/poktroll/x/shared/types" -) - -// replyWithError replies to the application with an error response and writes -// it to the writer provided. -// NOTE: This method is used to reply with an "internal" error that is related -// to the appgateserver itself and not to the relay request. -func (app *appGateServer) replyWithError( - replyError error, - poktHTTPRequest *sdktypes.POKTHTTPRequest, - serviceId string, - rpcType sharedtypes.RPCType, - writer http.ResponseWriter, -) { - relaysErrorsTotal.With("service_id", serviceId, "rpc_type", rpcType.String()).Add(1) - - errorResponse, _ := poktHTTPRequest.FormatError(replyError, false) - - // Write all the errorResponse headers to the writer. - // While the only header that errorResponse.Header contains is "Content-Type", - // errorResponse.Header are iterated over to future-proof against any additional - // headers that may be added in the future (e.g. compression or caching headers). - errorResponse.CopyToHTTPHeader(writer.Header()) - writer.WriteHeader(int(errorResponse.StatusCode)) - - if _, err := writer.Write(errorResponse.BodyBz); err != nil { - app.logger.Error().Err(err).Str("service_id", serviceId).Msg("failed writing relay response") - return - } -} diff --git a/pkg/appgateserver/errors.go b/pkg/appgateserver/errors.go deleted file mode 100644 index 8606b98b9..000000000 --- a/pkg/appgateserver/errors.go +++ /dev/null @@ -1,13 +0,0 @@ -package appgateserver - -import sdkerrors "cosmossdk.io/errors" - -var ( - codespace = "appgateserver" - ErrAppGateNoRelayEndpoints = sdkerrors.Register(codespace, 1, "no relay endpoints found") - ErrAppGateMissingAppAddress = sdkerrors.Register(codespace, 2, "missing application address") - ErrAppGateMissingSigningInformation = sdkerrors.Register(codespace, 3, "missing app client signing information") - ErrAppGateMissingListeningEndpoint = sdkerrors.Register(codespace, 4, "missing app client listening endpoint") - ErrAppGateHandleRelay = sdkerrors.Register(codespace, 5, "internal error handling relay request") - ErrAppGateUpstreamError = sdkerrors.Register(codespace, 6, "upstream error") -) diff --git a/pkg/appgateserver/metrics.go b/pkg/appgateserver/metrics.go deleted file mode 100644 index 4a80b260e..000000000 --- a/pkg/appgateserver/metrics.go +++ /dev/null @@ -1,50 +0,0 @@ -package appgateserver - -import ( - "github.com/go-kit/kit/metrics/prometheus" - stdprometheus "github.com/prometheus/client_golang/prometheus" -) - -const ( - appGateServerProcess = "appgateserver" - - relaysTotalMetric = "relay_requests_total" - relaysSuccessTotalMetric = "relay_requests_success_total" - relaysErrorsTotalMetric = "relay_requests_errors_total" -) - -var ( - // relaysTotal is a Counter metric for the total relays processed by the AppGate server. - // Crucial for understanding server workload and traffic, it increments monotonically. - // Labeled by 'service_id' and 'rpc_type', it facilitates nuanced analysis of relays - // across various services and RPC types. - // - // Usage: - // - Monitor aggregate load and relay rates. - // - Compare relay volumes by service and RPC type. - relaysTotal = prometheus.NewCounterFrom(stdprometheus.CounterOpts{ - Subsystem: appGateServerProcess, - Name: relaysTotalMetric, - }, []string{"service_id", "rpc_type"}) - - // relaysErrorsTotal is a Counter metric tracking errors on the AppGate server. - // Incrementing with each error, it's vital for server health and stability assessment. - // With 'service_id' and 'rpc_type' labels, it allows precise error rate analysis and troubleshooting - // across services and RPC types. - // - // Usage: - // - Monitor health and error rates by service and RPC type. - // - Identify and address high-error areas. - relaysErrorsTotal = prometheus.NewCounterFrom(stdprometheus.CounterOpts{ - Subsystem: appGateServerProcess, - Name: relaysErrorsTotalMetric, - }, []string{"service_id", "rpc_type"}) - - // relaysSuccessTotal is a Counter metric tracking successful relays on the AppGate server. - // It's essential for monitoring server reliability and performance. - // Labeled by 'service_id' and 'rpc_type', it enables detailed analysis of successful requests. - relaysSuccessTotal = prometheus.NewCounterFrom(stdprometheus.CounterOpts{ - Subsystem: appGateServerProcess, - Name: relaysSuccessTotalMetric, - }, []string{"service_id", "rpc_type"}) -) diff --git a/pkg/appgateserver/options.go b/pkg/appgateserver/options.go deleted file mode 100644 index 890e6c5bf..000000000 --- a/pkg/appgateserver/options.go +++ /dev/null @@ -1,17 +0,0 @@ -package appgateserver - -import "net/url" - -// WithSigningInformation sets the signing information for the appgate server. -func WithSigningInformation(signingInfo *SigningInformation) appGateServerOption { - return func(appGateServer *appGateServer) { - appGateServer.signingInformation = signingInfo - } -} - -// WithListeningUrl sets the listening URL for the appgate server. -func WithListeningUrl(listeningUrl *url.URL) appGateServerOption { - return func(appGateServer *appGateServer) { - appGateServer.listeningEndpoint = listeningUrl - } -} diff --git a/pkg/appgateserver/sdkadapter/block_client.go b/pkg/appgateserver/sdkadapter/block_client.go deleted file mode 100644 index 184b5c448..000000000 --- a/pkg/appgateserver/sdkadapter/block_client.go +++ /dev/null @@ -1,18 +0,0 @@ -package sdkadapter - -import ( - "cosmossdk.io/depinject" - - "github.com/pokt-network/poktroll/pkg/client" -) - -// NewBlockClient creates a new ShannonSDK compatible block client. -func NewBlockClient(deps depinject.Config) (client.BlockClient, error) { - blockClient := client.BlockClient(nil) - - if err := depinject.Inject(deps, &blockClient); err != nil { - return nil, err - } - - return blockClient, nil -} diff --git a/pkg/appgateserver/sdkadapter/sdk.go b/pkg/appgateserver/sdkadapter/sdk.go deleted file mode 100644 index 5bbe1d00c..000000000 --- a/pkg/appgateserver/sdkadapter/sdk.go +++ /dev/null @@ -1,145 +0,0 @@ -package sdkadapter - -import ( - "bytes" - "context" - "io" - "net/http" - - "cosmossdk.io/depinject" - cryptotypes "github.com/cosmos/cosmos-sdk/crypto/types" - shannonsdk "github.com/pokt-network/shannon-sdk" - - "github.com/pokt-network/poktroll/pkg/client" - "github.com/pokt-network/poktroll/pkg/client/query" - "github.com/pokt-network/poktroll/x/service/types" -) - -// ShannonSDK is a wrapper around the Shannon SDK that is used by the AppGateServer -// to encapsulate the SDK's functionality and dependencies. -type ShannonSDK struct { - blockClient client.BlockClient - sessionClient client.SessionQueryClient - appClient client.ApplicationQueryClient - accountClient client.AccountQueryClient - relayClient *http.Client - signer *shannonsdk.Signer -} - -// NewShannonSDK creates a new ShannonSDK instance with the given signing key and dependencies. -// It initializes the necessary clients and signer for the SDK. -func NewShannonSDK( - ctx context.Context, - signingKey cryptotypes.PrivKey, - deps depinject.Config, -) (*ShannonSDK, error) { - sessionClient, sessionClientErr := query.NewSessionQuerier(deps) - if sessionClientErr != nil { - return nil, sessionClientErr - } - - accountClient, accountClientErr := query.NewAccountQuerier(deps) - if accountClientErr != nil { - return nil, accountClientErr - } - - appClient, appClientErr := query.NewApplicationQuerier(deps) - if appClientErr != nil { - return nil, appClientErr - } - - blockClient := client.BlockClient(nil) - if depsErr := depinject.Inject(deps, &blockClient); depsErr != nil { - return nil, depsErr - } - - signer, signerErr := NewSigner(signingKey) - if signerErr != nil { - return nil, signerErr - } - - shannonSDK := &ShannonSDK{ - blockClient: blockClient, - sessionClient: sessionClient, - accountClient: accountClient, - appClient: appClient, - relayClient: http.DefaultClient, - signer: signer, - } - - return shannonSDK, nil -} - -// SendRelay builds a relay request from the given requestBz, signs it with the -// application address, then sends it to the given endpoint. -func (shannonSDK *ShannonSDK) SendRelay( - ctx context.Context, - appAddress string, - endpoint shannonsdk.Endpoint, - requestBz []byte, -) (*types.RelayResponse, error) { - relayRequest, err := shannonsdk.BuildRelayRequest(endpoint, requestBz) - if err != nil { - return nil, err - } - - application, err := shannonSDK.appClient.GetApplication(ctx, appAddress) - if err != nil { - return nil, err - } - - appRing := shannonsdk.ApplicationRing{ - PublicKeyFetcher: shannonSDK.accountClient, - Application: application, - } - - if _, err = shannonSDK.signer.Sign(ctx, relayRequest, appRing); err != nil { - return nil, err - } - - relayRequestBz, err := relayRequest.Marshal() - if err != nil { - return nil, err - } - - response, err := shannonSDK.relayClient.Post( - endpoint.Endpoint().Url, - "application/json", - bytes.NewReader(relayRequestBz), - ) - if err != nil { - return nil, err - } - defer response.Body.Close() - - responseBodyBz, err := io.ReadAll(response.Body) - if err != nil { - return nil, err - } - - return shannonsdk.ValidateRelayResponse( - ctx, - endpoint.Supplier(), - responseBodyBz, - shannonSDK.accountClient, - ) -} - -// GetSessionSupplierEndpoints returns the current session's supplier endpoints -// for the given appAddress and serviceId. -func (shannonSDK *ShannonSDK) GetSessionSupplierEndpoints( - ctx context.Context, - appAddress, serviceId string, -) (*shannonsdk.SessionFilter, error) { - currentHeight := shannonSDK.blockClient.LastBlock(ctx).Height() - session, err := shannonSDK.sessionClient.GetSession(ctx, appAddress, serviceId, currentHeight) - if err != nil { - return nil, err - } - - filteredSession := &shannonsdk.SessionFilter{ - Session: session, - } - - return filteredSession, nil -} diff --git a/pkg/appgateserver/sdkadapter/signer.go b/pkg/appgateserver/sdkadapter/signer.go deleted file mode 100644 index 90de84a76..000000000 --- a/pkg/appgateserver/sdkadapter/signer.go +++ /dev/null @@ -1,18 +0,0 @@ -package sdkadapter - -import ( - "encoding/hex" - - cryptotypes "github.com/cosmos/cosmos-sdk/crypto/types" - sdk "github.com/pokt-network/shannon-sdk" -) - -// NewSigner creates a new ShannonSDK compatible signer with the given signing key. -func NewSigner(signingKey cryptotypes.PrivKey) (*sdk.Signer, error) { - PrivateKeyHex := hex.EncodeToString(signingKey.Bytes()) - signer := &sdk.Signer{ - PrivateKeyHex: PrivateKeyHex, - } - - return signer, nil -} diff --git a/pkg/appgateserver/server.go b/pkg/appgateserver/server.go deleted file mode 100644 index ede1be059..000000000 --- a/pkg/appgateserver/server.go +++ /dev/null @@ -1,344 +0,0 @@ -package appgateserver - -import ( - "context" - "fmt" - "net" - "net/http" - "net/http/pprof" - "net/url" - "strings" - "sync" - - "cosmossdk.io/depinject" - sdktypes "github.com/pokt-network/shannon-sdk/types" - "github.com/prometheus/client_golang/prometheus/promhttp" - metrics "github.com/slok/go-http-metrics/metrics/prometheus" - metricsmiddleware "github.com/slok/go-http-metrics/middleware" - middlewarestd "github.com/slok/go-http-metrics/middleware/std" - - "github.com/pokt-network/poktroll/pkg/appgateserver/sdkadapter" - querytypes "github.com/pokt-network/poktroll/pkg/client/query/types" - "github.com/pokt-network/poktroll/pkg/polylog" - sharedtypes "github.com/pokt-network/poktroll/x/shared/types" -) - -// SigningInformation is a struct that holds information related to the signing -// of relay requests, used by the appGateServer to determine how they will sign -// relay requests (with either their own ring or the rign of the application). -type SigningInformation struct { - // SelfSigning indicates whether the server is running in self-signing mode - SelfSigning bool - - // SigningKeyName is the name of the key in the keyring that corresponds to the - // private key used to sign relay requests. - SigningKeyName string - - // AppAddress is the address of the application that the server is serving if - // If it is nil, then the application address must be included in each request via a query parameter. - AppAddress string -} - -// appGateServer is the server that listens for application requests and relays them to the supplier. -// It is responsible for maintaining the current session for the application, signing the requests, -// and verifying the response signatures. -// The appGateServer is the basis for both applications and gateways, depending on whether the application -// is running their own instance of the appGateServer or they are sending requests to a gateway running an -// instance of the appGateServer, they will need to either include the application address in the request or not. -type appGateServer struct { - logger polylog.Logger - - // signing information holds the signing key and application address for the server - signingInformation *SigningInformation - - // clientCtx is the client context for the application. - // It is used to query for the application's account to unmarshal the supplier's account - // and get the public key to verify the relay response signature. - clientCtx querytypes.Context - - // sdk is the ShannonSDK that the appGateServer uses to query for the current session - // and send relay requests to the supplier. - sdk *sdkadapter.ShannonSDK - - // listeningEndpoint is the endpoint that the appGateServer will listen on. - listeningEndpoint *url.URL - - // server is the HTTP server that will be used capture application requests - // so that they can be signed and relayed to the supplier. - server *http.Server - - // endpointSelectionIndexMu is a mutex that protects the endpointSelectionIndex - // from concurrent relay requests. - endpointSelectionIndexMu sync.Mutex - - // endpointSelectionIndex is the index of the last selected endpoint. - // It is used to cycle through the available endpoints using a round-robin strategy. - endpointSelectionIndex int -} - -// NewAppGateServer creates a new appGateServer instance with the given dependencies. -// -// Required dependencies: -// - polylog.Logger -// - sdkclient.Context -// - client.BlockClient -// - client.AccountQueryClient -// - crypto.RingCache -func NewAppGateServer( - deps depinject.Config, - opts ...appGateServerOption, -) (*appGateServer, error) { - app := &appGateServer{} - - if err := depinject.Inject( - deps, - &app.logger, - &app.clientCtx, - &app.sdk, - ); err != nil { - return nil, err - } - - for _, opt := range opts { - opt(app) - } - - if err := app.validateConfig(); err != nil { - return nil, err - } - - keyRecord, err := app.clientCtx.Keyring.Key(app.signingInformation.SigningKeyName) - if err != nil { - return nil, fmt.Errorf("failed to get key from keyring: %w", err) - } - - appAddress, err := keyRecord.GetAddress() - if err != nil { - return nil, fmt.Errorf("failed to get address from key: %w", err) - } - if app.signingInformation.SelfSigning { - app.signingInformation.AppAddress = appAddress.String() - } - - // TODO_IMPROVE: Use app.listeningEndpoint scheme to determine which - // kind of server to create (HTTP, HTTPS, TCP, UNIX, etc...) - // TODO_IMPROVE(#590): Currently, the communication between the AppGateServer and the - // RelayMiner uses HTTP. This could be changed to a more generic and performant - // one, such as pure TCP. - app.server = &http.Server{Addr: app.listeningEndpoint.Host} - - return app, nil -} - -// Start starts the appgate server and blocks until the context is done -// or the server returns an error. -func (app *appGateServer) Start(ctx context.Context) error { - // Shutdown the HTTP server when the context is done. - go func() { - <-ctx.Done() - _ = app.server.Shutdown(ctx) - }() - - // This hooks https://github.com/slok/go-http-metrics to the appgate server HTTP server. - mm := metricsmiddleware.New(metricsmiddleware.Config{ - Recorder: metrics.NewRecorder(metrics.Config{}), - }) - - // Set the HTTP handler. - app.server.Handler = middlewarestd.Handler("", mm, app) - - // Start the HTTP server. - return app.server.ListenAndServe() -} - -// Stop stops the appgate server and returns any error that occurred. -func (app *appGateServer) Stop(ctx context.Context) error { - return app.server.Shutdown(ctx) -} - -// ServeHTTP is the HTTP handler for the appgate server. -// It captures the application request, signs it, and sends it to the supplier. -// After receiving the response from the supplier, it verifies the response signature -// before returning the response to the application. -// The serviceId is extracted from the request path. -// The request's path should be of the form: -// -// "://host:port/serviceId[/other/path/segments]?applicationAddr=" -// -// where the serviceId is the id of the service that the application is requesting -// and the other (possible) path segments are the JSON RPC request path. -// TODO_TECHDEBT: Revisit the requestPath above based on the SDK that'll be exposed in the future. -func (app *appGateServer) ServeHTTP(writer http.ResponseWriter, request *http.Request) { - ctx := app.logger.WithContext(request.Context()) - - newUrlPath, serviceId := extractServiceId(request.URL.Path) - request.URL.Path = newUrlPath - - // Create an error logger with the common fields for error logging. - errorLogger := app.logger.With(). - Error(). - Str("service_id", serviceId). - Fields(map[string]interface{}{ - "method": request.Method, - "url": request.URL.String(), - "content_type": request.Header.Get("Content-Type"), - "content_length": request.ContentLength, - }) - - logger := app.logger.With().Debug().Str("service_id", serviceId) - - poktHTTPRequest, requestBz, err := sdktypes.SerializeHTTPRequest(request) - if err != nil { - // If the request cannot not be parsed, pass an empty POKTHTTPRequest and - // an UNKNOWN_RPC type to the replyWithError method. - emptyPOKTHTTPRequest := &sdktypes.POKTHTTPRequest{} - rpcType := sharedtypes.RPCType_UNKNOWN_RPC - errorReply := ErrAppGateHandleRelay.Wrapf("parsing request: %s", err) - - app.replyWithError(errorReply, emptyPOKTHTTPRequest, serviceId, rpcType, writer) - errorLogger.Err(err).Msg("failed parsing request") - - return - } - - logger.Msg("handling relay") - - // Get the type of the request by inspecting the request properties. - rpcType := poktHTTPRequest.GetRPCType() - - // Add newly available fields to the error logger. - errorLogger = errorLogger. - Str("rpc_type", rpcType.String()). - Str("payload", string(poktHTTPRequest.BodyBz)) - - logger = logger.Str("rpc_type", rpcType.String()) - logger.Msg("identified rpc type") - - // Determine the application address. - appAddress := app.signingInformation.AppAddress - if appAddress == "" { - appAddress = request.URL.Query().Get("applicationAddr") - } - if appAddress == "" { - // If no application address is provided, reply with an error response. - app.replyWithError(ErrAppGateMissingAppAddress, poktHTTPRequest, serviceId, rpcType, writer) - errorLogger.Err(ErrAppGateMissingAppAddress).Msg("no application address provided") - - return - } - - logger = logger.Str("application_addr", appAddress) - errorLogger = errorLogger.Str("application_addr", appAddress) - - // Create a requestInfo struct to pass to the handleSynchronousRelay method. - reqInfo := &requestInfo{ - appAddress: appAddress, - serviceId: serviceId, - rpcType: rpcType, - poktRequest: poktHTTPRequest, - requestBz: requestBz, - } - - // TODO_IMPROVE(@red-0ne, #40): Add support for asynchronous relays, and switch on - // the request type here. - if err := app.handleSynchronousRelay(ctx, reqInfo, writer); err != nil { - // Reply with an error response if there was an error handling the relay. - app.replyWithError(err, poktHTTPRequest, serviceId, rpcType, writer) - errorLogger.Err(err).Msg("failed handling relay") - - return - } - - logger.Msg("request serviced successfully") -} - -// validateConfig validates the appGateServer configuration. -func (app *appGateServer) validateConfig() error { - if app.signingInformation == nil { - return ErrAppGateMissingSigningInformation - } - if app.listeningEndpoint == nil { - return ErrAppGateMissingListeningEndpoint - } - return nil -} - -// Starts a metrics server on the given address. -func (app *appGateServer) ServeMetrics(addr string) error { - ln, err := net.Listen("tcp", addr) - if err != nil { - app.logger.Error().Err(err).Msg("failed to listen on address for metrics") - return err - } - - // If no error, start the server in a new goroutine - go func() { - app.logger.Info().Str("endpoint", addr).Msg("serving metrics") - if err := http.Serve(ln, promhttp.Handler()); err != nil { - app.logger.Error().Err(err).Msg("metrics server failed") - return - } - }() - - return nil -} - -// Starts a pprof server on the given address. -func (app *appGateServer) ServePprof(ctx context.Context, addr string) error { - pprofMux := http.NewServeMux() - pprofMux.HandleFunc("/debug/pprof/", pprof.Index) - pprofMux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline) - pprofMux.HandleFunc("/debug/pprof/profile", pprof.Profile) - pprofMux.HandleFunc("/debug/pprof/symbol", pprof.Symbol) - pprofMux.HandleFunc("/debug/pprof/trace", pprof.Trace) - - server := &http.Server{ - Addr: addr, - Handler: pprofMux, - } - - // If no error, start the server in a new goroutine - go func() { - app.logger.Info().Str("endpoint", addr).Msg("starting a pprof endpoint") - if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed { - app.logger.Error().Str("endpoint", addr).Msg("unable to start a pprof endpoint") - } - }() - - go func() { - <-ctx.Done() - app.logger.Info().Str("endpoint", addr).Msg("stopping a pprof endpoint") - _ = server.Shutdown(ctx) - }() - - return nil -} - -// extractServiceId extracts the serviceId from the request path and returns it -// along with the new request path that is stripped of the serviceId. -func extractServiceId(urlPath string) (newUrlPath string, serviceId string) { - // Extract the serviceId from the request path. - serviceId = strings.Split(urlPath, "/")[1] - - // Remove the serviceId from the request path which is specific AppGateServer business logic. - // The remaining path is the path of the request that will be serialized and - // sent to sent to the supplier within a RelayRequest. - // For example: - // * Assume a request to the Backend service has to be made with the path "/backend/relay" - // * The AppGateServer expects the request from the client to have the path - // "/serviceId/backend/relay" - // * The AppGateServer will remove the serviceId from the path, serialize the request - // and send it to the supplier with the path "/backend/relay" - // - // This is specific logic to how the AppGateServer functions. Other gateways - // may have different means or approaches of identifying the service that the - // request is for (e.g. POST data). - newUrlPath = strings.TrimPrefix(urlPath, fmt.Sprintf("/%s", serviceId)) - if newUrlPath == "" { - newUrlPath = "/" - } - - return newUrlPath, serviceId -} - -type appGateServerOption func(*appGateServer) diff --git a/pkg/appgateserver/synchronous.go b/pkg/appgateserver/synchronous.go deleted file mode 100644 index 8df1f8aea..000000000 --- a/pkg/appgateserver/synchronous.go +++ /dev/null @@ -1,91 +0,0 @@ -package appgateserver - -import ( - "context" - "net/http" - - sdktypes "github.com/pokt-network/shannon-sdk/types" - - sharedtypes "github.com/pokt-network/poktroll/x/shared/types" -) - -// requestInfo is a struct that holds the information needed to handle a relay request. -type requestInfo struct { - appAddress string - serviceId string - rpcType sharedtypes.RPCType - poktRequest *sdktypes.POKTHTTPRequest - requestBz []byte -} - -// handleSynchronousRelay handles relay requests for synchronous protocols, where -// there is a one-to-one correspondence between the request and response. -// It does everything from preparing, signing and sending the request. -// It then blocks on the response to come back and forward it to the provided writer. -func (app *appGateServer) handleSynchronousRelay( - ctx context.Context, - reqInfo *requestInfo, - writer http.ResponseWriter, -) error { - serviceId := reqInfo.serviceId - rpcType := reqInfo.rpcType - poktRequest := reqInfo.poktRequest - requestBz := reqInfo.requestBz - appAddress := reqInfo.appAddress - - relaysTotal. - With("service_id", serviceId, "rpc_type", rpcType.String()). - Add(1) - - sessionSuppliers, err := app.sdk.GetSessionSupplierEndpoints(ctx, appAddress, serviceId) - if err != nil { - return ErrAppGateHandleRelay.Wrapf("getting current session: %s", err) - } - - // Get a supplier URL and address for the given service and session. - supplierEndpoint, err := app.getRelayerUrl(rpcType, *sessionSuppliers, poktRequest.Url) - if err != nil { - return ErrAppGateHandleRelay.Wrapf("getting supplier URL: %s", err) - } - - relayResponse, err := app.sdk.SendRelay(ctx, appAddress, supplierEndpoint, requestBz) - // If the relayResponse is nil, it means that err is not nil and the error - // should be handled by the appGateServer. - if relayResponse == nil { - return err - } - // Here, neither the relayResponse nor the error are nil, so the relayResponse's - // contains the upstream service's error response. - if err != nil { - return ErrAppGateUpstreamError.Wrap(string(relayResponse.Payload)) - } - - // Deserialize the RelayResponse payload to get the serviceResponse that will - // be forwarded to the client. - serviceResponse, err := sdktypes.DeserializeHTTPResponse(relayResponse.Payload) - if err != nil { - return ErrAppGateHandleRelay.Wrapf("deserializing response: %s", err) - } - - app.logger.Debug(). - Str("relay_response_payload", string(serviceResponse.BodyBz)). - Msg("writing relay response payload") - - // Reply to the client with the service's response status code and headers. - // At this point the AppGateServer has not generated any internal errors, so - // the whole response will be forwarded to the client as is, including the - // status code and headers, be it an error or not. - serviceResponse.CopyToHTTPHeader(writer.Header()) - writer.WriteHeader(int(serviceResponse.StatusCode)) - - // Transmit the service's response body to the client. - if _, err := writer.Write(serviceResponse.BodyBz); err != nil { - return ErrAppGateHandleRelay.Wrapf("writing relay response payload: %s", err) - } - - relaysSuccessTotal. - With("service_id", serviceId, "rpc_type", rpcType.String()). - Add(1) - - return nil -} diff --git a/pkg/client/events/replay_client.go b/pkg/client/events/replay_client.go index ac6e9aa73..43b8bb965 100644 --- a/pkg/client/events/replay_client.go +++ b/pkg/client/events/replay_client.go @@ -26,12 +26,15 @@ const ( // eventsBytesRetryDelay is the delay between retry attempts when the events // bytes observable returns an error. eventsBytesRetryDelay = time.Second + + // TODO_MAINNET(@bryanchriswhite): Make this a customizable parameter in the + // Gateway & RelayMiner config files // eventsBytesRetryLimit is the maximum number of times to attempt to // re-establish the events query bytes subscription when the events bytes // observable returns an error or closes. - // TODO_TECHDEBT: to make this a customizable parameter in the appgateserver and relayminer config files. eventsBytesRetryLimit = 10 eventsBytesRetryResetTimeout = 10 * time.Second + // replayObsCacheBufferSize is the replay buffer size of the // replayObsCache replay observable which is used to cache the replay // observable that is notified of new events. diff --git a/pkg/deps/config/suppliers.go b/pkg/deps/config/suppliers.go index c7548f1cb..1d964caee 100644 --- a/pkg/deps/config/suppliers.go +++ b/pkg/deps/config/suppliers.go @@ -7,11 +7,9 @@ import ( "cosmossdk.io/depinject" sdkclient "github.com/cosmos/cosmos-sdk/client" cosmosflags "github.com/cosmos/cosmos-sdk/client/flags" - cryptotypes "github.com/cosmos/cosmos-sdk/crypto/types" "github.com/cosmos/gogoproto/grpc" "github.com/spf13/cobra" - "github.com/pokt-network/poktroll/pkg/appgateserver/sdkadapter" "github.com/pokt-network/poktroll/pkg/client/block" "github.com/pokt-network/poktroll/pkg/client/delegation" "github.com/pokt-network/poktroll/pkg/client/events" @@ -343,42 +341,6 @@ func NewSupplyRingCacheFn() SupplierFn { } } -// NewSupplyShannonSDKFn supplies a depinject config with a ShannonSDK given -// the signing key name. -func NewSupplyShannonSDKFn(signingKeyName string) SupplierFn { - return func( - ctx context.Context, - deps depinject.Config, - _ *cobra.Command, - ) (depinject.Config, error) { - var clientCtx sdkclient.Context - - // On a Cosmos environment we get the private key from the keyring - // Inject the client context, get the keyring from it then get the private key - if err := depinject.Inject(deps, &clientCtx); err != nil { - return nil, err - } - - keyRecord, err := clientCtx.Keyring.Key(signingKeyName) - if err != nil { - return nil, err - } - - privateKey, ok := keyRecord.GetLocal().PrivKey.GetCachedValue().(cryptotypes.PrivKey) - if !ok { - return nil, err - } - - shannonSDK, err := sdkadapter.NewShannonSDK(ctx, privateKey, deps) - if err != nil { - return nil, err - } - - // Supply the session querier to the provided deps - return depinject.Configs(deps, depinject.Supply(shannonSDK)), nil - } -} - // NewSupplySupplierClientsFn returns a function which constructs a // SupplierClientMap and returns a new depinject.Config which is // supplied with the given deps and the new SupplierClientMap. diff --git a/pkg/relayer/config/relayminer_configs_reader.go b/pkg/relayer/config/relayminer_configs_reader.go index 8d90344eb..1cc708908 100644 --- a/pkg/relayer/config/relayminer_configs_reader.go +++ b/pkg/relayer/config/relayminer_configs_reader.go @@ -16,7 +16,7 @@ func ParseRelayMinerConfigs(configContent []byte) (*RelayMinerConfig, error) { return nil, ErrRelayMinerConfigEmpty } - // Unmarshal the stake config file into a yamlAppGateConfig + // Unmarshal the stake config file into a yamlRelayMinerConfig if err := yaml.Unmarshal(configContent, &yamlRelayMinerConfig); err != nil { return nil, ErrRelayMinerConfigUnmarshalYAML.Wrap(err.Error()) } diff --git a/pkg/relayer/proxy/synchronous.go b/pkg/relayer/proxy/synchronous.go index 66cde7e6d..6d4d69266 100644 --- a/pkg/relayer/proxy/synchronous.go +++ b/pkg/relayer/proxy/synchronous.go @@ -57,7 +57,7 @@ type synchronousRPCServer struct { // relay requests and forwards them to the supported proxied service endpoint. // It takes the serviceId, endpointUrl, and the main RelayerProxy as arguments // and returns a RelayServer that listens to incoming RelayRequests. -// TODO_RESEARCH(#590): Currently, the communication between the AppGateServer and the +// TODO_RESEARCH(#590): Currently, the communication between the Gateway and the // RelayMiner uses HTTP. This could be changed to a more generic and performant // one, such as pure TCP. func NewSynchronousServer(