[Configs] Add foundation for RelayMiner operation configs. (#284)

* feat: Working SDK with AppGateServer integration

* chore: Address change requests

* fix: Chain depinject configs

* chore: Expose the full SessionSuppliers struct

* fix: Avoid nil sessions on GetSessionSupplierEndpoints

* chore: Explain SingleSupplierEndpoint struct usage

* fix: Rename client/event import as per main chainges

* fix: Add missing fmt package

* feat: Enable gRPC client connection

* feat: Have distinct JSON-RPC and gRPC urls

* chore: Trigger e2e tests

* chore: Rename config files entries and enable grpc for off-chain actors

* chore: Revert reviewdog

* chore: Address request changes

* chore: Fix temp assignment of FlagGRPC

* feat: [WIP] Implement new replayminer config

* feat: Adapt tests to new relay miner config

* chore: Add relay miner config and full example

* chore: add valid relay miner config

* chore: Address request changes

* fix: Correct tests error assertions

* chore: Address request changes

* refactor: Split relayminer config parsing and validation logic

* chore: Add TODO to make MarshalAndSend RPC-type agnostic

* chore: Strip-off comments form AppGateServer config, add documented example config

* chore: Strip comments out of relay miner config and reorder entries

* chore: Update flagNode variables and command documentation

* fix: Update sequencer host used in local net

* chore: Address change requests

* chore: Add UDP in TODO comment

* chore: Improve documentation about multiple hosts config

* chore: Add change to trigger e2e test

* chore: e2e bump

* fix: Use RPC url for tx broadcasting

localnet/kubernetes/values-appgateserver.yaml

@@ -1,2 +1,3 @@
 config:
-  query_node_url: tcp://sequencer-poktroll-sequencer:36657
+  query_node_rpc_url: tcp://sequencer-poktroll-sequencer:36657
+  query_node_grpc_url: tcp://sequencer-poktroll-sequencer:36658

localnet/kubernetes/values-relayminer.yaml

@@ -1,3 +1,20 @@
 config:
-  query_node_url: tcp://sequencer-poktroll-sequencer:36657
-  network_node_url: tcp://sequencer-poktroll-sequencer:36657
+  signing_key_name: supplier1
+  smt_store_path: smt_stores
+  pocket_node:
+    query_node_rpc_url: tcp://sequencer-poktroll-sequencer:36657
+    query_node_grpc_url: tcp://sequencer-poktroll-sequencer:36658
+    tx_node_rpc_url: tcp://sequencer-poktroll-sequencer:36657
+  proxies:
+    - proxy_name: http-proxy
+      type: http
+      host: 0.0.0.0:8545
+  suppliers:
+    - service_id: anvil
+      type: http
+      service_config:
+        url: http://anvil:8547/
+      proxy_names:
+        - http-proxy
+      hosts:
+        - tcp://relayminers:8545

localnet/poktrolld/config/appgate_server_config.yaml

@@ -1,8 +1,5 @@
-# Whether the server should sign all incoming requests with its own ring (for applications)
-self_signing: true
-# The name of the key (in the keyring) that will be used to sign relays
+query_node_rpc_url: tcp://sequencer-poktroll-sequencer:36657
+query_node_grpc_url: tcp://sequencer-poktroll-sequencer:36658
 signing_key: app1
-# The host and port that the appgate server will listen on
-listening_endpoint: http://localhost:42069
-# tcp://<host>:<port> to a full pocket node for reading data and listening for on-chain events
-query_node_url: tcp://127.0.0.1:36657
+self_signing: true
+listening_endpoint: http://localhost:42069

localnet/poktrolld/config/appgate_server_config_example.yaml

@@ -0,0 +1,11 @@
+# Pocket node URL that exposes CometBFT JSON-RPC API.
+# This can be used by the Cosmos client SDK, event subscriptions, etc...
+query_node_rpc_url: tcp://sequencer-poktroll-sequencer:36657
+# Pocket node URL that exposes the Cosmos gRPC service, dedicated to querying purposes.
+query_node_grpc_url: tcp://sequencer-poktroll-sequencer:36658
+# The name of the key (in the keyring) that will be used to sign relays
+signing_key: app1
+# Whether the server should sign all incoming requests with its own ring (for applications)
+self_signing: true
+# The host and port that the appgate server will listen on
+listening_endpoint: http://localhost:42069

localnet/poktrolld/config/relayminer_config.yaml

@@ -1,15 +1,19 @@
-# tcp://<host>:<port> to a full pocket node for reading data and listening for on-chain events
-query_node_url: tcp://localhost:36657
-# tcp://<host>:<port> to a pocket node that gossips transactions throughout the network (may or may not be the sequencer)
-network_node_url: tcp://127.0.0.1:36657
-# Name of the key (in the keyring) to sign transactions
 signing_key_name: supplier1
-# TODO_TECHDEBT(#137, #130): Once the `relayer.json` config file is implemented AND a local LLM RPC service
-# is supported on LocalNet, this needs to be expanded to include more than one service. The ability to support
-# multiple services is already in place but currently (as seen below) is hardcoded.
-# TODO_UPNEXT(@okdas): this hostname should be updated to match that of the in-tilt anvil service.
-proxied_service_endpoints:
-  anvil: http://anvil:8547
-  svc1: http://localhost:8082
-# Path to where the data backing SMT KV store exists on disk
 smt_store_path: smt_stores
+pocket_node:
+  query_node_rpc_url: tcp://sequencer-poktroll-sequencer:36657
+  query_node_grpc_url: tcp://sequencer-poktroll-sequencer:36658
+  tx_node_rpc_url: tcp://sequencer-poktroll-sequencer:36657
+proxies:
+  - proxy_name: http-proxy
+    type: http
+    host: 0.0.0.0:8545
+suppliers:
+  - service_id: anvil
+    type: http
+    service_config:
+      url: http://anvil:8547/
+    proxy_names:
+      - http-proxy
+    hosts:
+      - tcp://relayminers:8545

localnet/poktrolld/config/relayminer_config_full_example.yaml

@@ -0,0 +1,126 @@
+# TODO_CONSIDERATION: We don't need this now, but it would be beneficial if the
+# logic handling this config file could be designed in such a way that it allows for
+# "hot" config changes in the future, meaning changes without restarting a process.
+# This would be useful for adding a proxy or a supplier without interrupting the service.
+
+# Name of the key (in the keyring) to sign transactions.
+signing_key_name: supplier1
+# Relative path (on the relayminer's machine) to where the data backing
+# SMT KV store exists on disk.
+smt_store_path: smt_stores
+
+pocket_node:
+  # Pocket node URL that exposes CometBFT JSON-RPC API.
+  # This can be used by the Cosmos client SDK, event subscriptions, etc...
+  # If unspecified, defaults to `tx_node_rpc_url`.
+  query_node_rpc_url: tcp://sequencer-poktroll-sequencer:36657
+  # Pocket node URL that exposes the Cosmos gRPC service, dedicated to querying purposes.
+  query_node_grpc_url: tcp://sequencer-poktroll-sequencer:36658
+  # Pocket node URL that exposes the TendermintRPC service.
+  tx_node_rpc_url: tcp://sequencer-poktroll-sequencer:36658
+
+# Proxies are endpoints that expose different suppliers to the internet.
+proxies:
+    # Name of the proxy. It will be used to reference in a supplier.
+    # Must be unique.
+    # Required.
+    # TODO_CONSIDERATION: if we enforce DNS compliant names, it can potentially
+    # become handy in the future.
+    # More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#dns-subdomain-names
+  - proxy_name: http-example
+    # Type of proxy: currently only http is supported but will support more
+    # (https, tcp, quic ...) in the future.
+    # MUST match the type of the supplier.
+    # Required.
+    type: http
+    # Hostname to open port on.
+    # Use 0.0.0.0 in containerized environments.
+    # 127.0.0.1 with a reverse proxy when there's another process on localhost
+    # that can be used as a reverse proxy (nginx, apache, traefik, etc.).
+    # Required.
+    host: 127.0.0.1:8080
+
+  # TODO_IMPROVE: https is not currently supported, but this is how it could potentially look.
+  # - name: example-how-we-can-support-https
+  #   type: https
+  #   host: 0.0.0.0:8443
+  #   tls:
+  #     enabled: true
+  #     certificate: /path/to/crt
+  #     key: /path/to/key
+
+# Suppliers are different services that can be offered through RelayMiner.
+# When a supplier is configured to use a proxy and staked appropriately,
+# the relays will start flowing through RelayMiner.
+suppliers:
+    # The serviceId the supplier offered to the network .
+    # It must match the Service.Id of the service that has been staked for.
+    # Must be unique.
+    # Required.
+  - service_id: ethereum
+    # Type of how the supplier offers service through the network.
+    # Must match the type of the proxy the supplier is connected to.
+    # Required.
+    type: http
+    # Configuration of the service offered through RelayMiner.
+    service_config:
+      # URL RelayMiner proxies the requests to.
+      # Also known as the data node, or service node in some cases.
+      # Required.
+      url: http://anvil.servicer:8545
+      # Authentication for the service.
+      # HTTP Basic Auth: https://developer.mozilla.org/en-US/docs/Web/HTTP/Authentication
+      # Optional.
+      authentication:
+        username: user
+        password: pwd
+
+      # TODO_IMPROVE: This is not supported in code yet,
+      # but some services authenticate via a header.
+      # Example, if the service requires a header like `Authorization: Bearer <PASSWORD>`
+      # Authorization: Bearer <PASSWORD>
+      # Optional.
+      headers: {}
+
+    # A list of hosts the proxy is accepting requests from.
+    # When linked to the proxy, each host is going to be used to lookup the
+    # the Supplier.Service in Pocket Network.
+    # Each host in the list must match a Supplier.Service.Endpoint that the Supplier
+    # has advertised on-chain when staking for that Service.
+    # There are various reasons to having multiple hosts for the same supplier services,
+    # - The on-chain Supplier may provide the same Service on multiple domains
+    #   (e.g. for different regions).
+    # - The operator may want to route requests of different RPC types to
+    #   the same proxy.
+    # - Migrating from one domain to another. Where the operator could still
+    #   accept requests on the old domain while the new domain is being propagated.
+    # - The operator may want to have a different domain for internal requests.
+    # - The on-chain Service configuration accepts multiple endpoints.
+    # Must be unique within the proxy it is referenced in.
+    # Required.
+    hosts:
+      - ethereum.devnet1.poktroll.com
+      # The `service_id` of the supplier is automatically added to the hosts section
+      # for potential troubleshooting/debugging purposes such as:
+      # Having internal requests coming from non-FQDNs because of complex routing.
+      # Sending requests from k8s pods.
+      # Specify the `host` in curl requests when testing `curl -H "Host: ethereum" ...`
+      # and make the proxy server process the request without the need for an
+      # on-chain Endpoint entry.
+      # - ethereum # <- this part is added automatically.
+
+    # Names of proxies that this supplier is connected to.
+    # This MUST correspond to the `proxy_name` entry in the `proxies` section
+    # in order for the supplier to be available to the external network.
+    # Required.
+    proxy_names:
+      - http-example # when the RelayMiner server builder runs.
+  - service_id: 7b-llm-model
+    type: http
+    service_config:
+      url: http://llama-endpoint
+    hosts:
+      - 7b-llm-model.devnet1.poktroll.com
+      # - 7b-llm-model # <- this part is added automatically.
+    proxy_names:
+      - http-example

pkg/appgateserver/cmd/cmd.go

@@ -25,8 +25,13 @@ import (
 const omittedDefaultFlagValue = "explicitly omitting default"
 
 var (
+	// flagAppGateConfig is the variable containing the AppGate config filepath
+	// sourced from the `--config` flag.
 	flagAppGateConfig string
-	flagCosmosNodeURL string
+	// flagNodeRPCURL is the variable containing the Cosmos node RPC URL flag value.
+	flagNodeRPCURL string
+	// flagNodeGRPCURL is the variable containing the Cosmos node GRPC URL flag value.
+	flagNodeGRPCURL string
 )
 
 // AppGateServerCmd returns the Cobra command for running the AppGate server.
@@ -65,9 +70,11 @@ provided that:
 	cmd.Flags().StringVar(&flagAppGateConfig, "config", "", "The path to the appgate config file")
 
 	// Cosmos flags
+	// TODO_TECHDEBT(#256): Remove unneeded cosmos flags.
 	cmd.Flags().String(cosmosflags.FlagKeyringBackend, "", "Select keyring's backend (os|file|kwallet|pass|test)")
-	cmd.Flags().
-		StringVar(&flagCosmosNodeURL, cosmosflags.FlagNode, omittedDefaultFlagValue, "Register the default Cosmos node flag, which is needed to initialize the Cosmos query context correctly. It can be used to override the `QueryNodeUrl` field in the config file if specified.")
+	cmd.Flags().StringVar(&flagNodeRPCURL, cosmosflags.FlagNode, omittedDefaultFlagValue, "Register the default Cosmos node flag, which is needed to initialize the Cosmos query context correctly. It can be used to override the `QueryNodeUrl` field in the config file if specified.")
+	cmd.Flags().StringVar(&flagNodeGRPCURL, cosmosflags.FlagGRPC, omittedDefaultFlagValue, "Register the default Cosmos node grpc flag, which is needed to initialize the Cosmos query context with grpc correctly. It can be used to override the `QueryNodeGRPCUrl` field in the config file if specified.")
+	cmd.Flags().Bool(cosmosflags.FlagGRPCInsecure, true, "Used to initialize the Cosmos query context with grpc security options. It can be used to override the `QueryNodeGRPCInsecure` field in the config file if specified.")
 
 	return cmd
 }
@@ -145,24 +152,37 @@ func setupAppGateServerDependencies(
 	cmd *cobra.Command,
 	appGateConfig *appgateconfig.AppGateServerConfig,
 ) (_ depinject.Config, err error) {
-	queryNodeURL := appGateConfig.QueryNodeUrl
-	// Override the config file's `QueryNodeUrl` fields
+	queryNodeRPCURL := appGateConfig.QueryNodeRPCUrl
+	queryNodeGRPCURL := appGateConfig.QueryNodeGRPCUrl
+
+	// Override the config file's `QueryNodeGRPCUrl` field
+	// with the `--grpc-addr` flag if it was specified.
+	// TODO_TECHDEBT(#223) Remove this check once viper is used as SoT for overridable config values.
+	if flagNodeGRPCURL != omittedDefaultFlagValue {
+		queryNodeGRPCURL, err = url.Parse(flagNodeGRPCURL)
+		if err != nil {
+			return nil, fmt.Errorf("failed to parse grpc query URL: %w", err)
+		}
+	}
+
+	// Override the config file's `QueryNodeRPCURL` field
 	// with the `--node` flag if it was specified.
-	if flagCosmosNodeURL != omittedDefaultFlagValue {
-		queryNodeURL, err = url.Parse(flagCosmosNodeURL)
+	// TODO_TECHDEBT(#223) Remove this check once viper is used as SoT for overridable config values.
+	if flagNodeRPCURL != omittedDefaultFlagValue {
+		queryNodeRPCURL, err = url.Parse(flagNodeRPCURL)
 		if err != nil {
-			return nil, fmt.Errorf("failed to parse Cosmos node URL: %w", err)
+			return nil, fmt.Errorf("failed to parse rpc query URL: %w", err)
 		}
 	}
 
 	supplierFuncs := []config.SupplierFn{
 		config.NewSupplyLoggerFromCtx(ctx),
-		config.NewSupplyEventsQueryClientFn(queryNodeURL.Host),      // leaf
-		config.NewSupplyBlockClientFn(),                             // leaf
-		config.NewSupplyQueryClientContextFn(queryNodeURL.String()), // leaf
-		config.NewSupplyAccountQuerierFn(),                          // leaf
-		config.NewSupplyApplicationQuerierFn(),                      // leaf
-		config.NewSupplySessionQuerierFn(),                          // leaf
+		config.NewSupplyEventsQueryClientFn(queryNodeRPCURL),   // leaf
+		config.NewSupplyBlockClientFn(),                        // leaf
+		config.NewSupplyQueryClientContextFn(queryNodeGRPCURL), // leaf
+		config.NewSupplyAccountQuerierFn(),                     // leaf
+		config.NewSupplyApplicationQuerierFn(),                 // leaf
+		config.NewSupplySessionQuerierFn(),                     // leaf
 		config.NewSupplyRingCacheFn(),
 		config.NewSupplyPOKTRollSDKFn(appGateConfig.SigningKey),
 	}

pkg/appgateserver/config/appgate_configs_reader.go

@@ -9,58 +9,74 @@ import (
 // YAMLAppGateServerConfig is the structure used to unmarshal the AppGateServer config file
 // TODO_DOCUMENT(@red-0ne): Add proper README documentation for yaml config files.
 type YAMLAppGateServerConfig struct {
-	SelfSigning       bool   `yaml:"self_signing"`
+	QueryNodeRPCUrl   string `yaml:"query_node_rpc_url"`
+	QueryNodeGRPCUrl  string `yaml:"query_node_grpc_url"`
 	SigningKey        string `yaml:"signing_key"`
+	SelfSigning       bool   `yaml:"self_signing"`
 	ListeningEndpoint string `yaml:"listening_endpoint"`
-	QueryNodeUrl      string `yaml:"query_node_url"`
 }
 
 // AppGateServerConfig is the structure describing the AppGateServer config
 type AppGateServerConfig struct {
-	SelfSigning       bool
+	QueryNodeRPCUrl   *url.URL
+	QueryNodeGRPCUrl  *url.URL
 	SigningKey        string
+	SelfSigning       bool
 	ListeningEndpoint *url.URL
-	QueryNodeUrl      *url.URL
 }
 
 // ParseAppGateServerConfigs parses the stake config file into a AppGateConfig
 // NOTE: If SelfSigning is not defined in the config file, it will default to false
 func ParseAppGateServerConfigs(configContent []byte) (*AppGateServerConfig, error) {
 	var yamlAppGateServerConfig YAMLAppGateServerConfig
 
+	if len(configContent) == 0 {
+		return nil, ErrAppGateConfigEmpty
+	}
+
 	// Unmarshal the stake config file into a yamlAppGateConfig
 	if err := yaml.Unmarshal(configContent, &yamlAppGateServerConfig); err != nil {
-		return nil, ErrAppGateConfigUnmarshalYAML.Wrapf("%s", err)
+		return nil, ErrAppGateConfigUnmarshalYAML.Wrap(err.Error())
 	}
 
-	if yamlAppGateServerConfig.SigningKey == "" {
+	if len(yamlAppGateServerConfig.SigningKey) == 0 {
 		return nil, ErrAppGateConfigEmptySigningKey
 	}
 
-	if yamlAppGateServerConfig.ListeningEndpoint == "" {
+	if len(yamlAppGateServerConfig.ListeningEndpoint) == 0 {
 		return nil, ErrAppGateConfigInvalidListeningEndpoint
 	}
 
 	listeningEndpoint, err := url.Parse(yamlAppGateServerConfig.ListeningEndpoint)
 	if err != nil {
-		return nil, ErrAppGateConfigInvalidListeningEndpoint.Wrapf("%s", err)
+		return nil, ErrAppGateConfigInvalidListeningEndpoint.Wrap(err.Error())
 	}
 
-	if yamlAppGateServerConfig.QueryNodeUrl == "" {
-		return nil, ErrAppGateConfigInvalidQueryNodeUrl
+	if len(yamlAppGateServerConfig.QueryNodeGRPCUrl) == 0 {
+		return nil, ErrAppGateConfigInvalidQueryNodeGRPCUrl
 	}
 
-	queryNodeUrl, err := url.Parse(yamlAppGateServerConfig.QueryNodeUrl)
+	queryNodeGRPCUrl, err := url.Parse(yamlAppGateServerConfig.QueryNodeGRPCUrl)
 	if err != nil {
-		return nil, ErrAppGateConfigInvalidQueryNodeUrl.Wrapf("%s", err)
+		return nil, ErrAppGateConfigInvalidQueryNodeGRPCUrl.Wrap(err.Error())
+	}
+
+	if len(yamlAppGateServerConfig.QueryNodeRPCUrl) == 0 {
+		return nil, ErrAppGateConfigInvalidQueryNodeRPCUrl
+	}
+
+	queryNodeRPCUrl, err := url.Parse(yamlAppGateServerConfig.QueryNodeRPCUrl)
+	if err != nil {
+		return nil, ErrAppGateConfigInvalidQueryNodeRPCUrl.Wrap(err.Error())
 	}
 
 	// Populate the appGateServerConfig with the values from the yamlAppGateServerConfig
 	appGateServerConfig := &AppGateServerConfig{
-		SelfSigning:       yamlAppGateServerConfig.SelfSigning,
+		QueryNodeRPCUrl:   queryNodeRPCUrl,
+		QueryNodeGRPCUrl:  queryNodeGRPCUrl,
 		SigningKey:        yamlAppGateServerConfig.SigningKey,
+		SelfSigning:       yamlAppGateServerConfig.SelfSigning,
 		ListeningEndpoint: listeningEndpoint,
-		QueryNodeUrl:      queryNodeUrl,
 	}
 
 	return appGateServerConfig, nil