feat: wip: add support to remove database

.github/workflows/ci-e2e-no-metrics-tests.yml

@@ -0,0 +1,48 @@
+name: Continuous Integration (E2E Testing Checks without metrics database)
+
+on:
+  workflow_call:
+jobs:
+  e2e-test:
+    runs-on: ubuntu-latest
+    steps:
+      - name: checkout repo from current commit
+        uses: actions/checkout@v3
+      - name: set up Go
+        uses: actions/setup-go@v3
+        with:
+          go-version: "1.21"
+          check-latest: true
+          cache: true
+      - name: pull pre-built images
+        run: sudo docker compose -f ci.docker-compose.yml pull
+        env:
+          METRIC_DATABASE_ENABLED: "false"
+      # In this step, this action saves a list of existing images,
+      # the cache is created without them in the post run.
+      # It also restores the cache if it exists.
+      # TODO(yevhenii): this step failed with "No space left on device" error, debug it and enable back
+      # - name: cache docker images
+      #   uses: satackey/[email protected]
+        # Ignore the failure of a step and avoid terminating the job.
+        continue-on-error: true
+      - name: build and start proxy service and it's dependencies
+        run: sudo docker compose -f ci.docker-compose.yml up -d --build
+      - name: wait for proxy service to be running
+        run: bash ${GITHUB_WORKSPACE}/scripts/wait-for-proxy-service-running.sh
+        env:
+            PROXY_CONTAINER_PORT: 7777
+      - name: wait for proxy service metric partitions database tables to be created
+        run: bash ${GITHUB_WORKSPACE}/scripts/wait-for-proxy-service-database-metric-partitions.sh
+        env:
+            # needs to be 1 + number of partitions created by /clients/database/migrations/20230523101344_partition_proxied_request_metrics_table.up.sql
+            MINIMUM_REQUIRED_PARTITIONS: 30
+            PROXY_CONTAINER_PORT: 7777
+      - name: run e2e tests
+        run: make e2e-test-no-metrics
+      - name: print proxy service logs
+        run: sudo docker compose -f ci.docker-compose.yml logs proxy
+        # because we especially want the logs if the test(s) fail 😅
+        if: always()
+      # Finally, "Post Run jpribyl/[email protected]",
+      # which is the process of saving the cache, will be executed.

Makefile

@@ -42,8 +42,15 @@ unit-test:
 .PHONY: e2e-test
 # run tests that execute against a local or remote instance of the API
 e2e-test:
-	#go test -count=1 -v -cover -coverprofile cover.out --race ./... -run "^TestE2ETest*"
-	go test -count=1 -v -cover -coverprofile cover.out --race ./... -run "^TestE2ETestCachingMdwForGetTxByHashMethod"
+	PROXY_CONTAINER_PORT=7777 bash scripts/wait-for-proxy-service-running.sh
+	PROXY_CONTAINER_PORT=7777 MINIMUM_REQUIRED_PARTITIONS=30 bash scripts/wait-for-proxy-service-running.sh
+	go test -count=1 -v -cover -coverprofile cover.out --race ./... -run "^TestE2ETest*"
+	#go test -count=1 -v -cover -coverprofile cover.out --race ./... -run "^TestE2ETestCachingMdwForGetTxByHashMethod"
+
+.PHONY: e2e-test-no-metrics
+# run tests that execute against a local or remote instance of the API without database for metrics
+e2e-test-no-metrics:
+	go test -count=1 -v -cover -coverprofile cover.out --race ./... -run "^TestNoMetricsE2E*"
 
 .PHONY: ci-setup
 # set up your local environment such that running `make e2e-test` runs against testnet (like in CI)

clients/database/database.go

@@ -14,7 +14,11 @@ import (
 // that haven't been run on the database being used by the proxy service
 // returning error (if any) and a list of migrations that have been
 // run and any that were not
+// If db is nil, returns empty slice and nil error, as there is no database to migrate.
 func Migrate(ctx context.Context, db *bun.DB, migrations migrate.Migrations, logger *logging.ServiceLogger) (*migrate.MigrationSlice, error) {
+	if db == nil {
+		return &migrate.MigrationSlice{}, nil
+	}
 	// set up migration config
 	migrator := migrate.NewMigrator(db, &migrations)
 

clients/database/database_test.go

@@ -0,0 +1,14 @@
+package database
+
+import (
+	"context"
+	"github.com/stretchr/testify/require"
+	"github.com/uptrace/bun/migrate"
+	"testing"
+)
+
+func TestMigrateNoDatabase(t *testing.T) {
+	migrations, err := Migrate(context.Background(), nil, migrate.Migrations{}, nil)
+	require.NoError(t, err)
+	require.Empty(t, migrations)
+}

clients/database/postgres.go

@@ -16,12 +16,15 @@ import (
 // PostgresDatabaseConfig contains values for creating a
 // new connection to a postgres database
 type PostgresDatabaseConfig struct {
+	// DatabaseDisabled is used to disable the database, and it won't be used at all. All operations will be skipped.
+	DatabaseDisabled bool
+
 	DatabaseName                     string
 	DatabaseEndpointURL              string
 	DatabaseUsername                 string
 	DatabasePassword                 string
 	ReadTimeoutSeconds               int64
-	WriteTimeousSeconds              int64
+	WriteTimeoutsSeconds             int64
 	DatabaseMaxIdleConnections       int64
 	DatabaseConnectionMaxIdleSeconds int64
 	DatabaseMaxOpenConnections       int64
@@ -33,12 +36,19 @@ type PostgresDatabaseConfig struct {
 
 // PostgresClient wraps a connection to a postgres database
 type PostgresClient struct {
+	isDisabled bool
 	*bun.DB
 }
 
 // NewPostgresClient returns a new connection to the specified
 // postgres data and error (if any)
 func NewPostgresClient(config PostgresDatabaseConfig) (PostgresClient, error) {
+	if config.DatabaseDisabled {
+		return PostgresClient{
+			isDisabled: true,
+		}, nil
+	}
+
 	// configure postgres database connection options
 	var pgOptions *pgdriver.Connector
 
@@ -54,7 +64,7 @@ func NewPostgresClient(config PostgresDatabaseConfig) (PostgresClient, error) {
 				pgdriver.WithPassword(config.DatabasePassword),
 				pgdriver.WithDatabase(config.DatabaseName),
 				pgdriver.WithReadTimeout(time.Second*time.Duration(config.ReadTimeoutSeconds)),
-				pgdriver.WithWriteTimeout(time.Second*time.Duration(config.WriteTimeousSeconds)),
+				pgdriver.WithWriteTimeout(time.Second*time.Duration(config.WriteTimeoutsSeconds)),
 			)
 	} else {
 		pgOptions = pgdriver.NewConnector(
@@ -64,7 +74,7 @@ func NewPostgresClient(config PostgresDatabaseConfig) (PostgresClient, error) {
 			pgdriver.WithPassword(config.DatabasePassword),
 			pgdriver.WithDatabase(config.DatabaseName),
 			pgdriver.WithReadTimeout(time.Second*time.Duration(config.ReadTimeoutSeconds)),
-			pgdriver.WithWriteTimeout(time.Second*time.Duration(config.WriteTimeousSeconds)),
+			pgdriver.WithWriteTimeout(time.Second*time.Duration(config.WriteTimeoutsSeconds)),
 		)
 	}
 

clients/database/postgres_test.go

@@ -0,0 +1,15 @@
+package database
+
+import (
+	"github.com/stretchr/testify/require"
+	"testing"
+)
+
+func TestDisabledDBCreation(t *testing.T) {
+	config := PostgresDatabaseConfig{
+		DatabaseDisabled: true,
+	}
+	db, err := NewPostgresClient(config)
+	require.NoError(t, err)
+	require.True(t, db.isDisabled)
+}

clients/database/request_metric.go

@@ -34,8 +34,13 @@ type ProxiedRequestMetric struct {
 }
 
 // Save saves the current ProxiedRequestMetric to
-// the database, returning error (if any)
+// the database, returning error (if any).
+// If db is nil, returns nil error.
 func (prm *ProxiedRequestMetric) Save(ctx context.Context, db *bun.DB) error {
+	if db == nil {
+		return nil
+	}
+
 	_, err := db.NewInsert().Model(prm).Exec(ctx)
 
 	return err
@@ -44,8 +49,13 @@ func (prm *ProxiedRequestMetric) Save(ctx context.Context, db *bun.DB) error {
 // ListProxiedRequestMetricsWithPagination returns a page of max
 // `limit` ProxiedRequestMetrics from the offset specified by`cursor`
 // error (if any) along with a cursor to use to fetch the next page
-// if the cursor is 0 no more pages exists
+// if the cursor is 0 no more pages exists.
+// Uses only in tests. If db is nil, returns empty slice and 0 cursor.
 func ListProxiedRequestMetricsWithPagination(ctx context.Context, db *bun.DB, cursor int64, limit int) ([]ProxiedRequestMetric, int64, error) {
+	if db == nil {
+		return []ProxiedRequestMetric{}, 0, nil
+	}
+
 	var proxiedRequestMetrics []ProxiedRequestMetric
 	var nextCursor int64
 
@@ -62,8 +72,13 @@ func ListProxiedRequestMetricsWithPagination(ctx context.Context, db *bun.DB, cu
 
 // CountAttachedProxiedRequestMetricPartitions returns the current
 // count of attached partitions for the ProxiedRequestMetricsTableName
-// and error (if any)
+// and error (if any).
+// If db is nil, returns 0 and nil error.
 func CountAttachedProxiedRequestMetricPartitions(ctx context.Context, db *bun.DB) (int64, error) {
+	if db == nil {
+		return 0, nil
+	}
+
 	var count int64
 
 	countPartitionsQuery := fmt.Sprintf(`
@@ -88,7 +103,12 @@ func CountAttachedProxiedRequestMetricPartitions(ctx context.Context, db *bun.DB
 
 // GetLastCreatedAttachedProxiedRequestMetricsPartitionName gets the table name
 // for the last created (and attached) proxied request metrics partition
+// Used for status check. If db is nil, returns empty string and nil error.
 func GetLastCreatedAttachedProxiedRequestMetricsPartitionName(ctx context.Context, db *bun.DB) (string, error) {
+	if db == nil {
+		return "", nil
+	}
+
 	var lastCreatedAttachedPartitionName string
 
 	lastCreatedAttachedPartitionNameQuery := fmt.Sprintf(`
@@ -114,8 +134,13 @@ WHERE parent.relname='%s' order by child.oid desc limit 1;`, ProxiedRequestMetri
 
 // DeleteProxiedRequestMetricsOlderThanNDays deletes
 // all proxied request metrics older than the specified
-// days, returning error (if any)
+// days, returning error (if any).
+// Used during pruning process. If db is nil, returns nil error.
 func DeleteProxiedRequestMetricsOlderThanNDays(ctx context.Context, db *bun.DB, n int64) error {
+	if db == nil {
+		return nil
+	}
+
 	_, err := db.NewDelete().Model((*ProxiedRequestMetric)(nil)).Where(fmt.Sprintf("request_time < now() - interval '%d' day", n)).Exec(ctx)
 
 	return err

clients/database/request_metric_test.go

@@ -0,0 +1,37 @@
+package database
+
+import (
+	"context"
+	"github.com/stretchr/testify/require"
+	"testing"
+)
+
+func TestNoDatabaseSave(t *testing.T) {
+	prm := ProxiedRequestMetric{}
+	err := prm.Save(context.Background(), nil)
+	require.NoError(t, err)
+}
+
+func TestNoDatabaseListProxiedRequestMetricsWithPagination(t *testing.T) {
+	proxiedRequestMetrics, cursor, err := ListProxiedRequestMetricsWithPagination(context.Background(), nil, 0, 0)
+	require.NoError(t, err)
+	require.Empty(t, proxiedRequestMetrics)
+	require.Zero(t, cursor)
+}
+
+func TestNoDatabaseCountAttachedProxiedRequestMetricPartitions(t *testing.T) {
+	count, err := CountAttachedProxiedRequestMetricPartitions(context.Background(), nil)
+	require.NoError(t, err)
+	require.Zero(t, count)
+}
+
+func TestGetLastCreatedAttachedProxiedRequestMetricsPartitionName(t *testing.T) {
+	partitionName, err := GetLastCreatedAttachedProxiedRequestMetricsPartitionName(context.Background(), nil)
+	require.NoError(t, err)
+	require.Empty(t, partitionName)
+}
+
+func TestDeleteProxiedRequestMetricsOlderThanNDays(t *testing.T) {
+	err := DeleteProxiedRequestMetricsOlderThanNDays(context.Background(), nil, 0)
+	require.NoError(t, err)
+}

config/config.go

@@ -48,7 +48,7 @@ type Config struct {
 	MetricPruningRoutineInterval                  time.Duration
 	MetricPruningRoutineDelayFirstRun             time.Duration
 	MetricPruningMaxRequestMetricsHistoryDays     int
-	MetricOperationsEnabled                       bool
+	MetricDatabaseEnabled                         bool
 	CacheEnabled                                  bool
 	RedisEndpointURL                              string
 	RedisPassword                                 string
@@ -102,8 +102,8 @@ const (
 	DEFAULT_METRIC_PRUNING_ENABLED                              = true
 	METRIC_PRUNING_ROUTINE_INTERVAL_SECONDS_ENVIRONMENT_KEY     = "METRIC_PRUNING_ROUTINE_INTERVAL_SECONDS"
 	// 60 seconds * 60 minutes * 24 hours = 1 day
-	METRIC_OPERATIONS_ENABLED_ENVIRONMENT_KEY                         = "METRIC_OPERATIONS_ENABLED"
-	DEFAULT_METRIC_OPERATIONS_ENABLED                                 = true
+	METRIC_DATABASE_ENABLED_ENVIRONMENT_KEY                           = "METRIC_DATABASE_ENABLED"
+	DEFAULT_METRIC_DATABASE_ENABLED                                   = true
 	DEFAULT_METRIC_PRUNING_ROUTINE_INTERVAL_SECONDS                   = 86400
 	METRIC_PRUNING_ROUTINE_DELAY_FIRST_RUN_SECONDS_ENVIRONMENT_KEY    = "METRIC_PRUNING_ROUTINE_DELAY_FIRST_RUN_SECONDS"
 	DEFAULT_METRIC_PRUNING_ROUTINE_DELAY_FIRST_RUN_SECONDS            = 10
@@ -383,7 +383,7 @@ func ReadConfig() Config {
 		MetricPruningRoutineInterval:                  time.Duration(time.Duration(EnvOrDefaultInt(METRIC_PRUNING_ROUTINE_INTERVAL_SECONDS_ENVIRONMENT_KEY, DEFAULT_METRIC_PRUNING_ROUTINE_INTERVAL_SECONDS)) * time.Second),
 		MetricPruningRoutineDelayFirstRun:             time.Duration(time.Duration(EnvOrDefaultInt(METRIC_PRUNING_ROUTINE_DELAY_FIRST_RUN_SECONDS_ENVIRONMENT_KEY, DEFAULT_METRIC_PRUNING_ROUTINE_DELAY_FIRST_RUN_SECONDS)) * time.Second),
 		MetricPruningMaxRequestMetricsHistoryDays:     EnvOrDefaultInt(METRIC_PRUNING_MAX_REQUEST_METRICS_HISTORY_DAYS_ENVIRONMENT_KEY, DEFAULT_METRIC_PRUNING_MAX_REQUEST_METRICS_HISTORY_DAYS),
-		MetricOperationsEnabled:                       EnvOrDefaultBool(METRIC_OPERATIONS_ENABLED_ENVIRONMENT_KEY, DEFAULT_METRIC_OPERATIONS_ENABLED),
+		MetricDatabaseEnabled:                         EnvOrDefaultBool(METRIC_DATABASE_ENABLED_ENVIRONMENT_KEY, DEFAULT_METRIC_DATABASE_ENABLED),
 		CacheEnabled:                                  EnvOrDefaultBool(CACHE_ENABLED_ENVIRONMENT_KEY, false),
 		RedisEndpointURL:                              os.Getenv(REDIS_ENDPOINT_URL_ENVIRONMENT_KEY),
 		RedisPassword:                                 os.Getenv(REDIS_PASSWORD_ENVIRONMENT_KEY),

main.go

@@ -37,7 +37,7 @@ func init() {
 }
 
 func startMetricPartitioningRoutine(serviceConfig config.Config, service service.ProxyService, serviceLogger logging.ServiceLogger) <-chan error {
-	if !serviceConfig.MetricOperationsEnabled {
+	if !serviceConfig.MetricDatabaseEnabled {
 		serviceLogger.Info().Msg("skipping starting metric partitioning routine since it is disabled via config")
 
 		return nil
@@ -73,7 +73,7 @@ func startMetricPartitioningRoutine(serviceConfig config.Config, service service
 }
 
 func startMetricCompactionRoutine(serviceConfig config.Config, service service.ProxyService, serviceLogger logging.ServiceLogger) <-chan error {
-	if !serviceConfig.MetricOperationsEnabled {
+	if !serviceConfig.MetricDatabaseEnabled {
 		serviceLogger.Info().Msg("skipping starting metric compaction routine since it is disabled via config")
 
 		return nil
@@ -107,7 +107,7 @@ func startMetricCompactionRoutine(serviceConfig config.Config, service service.P
 }
 
 func startMetricPruningRoutine(serviceConfig config.Config, service service.ProxyService, serviceLogger logging.ServiceLogger) <-chan error {
-	if !serviceConfig.MetricPruningEnabled || !serviceConfig.MetricOperationsEnabled {
+	if !serviceConfig.MetricPruningEnabled || !serviceConfig.MetricDatabaseEnabled {
 		serviceLogger.Info().Msg("skipping starting metric pruning routine since it is disabled via config")
 
 		return make(<-chan error)