Add Redis and TiKV kv stores (guacsec#1502)

* Add Redis and TiKV kv stores Signed-off-by: Jeff Mendoza <[email protected]> * Only use TiKV on 64 bit Signed-off-by: Jeff Mendoza <[email protected]> * Also don't built tikv for darwin. Signed-off-by: Jeff Mendoza <[email protected]> --------- Signed-off-by: Jeff Mendoza <[email protected]>
desmax74 · Nov 15, 2023 · 1fb5ee9 · 1fb5ee9
1 parent 1e5a333
commit 1fb5ee9
Show file tree

Hide file tree

Showing 12 changed files with 368 additions and 74 deletions.
diff --git a/README.md b/README.md
@@ -75,10 +75,36 @@ The following are [GraphQL backends](pkg/assembler/backends) that are implemente
 3. Optimized: The backend has gone through a level of optimization to help improve performance.
 
 The enumerated backends are:
-- [keyvalue (supported, complete, optimized)](https://github.com/guacsec/guac/tree/main/pkg/assembler/backends/keyvalue): a non-persistent in-memory backend that doesn't require any additional infrastructure. Also acts as a conformance backend for API implementations. We recommend starting with this if you're just starting with GUAC!
-- [arangoDB (supported, incomplete, optimized)](https://github.com/guacsec/guac/tree/main/pkg/assembler/backends/arangodb): a persistent backend based on [ArangoDB](https://arangodb.com/)
-- [ent (supported, incomplete)](https://github.com/guacsec/guac/tree/main/pkg/assembler/backends/ent): a persistent backend based on [ent](https://entgo.io/) that can run on various SQL backends such as [PostgreSQL](https://www.postgresql.org/), [MySQL](https://www.mysql.com/) and [SQLite](https://www.sqlite.org/index.html).
-- [neo4j/openCypher (unsupported, incomplete)](https://github.com/guacsec/guac/tree/main/pkg/assembler/backends/neo4j): a persistent backend based on [neo4j](https://neo4j.com/) and [openCypher](https://opencypher.org/). This backend should work with any database that supported openCypher queries.
+
+- [keyvalue (supported, complete,
+  optimized)](https://github.com/guacsec/guac/tree/main/pkg/assembler/backends/keyvalue):
+  a non-persistent in-memory backend that doesn't require any additional
+  infrastructure. Also acts as a conformance backend for API
+  implementations. We recommend starting with this if you're just starting with
+  GUAC!
+
+- [arangoDB (supported, incomplete,
+  optimized)](https://github.com/guacsec/guac/tree/main/pkg/assembler/backends/arangodb):
+  a persistent backend based on [ArangoDB](https://arangodb.com/)
+
+- [ent (supported,
+  incomplete)](https://github.com/guacsec/guac/tree/main/pkg/assembler/backends/ent):
+  a persistent backend based on [ent](https://entgo.io/) that can run on
+  various SQL backends such as [PostgreSQL](https://www.postgresql.org/),
+  [MySQL](https://www.mysql.com/) and
+  [SQLite](https://www.sqlite.org/index.html).
+
+- [neo4j/openCypher (unsupported,
+  incomplete)](https://github.com/guacsec/guac/tree/main/pkg/assembler/backends/neo4j):
+  a persistent backend based on [neo4j](https://neo4j.com/) and
+  [openCypher](https://opencypher.org/). This backend should work with any
+  database that supported openCypher queries.
+
+- [keyvalue: Redis (experimental, complete)](/pkg/assembler/kv/redis): The
+  default keyvalue backend, but using Redis as storage.
+
+- [keyvalue: TiKV (experimental, complete)](/pkg/assembler/kv/tikv): The
+  default keyvalue backend, but using [TiKV](https://tikv.org/) as storage.
 
 ## Additional References
 

diff --git a/cmd/guacgql/cmd/ent.go b/cmd/guacgql/cmd/ent.go
@@ -18,6 +18,8 @@
 package cmd
 
 import (
+	"context"
+
 	"github.com/guacsec/guac/pkg/assembler/backends"
 	entbackend "github.com/guacsec/guac/pkg/assembler/backends/ent/backend"
 )
@@ -29,7 +31,7 @@ func init() {
 	getOpts[ent] = getEnt
 }
 
-func getEnt() backends.BackendArgs {
+func getEnt(_ context.Context) backends.BackendArgs {
 	return &entbackend.BackendOptions{
 		DriverName:  flags.dbDriver,
 		Address:     flags.dbAddress,

diff --git a/cmd/guacgql/cmd/root.go b/cmd/guacgql/cmd/root.go
@@ -58,6 +58,10 @@ var flags = struct {
 	neptuneRegion   string
 	neptuneUser     string
 	neptuneRealm    string
+
+	kvStore string
+	kvRedis string
+	kvTiKV  string
 }{}
 
 var rootCmd = &cobra.Command{
@@ -93,6 +97,10 @@ var rootCmd = &cobra.Command{
 		flags.neptuneUser = viper.GetString("neptune-user")
 		flags.neptuneRealm = viper.GetString("neptune-realm")
 
+		flags.kvStore = viper.GetString("kv-store")
+		flags.kvRedis = viper.GetString("kv-redis")
+		flags.kvTiKV = viper.GetString("kv-tikv")
+
 		startServer(cmd)
 	},
 }
@@ -106,6 +114,7 @@ func init() {
 		"neptune-endpoint", "neptune-port", "neptune-region", "neptune-user", "neptune-realm",
 		"gql-listen-port", "gql-tls-cert-file", "gql-tls-key-file", "gql-debug", "gql-backend", "gql-trace",
 		"db-address", "db-driver", "db-debug", "db-migrate",
+		"kv-store", "kv-redis", "kv-tikv",
 	})
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "failed to setup flag: %v", err)

diff --git a/cmd/guacgql/cmd/server.go b/cmd/guacgql/cmd/server.go
@@ -35,6 +35,8 @@ import (
 	"github.com/guacsec/guac/pkg/assembler/backends/neptune"
 	"github.com/guacsec/guac/pkg/assembler/graphql/generated"
 	"github.com/guacsec/guac/pkg/assembler/graphql/resolvers"
+	"github.com/guacsec/guac/pkg/assembler/kv"
+	"github.com/guacsec/guac/pkg/assembler/kv/redis"
 	"github.com/guacsec/guac/pkg/logging"
 	"github.com/spf13/cobra"
 	"golang.org/x/exp/maps"
@@ -48,7 +50,7 @@ const (
 	keyvalue = "keyvalue"
 )
 
-type optsFunc func() backends.BackendArgs
+type optsFunc func(context.Context) backends.BackendArgs
 
 var getOpts map[string]optsFunc
 
@@ -131,13 +133,16 @@ func validateFlags() error {
 	if !slices.Contains(backends.List(), flags.backend) {
 		return fmt.Errorf("invalid graphql backend specified: %v", flags.backend)
 	}
+	if !slices.Contains([]string{"memmap", "redis", "tikv"}, flags.kvStore) {
+		return fmt.Errorf("invalid kv store specified: %v", flags.kvStore)
+	}
 	return nil
 }
 
 func getGraphqlServer(ctx context.Context) (*handler.Server, error) {
 	var topResolver resolvers.Resolver
 
-	backend, err := backends.Get(flags.backend, ctx, getOpts[flags.backend]())
+	backend, err := backends.Get(flags.backend, ctx, getOpts[flags.backend](ctx))
 	if err != nil {
 		return nil, fmt.Errorf("Error creating %v backend: %w", flags.backend, err)
 	}
@@ -154,15 +159,15 @@ func healthHandler(w http.ResponseWriter, r *http.Request) {
 	_, _ = fmt.Fprint(w, "Server is healthy")
 }
 
-func getArango() backends.BackendArgs {
+func getArango(_ context.Context) backends.BackendArgs {
 	return &arangodb.ArangoConfig{
 		User:   flags.arangoUser,
 		Pass:   flags.arangoPass,
 		DBAddr: flags.arangoAddr,
 	}
 }
 
-func getNeo4j() backends.BackendArgs {
+func getNeo4j(_ context.Context) backends.BackendArgs {
 	return &neo4j.Neo4jConfig{
 		User:   flags.nUser,
 		Pass:   flags.nPass,
@@ -171,11 +176,34 @@ func getNeo4j() backends.BackendArgs {
 	}
 }
 
-func getKeyValue() backends.BackendArgs {
+var tikvGS func(context.Context, string) (kv.Store, error)
+
+func getKeyValue(ctx context.Context) backends.BackendArgs {
+	logger := logging.FromContext(ctx)
+	switch flags.kvStore {
+	case "memmap":
+		// default is memmap
+		return nil
+	case "redis":
+		s, err := redis.GetStore(flags.kvRedis)
+		if err != nil {
+			logger.Fatalf("error with Redis: %v", err)
+		}
+		return s
+	case "tikv":
+		if tikvGS == nil {
+			logger.Fatal("TiKV not supported on 32-bit")
+		}
+		s, err := tikvGS(ctx, flags.kvTiKV)
+		if err != nil {
+			logger.Fatalf("error with TiKV: %v", err)
+		}
+		return s
+	}
 	return nil
 }
 
-func getNeptune() backends.BackendArgs {
+func getNeptune(_ context.Context) backends.BackendArgs {
 	return &neptune.NeptuneConfig{
 		Endpoint: flags.neptuneEndpoint,
 		Port:     flags.neptunePort,

diff --git a/cmd/guacgql/cmd/tikv.go b/cmd/guacgql/cmd/tikv.go
@@ -0,0 +1,26 @@
+//
+// Copyright 2023 The GUAC Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build !(386 || arm || mips || darwin)
+
+package cmd
+
+import "github.com/guacsec/guac/pkg/assembler/kv/tikv"
+
+func init() {
+	// TiKV does not support 32 bit. Also darwin required CGO and cross compile
+	// using xcode...
+	tikvGS = tikv.GetStore
+}
diff --git a/go.mod b/go.mod
@@ -87,12 +87,18 @@ require (
 	github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect
 	github.com/chzyer/readline v1.5.1 // indirect
 	github.com/cloudflare/circl v1.3.3 // indirect
+	github.com/cloudfoundry/gosigar v1.3.6 // indirect
 	github.com/common-nighthawk/go-figure v0.0.0-20210622060536-734e95fb86be // indirect
 	github.com/containerd/stargz-snapshotter/estargz v0.14.3 // indirect
 	github.com/containerd/typeurl/v2 v2.1.1 // indirect
+	github.com/coreos/go-semver v0.3.1 // indirect
+	github.com/coreos/go-systemd/v22 v22.5.0 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
 	github.com/cyphar/filepath-securejoin v0.2.4 // indirect
+	github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
+	github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 // indirect
+	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
 	github.com/docker/cli v24.0.4+incompatible // indirect
 	github.com/docker/distribution v2.8.2+incompatible // indirect
 	github.com/docker/docker v24.0.7+incompatible // indirect
@@ -104,7 +110,7 @@ require (
 	github.com/gin-contrib/sse v0.1.0 // indirect
 	github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
 	github.com/go-git/go-billy/v5 v5.5.0 // indirect
-	github.com/go-logr/logr v1.2.4 // indirect
+	github.com/go-logr/logr v1.3.0 // indirect
 	github.com/go-openapi/inflect v0.19.0 // indirect
 	github.com/go-openapi/jsonpointer v0.20.0 // indirect
 	github.com/go-openapi/jsonreference v0.20.2 // indirect
@@ -118,10 +124,12 @@ require (
 	github.com/goccy/go-json v0.10.2 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang-jwt/jwt/v4 v4.5.0 // indirect
+	github.com/google/btree v1.1.2 // indirect
 	github.com/google/go-containerregistry v0.16.1 // indirect
 	github.com/google/go-github/v53 v53.2.0 // indirect
 	github.com/google/go-github/v56 v56.0.0 // indirect
 	github.com/google/go-querystring v1.1.0 // indirect
+	github.com/google/pprof v0.0.0-20231101202521-4ca4178f5c7a // indirect
 	github.com/google/renameio/v2 v2.0.0 // indirect
 	github.com/google/s2a-go v0.1.7 // indirect
 	github.com/google/wire v0.5.0 // indirect
@@ -158,16 +166,24 @@ require (
 	github.com/nats-io/nkeys v0.4.6 // indirect
 	github.com/nats-io/nuid v1.0.1 // indirect
 	github.com/olekukonko/tablewriter v0.0.5 // indirect
+	github.com/onsi/ginkgo/v2 v2.13.1 // indirect
+	github.com/onsi/gomega v1.29.0 // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/opencontainers/image-spec v1.1.0-rc3 // indirect
+	github.com/opentracing/opentracing-go v1.2.0 // indirect
 	github.com/owenrumney/go-sarif/v2 v2.3.0 // indirect
 	github.com/pelletier/go-toml/v2 v2.0.8 // indirect
 	github.com/pierrec/lz4/v4 v4.1.15 // indirect
+	github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c // indirect
+	github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c // indirect
+	github.com/pingcap/kvproto v0.0.0-20230904082117-ecdbf1f8c130 // indirect
+	github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 // indirect
 	github.com/pjbgf/sha1cd v0.3.0 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 // indirect
 	github.com/prometheus/common v0.44.0 // indirect
 	github.com/prometheus/procfs v0.11.1 // indirect
+	github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect
 	github.com/rhysd/actionlint v1.6.26 // indirect
 	github.com/rivo/uniseg v0.4.4 // indirect
 	github.com/robfig/cron/v3 v3.0.1 // indirect
@@ -184,8 +200,11 @@ require (
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
 	github.com/subosito/gotenv v1.4.2 // indirect
 	github.com/theupdateframework/go-tuf v0.5.2 // indirect
+	github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a // indirect
+	github.com/tikv/pd/client v0.0.0-20231115064546-181fdc95be65 // indirect
 	github.com/titanous/rocacheck v0.0.0-20171023193734-afe73141d399 // indirect
 	github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
+	github.com/twmb/murmur3 v1.1.3 // indirect
 	github.com/ugorji/go/codec v1.2.11 // indirect
 	github.com/ulikunitz/xz v0.5.11 // indirect
 	github.com/urfave/cli/v2 v2.25.7 // indirect
@@ -197,16 +216,21 @@ require (
 	github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
 	github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
 	github.com/zclconf/go-cty v1.10.0 // indirect
+	go.etcd.io/etcd/api/v3 v3.5.10 // indirect
+	go.etcd.io/etcd/client/pkg/v3 v3.5.10 // indirect
+	go.etcd.io/etcd/client/v3 v3.5.10 // indirect
+	go.uber.org/atomic v1.11.0 // indirect
 	gocloud.dev v0.34.0 // indirect
 	golang.org/x/arch v0.3.0 // indirect
-	golang.org/x/mod v0.13.0 // indirect
+	golang.org/x/mod v0.14.0 // indirect
 	golang.org/x/term v0.14.0 // indirect
 	golang.org/x/time v0.4.0 // indirect
-	golang.org/x/tools v0.14.0 // indirect
+	golang.org/x/tools v0.15.0 // indirect
 	golang.org/x/vuln v1.0.1 // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20231016165738-49dd2c1f3d0b // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20231030173426-d783a09b4405 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
+	gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
 	gopkg.in/square/go-jose.v2 v2.6.0 // indirect
 	gopkg.in/warnings.v0 v0.1.2 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
@@ -252,6 +276,7 @@ require (
 	github.com/package-url/packageurl-go v0.1.2
 	github.com/pkg/errors v0.9.1
 	github.com/prometheus/client_golang v1.17.0
+	github.com/redis/go-redis/v9 v9.3.0
 	github.com/regclient/regclient v0.5.3
 	github.com/segmentio/kafka-go v0.4.44
 	github.com/segmentio/ksuid v1.0.4
@@ -260,6 +285,7 @@ require (
 	github.com/spf13/viper v1.16.0
 	github.com/stretchr/testify v1.8.4
 	github.com/swaggo/swag v1.16.2
+	github.com/tikv/client-go/v2 v2.0.8-0.20231115083414-7c96dfd783fb
 	github.com/vektah/gqlparser/v2 v2.5.10
 	golang.org/x/exp v0.0.0-20231006140011-7918f672742d
 	gopkg.in/yaml.v3 v3.0.1