Merge pull request #4 from justinsb/wip_metal

Support static configuration

cmd/etcd-manager/BUILD.bazel

@@ -19,6 +19,7 @@ go_library(
         "//pkg/privateapi",
         "//pkg/privateapi/discovery",
         "//pkg/privateapi/discovery/vfs",
+        "//pkg/static",
         "//pkg/tlsconfig",
         "//pkg/urls",
         "//pkg/volumes",

cmd/etcd-manager/main.go

@@ -42,6 +42,7 @@ import (
 	"sigs.k8s.io/etcd-manager/pkg/privateapi"
 	"sigs.k8s.io/etcd-manager/pkg/privateapi/discovery"
 	vfsdiscovery "sigs.k8s.io/etcd-manager/pkg/privateapi/discovery/vfs"
+	"sigs.k8s.io/etcd-manager/pkg/static"
 	"sigs.k8s.io/etcd-manager/pkg/tlsconfig"
 	"sigs.k8s.io/etcd-manager/pkg/urls"
 	"sigs.k8s.io/etcd-manager/pkg/volumes"
@@ -87,6 +88,7 @@ func main() {
 	flag.StringVar(&o.BackupInterval, "backup-interval", o.BackupInterval, "interval for periodic backups")
 	flag.StringVar(&o.DiscoveryPollInterval, "discovery-poll-interval", o.DiscoveryPollInterval, "interval for discovery poll")
 	flag.StringVar(&o.DataDir, "data-dir", o.DataDir, "directory for storing etcd data")
+	flag.StringVar(&o.StaticConfig, "static-config", o.StaticConfig, "options for static cluster config")
 
 	flag.StringVar(&o.PKIDir, "pki-dir", o.PKIDir, "directory for PKI keys")
 	flag.BoolVar(&o.Insecure, "insecure", o.Insecure, "allow use of non-secure connections for etcd-manager")
@@ -146,6 +148,10 @@ type EtcdManagerOptions struct {
 	BackupInterval        string
 	DiscoveryPollInterval string
 
+	// StaticConfig can be provided to run with a static cluster configuration.
+	// Reconfiguration requires restarting etcd-manager externally.
+	StaticConfig string
+
 	// DNSSuffix is added to etcd member names and we then use internal client id discovery
 	DNSSuffix string
 
@@ -207,6 +213,15 @@ func RunEtcdManager(o *EtcdManagerOptions) error {
 		return fmt.Errorf("backup-store is required")
 	}
 
+	var staticConfig *static.Config
+	if o.StaticConfig != "" {
+		parsed, err := static.ParseStaticConfig(o.StaticConfig)
+		if err != nil {
+			return fmt.Errorf("invalid static-config: %w", err)
+		}
+		staticConfig = parsed
+	}
+
 	var networkCIDR *net.IPNet
 	if o.NetworkCIDR != "" {
 		if o.VolumeProviderID != "openstack" {
@@ -320,9 +335,13 @@ func RunEtcdManager(o *EtcdManagerOptions) error {
 			}
 			volumeProvider = externalVolumeProvider
 
-			// TODO: Allow this to be customized
-			seedDir := volumes.PathFor("/etc/kubernetes/etcd-manager/seeds")
-			discoveryProvider = external.NewExternalDiscovery(seedDir, externalVolumeProvider)
+			if staticConfig == nil {
+				// TODO: Allow this to be customized
+				seedDir := volumes.PathFor("/etc/kubernetes/etcd-manager/seeds")
+				discoveryProvider = external.NewExternalDiscovery(seedDir, externalVolumeProvider)
+			} else {
+				discoveryProvider = static.NewStaticDiscovery(staticConfig)
+			}
 
 		default:
 			fmt.Fprintf(os.Stderr, "unknown volume-provider %q\n", o.VolumeProviderID)
@@ -484,11 +503,16 @@ func RunEtcdManager(o *EtcdManagerOptions) error {
 		return fmt.Errorf("error initializing backup store: %v", err)
 	}
 
-	commandStore, err := commands.NewStore(o.BackupStorePath)
-	if err != nil {
-		klog.Fatalf("error initializing command store: %v", err)
+	var commandStore commands.Store
+	if staticConfig == nil {
+		store, err := commands.NewStore(o.BackupStorePath)
+		if err != nil {
+			klog.Fatalf("error initializing command store: %v", err)
+		}
+		commandStore = store
+	} else {
+		commandStore = static.NewStaticCommandStore(staticConfig, o.DataDir)
 	}
-
 	if _, err := legacy.ScanForExisting(o.DataDir, commandStore); err != nil {
 		klog.Fatalf("error performing scan for legacy data: %v", err)
 	}

pkg/static/BUILD.bazel

@@ -0,0 +1,18 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "static",
+    srcs = [
+        "commandstore.go",
+        "config.go",
+        "discovery.go",
+    ],
+    importpath = "sigs.k8s.io/etcd-manager/pkg/static",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//pkg/apis/etcd",
+        "//pkg/commands",
+        "//pkg/privateapi/discovery",
+        "//vendor/k8s.io/klog/v2:klog",
+    ],
+)

pkg/static/commandstore.go

@@ -0,0 +1,96 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package static
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"k8s.io/klog/v2"
+	protoetcd "sigs.k8s.io/etcd-manager/pkg/apis/etcd"
+	"sigs.k8s.io/etcd-manager/pkg/commands"
+)
+
+// newClusterMarkerFile is the name of the marker file for a new-cluster.
+// To avoid deleting data, we will only create a new cluster if this file exists.
+// After the cluster is created, we will remove this file.
+const newClusterMarkerFile = "please-create-new-cluster"
+
+func NewStaticCommandStore(config *Config, dataDir string) *StaticStore {
+	return &StaticStore{
+		config:  config,
+		dataDir: dataDir,
+	}
+}
+
+type StaticStore struct {
+	config *Config
+
+	// dataDir is the location of our data files, it is used for IsNewCluster
+	dataDir string
+}
+
+var _ commands.Store = &StaticStore{}
+
+func (s *StaticStore) AddCommand(cmd *protoetcd.Command) error {
+	return fmt.Errorf("StaticStore::AddCommand not supported")
+}
+
+func (s *StaticStore) ListCommands() ([]commands.Command, error) {
+	klog.Infof("StaticStore::ListCommands returning empty list")
+	return nil, nil
+}
+
+func (s *StaticStore) RemoveCommand(command commands.Command) error {
+	return fmt.Errorf("StaticStore::RemoveCommand not supported")
+}
+
+func (s *StaticStore) GetExpectedClusterSpec() (*protoetcd.ClusterSpec, error) {
+	spec := &protoetcd.ClusterSpec{
+		MemberCount: int32(len(s.config.Nodes)),
+		EtcdVersion: s.config.EtcdVersion,
+	}
+	return spec, nil
+}
+
+func (s *StaticStore) SetExpectedClusterSpec(spec *protoetcd.ClusterSpec) error {
+	klog.Warningf("ignoring SetExpectedClusterSpec %v", spec)
+	return nil
+}
+
+func (s *StaticStore) IsNewCluster() (bool, error) {
+	markerPath := filepath.Join(s.dataDir, newClusterMarkerFile)
+	_, err := os.Stat(markerPath)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return false, nil
+		}
+		return false, fmt.Errorf("checking for file %q: %w", markerPath, err)
+	}
+
+	// File exists so we can create a new cluster
+	return true, nil
+}
+
+func (s *StaticStore) MarkClusterCreated() error {
+	markerPath := filepath.Join(s.dataDir, newClusterMarkerFile)
+	if err := os.Remove(markerPath); err != nil {
+		return fmt.Errorf("deleting marker file %q: %w", markerPath, err)
+	}
+	return nil
+}

pkg/static/config.go

@@ -0,0 +1,40 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package static
+
+import (
+	"encoding/json"
+	"fmt"
+)
+
+type Config struct {
+	EtcdVersion string       `json:"etcdVersion"`
+	Nodes       []ConfigNode `json:"nodes"`
+}
+
+type ConfigNode struct {
+	ID string   `json:"id"`
+	IP []string `json:"ip"`
+}
+
+func ParseStaticConfig(config string) (*Config, error) {
+	c := &Config{}
+	if err := json.Unmarshal([]byte(config), c); err != nil {
+		return nil, fmt.Errorf("parsing config %q: %w", config, err)
+	}
+	return c, nil
+}

pkg/static/discovery.go

@@ -0,0 +1,54 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package static
+
+import (
+	"k8s.io/klog/v2"
+	"sigs.k8s.io/etcd-manager/pkg/privateapi/discovery"
+)
+
+// StaticDiscovery implements discovery.Interface using a fixed configuration
+type StaticDiscovery struct {
+	config *Config
+}
+
+var _ discovery.Interface = &StaticDiscovery{}
+
+func NewStaticDiscovery(config *Config) *StaticDiscovery {
+	d := &StaticDiscovery{
+		config: config,
+	}
+	return d
+}
+
+func (d *StaticDiscovery) Poll() (map[string]discovery.Node, error) {
+	nodes := make(map[string]discovery.Node)
+
+	for _, configNode := range d.config.Nodes {
+		node := discovery.Node{}
+		node.ID = configNode.ID
+		for _, ip := range configNode.IP {
+			node.Endpoints = append(node.Endpoints, discovery.NodeEndpoint{
+				IP: ip,
+			})
+		}
+		nodes[node.ID] = node
+	}
+
+	klog.Infof("static discovery poll => %+v", nodes)
+	return nodes, nil
+}

pkg/volumes/external/volumes.go

@@ -50,7 +50,7 @@ func NewExternalVolumes(clusterName string, basedir string, volumeTags []string)
 	}
 
 	if len(volumeTags) != 1 {
-		return nil, fmt.Errorf("baremetal expected a single volume tag (the prefix to use)")
+		return nil, fmt.Errorf("baremetal expected a single volume tag (the prefix to use), got %v", volumeTags)
 	}
 
 	a := &ExternalVolumes{
@@ -138,7 +138,7 @@ func (a *ExternalVolumes) FindVolumes() ([]*volumes.Volume, error) {
 
 		match := true
 		for _, tag := range a.volumeTags {
-			if strings.HasPrefix(f.Name(), tag) {
+			if !strings.HasPrefix(f.Name(), tag) {
 				match = false
 			}
 		}
@@ -150,6 +150,7 @@ func (a *ExternalVolumes) FindVolumes() ([]*volumes.Volume, error) {
 
 		p := filepath.Join(a.basedir, f.Name())
 
+		// TODO: Why the extra mnt?  Is this because we also have some metadata?
 		mntPath := filepath.Join(p, "mnt")
 		stat, err := os.Stat(mntPath)
 		if err != nil {