feat: add vector support (#86)

.chainsaw.yaml

@@ -6,12 +6,12 @@ spec:
   # namespace: chainsaw
   timeouts:
     apply: 300s
-    assert: 300s
+    assert: 600s
     cleanup: 120s
     delete: 120s
     error: 180s
     exec: 300s
-  # skipDelete: false
+  # skipDelete: true
   failFast: true
   forceTerminationGracePeriod: 10s
   parallel: 1

api/v1alpha1/master_types.go

@@ -43,7 +43,7 @@ type MasterConfigSpec struct {
 	GracefulShutdownTimeout *string `json:"gracefulShutdownTimeout,omitempty"`
 
 	// +kubebuilder:validation:Optional
-	Logging *commonsv1alpha1.LoggingConfigSpec `json:"logging,omitempty"`
+	Logging *LoggingSpec `json:"logging,omitempty"`
 
 	// +kubebuilder:validation:Optional
 	Resources *commonsv1alpha1.ResourcesSpec `json:"resources,omitempty"`

api/v1alpha1/productlogging.go

@@ -0,0 +1,10 @@
+package v1alpha1
+
+import commonsv1alpha1 "github.com/zncdatadev/operator-go/pkg/apis/commons/v1alpha1"
+
+type LoggingSpec struct {
+	// +kubebuilder:validation:Optional
+	EnableVectorAgent bool `json:"enableVectorAgent,omitempty"`
+	// +kubebuilder:validation:Optional
+	Containers map[string]*commonsv1alpha1.LoggingConfigSpec `json:"containers,omitempty"`
+}

api/v1alpha1/regionserver_types.go

@@ -42,7 +42,7 @@ type RegionConfigSpec struct {
 	GracefulShutdownTimeout *string `json:"gracefulShutdownTimeout,omitempty"`
 
 	// +kubebuilder:validation:Optional
-	Logging *commonsv1alpha1.LoggingConfigSpec `json:"logging,omitempty"`
+	Logging *LoggingSpec `json:"logging,omitempty"`
 
 	// +kubebuilder:validation:Optional
 	Resources *commonsv1alpha1.ResourcesSpec `json:"resources,omitempty"`

api/v1alpha1/restserver_types.go

@@ -43,7 +43,7 @@ type RestServerConfigSpec struct {
 	GracefulShutdownTimeout *string `json:"gracefulShutdownTimeout,omitempty"`
 
 	// +kubebuilder:validation:Optional
-	Logging *commonsv1alpha1.LoggingConfigSpec `json:"logging,omitempty"`
+	Logging *LoggingSpec `json:"logging,omitempty"`
 
 	// +kubebuilder:validation:Optional
 	Resources *commonsv1alpha1.ResourcesSpec `json:"resources,omitempty"`

internal/controller/common/configmap.go

@@ -3,15 +3,16 @@ package common
 import (
 	"context"
 	"fmt"
-	"path"
 	"strings"
 
 	corev1 "k8s.io/api/core/v1"
 	ctrl "sigs.k8s.io/controller-runtime"
 
+	commonsv1alpha1 "github.com/zncdatadev/operator-go/pkg/apis/commons/v1alpha1"
 	"github.com/zncdatadev/operator-go/pkg/builder"
 	"github.com/zncdatadev/operator-go/pkg/client"
 	"github.com/zncdatadev/operator-go/pkg/config/xml"
+	"github.com/zncdatadev/operator-go/pkg/productlogging"
 	"github.com/zncdatadev/operator-go/pkg/reconciler"
 	"github.com/zncdatadev/operator-go/pkg/util"
 
@@ -35,6 +36,9 @@ var _ builder.ConfigBuilder = &ConfigMapBuilder{}
 type ConfigMapBuilder struct {
 	builder.ConfigMapBuilder
 
+	LoggingConfig       *commonsv1alpha1.LoggingConfigSpec
+	VectorConfigMapName string
+
 	ClusterName   string
 	RoleName      string
 	RolegroupName string
@@ -45,6 +49,8 @@ type ConfigMapBuilder struct {
 func NewConfigMapBuilder(
 	client *client.Client,
 	name string,
+	loggingConfig *commonsv1alpha1.LoggingConfigSpec,
+	vectorConfigMapName string,
 	krb5SecretClass string,
 	tlsSecretClass string,
 	options builder.Options,
@@ -69,10 +75,12 @@ func NewConfigMapBuilder(
 			options.Labels,
 			options.Annotations,
 		),
-		ClusterName:   options.ClusterName,
-		RoleName:      options.RoleName,
-		RolegroupName: options.RoleGroupName,
-		krb5Config:    krb5Config,
+		ClusterName:         options.ClusterName,
+		RoleName:            options.RoleName,
+		RolegroupName:       options.RoleGroupName,
+		krb5Config:          krb5Config,
+		LoggingConfig:       loggingConfig,
+		VectorConfigMapName: vectorConfigMapName,
 	}
 }
 
@@ -103,26 +111,17 @@ func (b *ConfigMapBuilder) getJVMOPTS() string {
 	return fmt.Sprintf(`export HBASE_%s_OPTS="$HBASE_OPTS %s"`, b.RoleName, strings.Join(opts, " "))
 }
 
-func (b *ConfigMapBuilder) AddLog4jProperties() error {
-	log4j := `
-log4j.rootLogger=INFO, CONSOLE, FILE
-
-log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
-log4j.appender.CONSOLE.Threshold=INFO
-log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
-log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2}: %.1000m%n
-
-log4j.appender.FILE=org.apache.log4j.RollingFileAppender
-log4j.appender.FILE.Threshold=INFO
-log4j.appender.FILE.File=` + path.Join(HBaseConfigDir, "hbase.log4j.xml") + `
-log4j.appender.FILE.MaxFileSize=5MB
-log4j.appender.FILE.MaxBackupIndex=1
-log4j.appender.FILE.layout=org.apache.log4j.xml.XMLLayout
-
-`
-	b.AddData(map[string]string{LogKey: util.IndentTab4Spaces(log4j)})
-	logger.V(15).Info("Log4j", "log4j", log4j, "clusterName", b.ClusterName, "roleName", b.RoleName, "roleGroupName", b.RolegroupName)
-	return nil
+func (b *ConfigMapBuilder) AddLog4jProperties() {
+	l := productlogging.NewLog4jConfigGenerator(
+		b.LoggingConfig,
+		b.RoleName,
+		"%d{ISO8601} %-5p [%t] %c{2}: %.1000m%n",
+		nil,
+		"hbase.log4j.xml",
+		"",
+	)
+	s := l.Generate()
+	b.AddItem(LogKey, s)
 }
 
 func (b *ConfigMapBuilder) AddSSLClientXML() error {
@@ -191,6 +190,27 @@ func (b *ConfigMapBuilder) AddHbaseSite(znode *ZnodeConfiguration) error {
 	return nil
 }
 
+func (b *ConfigMapBuilder) AddVectorConfig(ctx context.Context) error {
+	if b.VectorConfigMapName == "" {
+		return nil
+	}
+	s, err := productlogging.MakeVectorYaml(
+		ctx,
+		b.Client.Client,
+		b.Client.GetOwnerNamespace(),
+		b.ClusterName,
+		b.RoleName,
+		b.RolegroupName,
+		b.VectorConfigMapName,
+	)
+	if err != nil {
+		return err
+	}
+
+	b.AddItem(builder.VectorConfigFile, s)
+	return nil
+}
+
 var _ reconciler.ResourceReconciler[*ConfigMapBuilder] = &ConfigMapReconciler[reconciler.AnySpec]{}
 
 type ConfigMapReconciler[T reconciler.AnySpec] struct {
@@ -224,9 +244,7 @@ func (r *ConfigMapReconciler[T]) Reconcile(ctx context.Context) (ctrl.Result, er
 		return ctrl.Result{}, err
 	}
 
-	if err := builder.AddLog4jProperties(); err != nil {
-		return ctrl.Result{}, err
-	}
+	builder.AddLog4jProperties()
 
 	if err := builder.AddSSLClientXML(); err != nil {
 		return ctrl.Result{}, err
@@ -236,6 +254,10 @@ func (r *ConfigMapReconciler[T]) Reconcile(ctx context.Context) (ctrl.Result, er
 		return ctrl.Result{}, err
 	}
 
+	if err := builder.AddVectorConfig(ctx); err != nil {
+		return ctrl.Result{}, err
+	}
+
 	resource, err := builder.Build(ctx)
 
 	if err != nil {
@@ -248,6 +270,7 @@ func (r *ConfigMapReconciler[T]) Reconcile(ctx context.Context) (ctrl.Result, er
 func NewConfigMapReconciler[T reconciler.AnySpec](
 	client *client.Client,
 	clusterConfig *hbasev1alph1.ClusterConfigSpec,
+	loggingConfig *commonsv1alpha1.LoggingConfigSpec,
 	options reconciler.RoleGroupInfo,
 	spec T,
 ) *ConfigMapReconciler[T] {
@@ -261,6 +284,8 @@ func NewConfigMapReconciler[T reconciler.AnySpec](
 	cmBuilder := NewConfigMapBuilder(
 		client,
 		options.GetFullName(),
+		loggingConfig,
+		clusterConfig.VectorAggregatorConfigMapName,
 		krb5SecretClass,
 		tlsSecretClass,
 		builder.Options{

internal/controller/common/statefulset.go

@@ -29,6 +29,9 @@ var (
 	HDFSConfigDir       = path.Join(constants.KubedoopConfigDir)
 	HBaseMountConfigDir = path.Join(constants.KubedoopConfigDirMount, "hbase")
 	HDFSMountConfigDir  = path.Join(constants.KubedoopConfigDirMount, "hdfs")
+
+	HHbaseConfigVolumeName = "hbase-config"
+	HbaseLogVolumeName     = "log"
 )
 
 var _ builder.StatefulSetBuilder = &StatefulSetBuilder{}
@@ -343,7 +346,25 @@ func (b *StatefulSetBuilder) Build(ctx context.Context) (ctrlclient.Object, erro
 		b.AddContainer(oidcContainer)
 	}
 
-	return b.GetObject()
+	obj, err := b.GetObject()
+	if err != nil {
+		return nil, err
+	}
+
+	if b.ClusterConfig.VectorAggregatorConfigMapName != "" {
+		d := builder.NewVectorDecorator(
+			obj,
+			b.GetImage(),
+			HbaseLogVolumeName,
+			HHbaseConfigVolumeName,
+			b.ClusterConfig.VectorAggregatorConfigMapName,
+		)
+		if err := d.Decorate(); err != nil {
+			return nil, err
+		}
+	}
+
+	return obj, nil
 }
 
 var _ reconciler.Reconciler = &StatefulSetReconciler{}

internal/controller/master/role.go

@@ -4,13 +4,15 @@ import (
 	"context"
 	"time"
 
-	hbasev1alph1 "github.com/zncdatadev/hbase-operator/api/v1alpha1"
-	"github.com/zncdatadev/hbase-operator/internal/controller/common"
+	commonsv1alpha1 "github.com/zncdatadev/operator-go/pkg/apis/commons/v1alpha1"
 	"github.com/zncdatadev/operator-go/pkg/builder"
 	"github.com/zncdatadev/operator-go/pkg/client"
 	"github.com/zncdatadev/operator-go/pkg/reconciler"
 	"github.com/zncdatadev/operator-go/pkg/util"
 	ctrl "sigs.k8s.io/controller-runtime"
+
+	hbasev1alph1 "github.com/zncdatadev/hbase-operator/api/v1alpha1"
+	"github.com/zncdatadev/hbase-operator/internal/controller/common"
 )
 
 var (
@@ -72,6 +74,11 @@ func (r *Reconciler) RegisterResources(ctx context.Context) error {
 func (r *Reconciler) RegisterResourceWithRoleGroup(_ context.Context, info reconciler.RoleGroupInfo, roleGroupSpec any) ([]reconciler.Reconciler, error) {
 	spec := roleGroupSpec.(*hbasev1alph1.MasterRoleGroupSpec)
 	var reconcilers []reconciler.Reconciler
+	var loggingConfig *commonsv1alpha1.LoggingConfigSpec
+
+	if spec.Config != nil && spec.Config.Logging != nil {
+		loggingConfig = spec.Config.Logging.Containers[info.RoleName]
+	}
 
 	options := builder.WorkloadOptions{
 		Options: builder.Options{
@@ -137,6 +144,7 @@ func (r *Reconciler) RegisterResourceWithRoleGroup(_ context.Context, info recon
 	configMapReconciler := common.NewConfigMapReconciler(
 		r.GetClient(),
 		r.ClusterConfig,
+		loggingConfig,
 		info,
 		roleGroupSpec.(*hbasev1alph1.MasterRoleGroupSpec),
 	)

internal/controller/regionserver/role.go

@@ -4,13 +4,15 @@ import (
 	"context"
 	"time"
 
-	hbasev1alph1 "github.com/zncdatadev/hbase-operator/api/v1alpha1"
-	"github.com/zncdatadev/hbase-operator/internal/controller/common"
+	commonsv1alpha1 "github.com/zncdatadev/operator-go/pkg/apis/commons/v1alpha1"
 	"github.com/zncdatadev/operator-go/pkg/builder"
 	"github.com/zncdatadev/operator-go/pkg/client"
 	"github.com/zncdatadev/operator-go/pkg/reconciler"
 	"github.com/zncdatadev/operator-go/pkg/util"
 	ctrl "sigs.k8s.io/controller-runtime"
+
+	hbasev1alph1 "github.com/zncdatadev/hbase-operator/api/v1alpha1"
+	"github.com/zncdatadev/hbase-operator/internal/controller/common"
 )
 
 var (
@@ -73,6 +75,11 @@ func (r *Reconciler) RegisterResources(ctx context.Context) error {
 func (r *Reconciler) RegisterResourceWithRoleGroup(_ context.Context, info reconciler.RoleGroupInfo, roleGroupSpec any) ([]reconciler.Reconciler, error) {
 	spec := roleGroupSpec.(*hbasev1alph1.RegionServerRoleGroupSpec)
 	var reconcilers []reconciler.Reconciler
+	var loggingConfig *commonsv1alpha1.LoggingConfigSpec
+
+	if spec.Config != nil && spec.Config.Logging != nil {
+		loggingConfig = spec.Config.Logging.Containers[info.RoleName]
+	}
 
 	options := builder.WorkloadOptions{
 		Options: builder.Options{
@@ -138,6 +145,7 @@ func (r *Reconciler) RegisterResourceWithRoleGroup(_ context.Context, info recon
 	configMapReconciler := common.NewConfigMapReconciler(
 		r.GetClient(),
 		r.ClusterConfig,
+		loggingConfig,
 		info,
 		roleGroupSpec.(*hbasev1alph1.RegionServerRoleGroupSpec),
 	)