From a63468ca74fc008400b1b0e6599643f0ada92c21 Mon Sep 17 00:00:00 2001 From: CeerDecy <1748788674@qq.com> Date: Fri, 6 Dec 2024 15:27:46 +0800 Subject: [PATCH 1/8] feat: runtime run as no-root user --- .../plugins/k8s/instanceinfosync/pod.go | 1 - .../scheduler/executor/plugins/k8s/k8s.go | 22 +++++++ .../executor/plugins/k8s/types/type.go | 5 ++ .../scheduler/executor/util/util.go | 66 +++++++++++++++++++ 4 files changed, 93 insertions(+), 1 deletion(-) diff --git a/internal/tools/orchestrator/scheduler/executor/plugins/k8s/instanceinfosync/pod.go b/internal/tools/orchestrator/scheduler/executor/plugins/k8s/instanceinfosync/pod.go index 53436ba43d6..dde28213991 100644 --- a/internal/tools/orchestrator/scheduler/executor/plugins/k8s/instanceinfosync/pod.go +++ b/internal/tools/orchestrator/scheduler/executor/plugins/k8s/instanceinfosync/pod.go @@ -498,7 +498,6 @@ func updatePodAndInstance(dbclient *instanceinfo.Client, podlist *corev1.PodList } } } else { - logrus.Infof("get [currentContainerID] from mainContainer") currentTerminatedContainer := mainContainer.State.Terminated if currentTerminatedContainer != nil { if len(strings.Split(mainContainer.ContainerID, "://")) == 2 { diff --git a/internal/tools/orchestrator/scheduler/executor/plugins/k8s/k8s.go b/internal/tools/orchestrator/scheduler/executor/plugins/k8s/k8s.go index 6ed74201327..52f0b473c64 100644 --- a/internal/tools/orchestrator/scheduler/executor/plugins/k8s/k8s.go +++ b/internal/tools/orchestrator/scheduler/executor/plugins/k8s/k8s.go @@ -789,6 +789,11 @@ func (k *Kubernetes) createOne(ctx context.Context, service *apistructs.Service, } } var err error + + if err = k.runAsDefaultUser(service); err != nil { + return err + } + switch service.WorkLoad { case types.ServicePerNode: err = k.createDaemonSet(ctx, service, sg) @@ -903,6 +908,12 @@ func (k *Kubernetes) updateOneByOne(ctx context.Context, sg *apistructs.ServiceG } for _, svc := range sg.Services { + svc := svc + + if err := k.runAsDefaultUser(&svc); err != nil { + return err + } + svc.Namespace = ns runtimeServiceName := util.GetDeployName(&svc) // Existing in the old service collection, do the put operation @@ -2058,3 +2069,14 @@ func (k *Kubernetes) DeployInEdgeCluster() bool { return true } + +// run as default user +func (k *Kubernetes) runAsDefaultUser(service *apistructs.Service) error { + values := map[string]any{ + "RunAsUser": &types.DefaultContainerUserId, + "RunAsGroup": &types.DefaultContainerGroupId, + } + util.MergeStructValue(service, values, "K8SSnippet", "Container", "SecurityContext") + logrus.Infof("after merge ======> %v", service.K8SSnippet.Container) + return nil +} diff --git a/internal/tools/orchestrator/scheduler/executor/plugins/k8s/types/type.go b/internal/tools/orchestrator/scheduler/executor/plugins/k8s/types/type.go index 668afac40ad..a880114276c 100644 --- a/internal/tools/orchestrator/scheduler/executor/plugins/k8s/types/type.go +++ b/internal/tools/orchestrator/scheduler/executor/plugins/k8s/types/type.go @@ -42,6 +42,11 @@ const ( DiceWorkSpace = "DICE_WORKSPACE" ) +var ( + DefaultContainerUserId int64 = 1000 // `dice` user + DefaultContainerGroupId int64 = 1000 // `dice` group +) + var EnvReg = regexp.MustCompile(`\$\{([^}]+?)\}`) type StatefulsetInfo struct { diff --git a/internal/tools/orchestrator/scheduler/executor/util/util.go b/internal/tools/orchestrator/scheduler/executor/util/util.go index 68fa5e1c249..ba250261263 100644 --- a/internal/tools/orchestrator/scheduler/executor/util/util.go +++ b/internal/tools/orchestrator/scheduler/executor/util/util.go @@ -17,6 +17,8 @@ package util import ( "encoding/base64" "fmt" + "reflect" + "slices" "sort" "strconv" "strings" @@ -369,3 +371,67 @@ func GetDeployName(service *apistructs.Service) string { } return service.Name } + +// MergeStructValue +// values is the data to be merged, format { field-name: field-value } +// needMerges is all the name of the `field` in every struct to be merged +func MergeStructValue(target any, values map[string]any, needMerges ...string) { + typ := reflect.TypeOf(target) + if typ.Kind() != reflect.Ptr { + return + } + typ = typ.Elem() + + if IsBaseType(typ) { + return + } + + val := reflect.ValueOf(target) + if val.Kind() != reflect.Ptr { + return + } + val = val.Elem() + for i := 0; i < typ.NumField(); i++ { + if typ.Field(i).Type.Kind() == reflect.Ptr && val.Field(i).IsNil() { + value, ok := values[typ.Field(i).Name] + if ok { + val.Field(i).Set(reflect.ValueOf(value)) + continue + } + + newType := val.Field(i).Type() + if newType.Kind() != reflect.Ptr { + continue + } + newType = newType.Elem() + if slices.Index(needMerges, typ.Field(i).Name) < 0 { + continue + } + val.Field(i).Set(reflect.New(newType)) + } + MergeStructValue(val.Field(i).Interface(), values, needMerges...) + } +} + +func IsBaseType(typ reflect.Type) bool { + switch typ.Kind() { + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return true + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + return true + case reflect.Float32, reflect.Float64: + return true + case reflect.Bool: + return true + case reflect.String: + return true + case reflect.Slice, reflect.Array: + return true + case reflect.Map: + return true + case reflect.Chan: + return true + default: + return false + } +} From 3a40eb91de0fdb99e86096b0cb066f6b39a03eb6 Mon Sep 17 00:00:00 2001 From: CeerDecy <1748788674@qq.com> Date: Fri, 6 Dec 2024 15:31:10 +0800 Subject: [PATCH 2/8] polish: remove useless code --- .../tools/orchestrator/scheduler/executor/plugins/k8s/k8s.go | 1 - 1 file changed, 1 deletion(-) diff --git a/internal/tools/orchestrator/scheduler/executor/plugins/k8s/k8s.go b/internal/tools/orchestrator/scheduler/executor/plugins/k8s/k8s.go index 52f0b473c64..cfcb8f721a4 100644 --- a/internal/tools/orchestrator/scheduler/executor/plugins/k8s/k8s.go +++ b/internal/tools/orchestrator/scheduler/executor/plugins/k8s/k8s.go @@ -2077,6 +2077,5 @@ func (k *Kubernetes) runAsDefaultUser(service *apistructs.Service) error { "RunAsGroup": &types.DefaultContainerGroupId, } util.MergeStructValue(service, values, "K8SSnippet", "Container", "SecurityContext") - logrus.Infof("after merge ======> %v", service.K8SSnippet.Container) return nil } From 5ca82c80ba85bd6cdec2a5e775b9e50d2e543385 Mon Sep 17 00:00:00 2001 From: CeerDecy <1748788674@qq.com> Date: Fri, 6 Dec 2024 15:40:04 +0800 Subject: [PATCH 3/8] feat: test coverage --- .../scheduler/executor/util/util_test.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/internal/tools/orchestrator/scheduler/executor/util/util_test.go b/internal/tools/orchestrator/scheduler/executor/util/util_test.go index a1ba82a868d..867755a8667 100644 --- a/internal/tools/orchestrator/scheduler/executor/util/util_test.go +++ b/internal/tools/orchestrator/scheduler/executor/util/util_test.go @@ -15,7 +15,9 @@ package util import ( + "encoding/json" "fmt" + "github.com/erda-project/erda/pkg/parser/diceyml" "strings" "testing" @@ -209,3 +211,14 @@ func Test_ParseAnnotationFromEnv(t *testing.T) { }) } } + +func TestMerge(t *testing.T) { + snippet := diceyml.K8SSnippet{} + values := map[string]any{ + "RunAsUser": new(int64), + "RunAsGroup": new(int64), + } + MergeStructValue(&snippet, values, "K8SSnippet", "Container", "SecurityContext") + marshal, _ := json.Marshal(snippet) + fmt.Println(string(marshal)) +} From 9489a67cc772bf710b98cea969962750a6eb4d36 Mon Sep 17 00:00:00 2001 From: CeerDecy <1748788674@qq.com> Date: Fri, 6 Dec 2024 15:43:02 +0800 Subject: [PATCH 4/8] polish: sort imports --- .../tools/orchestrator/scheduler/executor/util/util_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/tools/orchestrator/scheduler/executor/util/util_test.go b/internal/tools/orchestrator/scheduler/executor/util/util_test.go index 867755a8667..f13e2f4f615 100644 --- a/internal/tools/orchestrator/scheduler/executor/util/util_test.go +++ b/internal/tools/orchestrator/scheduler/executor/util/util_test.go @@ -17,13 +17,13 @@ package util import ( "encoding/json" "fmt" - "github.com/erda-project/erda/pkg/parser/diceyml" "strings" "testing" - "github.com/stretchr/testify/assert" - "github.com/erda-project/erda/apistructs" + "github.com/erda-project/erda/pkg/parser/diceyml" + + "github.com/stretchr/testify/assert" ) func TestParsePreserveProjects(t *testing.T) { From 464712a4ad5eedbce80f25f3f5b2a6b4c06f01a3 Mon Sep 17 00:00:00 2001 From: CeerDecy <1748788674@qq.com> Date: Fri, 6 Dec 2024 15:44:30 +0800 Subject: [PATCH 5/8] feat: test coverage --- .../tools/orchestrator/scheduler/executor/util/util_test.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/internal/tools/orchestrator/scheduler/executor/util/util_test.go b/internal/tools/orchestrator/scheduler/executor/util/util_test.go index f13e2f4f615..df6e72c12ea 100644 --- a/internal/tools/orchestrator/scheduler/executor/util/util_test.go +++ b/internal/tools/orchestrator/scheduler/executor/util/util_test.go @@ -212,7 +212,8 @@ func Test_ParseAnnotationFromEnv(t *testing.T) { } } -func TestMerge(t *testing.T) { +func TestMergeStructValue(t *testing.T) { + want := `{"container":{"securityContext":{"runAsUser":0,"runAsGroup":0}}}` snippet := diceyml.K8SSnippet{} values := map[string]any{ "RunAsUser": new(int64), @@ -220,5 +221,5 @@ func TestMerge(t *testing.T) { } MergeStructValue(&snippet, values, "K8SSnippet", "Container", "SecurityContext") marshal, _ := json.Marshal(snippet) - fmt.Println(string(marshal)) + assert.Equal(t, want, string(marshal)) } From 245963563306d17b49b7f8eb43ef31e1e66d68b4 Mon Sep 17 00:00:00 2001 From: CeerDecy <1748788674@qq.com> Date: Fri, 6 Dec 2024 16:07:00 +0800 Subject: [PATCH 6/8] feat: sort imports --- .../tools/orchestrator/scheduler/executor/util/util_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/tools/orchestrator/scheduler/executor/util/util_test.go b/internal/tools/orchestrator/scheduler/executor/util/util_test.go index df6e72c12ea..0a8ecd1a9c7 100644 --- a/internal/tools/orchestrator/scheduler/executor/util/util_test.go +++ b/internal/tools/orchestrator/scheduler/executor/util/util_test.go @@ -20,10 +20,10 @@ import ( "strings" "testing" + "github.com/stretchr/testify/assert" + "github.com/erda-project/erda/apistructs" "github.com/erda-project/erda/pkg/parser/diceyml" - - "github.com/stretchr/testify/assert" ) func TestParsePreserveProjects(t *testing.T) { From a04d2c07079e070a87a51f17783dde9aba993531 Mon Sep 17 00:00:00 2001 From: CeerDecy <1748788674@qq.com> Date: Mon, 9 Dec 2024 14:00:39 +0800 Subject: [PATCH 7/8] feat: update merge struct --- .../scheduler/executor/plugins/k8s/k8s.go | 22 +++-- .../scheduler/executor/util/util.go | 83 +++++++++++++------ .../scheduler/executor/util/util_test.go | 36 ++++++-- 3 files changed, 102 insertions(+), 39 deletions(-) diff --git a/internal/tools/orchestrator/scheduler/executor/plugins/k8s/k8s.go b/internal/tools/orchestrator/scheduler/executor/plugins/k8s/k8s.go index cfcb8f721a4..03d1a037c67 100644 --- a/internal/tools/orchestrator/scheduler/executor/plugins/k8s/k8s.go +++ b/internal/tools/orchestrator/scheduler/executor/plugins/k8s/k8s.go @@ -19,6 +19,7 @@ import ( "context" "encoding/json" "fmt" + "github.com/erda-project/erda/pkg/parser/diceyml" "strconv" "strings" "sync" @@ -794,6 +795,8 @@ func (k *Kubernetes) createOne(ctx context.Context, service *apistructs.Service, return err } + logrus.Infof("after %v", service.K8SSnippet.Container.SecurityContext) + switch service.WorkLoad { case types.ServicePerNode: err = k.createDaemonSet(ctx, service, sg) @@ -2072,10 +2075,19 @@ func (k *Kubernetes) DeployInEdgeCluster() bool { // run as default user func (k *Kubernetes) runAsDefaultUser(service *apistructs.Service) error { - values := map[string]any{ - "RunAsUser": &types.DefaultContainerUserId, - "RunAsGroup": &types.DefaultContainerGroupId, + snippet := diceyml.K8SSnippet{ + Container: &diceyml.ContainerSnippet{ + SecurityContext: &apiv1.SecurityContext{ + RunAsUser: &types.DefaultContainerUserId, + RunAsGroup: &types.DefaultContainerGroupId, + }, + }, } - util.MergeStructValue(service, values, "K8SSnippet", "Container", "SecurityContext") - return nil + + if service.K8SSnippet == nil { + service.K8SSnippet = &snippet + return nil + } + + return util.MergeStruct(service.K8SSnippet, &snippet) } diff --git a/internal/tools/orchestrator/scheduler/executor/util/util.go b/internal/tools/orchestrator/scheduler/executor/util/util.go index ba250261263..4dfacd7dfe8 100644 --- a/internal/tools/orchestrator/scheduler/executor/util/util.go +++ b/internal/tools/orchestrator/scheduler/executor/util/util.go @@ -18,7 +18,6 @@ import ( "encoding/base64" "fmt" "reflect" - "slices" "sort" "strconv" "strings" @@ -372,48 +371,78 @@ func GetDeployName(service *apistructs.Service) string { return service.Name } -// MergeStructValue -// values is the data to be merged, format { field-name: field-value } -// needMerges is all the name of the `field` in every struct to be merged -func MergeStructValue(target any, values map[string]any, needMerges ...string) { - typ := reflect.TypeOf(target) - if typ.Kind() != reflect.Ptr { - return +var ( + ErrorNotPointer = errors.New("must be a pointer") + ErrorNotSameType = errors.New("dst and src are not same type") + ErrorValueIsNil = errors.New("value is nil") +) + +func MergeStruct(dst, src any) error { + dstTpy := reflect.TypeOf(dst) + if dstTpy.Kind() != reflect.Ptr { + return ErrorNotPointer + } + dstTpy = dstTpy.Elem() + + srcTpy := reflect.TypeOf(src) + if srcTpy.Kind() != reflect.Ptr { + return ErrorNotPointer } - typ = typ.Elem() + srcTpy = srcTpy.Elem() - if IsBaseType(typ) { - return + if srcTpy != dstTpy { + return ErrorNotSameType } - val := reflect.ValueOf(target) - if val.Kind() != reflect.Ptr { - return + if isBaseType(srcTpy) { + return nil + } + + dstVal := reflect.ValueOf(dst) + if dstVal.IsNil() { + return ErrorValueIsNil + } + if dstVal.Kind() != reflect.Ptr { + return ErrorNotPointer } - val = val.Elem() - for i := 0; i < typ.NumField(); i++ { - if typ.Field(i).Type.Kind() == reflect.Ptr && val.Field(i).IsNil() { - value, ok := values[typ.Field(i).Name] - if ok { - val.Field(i).Set(reflect.ValueOf(value)) + dstVal = dstVal.Elem() + + srcVal := reflect.ValueOf(src) + if srcVal.IsNil() { + return ErrorValueIsNil + } + if srcVal.Kind() != reflect.Ptr { + return ErrorNotPointer + } + srcVal = srcVal.Elem() + + for i := 0; i < dstTpy.NumField(); i++ { + if dstTpy.Field(i).Type.Kind() == reflect.Ptr { + // source data is nil, skip merge + if srcVal.Field(i).IsNil() { continue } - newType := val.Field(i).Type() - if newType.Kind() != reflect.Ptr { + // dst and src are not nil, continue recursively + if !dstVal.Field(i).IsNil() && !srcVal.Field(i).IsNil() { + if err := MergeStruct(dstVal.Field(i).Interface(), srcVal.Field(i).Interface()); err != nil { + return err + } continue } - newType = newType.Elem() - if slices.Index(needMerges, typ.Field(i).Name) < 0 { + + // dst is nil and src is not nil, set value directly + if dstVal.Field(i).IsNil() && !srcVal.Field(i).IsNil() { + dstVal.Field(i).Set(srcVal.Field(i)) continue } - val.Field(i).Set(reflect.New(newType)) + } - MergeStructValue(val.Field(i).Interface(), values, needMerges...) } + return nil } -func IsBaseType(typ reflect.Type) bool { +func isBaseType(typ reflect.Type) bool { switch typ.Kind() { case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: return true diff --git a/internal/tools/orchestrator/scheduler/executor/util/util_test.go b/internal/tools/orchestrator/scheduler/executor/util/util_test.go index 0a8ecd1a9c7..6bbd6a832c2 100644 --- a/internal/tools/orchestrator/scheduler/executor/util/util_test.go +++ b/internal/tools/orchestrator/scheduler/executor/util/util_test.go @@ -17,6 +17,8 @@ package util import ( "encoding/json" "fmt" + "github.com/erda-project/erda/internal/tools/orchestrator/scheduler/executor/plugins/k8s/types" + apiv1 "k8s.io/api/core/v1" "strings" "testing" @@ -213,13 +215,33 @@ func Test_ParseAnnotationFromEnv(t *testing.T) { } func TestMergeStructValue(t *testing.T) { - want := `{"container":{"securityContext":{"runAsUser":0,"runAsGroup":0}}}` - snippet := diceyml.K8SSnippet{} - values := map[string]any{ - "RunAsUser": new(int64), - "RunAsGroup": new(int64), + + dstRunAsUser := int64(0) + want := `{"container":{"securityContext":{"runAsUser":0,"runAsGroup":1000}}}` + dst := apistructs.Service{ + K8SSnippet: &diceyml.K8SSnippet{ + Container: &diceyml.ContainerSnippet{ + SecurityContext: &apiv1.SecurityContext{ + RunAsUser: &dstRunAsUser, + }, + }, + }, + } + + src := apistructs.Service{ + K8SSnippet: &diceyml.K8SSnippet{ + Container: &diceyml.ContainerSnippet{ + SecurityContext: &apiv1.SecurityContext{ + RunAsUser: &types.DefaultContainerUserId, + RunAsGroup: &types.DefaultContainerGroupId, + }, + }, + }, } - MergeStructValue(&snippet, values, "K8SSnippet", "Container", "SecurityContext") - marshal, _ := json.Marshal(snippet) + err := MergeStruct(&dst, &src) + assert.NoError(t, err) + + marshal, _ := json.Marshal(dst.K8SSnippet) + fmt.Println(string(marshal)) assert.Equal(t, want, string(marshal)) } From 24f352bc7a365ba739950d225b8192c94522e9b9 Mon Sep 17 00:00:00 2001 From: CeerDecy <1748788674@qq.com> Date: Thu, 12 Dec 2024 17:41:43 +0800 Subject: [PATCH 8/8] sort imports --- .../tools/orchestrator/scheduler/executor/plugins/k8s/k8s.go | 2 +- .../tools/orchestrator/scheduler/executor/util/util_test.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/tools/orchestrator/scheduler/executor/plugins/k8s/k8s.go b/internal/tools/orchestrator/scheduler/executor/plugins/k8s/k8s.go index 03d1a037c67..64193d44f74 100644 --- a/internal/tools/orchestrator/scheduler/executor/plugins/k8s/k8s.go +++ b/internal/tools/orchestrator/scheduler/executor/plugins/k8s/k8s.go @@ -19,7 +19,6 @@ import ( "context" "encoding/json" "fmt" - "github.com/erda-project/erda/pkg/parser/diceyml" "strconv" "strings" "sync" @@ -85,6 +84,7 @@ import ( "github.com/erda-project/erda/pkg/k8sclient" k8sclientconfig "github.com/erda-project/erda/pkg/k8sclient/config" "github.com/erda-project/erda/pkg/k8sclient/scheme" + "github.com/erda-project/erda/pkg/parser/diceyml" "github.com/erda-project/erda/pkg/schedule/schedulepolicy/cpupolicy" "github.com/erda-project/erda/pkg/strutil" ) diff --git a/internal/tools/orchestrator/scheduler/executor/util/util_test.go b/internal/tools/orchestrator/scheduler/executor/util/util_test.go index 6bbd6a832c2..fde3b2130db 100644 --- a/internal/tools/orchestrator/scheduler/executor/util/util_test.go +++ b/internal/tools/orchestrator/scheduler/executor/util/util_test.go @@ -17,14 +17,14 @@ package util import ( "encoding/json" "fmt" - "github.com/erda-project/erda/internal/tools/orchestrator/scheduler/executor/plugins/k8s/types" - apiv1 "k8s.io/api/core/v1" "strings" "testing" "github.com/stretchr/testify/assert" + apiv1 "k8s.io/api/core/v1" "github.com/erda-project/erda/apistructs" + "github.com/erda-project/erda/internal/tools/orchestrator/scheduler/executor/plugins/k8s/types" "github.com/erda-project/erda/pkg/parser/diceyml" )