From 8b10adfbab196f0de402c3ac81722b3c8a60fd3b Mon Sep 17 00:00:00 2001 From: Norihiro Seto Date: Sun, 15 Dec 2019 02:23:22 +0900 Subject: [PATCH 1/2] Multiple pod delete. Add node affinity key: beta.kubernetes.io/arch Exclude toleration from pod scheduling check. --- .vscode/launch.json | 9 ++++++++ cmd/pod-rebalancer/main.go | 4 ++-- cmd/pod-rebalancer/rebalancer.go | 38 +++++++++++++++++++++++--------- evicted-cleaner.yaml | 5 +++++ go.sum | 4 ++++ pkg/k8sutils/rs.go | 2 +- pod-rebalancer.yaml | 5 +++++ 7 files changed, 54 insertions(+), 13 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 479c62c..8ec6b8f 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -4,6 +4,15 @@ // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 "version": "0.2.0", "configurations": [ + { + "name": "Launch", + "type": "go", + "request": "launch", + "mode": "auto", + "program": "${fileDirname}", + "env": {}, + "args": [] + }, { "name": "Microk8s", "type": "go", diff --git a/cmd/pod-rebalancer/main.go b/cmd/pod-rebalancer/main.go index 31f8902..8ef157d 100755 --- a/cmd/pod-rebalancer/main.go +++ b/cmd/pod-rebalancer/main.go @@ -1,7 +1,7 @@ package main -// Evicted Pod Cleaner -// Deletes all evicted pod. +// Pod Rebalancer +// Deletes pod scheduled biased node. import ( "fmt" diff --git a/cmd/pod-rebalancer/rebalancer.go b/cmd/pod-rebalancer/rebalancer.go index 9602f1c..ad669c2 100644 --- a/cmd/pod-rebalancer/rebalancer.go +++ b/cmd/pod-rebalancer/rebalancer.go @@ -18,10 +18,12 @@ type replicaState struct { type podState struct { pod v1.Pod node v1.Node + deleted bool } type rebalancer struct { current *replicaState + maxRate float32 } func (r *rebalancer) specReplicas() int32 { @@ -33,7 +35,7 @@ func (r *rebalancer) currentReplicas() int32 { } func newRebalancer(current *replicaState) *rebalancer { - return &rebalancer{current: current} + return &rebalancer{current: current, maxRate: .25} } func (r *rebalancer) Rebalance(c *kubernetes.Clientset) (bool, error) { @@ -46,21 +48,35 @@ func (r *rebalancer) Rebalance(c *kubernetes.Clientset) (bool, error) { return false, nil } - node, num := r.maxPodNode() - ave := float32(sr) / float32(nodeCount) - if len(node) > 0 && float32(num) >= ave+1.0 { - err := r.deleteNodePod(c, node) - return true, err + deleted := 0 + maxDel := int(float32(sr) * r.maxRate) + if maxDel < 1 { + maxDel = 1 } - return false, nil + for i := 0; i < maxDel; i++ { + node, num := r.maxPodNode() + ave := float32(sr) / float32(nodeCount) + if len(node) <= 0 || float32(num) < ave+1.0 { + return deleted > 0, nil + } + if err := r.deleteNodePod(c, node); err != nil { + return deleted > 0, err + } + deleted++ + } + + return deleted > 0, nil } // deleteNodePod deletes only one pod per replicaset. func (r *rebalancer) deleteNodePod(c *kubernetes.Clientset, node string) error { - for _, s := range r.current.podState { - if s.node.Name == node { + l := len(r.current.podState) + for i := 0; i < l; i++ { + s := &r.current.podState[i] + if s.node.Name == node && !s.deleted { log.Debug("Deleting pod " + s.pod.Name + " in " + node) + s.deleted = true return k8sutils.DeletePod(c, s.pod) } } @@ -73,7 +89,9 @@ func (r *rebalancer) maxPodNode() (string, int) { m[n.Name] = 0 } for _, s := range r.current.podState { - m[s.node.Name]++ + if !s.deleted { + m[s.node.Name]++ + } } maxVal := 0 diff --git a/evicted-cleaner.yaml b/evicted-cleaner.yaml index e6dd33c..bdb0d6d 100755 --- a/evicted-cleaner.yaml +++ b/evicted-cleaner.yaml @@ -49,6 +49,11 @@ spec: operator: In values: - amd64 + - matchExpressions: + - key: beta.kubernetes.io/arch + operator: In + values: + - amd64 containers: - name: evicted-cleaner image: docker.io/norseto/evicted-cleaner:0.1.1 diff --git a/go.sum b/go.sum index 89ca946..b3a4f2c 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,4 @@ +cloud.google.com/go v0.34.0 h1:eOI3/cP2VTU6uZLDYAoic+eyzzB9YyGmJ7eIjl8rOPg= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -23,6 +24,7 @@ github.com/json-iterator/go v1.1.8 h1:QiWkFLKq0T7mpzwOTu6BzNDbfTE8OLrYhVKYMLF46O github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -64,7 +66,9 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0 h1:/5xXl8Y5W96D+TtHSlonuFqGHIWVuyCkGJLwGh9JJFs= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= diff --git a/pkg/k8sutils/rs.go b/pkg/k8sutils/rs.go index 1fa4d1e..16685b8 100644 --- a/pkg/k8sutils/rs.go +++ b/pkg/k8sutils/rs.go @@ -61,7 +61,7 @@ func (u *rsowners) IsRollingUpdating(rs *appsv1.ReplicaSet) bool { // bool : True if pod of replicaset scheduling is limited. func IsPodScheduleLimeted(rs appsv1.ReplicaSet) bool { podSpec := rs.Spec.Template.Spec - return podSpec.Affinity != nil || len(podSpec.Tolerations) > 0 || len(podSpec.NodeSelector) > 0 + return podSpec.Affinity != nil || len(podSpec.NodeSelector) > 0 } // IsPodOwnedBy determins the owner of the pod is the specified replicaset diff --git a/pod-rebalancer.yaml b/pod-rebalancer.yaml index c1217b2..878173a 100755 --- a/pod-rebalancer.yaml +++ b/pod-rebalancer.yaml @@ -55,6 +55,11 @@ spec: operator: In values: - amd64 + - matchExpressions: + - key: beta.kubernetes.io/arch + operator: In + values: + - amd64 containers: - name: pod-rebalancer image: docker.io/norseto/pod-rebalancer:0.1.1 From 4beb5c251bc2255422ba68fa150d50b9beeb950c Mon Sep 17 00:00:00 2001 From: Norihiro Seto Date: Mon, 16 Dec 2019 02:49:59 +0900 Subject: [PATCH 2/2] Add node affinity key: beta.kubernetes.io/arch for older versions. --- README.md | 2 +- evicted-cleaner.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 1511018..e471874 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ This CronJos cleans "Evicted" pods. ### Installation ``` -kubectl apply -f https://github.com/norseto/k8s-watchdogs/releases/download/evicted-cleaner-v0.1.1/evicted-cleaner.yaml +kubectl apply -f https://github.com/norseto/k8s-watchdogs/releases/download/evicted-cleaner-v0.1.2/evicted-cleaner.yaml ``` ## Pod Rebalancer diff --git a/evicted-cleaner.yaml b/evicted-cleaner.yaml index bdb0d6d..dc87217 100755 --- a/evicted-cleaner.yaml +++ b/evicted-cleaner.yaml @@ -56,7 +56,7 @@ spec: - amd64 containers: - name: evicted-cleaner - image: docker.io/norseto/evicted-cleaner:0.1.1 + image: docker.io/norseto/evicted-cleaner:0.1.2 imagePullPolicy: IfNotPresent restartPolicy: OnFailure activeDeadlineSeconds: 60