feat: add new GC rules for release with version

internal/apps/dop/dicehub/release/db/release.go

@@ -286,3 +286,33 @@
 	}
 	return releases, nil
 }
+
+// GetGroupRelease list release group by project_id and application_id
+func (client *ReleaseConfigDB) GetGroupRelease() ([]Release, error) {
+	var releases []Release
+	if err := client.Group("project_id, application_id").
+		Find(&releases).Error; err != nil {
+		return nil, err
+	}
+	return releases, nil
+}
+
+// ListExpireReleaseWithVersion list release that has not been referenced before a given point in time and version is not empty
+func (client *ReleaseConfigDB) ListExpireReleaseWithVersion(projectID int64, applicationID int64, before time.Time) ([]Release, error) {
+	var releases []Release
+	if err := client.Where("reference <= ?", 0).Where("updated_at < ?", before).Where("version != ''").
+		Where("project_id = ?", projectID).Where("application_id = ?", applicationID).Where("is_project_release = 0 And is_formal=0").
+		Order("updated_at ASC").Find(&releases).Error; err != nil {
+		return nil, err
+	}
+	return releases, nil
+}
+
+// ListReleaseByAppAndProject list release by application_id and project_id, version is not empty
+func (client *ReleaseConfigDB) ListReleaseByAppAndProject(projectID int64, appID int64) ([]Release, error) {
+	var releases []Release
+	if err := client.Where("application_id = ? AND project_id =? AND version != ''", appID, projectID).Order("updated_at ASC").Find(&releases).Error; err != nil {
+		return nil, err
+	}
+	return releases, nil
+}

internal/apps/dop/dicehub/release/meta.go

@@ -15,7 +15,10 @@
 package release
 
 type releaseConfig struct {
-	MaxTimeReserved string
+	MaxTimeReserved         string
+	MaxReleaseLimit         int32 // Versioned release limit
+	MinReleaseLimit         int32
+	MaxTimeVersionedRelease string // Versioned release limit
 }
 
 // ResourceType Release type

internal/apps/dop/dicehub/release/provider.go

@@ -53,8 +53,11 @@
 )
 
 type config struct {
-	MaxTimeReserved string `file:"max_time_reserved" env:"RELEASE_MAX_TIME_RESERVED"`
-	GCSwitch        bool   `file:"gc_switch" env:"RELEASE_GC_SWITCH"`
+	MaxTimeReserved         string `file:"max_time_reserved" env:"RELEASE_MAX_TIME_RESERVED"`
+	GCSwitch                bool   `file:"gc_switch" env:"RELEASE_GC_SWITCH"`
+	MinReleaseLimit         int32  `file:"min_release_limit" env:"RELEASE_MIN_RELEASE_LIMIT" default:"10"`                    // 10
+	MaxReleaseLimit         int32  `file:"max_release_limit" env:"RELEASE_MAX_RELEASE_LIMIT" default:"30"`                    // 30
+	MaxTimeVersionedRelease string `file:"max_time_versioned_release" env:"RELEASE_MAX_TIME_VERSIONED_RELEASE" default:"168"` // 7 days = 168 h
 }
 
 // +provider
@@ -87,7 +90,10 @@
 		bdl:             p.bdl,
 		Etcd:            p.Etcd,
 		Config: &releaseConfig{
-			MaxTimeReserved: p.Cfg.MaxTimeReserved,
+			MaxTimeReserved:         p.Cfg.MaxTimeReserved,
+			MaxReleaseLimit:         p.Cfg.MaxReleaseLimit,
+			MinReleaseLimit:         p.Cfg.MinReleaseLimit,
+			MaxTimeVersionedRelease: p.Cfg.MaxTimeVersionedRelease,
 		},
 		ReleaseRule: release_rule.New(release_rule.WithDBClient(&dbclient.DBClient{
 			DBEngine: &dbengine.DBEngine{DB: p.DB},

internal/apps/dop/dicehub/release/release.service.go

@@ -1055,6 +1055,60 @@
 	return &latests, nil
 }
 
+func (s *ReleaseService) FindExpiredReleaseBefore(now time.Time) ([]db.Release, error) {
+	d, err := time.ParseDuration(strutil.Concat("-", s.Config.MaxTimeVersionedRelease, "h"))
+	before := now.Add(d)
+	releasesWithIDs, err := s.db.GetGroupRelease()
+	if err != nil {
+		return nil, err
+	}
+	// Save all releases that need to be deleted
+	deleteReleases := make([]db.Release, 0)
+	for _, releaseWithID := range releasesWithIDs {
+		// All release of current application
+		releases, err := s.db.ListReleaseByAppAndProject(releaseWithID.ProjectID, releaseWithID.ApplicationID)
+		if err != nil {
+			return nil, err
+		}
+		// expired release of current application
+		expiredReleases, err := s.db.ListExpireReleaseWithVersion(releaseWithID.ProjectID, releaseWithID.ApplicationID, before)
+
+		if err != nil {
+			return nil, err
+		}
+
+		releaseCount := int32(len(releases))
+		expiredCount := int32(len(expiredReleases))
+
+		minReleaseLimit := s.Config.MinReleaseLimit
+		maxReleaseLimit := s.Config.MaxReleaseLimit
+
+		if releaseCount <= minReleaseLimit {
+			// If the quantity of release is less than minReleaseLimit, there is no need to delete
+			continue
+		} else if releaseCount < maxReleaseLimit {
+			// delete releases utils the quantity of release is minReleaseLimit
+			if releaseCount-expiredCount < minReleaseLimit {
+				ableToDeleteCount := releaseCount - minReleaseLimit
+				deleteReleases = append(deleteReleases, expiredReleases[:ableToDeleteCount]...)
+			} else {
+				deleteReleases = append(deleteReleases, expiredReleases...)
+			}
+		} else {
+			// delete releases utils the quantity of release is minReleaseLimit~maxReleaseLimit
+			if releaseCount-expiredCount < minReleaseLimit {
+				deleteReleases = append(deleteReleases, expiredReleases[:releaseCount-minReleaseLimit]...)
+			} else {
+				deleteReleases = append(deleteReleases, expiredReleases...)
+				if releaseCount-expiredCount > maxReleaseLimit {
+					deleteReleases = append(deleteReleases, releases[:releaseCount-expiredCount-maxReleaseLimit]...)
+				}
+			}
+		}
+	}
+	return deleteReleases, nil
+}
+
 // RemoveDeprecatedsReleases Recycling expired release
 func (s *ReleaseService) RemoveDeprecatedsReleases(now time.Time) error {
 	d, err := time.ParseDuration(strutil.Concat("-", s.Config.MaxTimeReserved, "h"))
@@ -1071,13 +1125,25 @@
 	logrus.Infof("found %d releases that had no references before %s",
 		len(releases), before.Format("2006-01-02 15:04:05"))
 
+	// version is None will be deleted
+	var versionNoneRelease []db.Release
 	for i := range releases {
 		release := releases[i]
 		if release.Version != "" {
 			logrus.Debugf("release %s have been tagged, can't be recycled", release.ReleaseID)
 			continue
 		}
+		versionNoneRelease = append(versionNoneRelease, release)
+	}
+	// list of release with version need to be deleted
+	versionRelease, err := s.FindExpiredReleaseBefore(now)
+	if err != nil {
+		return err
+	}
+
+	deleteRelease := append(versionRelease, versionNoneRelease...)
 
+	for _, release := range deleteRelease {
 		images, err := s.imageDB.GetImagesByRelease(release.ReleaseID)
 		if err != nil {
 			logrus.Warnf(err.Error())
@@ -1118,6 +1184,7 @@
 			logrus.Infof("deleted release: %s", release.ReleaseID)
 		}
 	}
+
 	return nil
 }
 

internal/apps/dop/dicehub/release/release_test.go

@@ -27,6 +27,8 @@ import (
 	"bou.ke/monkey"
 	"github.com/DATA-DOG/go-sqlmock"
 	"github.com/jinzhu/gorm"
+	_ "github.com/jinzhu/gorm/dialects/sqlite"
+	"gopkg.in/stretchr/testify.v1/assert"
 
 	"github.com/erda-project/erda-proto-go/core/dicehub/release/pb"
 	"github.com/erda-project/erda/apistructs"
@@ -572,3 +574,86 @@ version: "2.0"
 		})
 	}
 }
+
+func TestReleaseServiceGC(t *testing.T) {
+	testCase := []struct {
+		name     string
+		dbDir    string
+		dbPath   string
+		dbEngine string
+		releases []db.Release
+	}{
+		{
+			name:     "baseTest",
+			dbDir:    "./testdata",
+			dbPath:   "./testdata/test.db",
+			dbEngine: "sqlite3",
+			releases: []db.Release{
+				{
+					ReleaseID:        "1",
+					ReleaseName:      "chenhaiqing",
+					Desc:             "",
+					Dice:             "",
+					Addon:            "",
+					Changelog:        "",
+					IsStable:         false,
+					IsFormal:         false,
+					IsProjectRelease: false,
+					Modes:            "",
+					Labels:           "",
+					GitBranch:        "",
+					Version:          "",
+					OrgID:            0,
+					ProjectID:        1,
+					ApplicationID:    2,
+					ProjectName:      "chenhaiqing",
+					ApplicationName:  "chenhaiqing",
+					UserID:           "",
+					ClusterName:      "",
+					Resources:        "",
+					Reference:        0,
+					CrossCluster:     false,
+					CreatedAt:        time.Time{},
+					UpdatedAt:        time.Time{},
+					IsLatest:         false,
+				},
+			},
+		},
+	}
+	for _, tt := range testCase {
+		t.Run(tt.name, func(t *testing.T) {
+			if _, err := os.Stat(tt.dbDir); os.IsNotExist(err) {
+				err = os.Mkdir(tt.dbDir, 0755)
+				assert.NoError(t, err)
+			}
+			if _, err := os.Stat(tt.dbPath); os.IsNotExist(err) {
+				_, err = os.Create(tt.dbPath)
+				assert.NoError(t, err)
+			}
+			dbClient, err := gorm.Open("sqlite3", "./testdata/test.db")
+			if err != nil {
+				t.Fatal(err)
+			}
+			if !dbClient.HasTable(&db.Release{}) {
+				dbClient.CreateTable(db.Release{})
+			}
+			rs := ReleaseService{
+				db: &db.ReleaseConfigDB{
+					DB: dbClient,
+				},
+				Config: &releaseConfig{MaxTimeReserved: "72"},
+			}
+			// insert data into db to test
+			for _, release := range tt.releases {
+				rs.db.Create(&release)
+			}
+			now := time.Now()
+			err = rs.RemoveDeprecatedsReleases(now)
+			assert.NoError(t, err)
+			dbClient.Exec("delete from dice_release")
+			// delete db dir
+			err = os.RemoveAll(tt.dbDir)
+			assert.NoError(t, err)
+		})
+	}
+}