diff --git a/applicationset/controllers/requeue_after_test.go b/applicationset/controllers/requeue_after_test.go index 6ecb792b18eee..7cc836ff24fa6 100644 --- a/applicationset/controllers/requeue_after_test.go +++ b/applicationset/controllers/requeue_after_test.go @@ -57,7 +57,7 @@ func TestRequeueAfter(t *testing.T) { }, } fakeDynClient := dynfake.NewSimpleDynamicClientWithCustomListKinds(runtime.NewScheme(), gvrToListKind, duckType) - scmConfig := generators.NewSCMConfig("", []string{""}, true, nil, true) + scmConfig := generators.NewSCMConfig("", []string{""}, true, nil, true, false, 0) terminalGenerators := map[string]generators.Generator{ "List": generators.NewListGenerator(), "Clusters": generators.NewClusterGenerator(ctx, k8sClient, appClientset, "argocd"), diff --git a/applicationset/generators/pull_request.go b/applicationset/generators/pull_request.go index 13256b9315b1b..5523fe7bf792a 100644 --- a/applicationset/generators/pull_request.go +++ b/applicationset/generators/pull_request.go @@ -214,7 +214,7 @@ func (g *PullRequestGenerator) github(ctx context.Context, cfg *argoprojiov1alph if err != nil { return nil, fmt.Errorf("error getting GitHub App secret: %w", err) } - return pullrequest.NewGithubAppService(*auth, cfg.API, cfg.Owner, cfg.Repo, cfg.Labels) + return pullrequest.NewGithubAppService(*auth, cfg.API, cfg.Owner, cfg.Repo, cfg.Labels, g.SCMConfig.GitHubClientCache) } // always default to token, even if not set (public access) @@ -222,5 +222,5 @@ func (g *PullRequestGenerator) github(ctx context.Context, cfg *argoprojiov1alph if err != nil { return nil, fmt.Errorf("error fetching Secret token: %w", err) } - return pullrequest.NewGithubService(ctx, token, cfg.API, cfg.Owner, cfg.Repo, cfg.Labels) + return pullrequest.NewGithubService(ctx, token, cfg.API, cfg.Owner, cfg.Repo, cfg.Labels, g.SCMConfig.GitHubClientCache) } diff --git a/applicationset/generators/pull_request_test.go b/applicationset/generators/pull_request_test.go index 5bde48ea1695d..b61a3106cb43d 100644 --- a/applicationset/generators/pull_request_test.go +++ b/applicationset/generators/pull_request_test.go @@ -283,7 +283,7 @@ func TestAllowedSCMProviderPullRequest(t *testing.T) { "gitea.myorg.com", "bitbucket.myorg.com", "azuredevops.myorg.com", - }, true, nil, true)) + }, true, nil, true, false, 0)) applicationSetInfo := argoprojiov1alpha1.ApplicationSet{ ObjectMeta: metav1.ObjectMeta{ @@ -306,7 +306,7 @@ func TestAllowedSCMProviderPullRequest(t *testing.T) { } func TestSCMProviderDisabled_PRGenerator(t *testing.T) { - generator := NewPullRequestGenerator(nil, NewSCMConfig("", []string{}, false, nil, true)) + generator := NewPullRequestGenerator(nil, NewSCMConfig("", []string{}, false, nil, true, false, 0)) applicationSetInfo := argoprojiov1alpha1.ApplicationSet{ ObjectMeta: metav1.ObjectMeta{ diff --git a/applicationset/generators/scm_provider.go b/applicationset/generators/scm_provider.go index 02d1a7905a476..17bbcb19fe012 100644 --- a/applicationset/generators/scm_provider.go +++ b/applicationset/generators/scm_provider.go @@ -7,6 +7,9 @@ import ( "strings" "time" + "github.com/aburan28/httpcache" + "github.com/aburan28/httpcache/lrucache" + "sigs.k8s.io/controller-runtime/pkg/client" log "github.com/sirupsen/logrus" @@ -35,17 +38,24 @@ type SCMConfig struct { allowedSCMProviders []string enableSCMProviders bool GitHubApps github_app_auth.Credentials + GitHubClientCache httpcache.Cache tokenRefStrictMode bool } -func NewSCMConfig(scmRootCAPath string, allowedSCMProviders []string, enableSCMProviders bool, gitHubApps github_app_auth.Credentials, tokenRefStrictMode bool) SCMConfig { - return SCMConfig{ +func NewSCMConfig(scmRootCAPath string, allowedSCMProviders []string, enableSCMProviders bool, gitHubApps github_app_auth.Credentials, tokenRefStrictMode bool, enableGithubCache bool, githubCacheSize int) SCMConfig { + scmConfig := SCMConfig{ scmRootCAPath: scmRootCAPath, allowedSCMProviders: allowedSCMProviders, enableSCMProviders: enableSCMProviders, GitHubApps: gitHubApps, tokenRefStrictMode: tokenRefStrictMode, } + + if enableGithubCache { + scmConfig.GitHubClientCache = lrucache.NewLRUCache(githubCacheSize) + } + + return scmConfig } func NewSCMProviderGenerator(client client.Client, scmConfig SCMConfig) Generator { @@ -282,6 +292,7 @@ func (g *SCMProviderGenerator) githubProvider(ctx context.Context, github *argop github.Organization, github.API, github.AllBranches, + g.GitHubClientCache, ) } @@ -289,5 +300,5 @@ func (g *SCMProviderGenerator) githubProvider(ctx context.Context, github *argop if err != nil { return nil, fmt.Errorf("error fetching Github token: %w", err) } - return scm_provider.NewGithubProvider(ctx, github.Organization, token, github.API, github.AllBranches) + return scm_provider.NewGithubProvider(ctx, github.Organization, token, github.API, github.AllBranches, g.GitHubClientCache) } diff --git a/applicationset/metrics/metrics.go b/applicationset/metrics/metrics.go index df75012c0a462..81ebbc81fade6 100644 --- a/applicationset/metrics/metrics.go +++ b/applicationset/metrics/metrics.go @@ -28,10 +28,19 @@ var ( descAppsetDefaultLabels, nil, ) + + descAppsetGithubCacheCounter = prometheus.NewDesc( + "argocd_appset_github_cached_items_count", + "Number of github api requests cached", + []string{}, + nil, + ) ) type ApplicationsetMetrics struct { - reconcileHistogram *prometheus.HistogramVec + reconcileHistogram *prometheus.HistogramVec + githubCacheHistogram *prometheus.HistogramVec + githubCacheCounter *prometheus.CounterVec } type appsetCollector struct { @@ -50,15 +59,33 @@ func NewApplicationsetMetrics(appsetLister applisters.ApplicationSetLister, apps }, descAppsetDefaultLabels, ) + githubCacheHistogram := prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "argocd_appset_github_cache_latency", + Help: "", + }, + descAppsetDefaultLabels, + ) + + githubCacheCounter := prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "argocd_appset_github_cached_items_count", + Help: "Number of items in the github cache", + }, + descAppsetDefaultLabels, + ) appsetCollector := newAppsetCollector(appsetLister, appsetLabels, appsetFilter) // Register collectors and metrics metrics.Registry.MustRegister(reconcileHistogram) metrics.Registry.MustRegister(appsetCollector) + metrics.Registry.MustRegister(githubCacheHistogram) + metrics.Registry.MustRegister(githubCacheCounter) return ApplicationsetMetrics{ reconcileHistogram: reconcileHistogram, + githubCacheCounter: githubCacheCounter, } } @@ -66,6 +93,15 @@ func (m *ApplicationsetMetrics) ObserveReconcile(appset *argoappv1.ApplicationSe m.reconcileHistogram.WithLabelValues(appset.Namespace, appset.Name).Observe(duration.Seconds()) } +func (m *ApplicationsetMetrics) ObserveGithubCacheLatency(appset *argoappv1.ApplicationSet, duration time.Duration) { + // TODO: come back to this + m.githubCacheHistogram.WithLabelValues().Observe(duration.Seconds()) +} + +func (m *ApplicationsetMetrics) IncGithubCacheItems() { + m.githubCacheCounter.WithLabelValues().Inc() +} + func newAppsetCollector(lister applisters.ApplicationSetLister, labels []string, filter func(appset *argoappv1.ApplicationSet) bool) *appsetCollector { descAppsetDefaultLabels = []string{"namespace", "name"} diff --git a/applicationset/metrics/metrics_test.go b/applicationset/metrics/metrics_test.go index 1db5b3efbef20..2f61f901ddf64 100644 --- a/applicationset/metrics/metrics_test.go +++ b/applicationset/metrics/metrics_test.go @@ -209,6 +209,25 @@ argocd_appset_owned_applications{name="test2",namespace="argocd"} 0 assert.NotContains(t, rr.Body.String(), `name="should-be-filtered-out"`) } +func IncGithubCacheItems(t *testing.T) { + appsetList := newFakeAppsets(fakeAppsetList) + client := initializeClient(appsetList) + metrics.Registry = prometheus.NewRegistry() + + appsetMetrics := NewApplicationsetMetrics(utils.NewAppsetLister(client), collectedLabels, filter) + + req, err := http.NewRequest(http.MethodGet, "/metrics", nil) + require.NoError(t, err) + rr := httptest.NewRecorder() + handler := promhttp.HandlerFor(metrics.Registry, promhttp.HandlerOpts{}) + appsetMetrics.IncGithubCacheItems() + handler.ServeHTTP(rr, req) + assert.Contains(t, rr.Body.String(), ` +argocd_appset_github_cached_items_count{} 1 +`) + +} + func TestObserveReconcile(t *testing.T) { appsetList := newFakeAppsets(fakeAppsetList) client := initializeClient(appsetList) diff --git a/applicationset/services/internal/github_app/client.go b/applicationset/services/internal/github_app/client.go index 4b9c8f0a6486e..54b75eff78410 100644 --- a/applicationset/services/internal/github_app/client.go +++ b/applicationset/services/internal/github_app/client.go @@ -4,6 +4,7 @@ import ( "fmt" "net/http" + "github.com/aburan28/httpcache" "github.com/bradleyfalzon/ghinstallation/v2" "github.com/google/go-github/v66/github" @@ -11,24 +12,38 @@ import ( ) // Client builds a github client for the given app authentication. -func Client(g github_app_auth.Authentication, url string) (*github.Client, error) { - rt, err := ghinstallation.New(http.DefaultTransport, g.Id, g.InstallationId, []byte(g.PrivateKey)) - if err != nil { - return nil, fmt.Errorf("failed to create github app install: %w", err) - } - if url == "" { - url = g.EnterpriseBaseURL - } +func Client(g github_app_auth.Authentication, url string, cache httpcache.Cache) (*github.Client, error) { + var httpClient *http.Client + var err error var client *github.Client + + if cache != nil { + tr := httpcache.NewTransport(cache) + at, err := ghinstallation.NewAppsTransport(tr, g.Id, []byte(g.PrivateKey)) + if err != nil { + return nil, fmt.Errorf("failed to create github app transport: %w", err) + } + + httpClient = &http.Client{ + Transport: at, + } + } else { + rt, err := ghinstallation.New(http.DefaultTransport, g.Id, g.InstallationId, []byte(g.PrivateKey)) + if err != nil { + return nil, fmt.Errorf("failed to create github app install: %w", err) + } + httpClient = &http.Client{ + Transport: rt, + } + + } + if url == "" { - httpClient := http.Client{Transport: rt} - client = github.NewClient(&httpClient) + client = github.NewClient(httpClient) } else { - rt.BaseURL = url - httpClient := http.Client{Transport: rt} - client, err = github.NewClient(&httpClient).WithEnterpriseURLs(url, url) + client, err = github.NewClient(httpClient).WithEnterpriseURLs(url, url) if err != nil { - return nil, fmt.Errorf("failed to create github enterprise client: %w", err) + return nil, fmt.Errorf("failed to create github client: %w", err) } } return client, nil diff --git a/applicationset/services/pull_request/github.go b/applicationset/services/pull_request/github.go index eaee02ffb171d..5425ad43b6b78 100644 --- a/applicationset/services/pull_request/github.go +++ b/applicationset/services/pull_request/github.go @@ -3,10 +3,11 @@ package pull_request import ( "context" "fmt" + "net/http" "os" + "github.com/aburan28/httpcache" "github.com/google/go-github/v66/github" - "golang.org/x/oauth2" ) type GithubService struct { @@ -18,21 +19,27 @@ type GithubService struct { var _ PullRequestService = (*GithubService)(nil) -func NewGithubService(ctx context.Context, token, url, owner, repo string, labels []string) (PullRequestService, error) { - var ts oauth2.TokenSource +func NewGithubService(ctx context.Context, token, url, owner, repo string, labels []string, cache httpcache.Cache) (PullRequestService, error) { + // var ts oauth2.TokenSource // Undocumented environment variable to set a default token, to be used in testing to dodge anonymous rate limits. if token == "" { token = os.Getenv("GITHUB_TOKEN") } - if token != "" { - ts = oauth2.StaticTokenSource( - &oauth2.Token{AccessToken: token}, - ) - } - httpClient := oauth2.NewClient(ctx, ts) + var httpClient *http.Client var client *github.Client + + if cache != nil { + httpClient = &http.Client{ + Transport: &httpcache.Transport{ + Cache: cache, + }, + } + } else { + httpClient = &http.Client{} + } + if url == "" { - client = github.NewClient(httpClient) + client = github.NewClient(httpClient).WithAuthToken(token) } else { var err error client, err = github.NewClient(httpClient).WithEnterpriseURLs(url, url) diff --git a/applicationset/services/pull_request/github_app.go b/applicationset/services/pull_request/github_app.go index 3eeea1853f70b..da627ac92691c 100644 --- a/applicationset/services/pull_request/github_app.go +++ b/applicationset/services/pull_request/github_app.go @@ -1,12 +1,13 @@ package pull_request import ( + "github.com/aburan28/httpcache" "github.com/argoproj/argo-cd/v3/applicationset/services/github_app_auth" "github.com/argoproj/argo-cd/v3/applicationset/services/internal/github_app" ) -func NewGithubAppService(g github_app_auth.Authentication, url, owner, repo string, labels []string) (PullRequestService, error) { - client, err := github_app.Client(g, url) +func NewGithubAppService(g github_app_auth.Authentication, url, owner, repo string, labels []string, cache httpcache.Cache) (PullRequestService, error) { + client, err := github_app.Client(g, url, cache) if err != nil { return nil, err } diff --git a/applicationset/services/pull_request/github_test.go b/applicationset/services/pull_request/github_test.go index fb35ad20b0b99..e7f5af055d542 100644 --- a/applicationset/services/pull_request/github_test.go +++ b/applicationset/services/pull_request/github_test.go @@ -1,6 +1,7 @@ package pull_request import ( + "context" "testing" "github.com/google/go-github/v66/github" @@ -11,6 +12,29 @@ func toPtr(s string) *string { return &s } +func TestGithubToken(t *testing.T) { + ctx := context.Background() + + testCases := []struct { + name string + token string + }{ + { + name: "No token", + }, + { + name: "With token", + token: "test", + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + _, err := NewGithubService(ctx, tc.token, "", "test", "test", []string{}, nil) + require.NoError(t, err) + }) + } +} + func TestContainLabels(t *testing.T) { cases := []struct { Name string diff --git a/applicationset/services/scm_provider/github.go b/applicationset/services/scm_provider/github.go index 2e2f2a7faf3d4..3bdd2237c2c4a 100644 --- a/applicationset/services/scm_provider/github.go +++ b/applicationset/services/scm_provider/github.go @@ -6,8 +6,8 @@ import ( "net/http" "os" + "github.com/aburan28/httpcache" "github.com/google/go-github/v66/github" - "golang.org/x/oauth2" ) type GithubProvider struct { @@ -18,19 +18,29 @@ type GithubProvider struct { var _ SCMProviderService = &GithubProvider{} -func NewGithubProvider(ctx context.Context, organization string, token string, url string, allBranches bool) (*GithubProvider, error) { - var ts oauth2.TokenSource +func NewGithubProvider(ctx context.Context, organization string, token string, url string, allBranches bool, cache httpcache.Cache) (*GithubProvider, error) { // Undocumented environment variable to set a default token, to be used in testing to dodge anonymous rate limits. + var client *github.Client + if token == "" { token = os.Getenv("GITHUB_TOKEN") } + + httpClient := &http.Client{} + if cache != nil { + httpClient = &http.Client{ + Transport: &httpcache.Transport{ + Cache: cache, + }, + } + } else { + httpClient = &http.Client{} + } + if token != "" { - ts = oauth2.StaticTokenSource( - &oauth2.Token{AccessToken: token}, - ) + client.WithAuthToken(token) } - httpClient := oauth2.NewClient(ctx, ts) - var client *github.Client + if url == "" { client = github.NewClient(httpClient) } else { diff --git a/applicationset/services/scm_provider/github_app.go b/applicationset/services/scm_provider/github_app.go index 8ee26cec41701..1a9d6b4cb8bfe 100644 --- a/applicationset/services/scm_provider/github_app.go +++ b/applicationset/services/scm_provider/github_app.go @@ -1,12 +1,13 @@ package scm_provider import ( + "github.com/aburan28/httpcache" "github.com/argoproj/argo-cd/v3/applicationset/services/github_app_auth" "github.com/argoproj/argo-cd/v3/applicationset/services/internal/github_app" ) -func NewGithubAppProviderFor(g github_app_auth.Authentication, organization string, url string, allBranches bool) (*GithubProvider, error) { - client, err := github_app.Client(g, url) +func NewGithubAppProviderFor(g github_app_auth.Authentication, organization string, url string, allBranches bool, cache httpcache.Cache) (*GithubProvider, error) { + client, err := github_app.Client(g, url, cache) if err != nil { return nil, err } diff --git a/applicationset/services/scm_provider/github_test.go b/applicationset/services/scm_provider/github_test.go index f1fb0e412abb6..132ade662b108 100644 --- a/applicationset/services/scm_provider/github_test.go +++ b/applicationset/services/scm_provider/github_test.go @@ -7,6 +7,7 @@ import ( "net/http/httptest" "testing" + "github.com/aburan28/httpcache" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -242,8 +243,9 @@ func TestGithubListRepos(t *testing.T) { })) defer ts.Close() for _, c := range cases { + t.Run(c.name, func(t *testing.T) { - provider, _ := NewGithubProvider(context.Background(), "argoproj", "", ts.URL, c.allBranches) + provider, _ := NewGithubProvider(context.Background(), "argoproj", "", ts.URL, c.allBranches, nil) rawRepos, err := ListRepos(context.Background(), provider, c.filters, c.proto) if c.hasError { require.Error(t, err) @@ -273,7 +275,9 @@ func TestGithubHasPath(t *testing.T) { githubMockHandler(t)(w, r) })) defer ts.Close() - host, _ := NewGithubProvider(context.Background(), "argoproj", "", ts.URL, false) + cache := httpcache.Cache{} + + host, _ := NewGithubProvider(context.Background(), "argoproj", "", ts.URL, false, cache) repo := &Repository{ Organization: "argoproj", Repository: "argo-cd", @@ -293,7 +297,9 @@ func TestGithubGetBranches(t *testing.T) { githubMockHandler(t)(w, r) })) defer ts.Close() - host, _ := NewGithubProvider(context.Background(), "argoproj", "", ts.URL, false) + cache := httpcache.Cache{} + + host, _ := NewGithubProvider("argoproj", "", ts.URL, false, cache) repo := &Repository{ Organization: "argoproj", Repository: "argo-cd", diff --git a/cmd/argocd-applicationset-controller/commands/applicationset_controller.go b/cmd/argocd-applicationset-controller/commands/applicationset_controller.go index 943b6653ffb49..0eb6da665ccce 100644 --- a/cmd/argocd-applicationset-controller/commands/applicationset_controller.go +++ b/cmd/argocd-applicationset-controller/commands/applicationset_controller.go @@ -54,6 +54,8 @@ func NewCommand() *cobra.Command { probeBindAddr string webhookAddr string enableLeaderElection bool + enableGithubCache bool + githubCacheSize int applicationSetNamespaces []string argocdRepoServer string policy string @@ -171,7 +173,7 @@ func NewCommand() *cobra.Command { argoSettingsMgr := argosettings.NewSettingsManager(ctx, k8sClient, namespace) argoCDDB := db.NewDB(namespace, argoSettingsMgr, k8sClient) - scmConfig := generators.NewSCMConfig(scmRootCAPath, allowedScmProviders, enableScmProviders, github_app.NewAuthCredentials(argoCDDB.(db.RepoCredsDB)), tokenRefStrictMode) + scmConfig := generators.NewSCMConfig(scmRootCAPath, allowedScmProviders, enableScmProviders, github_app.NewAuthCredentials(argoCDDB.(db.RepoCredsDB)), tokenRefStrictMode, enableGithubCache, githubCacheSize) tlsConfig := apiclient.TLSConfiguration{ DisableTLS: repoServerPlaintext, @@ -269,6 +271,8 @@ func NewCommand() *cobra.Command { command.Flags().StringSliceVar(&globalPreservedLabels, "preserved-labels", env.StringsFromEnv("ARGOCD_APPLICATIONSET_CONTROLLER_GLOBAL_PRESERVED_LABELS", []string{}, ","), "Sets global preserved field values for labels") command.Flags().IntVar(&webhookParallelism, "webhook-parallelism-limit", env.ParseNumFromEnv("ARGOCD_APPLICATIONSET_CONTROLLER_WEBHOOK_PARALLELISM_LIMIT", 50, 1, 1000), "Number of webhook requests processed concurrently") command.Flags().StringSliceVar(&metricsAplicationsetLabels, "metrics-applicationset-labels", []string{}, "List of Application labels that will be added to the argocd_applicationset_labels metric") + command.Flags().BoolVar(&enableGithubCache, "enable-github-cache", env.ParseBoolFromEnv("ARGOCD_APPLICATIONSET_CONTROLLER_ENABLE_GITHUB_CACHE", false), "Enable caching for GitHub client") + command.Flags().IntVar(&githubCacheSize, "github-cache-size", env.ParseNumFromEnv("ARGOCD_APPLICATIONSET_CONTROLLER_GITHUB_CACHE_SIZE", 5000, 1000, 100000), "Max items to hold in the cache for github") return &command } diff --git a/cmd/argocd-server/commands/argocd_server.go b/cmd/argocd-server/commands/argocd_server.go index 9a0c6e17e959c..caf8736ec55a8 100644 --- a/cmd/argocd-server/commands/argocd_server.go +++ b/cmd/argocd-server/commands/argocd_server.go @@ -94,6 +94,8 @@ func NewCommand() *cobra.Command { scmRootCAPath string allowedScmProviders []string enableScmProviders bool + enableGithubCache bool + githubCacheSize int // argocd k8s event logging flag enableK8sEvent []string @@ -330,6 +332,8 @@ func NewCommand() *cobra.Command { command.Flags().BoolVar(&enableScmProviders, "appset-enable-scm-providers", env.ParseBoolFromEnv("ARGOCD_APPLICATIONSET_CONTROLLER_ENABLE_SCM_PROVIDERS", true), "Enable retrieving information from SCM providers, used by the SCM and PR generators (Default: true)") command.Flags().StringSliceVar(&allowedScmProviders, "appset-allowed-scm-providers", env.StringsFromEnv("ARGOCD_APPLICATIONSET_CONTROLLER_ALLOWED_SCM_PROVIDERS", []string{}, ","), "The list of allowed custom SCM provider API URLs. This restriction does not apply to SCM or PR generators which do not accept a custom API URL. (Default: Empty = all)") command.Flags().BoolVar(&enableNewGitFileGlobbing, "appset-enable-new-git-file-globbing", env.ParseBoolFromEnv("ARGOCD_APPLICATIONSET_CONTROLLER_ENABLE_NEW_GIT_FILE_GLOBBING", false), "Enable new globbing in Git files generator.") + command.Flags().BoolVar(&enableGithubCache, "enabled-github-cache", env.ParseBoolFromEnv("ARGOCD_APPLICATIONSET_CONTROLLER_ENABLED_GITHUB_CACHE", false), "Enable caching for GitHub client") + command.Flags().IntVar(&githubCacheSize, "github-cache-size", env.ParseNumFromEnv("ARGOCD_APPLICATIONSET_CONTROLLER_GITHUB_CACHE_SIZE", 5000, 1000, 100000), "Max items to hold in the cache for github") tlsConfigCustomizerSrc = tls.AddTLSFlagsToCmd(command) cacheSrc = servercache.AddCacheFlagsToCmd(command, cacheutil.Options{ diff --git a/docs/operator-manual/argocd-cmd-params-cm.yaml b/docs/operator-manual/argocd-cmd-params-cm.yaml index 59382a30e9db7..0e06c3d74bf3d 100644 --- a/docs/operator-manual/argocd-cmd-params-cm.yaml +++ b/docs/operator-manual/argocd-cmd-params-cm.yaml @@ -272,7 +272,10 @@ data: applicationsetcontroller.requeue.after: "3m" # Enable strict mode for tokenRef in ApplicationSet resources. When enabled, the referenced secret must have a label `argocd.argoproj.io/secret-type` with value `scm-creds`. applicationsetcontroller.enable.tokenref.strict.mode: "false" - + # Enable caching of Github API requests for scm and pull request generators + applicationsetcontroller.enable.github.cache: "false" + # Max number of items to store in the in-memory cache for Github API requests. Note that if enabled, to adjust the resource requests/limits accordingly. + applicationsetcontroller.github.cache.size: "1000" ## Argo CD Notifications Controller Properties # Set the logging level. One of: debug|info|warn|error (default "info") notificationscontroller.log.level: "info" diff --git a/docs/operator-manual/server-commands/argocd-server.md b/docs/operator-manual/server-commands/argocd-server.md index fe284a5940733..1fc7547055fca 100644 --- a/docs/operator-manual/server-commands/argocd-server.md +++ b/docs/operator-manual/server-commands/argocd-server.md @@ -53,6 +53,8 @@ argocd-server [flags] --enable-gzip Enable GZIP compression (default true) --enable-k8s-event none Enable ArgoCD to use k8s event. For disabling all events, set the value as none. (e.g --enable-k8s-event=none), For enabling specific events, set the value as `event reason`. (e.g --enable-k8s-event=StatusRefreshed,ResourceCreated) (default [all]) --enable-proxy-extension Enable Proxy Extension feature + --enabled-github-cache Enable caching for GitHub client + --github-cache-size int Max items to hold in the cache for github (default 5000) --gloglevel int Set the glog logging level -h, --help help for argocd-server --hydrator-enabled Feature flag to enable Hydrator. Default ("false") diff --git a/go.mod b/go.mod index 9d7cff28a34f2..878ce368b27fc 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/argoproj/argo-cd/v3 -go 1.22.0 +go 1.23.3 require ( code.gitea.io/sdk/gitea v0.20.0 @@ -9,6 +9,7 @@ require ( github.com/Masterminds/semver/v3 v3.3.1 github.com/Masterminds/sprig/v3 v3.3.0 github.com/TomOnTime/utfutil v0.0.0-20180511104225-09c41003ee1d + github.com/aburan28/httpcache v0.0.1 github.com/alicebob/miniredis/v2 v2.34.0 github.com/antonmedv/expr v1.15.1 github.com/argoproj/gitops-engine v0.7.1-0.20241216155226-54992bf42431 @@ -143,6 +144,8 @@ require ( github.com/google/s2a-go v0.1.7 // indirect github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect github.com/googleapis/gax-go/v2 v2.12.3 // indirect + github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 // indirect + github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect github.com/imdario/mergo v0.3.16 // indirect github.com/kylelemons/godebug v1.1.0 // indirect github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect @@ -218,7 +221,6 @@ require ( github.com/google/gofuzz v1.2.0 // indirect github.com/gosimple/unidecode v1.0.1 // indirect github.com/gregdel/pushover v1.2.1 // indirect - github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.23.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/hashicorp/go-version v1.6.0 // indirect diff --git a/go.sum b/go.sum index dac1d1736f672..efe76a33ec9a0 100644 --- a/go.sum +++ b/go.sum @@ -72,6 +72,8 @@ github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMx github.com/TomOnTime/utfutil v0.0.0-20180511104225-09c41003ee1d h1:WtAMR0fPCOfK7TPGZ8ZpLLY18HRvL7XJ3xcs0wnREgo= github.com/TomOnTime/utfutil v0.0.0-20180511104225-09c41003ee1d/go.mod h1:WML6KOYjeU8N6YyusMjj2qRvaPNUEvrQvaxuFcMRFJY= github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g= +github.com/aburan28/httpcache v0.0.1 h1:Ow0+bnxO0KYeF4CpU8E9RaASrfcfW0dw36uy/Dou7uE= +github.com/aburan28/httpcache v0.0.1/go.mod h1:9l7da/RFyyILtLd4HoDzCpk2Dl2F5AEvuHx5APrFd/M= github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= @@ -528,6 +530,8 @@ github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09 github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= +github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= @@ -932,6 +936,8 @@ github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/tj/assert v0.0.3 h1:Df/BlaZ20mq6kuai7f5z2TvPFiwC3xaWJSDQNiIS3Rk= +github.com/tj/assert v0.0.3/go.mod h1:Ne6X72Q+TB1AteidzQncjw9PabbMp4PBMZ1k+vd1Pvk= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= diff --git a/server/applicationset/applicationset.go b/server/applicationset/applicationset.go index ac163282959fc..86e7fdbafc167 100644 --- a/server/applicationset/applicationset.go +++ b/server/applicationset/applicationset.go @@ -66,6 +66,8 @@ type Server struct { ScmRootCAPath string AllowedScmProviders []string EnableScmProviders bool + EnableGithubCache bool + GithubCacheSize int } // NewServer returns a new instance of the ApplicationSet service @@ -266,7 +268,7 @@ func (s *Server) Create(ctx context.Context, q *applicationset.ApplicationSetCre func (s *Server) generateApplicationSetApps(ctx context.Context, logEntry *log.Entry, appset v1alpha1.ApplicationSet, namespace string) ([]v1alpha1.Application, error) { argoCDDB := s.db - scmConfig := generators.NewSCMConfig(s.ScmRootCAPath, s.AllowedScmProviders, s.EnableScmProviders, github_app.NewAuthCredentials(argoCDDB.(db.RepoCredsDB)), true) + scmConfig := generators.NewSCMConfig(s.ScmRootCAPath, s.AllowedScmProviders, s.EnableScmProviders, github_app.NewAuthCredentials(argoCDDB.(db.RepoCredsDB)), true, s.EnableGithubCache, s.GithubCacheSize) getRepository := func(ctx context.Context, url, project string) (*v1alpha1.Repository, error) { return s.db.GetRepository(ctx, url, project)