Skip to content

Commit

Permalink
metrics: add statsd instrumentation (#3)
Browse files Browse the repository at this point in the history
  • Loading branch information
danfaizer authored Feb 5, 2025
1 parent 4a0bd3c commit 871addd
Show file tree
Hide file tree
Showing 7 changed files with 261 additions and 14 deletions.
8 changes: 7 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -50,8 +50,14 @@ The `ghe-reposec` tool can be configured using environment variables. Below are
- `REPOSEC_LAVA_CONCURRENCY`: The number of concurrent Lava scans (default: `10`).
- `REPOSEC_LAVA_BINARY_PATH`: The path to the Lava binary (default: `/usr/bin/lava`).
- `REPOSEC_LAVA_CHECK_IMAGE`: The Lava check image (default: `vulcansec/vulcan-repository-sctrl:a20516f-4aae88d`).
- `LAVA_RESULTS_PATH`: The path where Lava results (stdout and stderr) will be stored if specified.
- `REPOSEC_LAVA_RESULTS_PATH`: The path where Lava results (stdout and stderr) will be stored if specified.

### Metrics Configuration

- `REPOSEC_METRICS_ENABLED`: Enable metrics (default: `false`).
- `REPOSEC_METRICS_ADDRESS`: The statsd listener address (default: `localhost:8125`).
- `REPOSEC_METRICS_NAMESPACE`: The metrics namespace (default: `ghereposec`).
- `REPOSEC_METRICS_TAGS`: The metrics tags (default: `ghereposec:metrics`). Multiple tags can be specified separated by commas.

## Contributing

Expand Down
8 changes: 7 additions & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,15 @@ module github.com/adevinta/ghe-reposec
go 1.23

require (
github.com/DataDog/datadog-go v4.8.3+incompatible
github.com/adevinta/vulcan-report v1.0.0
github.com/caarlos0/env/v11 v11.3.1
github.com/google/go-github/v67 v67.0.0
)

require github.com/google/go-querystring v1.1.0 // indirect
require (
github.com/Microsoft/go-winio v0.6.2 // indirect
github.com/google/go-querystring v1.1.0 // indirect
github.com/stretchr/testify v1.10.0 // indirect
golang.org/x/sys v0.10.0 // indirect
)
16 changes: 16 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
@@ -1,12 +1,28 @@
github.com/DataDog/datadog-go v4.8.3+incompatible h1:fNGaYSuObuQb5nzeTQqowRAd9bpDIRRV4/gUtIBjh8Q=
github.com/DataDog/datadog-go v4.8.3+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ=
github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY=
github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
github.com/adevinta/vulcan-report v1.0.0 h1:44aICPZ+4svucgCSA5KmjlT3ZGzrvZXiSnkbnj6AC2k=
github.com/adevinta/vulcan-report v1.0.0/go.mod h1:k34KaeoXc3H77WNMwI9F4F1G28hBjB95PeMUp9oHbEE=
github.com/caarlos0/env/v11 v11.3.1 h1:cArPWC15hWmEt+gWk7YBi7lEXTXCvpaSdCiZE2X5mCA=
github.com/caarlos0/env/v11 v11.3.1/go.mod h1:qupehSf/Y0TUTsxKywqRt/vJjN5nz6vauiYEUUr8P4U=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/go-github/v67 v67.0.0 h1:g11NDAmfaBaCO8qYdI9fsmbaRipHNWRIU/2YGvlh4rg=
github.com/google/go-github/v67 v67.0.0/go.mod h1:zH3K7BxjFndr9QSeFibx4lTKkYS3K9nDanoI1NjaOtY=
github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8=
github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=
github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA=
golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
13 changes: 11 additions & 2 deletions internal/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,14 @@ type LavaConfig struct {
ResultsPath string `env:"LAVA_RESULTS_PATH"`
}

// MetricsConfig represents the metrics configuration.
type MetricsConfig struct {
Enabled bool `env:"METRICS_ENABLED" envDefault:"false"`
Address string `env:"METRICS_ADDRESS" envDefault:"localhost:8125"`
Namespace string `env:"METRICS_NAMESPACE" envDefault:"ghereposec"`
Tags []string `env:"METRICS_TAGS" envSeparator:"," envDefault:"ghereposec:metrics"`
}

// Config represents the ghe-reposec configuration.
type Config struct {
LogLevel string `env:"LOG_LEVEL" envDefault:"info"`
Expand All @@ -52,8 +60,9 @@ type Config struct {
OutputFilePath string `env:"OUTPUT_FILE" envDefault:"/tmp/reposec.csv"`
OutputFormat string `env:"OUTPUT_FORMAT" envDefault:"csv"`

GHECfg GHEConfig
LavaCfg LavaConfig
GHECfg GHEConfig
LavaCfg LavaConfig
MetricsCfg MetricsConfig
}

// Redacted returns a secret redacted version of the configuration.
Expand Down
43 changes: 34 additions & 9 deletions internal/github/github.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import (
gh "github.com/google/go-github/v67/github"

"github.com/adevinta/ghe-reposec/internal/config"
"github.com/adevinta/ghe-reposec/internal/metrics"
)

var (
Expand All @@ -25,14 +26,15 @@ var (

// Client is a GitHub client wrapper.
type Client struct {
cfg config.GHEConfig
client *gh.Client
logger *slog.Logger
ctx context.Context
cfg config.GHEConfig
client *gh.Client
logger *slog.Logger
metrics *metrics.Client
ctx context.Context
}

// NewClient creates a new GitHub Enterprise client.
func NewClient(ctx context.Context, logger *slog.Logger, cfg config.GHEConfig) (*Client, error) {
func NewClient(ctx context.Context, logger *slog.Logger, m *metrics.Client, cfg config.GHEConfig) (*Client, error) {
if cfg.Token == "" {
return nil, ErrTokenRequired
}
Expand All @@ -59,10 +61,11 @@ func NewClient(ctx context.Context, logger *slog.Logger, cfg config.GHEConfig) (
logger.Debug("GitHub Enterprise token", "owner", user.GetLogin())

return &Client{
cfg: cfg,
logger: logger,
client: client,
ctx: ctx,
cfg: cfg,
logger: logger,
client: client,
metrics: m,
ctx: ctx,
}, nil
}

Expand Down Expand Up @@ -108,6 +111,7 @@ func (c *Client) Repositories(targetOrg string) ([]string, error) {
return []string{}, fmt.Errorf("failed to list organizations: %w", err)
}
}
c.metrics.Gauge("organizations", len(orgs), []string{})

c.logger.Debug("listing repositories")
sem := make(chan struct{}, c.cfg.Concurrency)
Expand Down Expand Up @@ -140,6 +144,16 @@ func orgRepositories(c *Client, org string, wg *sync.WaitGroup, sem chan struct{

c.logger.Debug("obtaining repositories for organization", "organization", org)

repoMetrics := map[string]int{
"too_big": 0,
"empty": 0,
"archived": 0,
"disabled": 0,
"fork": 0,
"template": 0,
"inactive": 0,
"selected": 0,
}
allRepos := []string{}
listOpts := &gh.RepositoryListByOrgOptions{ListOptions: gh.ListOptions{PerPage: 100}}
for {
Expand All @@ -161,31 +175,37 @@ func orgRepositories(c *Client, org string, wg *sync.WaitGroup, sem chan struct{
// If repository is too big, skip it.
if repo.Size != nil && *repo.Size > c.cfg.RepositorySizeLimit {
c.logger.Warn("repository is too big, skipping", "size_kb", *repo.Size, "repository", repo.GetFullName())
repoMetrics["too_big"]++
continue
}
// If repository is empty, skip it.
if (repo.Size != nil && *repo.Size == 0) && !c.cfg.IncludeEmpty {
c.logger.Warn("repository is empty, skipping", "repository", repo.GetFullName())
repoMetrics["empty"]++
continue
}
// If repository is archived, skip it.
if (repo.Archived != nil && *repo.Archived) && !c.cfg.IncludeArchived {
c.logger.Warn("repository is archived, skipping", "repository", repo.GetFullName())
repoMetrics["archived"]++
continue
}
// If repository is disabled, skip it.
if (repo.Disabled != nil && *repo.Disabled) && !c.cfg.IncludeDisabled {
c.logger.Warn("repository is disabled, skipping", "repository", repo.GetFullName())
repoMetrics["disabled"]++
continue
}
// If repository is a fork, skip it.
if (repo.Fork != nil && *repo.Fork) && !c.cfg.IncludeForks {
c.logger.Warn("repository is a fork, skipping", "repository", repo.GetFullName())
repoMetrics["fork"]++
continue
}
// If repository is a template, skip it.
if (repo.IsTemplate != nil && *repo.IsTemplate) && !c.cfg.IncludeTemplates {
c.logger.Warn("repository is a template, skipping", "repository", repo.GetFullName())
repoMetrics["template"]++
continue
}
// If repository hadn't been active for a while, skip it.
Expand All @@ -196,10 +216,12 @@ func orgRepositories(c *Client, org string, wg *sync.WaitGroup, sem chan struct{

if isUpdatedInactive && isPushedInactive {
c.logger.Warn("repository has not been active for a while, skipping", "repository", repo.GetFullName())
repoMetrics["inactive"]++
continue
}
}
allRepos = append(allRepos, *repo.CloneURL)
repoMetrics["selected"]++
}
if resp.NextPage == 0 {
break
Expand All @@ -208,6 +230,9 @@ func orgRepositories(c *Client, org string, wg *sync.WaitGroup, sem chan struct{
}

c.logger.Debug("organization repository listing completed", "organization", org, "repositories", len(allRepos))
for k, v := range repoMetrics {
c.metrics.Gauge("repositories", v, []string{"status:" + k})
}

resultChan <- allRepos
}
135 changes: 135 additions & 0 deletions internal/metrics/metrics.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,135 @@
// Copyright 2025 Adevinta

// Package metrics provides a wrapper to interact with StatsD.
package metrics

import (
"context"
"fmt"
"log/slog"

"github.com/DataDog/datadog-go/statsd"

"github.com/adevinta/ghe-reposec/internal/config"
)

var (
// ClientNotInitializedMsg is logged when the metrics client is not
// initialized and metrics are enabled.
ClientNotInitializedMsg = "metrics client not initialized"
)

const (
// DefaultMetricsClientAddr is the default metrics client address.
DefaultMetricsClientAddr = "localhost:8125"
)

// Client represents a metrics service client.
type Client struct {
cfg config.MetricsConfig
client *statsd.Client
logger *slog.Logger
ctx context.Context
}

// NewClient creates a new metrics client based on environment variables config.
func NewClient(ctx context.Context, logger *slog.Logger, cfg config.MetricsConfig) (*Client, error) {
if !cfg.Enabled {
logger.Info("metrics reporting disabled")
return &Client{}, nil
}
address := cfg.Address
if address == "" {
logger.Warn("metrics address not provided, using default", "address", DefaultMetricsClientAddr)
address = DefaultMetricsClientAddr
}

statsd, err := statsd.New(address)
if err != nil {
return nil, err
}

return &Client{
cfg: cfg,
client: statsd,
logger: logger,
ctx: ctx,
}, nil
}

// Gauge sends a gauge metric to the metrics service.
func (c *Client) Gauge(name string, value int, tags []string) {
if !c.cfg.Enabled {
return
}
if c.client == nil {
c.logger.Warn(ClientNotInitializedMsg)
return
}
tags = append(tags, c.cfg.Tags...)
name = fmt.Sprintf("%s.%s", c.cfg.Namespace, name)
err := c.client.Gauge(name, float64(value), tags, 1)
if err != nil {
c.logger.Error("gauge metric push error", "error", err)
return
}
c.logger.Debug("gauge metric pushed", "name", name, "value", value, "tags", tags)
}

// ServiceCheck sends a service satus signal to the metrics service.
func (c *Client) ServiceCheck(status byte, message string, tags []string) {
if !c.cfg.Enabled {
return
}
if c.client == nil {
c.logger.Warn(ClientNotInitializedMsg)
return
}
tags = append(tags, c.cfg.Tags...)
name := fmt.Sprintf("%s.service_check", c.cfg.Namespace)
err := c.client.ServiceCheck(&statsd.ServiceCheck{
Name: name,
Status: statsd.ServiceCheckStatus(status),
Tags: tags,
Message: message,
})
if err != nil {
c.logger.Error("service check push error", "error", err)
return
}
c.logger.Debug("service check pushed", "status", status, "message", message)
}

// Close closes the metrics client.
func (c *Client) Close() {
if !c.cfg.Enabled {
return
}
if c.client == nil {
c.logger.Warn(ClientNotInitializedMsg)
return
}
err := c.client.Close()
if err != nil {
c.logger.Error("metrics client close error", "error", err)
return
}
c.logger.Debug("metrics client closed")
}

// Flush flushes the metrics client.
func (c *Client) Flush() {
if !c.cfg.Enabled {
return
}
if c.client == nil {
c.logger.Warn(ClientNotInitializedMsg)
return
}
err := c.client.Flush()
if err != nil {
c.logger.Error("metrics client flush error", "error", err)
return
}
c.logger.Debug("metrics client flushed")
}
Loading

0 comments on commit 871addd

Please sign in to comment.