Pull all projects and all branches

[hailsonsilva]

I'm trying to pull all my GitLab projects metrics using the configuration bellow, but it's not bringing anything.... Is anything wrong with my configuration?

config:
  log:
    level: trace
    format: text
  gitlab:
    url: https://gitlab.com
    token: [TOKEN]
    enable_health_check: true
    enable_tls_verify: true
  project_defaults:
    output_sparse_status_metrics: true
    pull:
      environments:
        enabled: true
        regexp: ".*"
        exclude_stopped: false
      refs:
        branches:
          enabled: true
          regexp: ".*"
          most_recent: 0
          max_age_seconds: 0
          exclude_deleted: true
        tags:
          enabled: true
          regexp: ".*"
          most_recent: 0
          max_age_seconds: 0
          exclude_deleted: true
        merge_requests:
          enabled: true
          most_recent: 0
          max_age_seconds: 0
      pipeline:
        jobs:
          enabled: true
          from_child_pipelines:
            enabled: true
          runner_description:
            enabled: true
            aggregation_regexp: shared-runners-manager-(\d*)\.gitlab\.com
        variables:
          enabled: false
          regexp: ".*"
        test_reports:
          enabled: false
          test_cases:
            enabled: false
  wildcards:
    - {}

Logs:

  time="2024-08-07T22:42:33Z" level=info msg=configured gitlab-endpoint="https://gitlab.com" gitlab-rate-limit=5rps
time="2024-08-07T22:42:33Z" level=info msg="pull projects from wildcards" on-init=yes scheduled="every 1800s"
time="2024-08-07T22:42:33Z" level=info msg="pull environments from projects" on-init=yes scheduled="every 1800s"
time="2024-08-07T22:42:33Z" level=info msg="pull refs from projects" on-init=yes scheduled="every 300s"
time="2024-08-07T22:42:33Z" level=info msg="pull metrics" on-init=yes scheduled="every 30s"
time="2024-08-07T22:42:33Z" level=info msg="garbage collect projects" on-init=yes scheduled="every 14400s"
time="2024-08-07T22:42:33Z" level=info msg="garbage collect environments" on-init=yes scheduled="every 14400s"
time="2024-08-07T22:42:33Z" level=info msg="garbage collect refs" on-init=yes scheduled="every 1800s"
time="2024-08-07T22:42:33Z" level=info msg="garbage collect metrics" on-init=yes scheduled="every 600s"
time="2024-08-07T22:42:33Z" level=debug msg="opentelemetry.grpc_endpoint is not configured, skipping open telemetry support"
time="2024-08-07T22:42:33Z" level=info msg="redis url configured, initializing connection.."
time="2024-08-07T22:42:33Z" level=info msg="connected to redis"
time="2024-08-07T22:42:33Z" level=debug msg="task scheduled" interval_seconds=1800 task=PullEnvironmentsFromProjects
time="2024-08-07T22:42:33Z" level=debug msg="task scheduled" interval_seconds=300 task=PullRefsFromProjects
time="2024-08-07T22:42:33Z" level=debug msg="task scheduled" interval_seconds=30 task=PullMetrics
time="2024-08-07T22:42:33Z" level=info msg="scheduling refs from projects pull" projects-count=0
time="2024-08-07T22:42:33Z" level=debug msg="task scheduled" interval_seconds=14400 task=GarbageCollectProjects
time="2024-08-07T22:42:33Z" level=info msg="starting 'projects' garbage collection"
time="2024-08-07T22:42:33Z" level=debug msg="task scheduled" interval_seconds=14400 task=GarbageCollectEnvironments
time="2024-08-07T22:42:33Z" level=debug msg="listing all projects from wildcard" wildcard-archived=false wildcard-owner-include-subgroups=false wildcard-owner-kind= wildcard-owner-name= wildcard-search=
time="2024-08-07T22:42:33Z" level=debug msg="task scheduled" interval_seconds=1800 task=GarbageCollectRefs
time="2024-08-07T22:42:33Z" level=info msg="starting 'refs' garbage collection"
time="2024-08-07T22:42:33Z" level=debug msg="task scheduled" interval_seconds=600 task=GarbageCollectMetrics
time="2024-08-07T22:42:33Z" level=debug msg="task scheduled" interval_seconds=1800 task=PullProjectsFromWildcards
time="2024-08-07T22:42:33Z" level=info msg="ending 'refs' garbage collection"
time="2024-08-07T22:42:33Z" level=info msg="internal monitoring listener set" host="127.0.0.1:8082" path= scheme=tcp
time="2024-08-07T22:42:33Z" level=info msg="http server started" controller-uuid=10b30980-7c7d-41a7-9dec-eda2dbdc2a75 listen-address=":8080" metrics-endpoint-enabled=true openmetrics-encoding-enabled=false pprof-endpoint-enabled=false webhook-endpoint-enabled=false
time="2024-08-07T22:42:33Z" level=info msg="scheduling projects from wildcards pull" wildcards-count=1
time="2024-08-07T22:43:03Z" level=info msg="scheduling metrics pull" environments-count=0 refs-count=0
time="2024-08-07T22:43:33Z" level=info msg="scheduling metrics pull" environments-count=0 refs-count=0
time="2024-08-07T22:44:03Z" level=info msg="scheduling metrics pull" environments-count=0 refs-count=0
time="2024-08-07T22:44:33Z" level=info msg="scheduling metrics pull" environments-count=0 refs-count=0
time="2024-08-07T22:45:03Z" level=info msg="scheduling metrics pull" environments-count=0 refs-count=0
time="2024-08-07T22:45:33Z" level=info msg="scheduling metrics pull" environments-count=0 refs-count=0
time="2024-08-07T22:46:03Z" level=info msg="scheduling metrics pull" environments-count=0 refs-count=0
time="2024-08-07T22:46:33Z" level=info msg="scheduling metrics pull" environments-count=0 refs-count=0
time="2024-08-07T22:47:03Z" level=info msg="scheduling metrics pull" environments-count=0 refs-count=0
time="2024-08-07T22:47:33Z" level=info msg="scheduling refs from projects pull" projects-count=0
time="2024-08-07T22:47:33Z" level=info msg="scheduling metrics pull" environments-count=0 refs-count=0
time="2024-08-07T22:48:03Z" level=info msg="scheduling metrics pull" environments-count=0 refs-count=0

Metrics:

curl http://gitlab-ci-pipelines-exporter.monitoring.svc.cluster.local:8080/metrics
# HELP gcpe_currently_queued_tasks_count Number of tasks in the queue
# TYPE gcpe_currently_queued_tasks_count gauge
gcpe_currently_queued_tasks_count 2
# HELP gcpe_environments_count Number of GitLab environments being exported
# TYPE gcpe_environments_count gauge
gcpe_environments_count 0
# HELP gcpe_executed_tasks_count Number of tasks executed
# TYPE gcpe_executed_tasks_count gauge
gcpe_executed_tasks_count 107
# HELP gcpe_gitlab_api_requests_count GitLab API requests count
# TYPE gcpe_gitlab_api_requests_count gauge
gcpe_gitlab_api_requests_count 186
# HELP gcpe_gitlab_api_requests_limit GitLab API requests available in the api limit
# TYPE gcpe_gitlab_api_requests_limit gauge
gcpe_gitlab_api_requests_limit 0
# HELP gcpe_gitlab_api_requests_remaining GitLab API requests remaining in the api limit
# TYPE gcpe_gitlab_api_requests_remaining gauge
gcpe_gitlab_api_requests_remaining 0
# HELP gcpe_metrics_count Number of GitLab pipelines metrics being exported
# TYPE gcpe_metrics_count gauge
gcpe_metrics_count 0
# HELP gcpe_projects_count Number of GitLab projects being exported
# TYPE gcpe_projects_count gauge
gcpe_projects_count 0
# HELP gcpe_refs_count Number of GitLab refs being exported
# TYPE gcpe_refs_count gauge
gcpe_refs_count 0
# HELP promhttp_metric_handler_errors_total Total number of internal errors encountered by the promhttp metric handler.
# TYPE promhttp_metric_handler_errors_total counter
promhttp_metric_handler_errors_total{cause="encoding"} 0
promhttp_metric_handler_errors_total{cause="gathering"} 0

[sibzz]

I have the same issue. nothing is pulled if i use the wildcards: -{} approach

[ynsdll]

I have same issue.
I tried two different type of wildcards - {} and adding owner. it doesnt work for me

First,
platform is owner for all projects. I can see as a owner of all projects members section. But doesnt work.

# config -- configuration of the exporter
config:
  gitlab:
    url: https://gitlab.com
    maximum_jobs_queue_size: 10000
  project_defaults:
    pull:
      environments:
        enabled: true
      pipeline:
        jobs:
          enabled: true
  wildcards:
    - owner:
        name: platform
        kind: group
        include_subgroups: true

# config -- configuration of the exporter
config:
  gitlab:
    url: https://gitlab.com
    maximum_jobs_queue_size: 10000
  project_defaults:
    pull:
      environments:
        enabled: true
      pipeline:
        jobs:
          enabled: true
  wildcards:
    - {}