Create ServiceMonitor for operator metrics programmatically

Signed-off-by: Israel Blancas <[email protected]>
open-telemetry · Oct 18, 2024 · 19f5dbc · 19f5dbc
1 parent b703b78
commit 19f5dbc
Show file tree

Hide file tree

Showing 21 changed files with 352 additions and 33 deletions.
diff --git a/.chloggen/3370-create-dynamic-sm.yaml b/.chloggen/3370-create-dynamic-sm.yaml
@@ -0,0 +1,19 @@
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: enhancement
+
+# The name of the component, or a single word describing the area of concern, (e.g. collector, target allocator, auto-instrumentation, opamp, github action)
+component: operator
+
+# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Programmatically create the `ServiceMonitor` for the operator metrics endpoint, ensuring correct namespace handling and dynamic configuration.
+
+# One or more tracking issues related to the change
+issues: [3370]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext: |
+  Previously, the `ServiceMonitor` was created statically from a manifest file, causing failures when the
+  operator was deployed in a non-default namespace. This enhancement ensures automatic adjustment of the
+  `serverName` and seamless metrics scraping.
diff --git a/...shift/manifests/opentelemetry-operator-controller-manager-metrics-service_v1_service.yaml b/...shift/manifests/opentelemetry-operator-controller-manager-metrics-service_v1_service.yaml
@@ -1,9 +1,13 @@
 apiVersion: v1
 kind: Service
 metadata:
+  annotations:
+    service.beta.openshift.io/serving-cert-secret-name: opentelemetry-operator-metrics
   creationTimestamp: null
   labels:
+    app.kubernetes.io/managed-by: operator-lifecycle-manager
     app.kubernetes.io/name: opentelemetry-operator
+    app.kubernetes.io/part-of: opentelemetry-operator
     control-plane: controller-manager
   name: opentelemetry-operator-controller-manager-metrics-service
 spec:
@@ -13,7 +17,9 @@ spec:
     protocol: TCP
     targetPort: https
   selector:
+    app.kubernetes.io/managed-by: operator-lifecycle-manager
     app.kubernetes.io/name: opentelemetry-operator
+    app.kubernetes.io/part-of: opentelemetry-operator
     control-plane: controller-manager
 status:
   loadBalancer: {}
diff --git a/...fests/opentelemetry-operator-metrics-reader_rbac.authorization.k8s.io_v1_clusterrole.yaml b/...fests/opentelemetry-operator-metrics-reader_rbac.authorization.k8s.io_v1_clusterrole.yaml
@@ -3,7 +3,9 @@ kind: ClusterRole
 metadata:
   creationTimestamp: null
   labels:
+    app.kubernetes.io/managed-by: operator-lifecycle-manager
     app.kubernetes.io/name: opentelemetry-operator
+    app.kubernetes.io/part-of: opentelemetry-operator
   name: opentelemetry-operator-metrics-reader
 rules:
 - nonResourceURLs:

diff --git a/...ests/opentelemetry-operator-prometheus-rules_monitoring.coreos.com_v1_prometheusrule.yaml b/...ests/opentelemetry-operator-prometheus-rules_monitoring.coreos.com_v1_prometheusrule.yaml
@@ -0,0 +1,24 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  labels:
+    app.kubernetes.io/managed-by: operator-lifecycle-manager
+    app.kubernetes.io/name: opentelemetry-operator
+    app.kubernetes.io/part-of: opentelemetry-operator
+  name: opentelemetry-operator-prometheus-rules
+spec:
+  groups:
+  - name: opentelemetry-operator-monitoring.rules
+    rules:
+    - expr: sum by (type) (opentelemetry_collector_receivers)
+      record: type:opentelemetry_collector_receivers:sum
+    - expr: sum by (type) (opentelemetry_collector_exporters)
+      record: type:opentelemetry_collector_exporters:sum
+    - expr: sum by (type) (opentelemetry_collector_processors)
+      record: type:opentelemetry_collector_processors:sum
+    - expr: sum by (type) (opentelemetry_collector_extensions)
+      record: type:opentelemetry_collector_extensions:sum
+    - expr: sum by (type) (opentelemetry_collector_connectors)
+      record: type:opentelemetry_collector_connectors:sum
+    - expr: sum by (type) (opentelemetry_collector_info)
+      record: type:opentelemetry_collector_info:sum
diff --git a/...nshift/manifests/opentelemetry-operator-prometheus_rbac.authorization.k8s.io_v1_role.yaml b/...nshift/manifests/opentelemetry-operator-prometheus_rbac.authorization.k8s.io_v1_role.yaml
@@ -0,0 +1,22 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  annotations:
+    include.release.openshift.io/self-managed-high-availability: "true"
+    include.release.openshift.io/single-node-developer: "true"
+  labels:
+    app.kubernetes.io/managed-by: operator-lifecycle-manager
+    app.kubernetes.io/name: opentelemetry-operator
+    app.kubernetes.io/part-of: opentelemetry-operator
+  name: opentelemetry-operator-prometheus
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - services
+  - endpoints
+  - pods
+  verbs:
+  - get
+  - list
+  - watch
diff --git a/...manifests/opentelemetry-operator-prometheus_rbac.authorization.k8s.io_v1_rolebinding.yaml b/...manifests/opentelemetry-operator-prometheus_rbac.authorization.k8s.io_v1_rolebinding.yaml
@@ -0,0 +1,19 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  annotations:
+    include.release.openshift.io/self-managed-high-availability: "true"
+    include.release.openshift.io/single-node-developer: "true"
+  labels:
+    app.kubernetes.io/managed-by: operator-lifecycle-manager
+    app.kubernetes.io/name: opentelemetry-operator
+    app.kubernetes.io/part-of: opentelemetry-operator
+  name: opentelemetry-operator-prometheus
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: opentelemetry-operator-prometheus
+subjects:
+- kind: ServiceAccount
+  name: prometheus-k8s
+  namespace: openshift-monitoring
diff --git a/bundle/openshift/manifests/opentelemetry-operator-webhook-service_v1_service.yaml b/bundle/openshift/manifests/opentelemetry-operator-webhook-service_v1_service.yaml
@@ -3,15 +3,19 @@ kind: Service
 metadata:
   creationTimestamp: null
   labels:
+    app.kubernetes.io/managed-by: operator-lifecycle-manager
     app.kubernetes.io/name: opentelemetry-operator
+    app.kubernetes.io/part-of: opentelemetry-operator
   name: opentelemetry-operator-webhook-service
 spec:
   ports:
   - port: 443
     protocol: TCP
     targetPort: 9443
   selector:
+    app.kubernetes.io/managed-by: operator-lifecycle-manager
     app.kubernetes.io/name: opentelemetry-operator
+    app.kubernetes.io/part-of: opentelemetry-operator
     control-plane: controller-manager
 status:
   loadBalancer: {}
diff --git a/bundle/openshift/manifests/opentelemetry-operator.clusterserviceversion.yaml b/bundle/openshift/manifests/opentelemetry-operator.clusterserviceversion.yaml
@@ -99,7 +99,7 @@ metadata:
     categories: Logging & Tracing,Monitoring
     certified: "false"
     containerImage: ghcr.io/open-telemetry/opentelemetry-operator/opentelemetry-operator
-    createdAt: "2024-10-16T10:10:50Z"
+    createdAt: "2024-10-18T17:14:07Z"
     description: Provides the OpenTelemetry components, including the Collector
     operators.operatorframework.io/builder: operator-sdk-v1.29.0
     operators.operatorframework.io/project_layout: go.kubebuilder.io/v3
@@ -453,20 +453,26 @@ spec:
         serviceAccountName: opentelemetry-operator-controller-manager
       deployments:
       - label:
+          app.kubernetes.io/managed-by: operator-lifecycle-manager
           app.kubernetes.io/name: opentelemetry-operator
+          app.kubernetes.io/part-of: opentelemetry-operator
           control-plane: controller-manager
         name: opentelemetry-operator-controller-manager
         spec:
           replicas: 1
           selector:
             matchLabels:
+              app.kubernetes.io/managed-by: operator-lifecycle-manager
               app.kubernetes.io/name: opentelemetry-operator
+              app.kubernetes.io/part-of: opentelemetry-operator
               control-plane: controller-manager
           strategy: {}
           template:
             metadata:
               labels:
+                app.kubernetes.io/managed-by: operator-lifecycle-manager
                 app.kubernetes.io/name: opentelemetry-operator
+                app.kubernetes.io/part-of: opentelemetry-operator
                 control-plane: controller-manager
             spec:
               containers:
@@ -477,9 +483,9 @@ spec:
                 - --zap-time-encoding=rfc3339nano
                 - --enable-nginx-instrumentation=true
                 - --enable-go-instrumentation=true
-                - --enable-multi-instrumentation=true
                 - --openshift-create-dashboard=true
                 - --feature-gates=+operator.observability.prometheus
+                - --enable-cr-metrics=true
                 env:
                 - name: SERVICE_ACCOUNT_NAME
                   valueFrom:
@@ -516,6 +522,10 @@ spec:
                 - --upstream=http://127.0.0.1:8080/
                 - --logtostderr=true
                 - --v=0
+                - --tls-cert-file=/var/run/tls/server/tls.crt
+                - --tls-private-key-file=/var/run/tls/server/tls.key
+                - --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256,TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_CBC_SHA256
+                - --tls-min-version=VersionTLS12
                 image: gcr.io/kubebuilder/kube-rbac-proxy:v0.13.1
                 name: kube-rbac-proxy
                 ports:
@@ -529,9 +539,16 @@ spec:
                   requests:
                     cpu: 5m
                     memory: 64Mi
+                volumeMounts:
+                - mountPath: /var/run/tls/server
+                  name: opentelemetry-operator-metrics-cert
               serviceAccountName: opentelemetry-operator-controller-manager
               terminationGracePeriodSeconds: 10
               volumes:
+              - name: opentelemetry-operator-metrics-cert
+                secret:
+                  defaultMode: 420
+                  secretName: opentelemetry-operator-metrics
               - name: cert
                 secret:
                   defaultMode: 420

diff --git a/bundle/openshift/manifests/opentelemetry.io_instrumentations.yaml b/bundle/openshift/manifests/opentelemetry.io_instrumentations.yaml
@@ -5,7 +5,9 @@ metadata:
     controller-gen.kubebuilder.io/version: v0.16.1
   creationTimestamp: null
   labels:
+    app.kubernetes.io/managed-by: operator-lifecycle-manager
     app.kubernetes.io/name: opentelemetry-operator
+    app.kubernetes.io/part-of: opentelemetry-operator
   name: instrumentations.opentelemetry.io
 spec:
   group: opentelemetry.io

diff --git a/bundle/openshift/manifests/opentelemetry.io_opampbridges.yaml b/bundle/openshift/manifests/opentelemetry.io_opampbridges.yaml
@@ -6,7 +6,9 @@ metadata:
     controller-gen.kubebuilder.io/version: v0.16.1
   creationTimestamp: null
   labels:
+    app.kubernetes.io/managed-by: operator-lifecycle-manager
     app.kubernetes.io/name: opentelemetry-operator
+    app.kubernetes.io/part-of: opentelemetry-operator
   name: opampbridges.opentelemetry.io
 spec:
   group: opentelemetry.io

diff --git a/bundle/openshift/manifests/opentelemetry.io_opentelemetrycollectors.yaml b/bundle/openshift/manifests/opentelemetry.io_opentelemetrycollectors.yaml
@@ -6,7 +6,9 @@ metadata:
     controller-gen.kubebuilder.io/version: v0.16.1
   creationTimestamp: null
   labels:
+    app.kubernetes.io/managed-by: operator-lifecycle-manager
     app.kubernetes.io/name: opentelemetry-operator
+    app.kubernetes.io/part-of: opentelemetry-operator
   name: opentelemetrycollectors.opentelemetry.io
 spec:
   conversion:

diff --git a/config/default/kustomization.yaml b/config/default/kustomization.yaml
@@ -18,8 +18,6 @@ bases:
 - ../manager
 - ../webhook
 - ../certmanager
-# [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'. 
-#- ../prometheus
 
 patchesStrategicMerge:
   # Protect the /metrics endpoint by putting it behind auth.

diff --git a/config/overlays/openshift/kustomization.yaml b/config/overlays/openshift/kustomization.yaml
@@ -1,10 +1,21 @@
 resources:
 - ../../default
 
+labels:
+- pairs:
+    app.kubernetes.io/name: opentelemetry-operator
+    app.kubernetes.io/part-of: opentelemetry-operator
+    app.kubernetes.io/managed-by: operator-lifecycle-manager
+  includeSelectors: true
+
 patches:
  - target:
       group: apps
       version: v1
       kind: Deployment
       name: controller-manager
    path: manager-patch.yaml
+
+patchesStrategicMerge:
+- metrics_service_tls_patch.yaml
+- manager_auth_proxy_tls_patch.yaml
diff --git a/config/overlays/openshift/manager-patch.yaml b/config/overlays/openshift/manager-patch.yaml
@@ -7,6 +7,6 @@
     - --zap-time-encoding=rfc3339nano
     - --enable-nginx-instrumentation=true
     - '--enable-go-instrumentation=true'
-    - '--enable-multi-instrumentation=true'
     - '--openshift-create-dashboard=true'
     - '--feature-gates=+operator.observability.prometheus'
+    - '--enable-cr-metrics=true'
diff --git a/config/overlays/openshift/manager_auth_proxy_tls_patch.yaml b/config/overlays/openshift/manager_auth_proxy_tls_patch.yaml
@@ -0,0 +1,29 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: controller-manager
+  namespace: system
+spec:
+  template:
+    spec:
+      containers:
+      - name: manager # without this line, kustomize reorders the containers, making kube-rbac-proxy the default container
+      - name: kube-rbac-proxy
+        args:
+        - "--secure-listen-address=0.0.0.0:8443"
+        - "--upstream=http://127.0.0.1:8080/"
+        - "--logtostderr=true"
+        - "--v=0"
+        - "--tls-cert-file=/var/run/tls/server/tls.crt"
+        - "--tls-private-key-file=/var/run/tls/server/tls.key"
+        - "--tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256,TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_CBC_SHA256"
+        - "--tls-min-version=VersionTLS12"
+        volumeMounts:
+        - mountPath: /var/run/tls/server
+          name: opentelemetry-operator-metrics-cert
+      volumes:
+      - name: opentelemetry-operator-metrics-cert
+        secret:
+          defaultMode: 420
+          # secret generated by the 'service.beta.openshift.io/serving-cert-secret-name' annotation on the metrics-service
+          secretName: opentelemetry-operator-metrics
diff --git a/config/overlays/openshift/metrics_service_tls_patch.yaml b/config/overlays/openshift/metrics_service_tls_patch.yaml
@@ -0,0 +1,7 @@
+apiVersion: v1
+kind: Service
+metadata:
+  annotations:
+    service.beta.openshift.io/serving-cert-secret-name: opentelemetry-operator-metrics
+  name: controller-manager-metrics-service
+  namespace: system
diff --git a/config/prometheus/kustomization.yaml b/config/prometheus/kustomization.yaml
diff --git a/config/prometheus/monitor.yaml b/config/prometheus/monitor.yaml