From 9bd9bcb71c1cf61a6fc9148e9aab70e5b00c156c Mon Sep 17 00:00:00 2001 From: Joshua MacDonald Date: Thu, 4 Jan 2024 10:24:12 -0800 Subject: [PATCH 1/5] delete and move --- arrow/otelcolarrow-build.yaml | 172 +++++++++++++++++++++++++--------- gateway-build.yaml | 142 ---------------------------- 2 files changed, 128 insertions(+), 186 deletions(-) delete mode 100644 gateway-build.yaml diff --git a/arrow/otelcolarrow-build.yaml b/arrow/otelcolarrow-build.yaml index 056fd3a..5233714 100644 --- a/arrow/otelcolarrow-build.yaml +++ b/arrow/otelcolarrow-build.yaml @@ -1,58 +1,142 @@ -# This file describes a sample OpenTelemetry Collector build -# containing the primary OpenTelemetry Protocol with Apache Arrow -# components and a few generally useful components from the -# OpenTelemetry Collector Contrib repository that have proven useful. +# This is a sample OpenTelemetry Collector builder configuration file +# meant to be used by ServiceNow Cloud Observability customers in +# setting up gateway-mode telemetry collectors and sending to our SaaS +# using the new OTel-Arrow protocol. # -# The full list of components is detailed below. Many of these -# components are optional and meant to assist with experimentation -# and validation of the primary components. +# This configuration file describes the components that will be built +# into a custom OpenTelemetry collector. For an overview of this +# process, see: # -# This configuration file be used as a starting point for building a -# custom gateway collector for deploying OpenTelemetry Protocol with -# Apache Arrow in production. See the instructions in BUILDING.md. +# https://opentelemetry.io/docs/collector/custom-collector/ +# +# Note: This is a relatively advanced operation, as it requires installing a Golang +# toolchain to execute and build the collector this way. When you have this file +# configured to your liking, +# +# go install go.opentelemetry.io/collector/cmd/builder@latest +# builder --config build-config.yaml +# +# the built collector artifact is placed in the `./dist` directory. dist: - module: github.com/open-telemetry/otel-arrow/collector/cmd/otelarrowcol - # Project-internal use: Directory path required for the `make - # genotelarrowcol`, which the Dockerfile also recognizes. + # See https://github.com/open-telemetry/opentelemetry-collector/blob/main/cmd/builder/README.md + # for more detail on these options. # - # Users: This can be customized for integration into your CI/CD system. - output_path: dist/ - name: otelarrowcol - - description: OpenTelemetry Protocol with Apache Arrow development collector, for testing and evaluation - - # Note: this version number is replaced to match the current release using `sed` - # during the release process, see ../../../RELEASING.md. - version: 0.10.0 - - # Note: This should match the version of the core and contrib - # collector components used below (e.g., the debugexporter and - # otlphttpexporter versions below). + # This is the name of the `go.mod` module file generated by the + # builder, which may be visible in a CI/CD pipeline, for example. + module: github.com/myorg/telemetrygateway/dist + # The collector name shows in startup logs, for example, and other + # internal telemetry (optional). + name: myorgtelemetrygateway + # The version of the collector (optional). + version: 0.1.0 + # This indicates which version of the core collector components as + # well as the builder version. At the time of this writing, we + # recommend using at least the current release. otelcol_version: 0.91.0 +# We recommend building in both the OTLP exporter and the OTel-Arrow +# exporter. exporters: - # This is the core OpenTelemetry Protocol with Apache Arrow exporter, - # recommended for exporting to ServiceNow Cloud Observability. - - import: github.com/open-telemetry/otel-arrow/collector/exporter/otelarrowexporter - gomod: github.com/open-telemetry/otel-arrow/collector v0.13.0 - - # The following components may be useful for debugging. - - # The debug exporter, useful for printing telemetry to the console. - - gomod: go.opentelemetry.io/collector/exporter/debugexporter v0.91.0 - # This is the core OTLP/HTTP exporter, useful for forwarding OTLP/HTTP - - gomod: go.opentelemetry.io/collector/exporter/otlphttpexporter v0.91.0 - # This is the core OTLP/gRPC exporter, useful for forwarding OTLP/gRPC. + # We have submitted the OTel-Arrow exporter to be included in the + # OpenTelemetry Collector-Contrib release, so that its functionality + # becomes available in the public contrib collector build. Until the + # submission is finished, the component can be imported directly from + # the otel-arrow repository. + # + # Use this exporter stanza: + - gomod: github.com/open-telemetry/otel-arrow/collector v0.13.0 + import: github.com/open-telemetry/otel-arrow/collector/exporter/otelarrowexporter + # Note that while OTel-Arrow supports traces, metrics, and logs. + # ServiceNow's initial release is limited to OTel-Arrow traces + # support. Contact your ServiceNow representative to discuss our + # deployment of OTel-Arrow for logs and metrics. + + # As an alternate to the OTel-Arrow exporter (e.g., in case + # malfunction or performance issue), we recommend building with the + # core OTLP exporter as a fallback. The OTel-Arrow component is + # derived from the OTLP component, so it supports compatible + # configuration. Generally, remove the `arrow` subection of the + # `otelarrow` exporter configuration and it will apply to the OTLP + # exporter. - gomod: go.opentelemetry.io/collector/exporter/otlpexporter v0.91.0 + # In case you want to record an OTLP telemetry session to a JSON + # file, we recommend this additional utility. See also the + # obfuscation processor, listed below. The OTel-Arrow provides + # offline tools that can help explain poor compression performance + # using inputs generated by this exporter, for example. + - gomod: github.com/open-telemetry/otel-arrow/collector v0.13.0 + import: github.com/open-telemetry/otel-arrow/collector/exporter/fileexporter + receivers: - # This is the core OpenTelemetry Protocol with Apache Arrow receiver, - # recommended for receiving OTLP/gRPC and OTel-Arrow. - - import: github.com/open-telemetry/otel-arrow/collector/receiver/otelarrowreceiver - gomod: github.com/open-telemetry/otel-arrow/collector v0.13.0 - # This is the core OTLP receiver, necessary for receiving OTLP/HTTP. + # The OTel-Arrow receiver supports multiple protocols including OTel-Arrow, OTLP gRPC, + # OTLP HTTP/proto, and OTLP HTTP/json. We recommend using this component to receive + # OTLP (all forms) and OTel-Arrow in a single component. + # + # This enables other OTel Collectors or experimental SDKs that support OTel-Arrow to + # send to this collector, such as this one for the OTel-Go Trace SDK: + # https://github.com/lightstep/otel-launcher-go/tree/main/lightstep/sdk/trace/exporters/otlp/otelcol + - gomod: github.com/open-telemetry/otel-arrow/collector v0.13.0 + import: github.com/open-telemetry/otel-arrow/collector/receiver/otelarrowreceiver + # As an alternate to the OTel-Arrow receiver (e.g., in case malfunction or performance + # issue), we recommend building with the core OTLP receiver as a fallback. The + # OTel-Arrow component is derived from the OTLP component, so it supports compatible + # configuration. Generally, remove the `arrow` subection of the `otelarrow` receiver + # configuration and it will apply to the OTLP exporter. - gomod: go.opentelemetry.io/collector/receiver/otlpreceiver v0.91.0 + # You may wish to enable other receivers from the Collector-Contrib repository or + # elsewhere. Here are some that might be useful: + # + # - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/prometheusreceiver v0.91.0 + # - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/hostmetricsreceiver v0.91.0 + # - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/kubeletstatsreceiver v0.91.0 + # - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/k8sclusterreceiver v0.91.0 + +processors: + # We recommend use of the batch processor. + - gomod: go.opentelemetry.io/collector/processor/batchprocessor v0.91.0 + + # We recommend building with the follow processor for obfuscation, in case you want to + # record telemetry sessions for offline analysis. + - gomod: github.com/open-telemetry/otel-arrow/collector v0.13.0 + import: github.com/open-telemetry/otel-arrow/collector/processor/obfuscationprocessor + + # We emphatically DO NOT recommend use of the memory limiter processor, i.e., do not + # build with go.opentelemetry.io/collector/processor/memorylimiter. + + # These Collector-Contrib components are referred to in the charts in this package. + - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/resourcedetectionprocessor v0.91.0 + - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/k8sattributesprocessor v0.91.0 + - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/resourceprocessor v0.91.0 + - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/metricstransformprocessor v0.91.0 + + # ServiceNow does not at this time recommend the probabilistic sampler processor, + # because it leads to undercounting of spans. We will update this recommendation + # when the sampler is fully supported. + # - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/probabilisticsamplerprocessor v0.91.0 + + # TODO: Note: We will recommend use of a new uncompressed-size processor + # when it is ready. + + # You may be interested in other transform components in the + # Collector-Contrib repository. Here are some that might be useful: + # + # - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/attributesprocessor v0.91.0 extensions: - - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/extension/headerssetterextension v0.91.0 + # You may be interested in externsions from the Collector-Contrib repository. None are + # required. Here are some that might be useful: + - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/extension/healthcheckextension v0.91.0 + - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/extension/pprofextension v0.91.0 + + # We do not support use of headersetterextension to apply the lightstep-access-token + # header on a per-request basis. Contact ServiceNow if you are interested in + # multi-tenancy with OTel-Arrow. Do not build with + # github.com/open-telemetry/opentelemetry-collector-contrib/extension/headerssetterextension + +# If you are building this collector in an Golang monorepo, or to apply security patches +# and other overrides to the go.mod file, use this section. +replaces: + # For example: + # - github.com/at-risk/code/module => github.com/at-risk/code/module v2.2.2 diff --git a/gateway-build.yaml b/gateway-build.yaml deleted file mode 100644 index 5233714..0000000 --- a/gateway-build.yaml +++ /dev/null @@ -1,142 +0,0 @@ -# This is a sample OpenTelemetry Collector builder configuration file -# meant to be used by ServiceNow Cloud Observability customers in -# setting up gateway-mode telemetry collectors and sending to our SaaS -# using the new OTel-Arrow protocol. -# -# This configuration file describes the components that will be built -# into a custom OpenTelemetry collector. For an overview of this -# process, see: -# -# https://opentelemetry.io/docs/collector/custom-collector/ -# -# Note: This is a relatively advanced operation, as it requires installing a Golang -# toolchain to execute and build the collector this way. When you have this file -# configured to your liking, -# -# go install go.opentelemetry.io/collector/cmd/builder@latest -# builder --config build-config.yaml -# -# the built collector artifact is placed in the `./dist` directory. -dist: - # See https://github.com/open-telemetry/opentelemetry-collector/blob/main/cmd/builder/README.md - # for more detail on these options. - # - # This is the name of the `go.mod` module file generated by the - # builder, which may be visible in a CI/CD pipeline, for example. - module: github.com/myorg/telemetrygateway/dist - # The collector name shows in startup logs, for example, and other - # internal telemetry (optional). - name: myorgtelemetrygateway - # The version of the collector (optional). - version: 0.1.0 - # This indicates which version of the core collector components as - # well as the builder version. At the time of this writing, we - # recommend using at least the current release. - otelcol_version: 0.91.0 - -# We recommend building in both the OTLP exporter and the OTel-Arrow -# exporter. -exporters: - # We have submitted the OTel-Arrow exporter to be included in the - # OpenTelemetry Collector-Contrib release, so that its functionality - # becomes available in the public contrib collector build. Until the - # submission is finished, the component can be imported directly from - # the otel-arrow repository. - # - # Use this exporter stanza: - - gomod: github.com/open-telemetry/otel-arrow/collector v0.13.0 - import: github.com/open-telemetry/otel-arrow/collector/exporter/otelarrowexporter - # Note that while OTel-Arrow supports traces, metrics, and logs. - # ServiceNow's initial release is limited to OTel-Arrow traces - # support. Contact your ServiceNow representative to discuss our - # deployment of OTel-Arrow for logs and metrics. - - # As an alternate to the OTel-Arrow exporter (e.g., in case - # malfunction or performance issue), we recommend building with the - # core OTLP exporter as a fallback. The OTel-Arrow component is - # derived from the OTLP component, so it supports compatible - # configuration. Generally, remove the `arrow` subection of the - # `otelarrow` exporter configuration and it will apply to the OTLP - # exporter. - - gomod: go.opentelemetry.io/collector/exporter/otlpexporter v0.91.0 - - # In case you want to record an OTLP telemetry session to a JSON - # file, we recommend this additional utility. See also the - # obfuscation processor, listed below. The OTel-Arrow provides - # offline tools that can help explain poor compression performance - # using inputs generated by this exporter, for example. - - gomod: github.com/open-telemetry/otel-arrow/collector v0.13.0 - import: github.com/open-telemetry/otel-arrow/collector/exporter/fileexporter - -receivers: - # The OTel-Arrow receiver supports multiple protocols including OTel-Arrow, OTLP gRPC, - # OTLP HTTP/proto, and OTLP HTTP/json. We recommend using this component to receive - # OTLP (all forms) and OTel-Arrow in a single component. - # - # This enables other OTel Collectors or experimental SDKs that support OTel-Arrow to - # send to this collector, such as this one for the OTel-Go Trace SDK: - # https://github.com/lightstep/otel-launcher-go/tree/main/lightstep/sdk/trace/exporters/otlp/otelcol - - gomod: github.com/open-telemetry/otel-arrow/collector v0.13.0 - import: github.com/open-telemetry/otel-arrow/collector/receiver/otelarrowreceiver - # As an alternate to the OTel-Arrow receiver (e.g., in case malfunction or performance - # issue), we recommend building with the core OTLP receiver as a fallback. The - # OTel-Arrow component is derived from the OTLP component, so it supports compatible - # configuration. Generally, remove the `arrow` subection of the `otelarrow` receiver - # configuration and it will apply to the OTLP exporter. - - gomod: go.opentelemetry.io/collector/receiver/otlpreceiver v0.91.0 - - # You may wish to enable other receivers from the Collector-Contrib repository or - # elsewhere. Here are some that might be useful: - # - # - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/prometheusreceiver v0.91.0 - # - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/hostmetricsreceiver v0.91.0 - # - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/kubeletstatsreceiver v0.91.0 - # - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/k8sclusterreceiver v0.91.0 - -processors: - # We recommend use of the batch processor. - - gomod: go.opentelemetry.io/collector/processor/batchprocessor v0.91.0 - - # We recommend building with the follow processor for obfuscation, in case you want to - # record telemetry sessions for offline analysis. - - gomod: github.com/open-telemetry/otel-arrow/collector v0.13.0 - import: github.com/open-telemetry/otel-arrow/collector/processor/obfuscationprocessor - - # We emphatically DO NOT recommend use of the memory limiter processor, i.e., do not - # build with go.opentelemetry.io/collector/processor/memorylimiter. - - # These Collector-Contrib components are referred to in the charts in this package. - - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/resourcedetectionprocessor v0.91.0 - - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/k8sattributesprocessor v0.91.0 - - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/resourceprocessor v0.91.0 - - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/metricstransformprocessor v0.91.0 - - # ServiceNow does not at this time recommend the probabilistic sampler processor, - # because it leads to undercounting of spans. We will update this recommendation - # when the sampler is fully supported. - # - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/probabilisticsamplerprocessor v0.91.0 - - # TODO: Note: We will recommend use of a new uncompressed-size processor - # when it is ready. - - # You may be interested in other transform components in the - # Collector-Contrib repository. Here are some that might be useful: - # - # - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/attributesprocessor v0.91.0 - -extensions: - # You may be interested in externsions from the Collector-Contrib repository. None are - # required. Here are some that might be useful: - - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/extension/healthcheckextension v0.91.0 - - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/extension/pprofextension v0.91.0 - - # We do not support use of headersetterextension to apply the lightstep-access-token - # header on a per-request basis. Contact ServiceNow if you are interested in - # multi-tenancy with OTel-Arrow. Do not build with - # github.com/open-telemetry/opentelemetry-collector-contrib/extension/headerssetterextension - -# If you are building this collector in an Golang monorepo, or to apply security patches -# and other overrides to the go.mod file, use this section. -replaces: - # For example: - # - github.com/at-risk/code/module => github.com/at-risk/code/module v2.2.2 From fac6d3ce1ed06b925fbe22c425b0b792f11b174c Mon Sep 17 00:00:00 2001 From: Joshua MacDonald Date: Thu, 4 Jan 2024 10:42:57 -0800 Subject: [PATCH 2/5] it builds --- .github/workflows/publish-image.yml | 3 +- arrow/Dockerfile | 2 +- arrow/otelcolarrow-build.yaml | 82 ++++++++++++++++------------- 3 files changed, 47 insertions(+), 40 deletions(-) diff --git a/.github/workflows/publish-image.yml b/.github/workflows/publish-image.yml index 5fbdf75..2161ce9 100644 --- a/.github/workflows/publish-image.yml +++ b/.github/workflows/publish-image.yml @@ -38,6 +38,7 @@ jobs: id: meta uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7 with: + # TODO: remove -EXPERIMENTAL below. images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-EXPERIMENTAL tags: | # Set the OTELCOL_VERSION here @@ -67,4 +68,4 @@ jobs: tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache \ No newline at end of file + cache-to: type=local,dest=/tmp/.buildx-cache diff --git a/arrow/Dockerfile b/arrow/Dockerfile index 6dd0977..288574e 100644 --- a/arrow/Dockerfile +++ b/arrow/Dockerfile @@ -6,7 +6,7 @@ ENV CGO_ENABLED=0 # Note the version MUST MATCH otelarrowcol-build.yaml # Future optimization - curl the release. -RUN go install go.opentelemetry.io/collector/cmd/builder@v0.89.0 +RUN go install go.opentelemetry.io/collector/cmd/builder@v0.91.0 # This command generates main.go, go.mod but does not update deps. RUN builder --config=/otelarrowcol/otelcolarrow-build.yaml diff --git a/arrow/otelcolarrow-build.yaml b/arrow/otelcolarrow-build.yaml index 5233714..25544e3 100644 --- a/arrow/otelcolarrow-build.yaml +++ b/arrow/otelcolarrow-build.yaml @@ -14,21 +14,32 @@ # configured to your liking, # # go install go.opentelemetry.io/collector/cmd/builder@latest -# builder --config build-config.yaml +# builder --config otelarrowcol-build.yaml # -# the built collector artifact is placed in the `./dist` directory. dist: # See https://github.com/open-telemetry/opentelemetry-collector/blob/main/cmd/builder/README.md # for more detail on these options. # # This is the name of the `go.mod` module file generated by the # builder, which may be visible in a CI/CD pipeline, for example. - module: github.com/myorg/telemetrygateway/dist + # + # module: github.com/myorg/telemetrygateway/dist + # + module: github.com/lightstep/otel-collector-charts/arrow/dist + + # the built collector artifact is placed in the `./dist` directory. + output_path: dist/ + # The collector name shows in startup logs, for example, and other - # internal telemetry (optional). - name: myorgtelemetrygateway - # The version of the collector (optional). - version: 0.1.0 + # internal telemetry (optional). You may wish to use, e.g., + # + # name: myorgtelemetrygateway + # description: My Organization telemetry collector + # + name: otelarrowcol + version: 0.10.0 + description: ServiceNow Cloud Observability OpenTelemetry Protocol with Apache Arrow gateway collector + # This indicates which version of the core collector components as # well as the builder version. At the time of this writing, we # recommend using at least the current release. @@ -37,29 +48,23 @@ dist: # We recommend building in both the OTLP exporter and the OTel-Arrow # exporter. exporters: - # We have submitted the OTel-Arrow exporter to be included in the - # OpenTelemetry Collector-Contrib release, so that its functionality - # becomes available in the public contrib collector build. Until the - # submission is finished, the component can be imported directly from - # the otel-arrow repository. - # - # Use this exporter stanza: + # This is the core OpenTelemetry Protocol with Apache Arrow exporter, + # recommended for exporting to ServiceNow Cloud Observability using + # either OTel-Arrow or standard OTLP over gRPC. - gomod: github.com/open-telemetry/otel-arrow/collector v0.13.0 import: github.com/open-telemetry/otel-arrow/collector/exporter/otelarrowexporter - # Note that while OTel-Arrow supports traces, metrics, and logs. - # ServiceNow's initial release is limited to OTel-Arrow traces - # support. Contact your ServiceNow representative to discuss our - # deployment of OTel-Arrow for logs and metrics. - - # As an alternate to the OTel-Arrow exporter (e.g., in case - # malfunction or performance issue), we recommend building with the - # core OTLP exporter as a fallback. The OTel-Arrow component is - # derived from the OTLP component, so it supports compatible - # configuration. Generally, remove the `arrow` subection of the - # `otelarrow` exporter configuration and it will apply to the OTLP - # exporter. + + # The following components may be useful for debugging. + + # As an alternate to the OTel-Arrow exporter, we recommend building + # with the core OTLP exporter as a fallback. These components use + # compatible configuration. - gomod: go.opentelemetry.io/collector/exporter/otlpexporter v0.91.0 + # In case OTLP/HTTP export is required, as opposed to the two + # gRPC options above. + - gomod: go.opentelemetry.io/collector/exporter/otlphttpexporter v0.91.0 + # In case you want to record an OTLP telemetry session to a JSON # file, we recommend this additional utility. See also the # obfuscation processor, listed below. The OTel-Arrow provides @@ -68,6 +73,9 @@ exporters: - gomod: github.com/open-telemetry/otel-arrow/collector v0.13.0 import: github.com/open-telemetry/otel-arrow/collector/exporter/fileexporter + # The debug exporter, useful for printing telemetry to the console. + - gomod: go.opentelemetry.io/collector/exporter/debugexporter v0.91.0 + receivers: # The OTel-Arrow receiver supports multiple protocols including OTel-Arrow, OTLP gRPC, # OTLP HTTP/proto, and OTLP HTTP/json. We recommend using this component to receive @@ -78,11 +86,8 @@ receivers: # https://github.com/lightstep/otel-launcher-go/tree/main/lightstep/sdk/trace/exporters/otlp/otelcol - gomod: github.com/open-telemetry/otel-arrow/collector v0.13.0 import: github.com/open-telemetry/otel-arrow/collector/receiver/otelarrowreceiver - # As an alternate to the OTel-Arrow receiver (e.g., in case malfunction or performance - # issue), we recommend building with the core OTLP receiver as a fallback. The - # OTel-Arrow component is derived from the OTLP component, so it supports compatible - # configuration. Generally, remove the `arrow` subection of the `otelarrow` receiver - # configuration and it will apply to the OTLP exporter. + + # To support receiving OTLP/HTTP. - gomod: go.opentelemetry.io/collector/receiver/otlpreceiver v0.91.0 # You may wish to enable other receivers from the Collector-Contrib repository or @@ -94,16 +99,20 @@ receivers: # - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/k8sclusterreceiver v0.91.0 processors: - # We recommend use of the batch processor. - - gomod: go.opentelemetry.io/collector/processor/batchprocessor v0.91.0 + # We recommend use of the OTel-Arrow concurrent batch processor. + - gomod: github.com/open-telemetry/otel-arrow/collector v0.13.0 + import: github.com/open-telemetry/otel-arrow/collector/processor/concurrentbatchprocessor # We recommend building with the follow processor for obfuscation, in case you want to # record telemetry sessions for offline analysis. - gomod: github.com/open-telemetry/otel-arrow/collector v0.13.0 import: github.com/open-telemetry/otel-arrow/collector/processor/obfuscationprocessor - # We emphatically DO NOT recommend use of the memory limiter processor, i.e., do not - # build with go.opentelemetry.io/collector/processor/memorylimiter. + # We emphatically DO NOT recommend use of the memory limiter + # processor or the core batch processor, i.e., do not build with + # either of: + # - go.opentelemetry.io/collector/processor/memorylimiterprocessor. + # - go.opentelemetry.io/collector/processor/batchprocessor. # These Collector-Contrib components are referred to in the charts in this package. - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/resourcedetectionprocessor v0.91.0 @@ -116,9 +125,6 @@ processors: # when the sampler is fully supported. # - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/probabilisticsamplerprocessor v0.91.0 - # TODO: Note: We will recommend use of a new uncompressed-size processor - # when it is ready. - # You may be interested in other transform components in the # Collector-Contrib repository. Here are some that might be useful: # From 02f77e4c3dd83b71a6b52409478be317928a7982 Mon Sep 17 00:00:00 2001 From: Joshua MacDonald Date: Thu, 4 Jan 2024 11:29:50 -0800 Subject: [PATCH 3/5] Update OTel-Arrow build config / example config --- README.md | 14 +-- arrow/Makefile | 4 +- arrow/config/gateway-collector.yaml | 127 ++++++++++++++++++++++++++++ arrow/config/saas-collector.yaml | 65 -------------- 4 files changed, 138 insertions(+), 72 deletions(-) create mode 100644 arrow/config/gateway-collector.yaml delete mode 100644 arrow/config/saas-collector.yaml diff --git a/README.md b/README.md index 1fef68a..acb0042 100644 --- a/README.md +++ b/README.md @@ -17,10 +17,10 @@ This is the repository for recommended [Helm](https://helm.sh/) charts for runni In order to use an arrow trace collector, you can use (1) the prebuilt image available via the Github Container Registry (GHCR) or you may (2) build your own custom image. ### 1. Use the prebuilt Docker image -1. We have built a Docker image using the example [build config](https://github.com/lightstep/otel-collector-charts/blob/main/arrow/otelcolarrow-build.yaml) +1. We have built a Docker image using the recommended [build config](https://github.com/lightstep/otel-collector-charts/blob/main/arrow/otelcolarrow-build.yaml) 2. This Docker [image](https://github.com/lightstep/otel-collector-charts/pkgs/container/otel-collector-charts%2Fotelarrowcol-experimental) can be pulled by running: `docker pull ghcr.io/lightstep/otel-collector-charts/otelarrowcol-experimental:latest` -3. You can use the example collector config (`/arrow/config/saas-config.yaml`) by running: -`docker run -it -v ./config/:/config --entrypoint /otelarrowcol ghcr.io/lightstep/otel-collector-charts/otelarrowcol-experimental:latest --config=/config/saas-collector.yaml` +3. You can use the collector config (`/arrow/config/gateway-config.yaml`) by running: +`docker run -it -v $(PWD)/config/:/config --entrypoint /otelarrowcol ghcr.io/lightstep/otel-collector-charts/otelarrowcol-experimental:latest --config=/config/gateway-collector.yaml` ### 2. Build your own custom image @@ -33,5 +33,9 @@ In order to use an arrow trace collector, you can use (1) the prebuilt image av Some of the features available in these charts are optional because they rely on components that have not been released in the OpenTelemetry Contrib Collector. Specifically, to make use of the new -OTel-Arrow protocol requires building a customer collector at this -time. See a [recommended custom collector build configuration](./gateway-build.yaml). +OpenTelemetry Protocol With Apache Arrow support requires using either +the prebuilt image or a customer collector build at this time. + +See the [recommended custom collector build +configuration](./arrow/otelcolarrow-build.yaml.yaml) as a starting +point. diff --git a/arrow/Makefile b/arrow/Makefile index a2e7791..a783d61 100644 --- a/arrow/Makefile +++ b/arrow/Makefile @@ -5,7 +5,7 @@ build-arm64: docker build . -t otelarrowcol-arm64 --platform linux/arm64 run-amd64: - docker run -it -v ./config/:/config --entrypoint /otelarrowcol otelarrowcol-amd64 --config=/config/saas-collector.yaml + docker run -it -v `pwd`/config/:/config --entrypoint /otelarrowcol otelarrowcol-amd64 --config=/config/gateway-collector.yaml run-arm64: - docker run -it -v ./config/:/config --entrypoint /otelarrowcol otelarrowcol-arm64 --config=/config/saas-collector.yaml \ No newline at end of file + docker run -it -v `pwd`/config/:/config --entrypoint /otelarrowcol otelarrowcol-arm64 --config=/config/gateway-collector.yaml diff --git a/arrow/config/gateway-collector.yaml b/arrow/config/gateway-collector.yaml new file mode 100644 index 0000000..32e37b8 --- /dev/null +++ b/arrow/config/gateway-collector.yaml @@ -0,0 +1,127 @@ +# This is a simple but functional recommended gateway configuration. +# +# Depending on the number of senders and size of data, the amount of +# resources required will vary. +# +# These configurations have been tested in a small configuration: +# +# resources: +# requests: +# cpu: 2 +# memory: 3Gi +# limits: +# cpu: 2 +# memory: 8Gi +# +# and a large configuration +# +# resources: +# requests: +# cpu: 8 +# memory: 6Gi +# limits: +# cpu: 8 +# memory: 24Gi +# +# In the larger configuration tested, we used `max_in_flight_size_mib: 128`. + +receivers: + # otelarrow is the OpenTelemetry Protocol with Apache Arrow receiver + # which combines support for standard OTLP/gRPC. + otelarrow: + protocols: + grpc: + # This is the default OTLP/gRPC port. OTel-Arrow is served on the + # same port. + endpoint: "0.0.0.0:4317" + + # allow large arriving telemetry payloads. they will be split + # into a reasonable size by the batch processor. + max_recv_msg_size_mib: 128 + + # Limit received OTel-Arrow stream length to 3m in total. + keepalive: + server_parameters: + max_connection_age: 30s + max_connection_age_grace: 2m30s + + # otlp is the core OTLP exporter, which we enable to receive + # OTLP/HTTP data. + otlp: + protocols: + http: + endpoint: "0.0.0.0:4318" + +processors: + # The concurrent batch processor is recommended instead of the + # core `batchprocessor` component, when available. + concurrentbatch: + send_batch_max_size: 1500 + send_batch_size: 1000 + timeout: 1s + + # For larger configurations, consider raising this parameter. + max_in_flight_size_mib: 128 + +exporters: + otelarrow: + endpoint: "ingest.lightstep.com:443" + headers: + "lightstep-access-token": "${LS_TOKEN}" + + arrow: + disabled: false + max_stream_lifetime: 2m + num_streams: 6 + + # The pipeline will continue trying requests until they timeout. + # Timeout and retry settings are independent. If retry_on_failure + # is also enabled, each (retried) request will also have this + # timeout. + timeout: 30s + + # Retries are disabled by default. Since the most likely reason + # for failure is timeout, having retry-on-failure enabled implies + # dedicating a significant amount of additional memory to the task. + retry_on_failure: + enabled: false + + # Do not enable the sending queue. The concurrent batch processor + # is a better way to parallelize the export. + sending_queue: + enabled: false + +service: + pipelines: + traces: + receivers: [otelarrow, otlp] + processors: [concurrentbatch] + exporters: [otelarrow] + + metrics: + receivers: [otelarrow, otlp] + processors: [concurrentbatch] + exporters: [otelarrow] + + telemetry: + metrics: + level: detailed + readers: + - periodic: + exporter: + otlp: + protocol: grpc/protobuf + endpoint: https://ingest.lightstep.com:443 + headers: + lightstep-access-token: "${LS_TOKEN}" + traces: + processors: + - batch: + exporter: + otlp: + protocol: grpc/protobuf + endpoint: https://ingest.lightstep.com:443 + headers: + lightstep-access-token: "${LS_TOKEN}" + resource: + service.name: otelarrow-gateway-collector diff --git a/arrow/config/saas-collector.yaml b/arrow/config/saas-collector.yaml deleted file mode 100644 index 3323acb..0000000 --- a/arrow/config/saas-collector.yaml +++ /dev/null @@ -1,65 +0,0 @@ -# This configuration is used to sanity check the build configuration -# in the directory above. This is not a useful example configuration -# for sending to ServiceNow Cloud Observability; for a useful example -# configuration of the OTel-Arrow components, see ../../charts/*/arrow.yaml. - -receivers: - # otelarrow is an OTel Arrow receiver that will operate as the SaaS-side - # of the bridge. - otelarrow: - protocols: - grpc: - # Port 5000 is the endpoint used in edge-collector. - endpoint: 127.0.0.1:8100 - - # Include metadata so that the exporter can copy it - # to the next hop. - include_metadata: true - - keepalive: - server_parameters: - max_connection_age: 10s - max_connection_age_grace: 10s - -exporters: - debug: - - otlphttp: - # You can use an HTTP listener on port 5001 to see the headers - # and raw data. - endpoint: http://127.0.0.1:8101 - compression: none - - # Associate the headers_setter extension with this exporter - # so that it passes through headers set on the edge collector. - auth: - authenticator: headers_setter - -extensions: - # Configure the headers_setter extension to propagate the - # X-Scope-OrgID property in the outgoing context. - headers_setter: - headers: - - key: X-Scope-OrgID - from_context: X-Scope-OrgID - -service: - extensions: [headers_setter] - pipelines: - traces: - receivers: [otelarrow] - - # Note there is no need to re-apply the batch processor on the - # SaaS-side of a bridge. - processors: [] - exporters: [debug, otlphttp] - - metrics: - receivers: [otelarrow] - processors: [] - exporters: [debug, otlphttp] - - telemetry: - metrics: - address: 127.0.0.1:8889 - level: normal From c4a0be809826a7832a2fdb7d2577d5d4dbf62e8d Mon Sep 17 00:00:00 2001 From: Joshua MacDonald Date: Thu, 4 Jan 2024 11:37:54 -0800 Subject: [PATCH 4/5] more idiomatic --- arrow/Makefile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/arrow/Makefile b/arrow/Makefile index a783d61..c342582 100644 --- a/arrow/Makefile +++ b/arrow/Makefile @@ -1,3 +1,5 @@ +current_dir = $(shell pwd) + build-amd64: docker build . -t otelarrowcol-amd64 --platform linux/amd64 @@ -5,7 +7,7 @@ build-arm64: docker build . -t otelarrowcol-arm64 --platform linux/arm64 run-amd64: - docker run -it -v `pwd`/config/:/config --entrypoint /otelarrowcol otelarrowcol-amd64 --config=/config/gateway-collector.yaml + docker run -it -v $(current_dir)/config/:/config --entrypoint /otelarrowcol otelarrowcol-amd64 --config=/config/gateway-collector.yaml run-arm64: - docker run -it -v `pwd`/config/:/config --entrypoint /otelarrowcol otelarrowcol-arm64 --config=/config/gateway-collector.yaml + docker run -it -v $(current_dir)/config/:/config --entrypoint /otelarrowcol otelarrowcol-arm64 --config=/config/gateway-collector.yaml From 8a3275c631c012905683f6a1204b1f35d079aae1 Mon Sep 17 00:00:00 2001 From: Joshua MacDonald Date: Thu, 4 Jan 2024 11:48:55 -0800 Subject: [PATCH 5/5] fix comment --- arrow/config/gateway-collector.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arrow/config/gateway-collector.yaml b/arrow/config/gateway-collector.yaml index 32e37b8..1166a15 100644 --- a/arrow/config/gateway-collector.yaml +++ b/arrow/config/gateway-collector.yaml @@ -23,7 +23,7 @@ # cpu: 8 # memory: 24Gi # -# In the larger configuration tested, we used `max_in_flight_size_mib: 128`. +# In the larger configuration tested, we used `max_in_flight_size_mib: 256`. receivers: # otelarrow is the OpenTelemetry Protocol with Apache Arrow receiver