From 16dce7588ae6435bade23f48f6f8475312935445 Mon Sep 17 00:00:00 2001 From: Daniyar Itegulov Date: Mon, 25 Mar 2024 13:27:39 +0000 Subject: [PATCH] feat(prover): export prover traces through OTLP (#1427) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What ❔ This PR adds an optional `tracing` layer to `vlog` that exports all spans to a configurable OTEL collector. This layer is then used in most prover components to export spans with structured information about the block number that is currently being processed. ## Why ❔ Better observability for prover and tracing for logs with a specific block number ## Checklist - [x] PR title corresponds to the body of PR (we generate changelog entries from PRs). - [ ] Tests for the changes have been added / updated. - [ ] Documentation comments have been added / updated. - [x] Code has been formatted via `zk fmt` and `zk lint`. - [x] Spellcheck has been run via `zk spellcheck`. - [x] Linkcheck has been run via `zk linkcheck`. --- Cargo.lock | 236 ++++++++++++- Cargo.toml | 4 + checks-config/era.dic | 1 + core/bin/contract-verifier/src/verifier.rs | 1 + .../external_node/src/config/observability.rs | 1 + core/lib/config/src/configs/mod.rs | 2 +- core/lib/config/src/configs/observability.rs | 10 + core/lib/config/src/testonly.rs | 10 + core/lib/env_config/src/observability.rs | 9 +- core/lib/protobuf_config/src/observability.rs | 26 +- .../src/proto/observability.proto | 6 + core/lib/queued_job_processor/src/lib.rs | 3 +- core/lib/vlog/Cargo.toml | 7 + core/lib/vlog/src/lib.rs | 173 +++++++++- .../src/basic_witness_input_producer/mod.rs | 1 + prover/Cargo.lock | 320 +++++++++++++++++- prover/proof_fri_compressor/src/compressor.rs | 3 + prover/proof_fri_compressor/src/main.rs | 9 + .../src/gpu_prover_job_processor.rs | 3 + prover/prover_fri/src/main.rs | 10 + prover/prover_fri/src/prover_job_processor.rs | 3 + .../witness_generator/src/basic_circuits.rs | 4 + .../witness_generator/src/leaf_aggregation.rs | 7 +- prover/witness_generator/src/main.rs | 9 + .../witness_generator/src/node_aggregation.rs | 7 +- prover/witness_generator/src/scheduler.rs | 7 +- .../witness_vector_generator/src/generator.rs | 3 + prover/witness_vector_generator/src/main.rs | 9 + 28 files changed, 842 insertions(+), 42 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4d9057bf064c..f7c41b39491a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2737,6 +2737,18 @@ dependencies = [ "tokio-rustls 0.24.1", ] +[[package]] +name = "hyper-timeout" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" +dependencies = [ + "hyper", + "pin-project-lite", + "tokio", + "tokio-io-timeout", +] + [[package]] name = "hyper-tls" version = "0.5.0" @@ -3992,6 +4004,110 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "opentelemetry" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9591d937bc0e6d2feb6f71a559540ab300ea49955229c347a517a28d27784c54" +dependencies = [ + "opentelemetry_api", + "opentelemetry_sdk", +] + +[[package]] +name = "opentelemetry-http" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7594ec0e11d8e33faf03530a4c49af7064ebba81c1480e01be67d90b356508b" +dependencies = [ + "async-trait", + "bytes", + "http", + "opentelemetry_api", + "reqwest", +] + +[[package]] +name = "opentelemetry-otlp" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e5e5a5c4135864099f3faafbe939eb4d7f9b80ebf68a8448da961b32a7c1275" +dependencies = [ + "async-trait", + "futures-core", + "http", + "opentelemetry-http", + "opentelemetry-proto", + "opentelemetry-semantic-conventions", + "opentelemetry_api", + "opentelemetry_sdk", + "prost 0.11.9", + "reqwest", + "thiserror", + "tokio", + "tonic", +] + +[[package]] +name = "opentelemetry-proto" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1e3f814aa9f8c905d0ee4bde026afd3b2577a97c10e1699912e3e44f0c4cbeb" +dependencies = [ + "opentelemetry_api", + "opentelemetry_sdk", + "prost 0.11.9", + "tonic", +] + +[[package]] +name = "opentelemetry-semantic-conventions" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73c9f9340ad135068800e7f1b24e9e09ed9e7143f5bf8518ded3d3ec69789269" +dependencies = [ + "opentelemetry", +] + +[[package]] +name = "opentelemetry_api" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a81f725323db1b1206ca3da8bb19874bbd3f57c3bcd59471bfb04525b265b9b" +dependencies = [ + "futures-channel", + "futures-util", + "indexmap 1.9.3", + "js-sys", + "once_cell", + "pin-project-lite", + "thiserror", + "urlencoding", +] + +[[package]] +name = "opentelemetry_sdk" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa8e705a0612d48139799fcbaba0d4a90f06277153e43dd2bdc16c6f0edd8026" +dependencies = [ + "async-trait", + "crossbeam-channel 0.5.8", + "futures-channel", + "futures-executor", + "futures-util", + "once_cell", + "opentelemetry_api", + "ordered-float 3.9.2", + "percent-encoding", + "rand 0.8.5", + "regex", + "serde_json", + "thiserror", + "tokio", + "tokio-stream", +] + [[package]] name = "ordered-float" version = "2.10.1" @@ -4001,6 +4117,15 @@ dependencies = [ "num-traits", ] +[[package]] +name = "ordered-float" +version = "3.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1e1c390732d15f1d48471625cd92d154e66db2c56645e29a9cd26f4699f72dc" +dependencies = [ + "num-traits", +] + [[package]] name = "os_info" version = "3.7.0" @@ -4497,6 +4622,16 @@ dependencies = [ "vise-exporter", ] +[[package]] +name = "prost" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b82eaa1d779e9a4bc1c3217db8ffbeabaae1dca241bf70183242128d48681cd" +dependencies = [ + "bytes", + "prost-derive 0.11.9", +] + [[package]] name = "prost" version = "0.12.1" @@ -4504,7 +4639,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f4fdd22f3b9c31b53c060df4a0613a1c7f062d4115a2b984dd15b1858f7e340d" dependencies = [ "bytes", - "prost-derive", + "prost-derive 0.12.1", ] [[package]] @@ -4521,7 +4656,7 @@ dependencies = [ "once_cell", "petgraph", "prettyplease", - "prost", + "prost 0.12.1", "prost-types", "regex", "syn 2.0.38", @@ -4529,6 +4664,19 @@ dependencies = [ "which", ] +[[package]] +name = "prost-derive" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5d2d8d10f3c6ded6da8b05b5fb3b8a5082514344d56c9f871412d29b4e075b4" +dependencies = [ + "anyhow", + "itertools 0.10.5", + "proc-macro2 1.0.69", + "quote 1.0.33", + "syn 1.0.109", +] + [[package]] name = "prost-derive" version = "0.12.1" @@ -4552,7 +4700,7 @@ dependencies = [ "logos", "miette", "once_cell", - "prost", + "prost 0.12.1", "prost-types", "serde", "serde-value", @@ -4564,7 +4712,7 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e081b29f63d83a4bc75cfc9f3fe424f9156cf92d8a4f0c9407cce9a1b67327cf" dependencies = [ - "prost", + "prost 0.12.1", ] [[package]] @@ -4575,7 +4723,7 @@ checksum = "00bb76c5f6221de491fe2c8f39b106330bbd9762c6511119c07940e10eb9ff11" dependencies = [ "bytes", "miette", - "prost", + "prost 0.12.1", "prost-reflect", "prost-types", "protox-parse", @@ -5494,7 +5642,7 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f3a1a3341211875ef120e117ea7fd5228530ae7e7036a779fdc9117be6b3282c" dependencies = [ - "ordered-float", + "ordered-float 2.10.1", "serde", ] @@ -6513,6 +6661,16 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "tokio-io-timeout" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30b74022ada614a1b4834de765f9bb43877f910cc8ce4be40e89042c9223a8bf" +dependencies = [ + "pin-project-lite", + "tokio", +] + [[package]] name = "tokio-macros" version = "2.2.0" @@ -6609,6 +6767,34 @@ dependencies = [ "winnow", ] +[[package]] +name = "tonic" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3082666a3a6433f7f511c7192923fa1fe07c69332d3c6a2e6bb040b569199d5a" +dependencies = [ + "async-trait", + "axum", + "base64 0.21.5", + "bytes", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "hyper", + "hyper-timeout", + "percent-encoding", + "pin-project", + "prost 0.11.9", + "tokio", + "tokio-stream", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + [[package]] name = "tower" version = "0.4.13" @@ -6716,6 +6902,22 @@ dependencies = [ "tracing-core", ] +[[package]] +name = "tracing-opentelemetry" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75327c6b667828ddc28f5e3f169036cb793c3f588d83bf0f262a7f062ffed3c8" +dependencies = [ + "once_cell", + "opentelemetry", + "opentelemetry_sdk", + "smallvec", + "tracing", + "tracing-core", + "tracing-log", + "tracing-subscriber", +] + [[package]] name = "tracing-serde" version = "0.1.3" @@ -7016,9 +7218,13 @@ name = "vlog" version = "0.1.0" dependencies = [ "chrono", + "opentelemetry", + "opentelemetry-otlp", + "opentelemetry-semantic-conventions", "sentry", "serde_json", "tracing", + "tracing-opentelemetry", "tracing-subscriber", ] @@ -7866,7 +8072,7 @@ dependencies = [ "im", "once_cell", "pin-project", - "prost", + "prost 0.12.1", "rand 0.8.5", "snow", "thiserror", @@ -7889,7 +8095,7 @@ dependencies = [ "anyhow", "bit-vec", "hex", - "prost", + "prost 0.12.1", "rand 0.8.5", "serde", "thiserror", @@ -7908,7 +8114,7 @@ source = "git+https://github.com/matter-labs/era-consensus.git?rev=5329a809cfc06 dependencies = [ "anyhow", "async-trait", - "prost", + "prost 0.12.1", "rand 0.8.5", "thiserror", "tracing", @@ -8009,7 +8215,7 @@ dependencies = [ "once_cell", "pin-project-lite", "prometheus_exporter", - "prost", + "prost 0.12.1", "prover_dal", "rand 0.8.5", "reqwest", @@ -8088,7 +8294,7 @@ dependencies = [ "chrono", "hex", "itertools 0.10.5", - "prost", + "prost 0.12.1", "rand 0.8.5", "serde", "serde_json", @@ -8336,7 +8542,7 @@ dependencies = [ "google-cloud-auth", "google-cloud-storage", "http", - "prost", + "prost 0.12.1", "serde_json", "tempdir", "tokio", @@ -8355,7 +8561,7 @@ dependencies = [ "anyhow", "bit-vec", "once_cell", - "prost", + "prost 0.12.1", "prost-reflect", "quick-protobuf", "rand 0.8.5", @@ -8387,7 +8593,7 @@ version = "0.1.0" dependencies = [ "anyhow", "pretty_assertions", - "prost", + "prost 0.12.1", "rand 0.8.5", "serde_json", "serde_yaml", @@ -8537,7 +8743,7 @@ dependencies = [ "num", "num_enum 0.7.2", "once_cell", - "prost", + "prost 0.12.1", "rlp", "secp256k1", "serde", diff --git a/Cargo.toml b/Cargo.toml index 5620dcbf225d..fe863832ec7b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -111,6 +111,9 @@ num = "0.4.0" num_cpus = "1.13" num_enum = "0.7.2" once_cell = "1" +opentelemetry = "0.20.0" +opentelemetry-otlp = "0.13.0" +opentelemetry-semantic-conventions = "0.12.0" pin-project-lite = "0.2.13" pretty_assertions = "1" prost = "0.12.1" @@ -145,6 +148,7 @@ tower = "0.4.13" tower-http = "0.4.1" tracing = "0.1" tracing-subscriber = "0.3" +tracing-opentelemetry = "0.21.0" url = "2" web3 = "0.19.0" diff --git a/checks-config/era.dic b/checks-config/era.dic index fda3689015ad..8707fc0aa3d5 100644 --- a/checks-config/era.dic +++ b/checks-config/era.dic @@ -921,3 +921,4 @@ p2p StorageProcessor StorageMarker SIGINT +opentelemetry diff --git a/core/bin/contract-verifier/src/verifier.rs b/core/bin/contract-verifier/src/verifier.rs index c6c70b32ae99..66e7b6deebbb 100644 --- a/core/bin/contract-verifier/src/verifier.rs +++ b/core/bin/contract-verifier/src/verifier.rs @@ -506,6 +506,7 @@ impl JobProcessor for ContractVerifier { #[allow(clippy::async_yields_async)] async fn process_job( &self, + _job_id: &Self::JobId, job: VerificationRequest, started_at: Instant, ) -> tokio::task::JoinHandle> { diff --git a/core/bin/external_node/src/config/observability.rs b/core/bin/external_node/src/config/observability.rs index ddc2256e78bc..388e75e29479 100644 --- a/core/bin/external_node/src/config/observability.rs +++ b/core/bin/external_node/src/config/observability.rs @@ -35,5 +35,6 @@ pub fn observability_config_from_env() -> anyhow::Result { sentry_url, sentry_environment, log_format, + opentelemetry: None, }) } diff --git a/core/lib/config/src/configs/mod.rs b/core/lib/config/src/configs/mod.rs index f9ad80dbe344..866aa25fa895 100644 --- a/core/lib/config/src/configs/mod.rs +++ b/core/lib/config/src/configs/mod.rs @@ -14,7 +14,7 @@ pub use self::{ fri_witness_generator::FriWitnessGeneratorConfig, fri_witness_vector_generator::FriWitnessVectorGeneratorConfig, object_store::ObjectStoreConfig, - observability::ObservabilityConfig, + observability::{ObservabilityConfig, OpentelemetryConfig}, proof_data_handler::ProofDataHandlerConfig, snapshots_creator::SnapshotsCreatorConfig, utils::PrometheusConfig, diff --git a/core/lib/config/src/configs/observability.rs b/core/lib/config/src/configs/observability.rs index b492b0ffa471..ce7a289f0ccd 100644 --- a/core/lib/config/src/configs/observability.rs +++ b/core/lib/config/src/configs/observability.rs @@ -6,7 +6,17 @@ pub struct ObservabilityConfig { pub sentry_url: Option, /// Name of the environment to use in Sentry. pub sentry_environment: Option, + /// Opentelemetry configuration. + pub opentelemetry: Option, /// Format of the logs as expected by the `vlog` crate. /// Currently must be either `plain` or `json`. pub log_format: String, } + +#[derive(Debug, Clone, PartialEq)] +pub struct OpentelemetryConfig { + /// Enables export of span data of specified level (and above) using opentelemetry exporters. + pub level: String, + /// Opentelemetry HTTP collector endpoint. + pub endpoint: String, +} diff --git a/core/lib/config/src/testonly.rs b/core/lib/config/src/testonly.rs index 1a69cf6247ee..4b3ff93ed592 100644 --- a/core/lib/config/src/testonly.rs +++ b/core/lib/config/src/testonly.rs @@ -754,6 +754,16 @@ impl RandomConfig for configs::ObservabilityConfig { sentry_url: g.gen(), sentry_environment: g.gen(), log_format: g.gen(), + opentelemetry: g.gen(), + } + } +} + +impl RandomConfig for configs::OpentelemetryConfig { + fn sample(g: &mut Gen) -> Self { + Self { + level: g.gen(), + endpoint: g.gen(), } } } diff --git a/core/lib/env_config/src/observability.rs b/core/lib/env_config/src/observability.rs index a168c664b227..48b9c40b96f9 100644 --- a/core/lib/env_config/src/observability.rs +++ b/core/lib/env_config/src/observability.rs @@ -1,4 +1,4 @@ -use zksync_config::configs::ObservabilityConfig; +use zksync_config::configs::{ObservabilityConfig, OpentelemetryConfig}; use crate::FromEnv; @@ -33,11 +33,18 @@ impl FromEnv for ObservabilityConfig { } else { "plain".to_string() }; + let opentelemetry_level = std::env::var("OPENTELEMETRY_LEVEL").ok(); + let otlp_endpoint = std::env::var("OTLP_ENDPOINT").ok(); + let opentelemetry = match (opentelemetry_level, otlp_endpoint) { + (Some(level), Some(endpoint)) => Some(OpentelemetryConfig { level, endpoint }), + _ => None, + }; Ok(ObservabilityConfig { sentry_url, sentry_environment, log_format, + opentelemetry, }) } } diff --git a/core/lib/protobuf_config/src/observability.rs b/core/lib/protobuf_config/src/observability.rs index b2ac1d0b2f8d..6ef67ea0beb0 100644 --- a/core/lib/protobuf_config/src/observability.rs +++ b/core/lib/protobuf_config/src/observability.rs @@ -1,5 +1,5 @@ use anyhow::Context as _; -use zksync_config::configs; +use zksync_config::configs::{self}; use zksync_protobuf::{required, ProtoRepr}; use crate::proto::observability as proto; @@ -11,6 +11,11 @@ impl ProtoRepr for proto::Observability { sentry_url: self.sentry_url.clone(), sentry_environment: self.sentry_environment.clone(), log_format: required(&self.log_format).context("log_format")?.clone(), + opentelemetry: self + .opentelemetry + .as_ref() + .map(|cfg| cfg.read().context("opentelemetry")) + .transpose()?, }) } @@ -19,6 +24,25 @@ impl ProtoRepr for proto::Observability { sentry_url: this.sentry_url.clone(), sentry_environment: this.sentry_environment.clone(), log_format: Some(this.log_format.clone()), + opentelemetry: this.opentelemetry.as_ref().map(ProtoRepr::build), + } + } +} + +impl ProtoRepr for proto::Opentelemetry { + type Type = configs::OpentelemetryConfig; + + fn read(&self) -> anyhow::Result { + Ok(Self::Type { + level: required(&self.level).context("level")?.clone(), + endpoint: required(&self.endpoint).context("endpoint")?.clone(), + }) + } + + fn build(this: &Self::Type) -> Self { + Self { + level: Some(this.level.clone()), + endpoint: Some(this.endpoint.clone()), } } } diff --git a/core/lib/protobuf_config/src/proto/observability.proto b/core/lib/protobuf_config/src/proto/observability.proto index 8a09a2a22854..04b1c059fb5f 100644 --- a/core/lib/protobuf_config/src/proto/observability.proto +++ b/core/lib/protobuf_config/src/proto/observability.proto @@ -6,4 +6,10 @@ message Observability { optional string sentry_url = 1; // optional optional string sentry_environment = 2; // optional optional string log_format = 3; // required + optional Opentelemetry opentelemetry = 4; // optional +} + +message Opentelemetry { + optional string level = 1; // required + optional string endpoint = 2; // required } diff --git a/core/lib/queued_job_processor/src/lib.rs b/core/lib/queued_job_processor/src/lib.rs index 49ec8b348ee2..569a2b7f59da 100644 --- a/core/lib/queued_job_processor/src/lib.rs +++ b/core/lib/queued_job_processor/src/lib.rs @@ -46,6 +46,7 @@ pub trait JobProcessor: Sync + Send { /// Function that processes a job async fn process_job( &self, + job_id: &Self::JobId, job: Self::Job, started_at: Instant, ) -> JoinHandle>; @@ -83,7 +84,7 @@ pub trait JobProcessor: Sync + Send { Self::SERVICE_NAME, job_id ); - let task = self.process_job(job, started_at).await; + let task = self.process_job(&job_id, job, started_at).await; self.wait_for_task(job_id, started_at, task) .await diff --git a/core/lib/vlog/Cargo.toml b/core/lib/vlog/Cargo.toml index 7b0b96d6e38a..b32b4694137e 100644 --- a/core/lib/vlog/Cargo.toml +++ b/core/lib/vlog/Cargo.toml @@ -19,5 +19,12 @@ tracing-subscriber = { workspace = true, features = [ "time", "json", ] } +tracing-opentelemetry.workspace = true sentry.workspace = true serde_json.workspace = true +opentelemetry = { workspace = true, features = ["rt-tokio", "trace"] } +opentelemetry-otlp = { workspace = true, features = [ + "http-proto", + "reqwest-client", +] } +opentelemetry-semantic-conventions.workspace = true diff --git a/core/lib/vlog/src/lib.rs b/core/lib/vlog/src/lib.rs index c8e4761744eb..a610cdd09048 100644 --- a/core/lib/vlog/src/lib.rs +++ b/core/lib/vlog/src/lib.rs @@ -5,9 +5,30 @@ use std::{backtrace::Backtrace, borrow::Cow, panic::PanicInfo, str::FromStr}; // Temporary re-export of `sentry::capture_message` aiming to simplify the transition from `vlog` to using // crates directly. +use opentelemetry::{ + sdk::{ + propagation::TraceContextPropagator, + trace::{self, RandomIdGenerator, Sampler, Tracer}, + Resource, + }, + KeyValue, +}; +use opentelemetry_otlp::WithExportConfig; +use opentelemetry_semantic_conventions::resource::SERVICE_NAME; pub use sentry::{capture_message, Level as AlertLevel}; use sentry::{types::Dsn, ClientInitGuard}; -use tracing_subscriber::{fmt, layer::SubscriberExt, util::SubscriberInitExt}; +use tracing_opentelemetry::OpenTelemetryLayer; +use tracing_subscriber::{ + filter::Filtered, + fmt, + layer::{Layered, SubscriberExt}, + registry::LookupSpan, + util::SubscriberInitExt, + EnvFilter, Layer, +}; + +type TracingLayer = + Layered, EnvFilter, Inner>, Inner>; /// Specifies the format of the logs in stdout. #[derive(Debug, Clone, Copy, Default)] @@ -40,6 +61,63 @@ impl FromStr for LogFormat { } } +// Doesn't define WARN and ERROR, because the highest verbosity of spans is INFO. +#[derive(Copy, Clone, Debug, Default)] +pub enum OpenTelemetryLevel { + #[default] + OFF, + INFO, + DEBUG, + TRACE, +} + +#[derive(Debug)] +pub struct OpenTelemetryLevelFormatError; + +impl std::fmt::Display for OpenTelemetryLevelFormatError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "Invalid OpenTelemetry level format") + } +} + +impl std::error::Error for OpenTelemetryLevelFormatError {} + +impl FromStr for OpenTelemetryLevel { + type Err = OpenTelemetryLevelFormatError; + + fn from_str(s: &str) -> Result { + match s { + "off" => Ok(OpenTelemetryLevel::OFF), + "info" => Ok(OpenTelemetryLevel::INFO), + "debug" => Ok(OpenTelemetryLevel::DEBUG), + "trace" => Ok(OpenTelemetryLevel::TRACE), + _ => Err(OpenTelemetryLevelFormatError), + } + } +} + +impl std::fmt::Display for OpenTelemetryLevel { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let str = match self { + OpenTelemetryLevel::OFF => "off", + OpenTelemetryLevel::INFO => "info", + OpenTelemetryLevel::DEBUG => "debug", + OpenTelemetryLevel::TRACE => "trace", + }; + write!(f, "{}", str) + } +} + +#[derive(Clone, Debug)] +pub struct OpenTelemetryOptions { + /// Enables export of span data of specified level (and above) using opentelemetry exporters. + pub opentelemetry_level: OpenTelemetryLevel, + /// Opentelemetry HTTP collector endpoint. + pub otlp_endpoint: String, + /// Logical service name to be used for exported events. See [`SERVICE_NAME`]. + pub service_name: String, +} + /// Builder for the observability subsystem. /// Currently capable of configuring logging output and sentry integration. #[derive(Debug, Default)] @@ -47,6 +125,7 @@ pub struct ObservabilityBuilder { log_format: LogFormat, sentry_url: Option, sentry_environment: Option, + opentelemetry_options: Option, } /// Guard for the observability subsystem. @@ -92,19 +171,89 @@ impl ObservabilityBuilder { self } + pub fn with_opentelemetry( + mut self, + opentelemetry_level: &str, + otlp_endpoint: String, + service_name: String, + ) -> Result { + self.opentelemetry_options = Some(OpenTelemetryOptions { + opentelemetry_level: opentelemetry_level.parse()?, + otlp_endpoint, + service_name, + }); + Ok(self) + } + + fn add_opentelemetry_layer( + opentelemetry_level: OpenTelemetryLevel, + otlp_endpoint: String, + service_name: String, + subscriber: S, + ) -> TracingLayer + where + S: tracing::Subscriber + for<'span> LookupSpan<'span> + Send + Sync, + { + let filter = match opentelemetry_level { + OpenTelemetryLevel::OFF => EnvFilter::new("off"), + OpenTelemetryLevel::INFO => EnvFilter::new("info"), + OpenTelemetryLevel::DEBUG => EnvFilter::new("debug"), + OpenTelemetryLevel::TRACE => EnvFilter::new("trace"), + }; + // `otel::tracing` should be a level info to emit opentelemetry trace & span + // `otel` set to debug to log detected resources, configuration read and inferred + let filter = filter + .add_directive("otel::tracing=trace".parse().unwrap()) + .add_directive("otel=debug".parse().unwrap()); + + let resource = vec![KeyValue::new(SERVICE_NAME, service_name)]; + + let tracer = opentelemetry_otlp::new_pipeline() + .tracing() + .with_exporter( + opentelemetry_otlp::new_exporter() + .http() + .with_endpoint(otlp_endpoint), + ) + .with_trace_config( + trace::config() + .with_sampler(Sampler::AlwaysOn) + .with_id_generator(RandomIdGenerator::default()) + .with_resource(Resource::new(resource)), + ) + .install_batch(opentelemetry::runtime::Tokio) + .unwrap(); + + opentelemetry::global::set_text_map_propagator(TraceContextPropagator::new()); + let layer = tracing_opentelemetry::layer() + .with_tracer(tracer) + .with_filter(filter); + subscriber.with(layer) + } + /// Initializes the observability subsystem. pub fn build(self) -> ObservabilityGuard { // Initialize logs. match self.log_format { LogFormat::Plain => { - tracing_subscriber::registry() + let subscriber = tracing_subscriber::registry() .with(tracing_subscriber::EnvFilter::from_default_env()) - .with(fmt::Layer::default()) - .init(); + .with(fmt::Layer::default()); + if let Some(opts) = self.opentelemetry_options { + let subscriber = Self::add_opentelemetry_layer( + opts.opentelemetry_level, + opts.otlp_endpoint, + opts.service_name, + subscriber, + ); + subscriber.init() + } else { + subscriber.init() + } } LogFormat::Json => { let timer = tracing_subscriber::fmt::time::UtcTime::rfc_3339(); - tracing_subscriber::registry() + let subscriber = tracing_subscriber::registry() .with(tracing_subscriber::EnvFilter::from_default_env()) .with( fmt::Layer::default() @@ -112,8 +261,18 @@ impl ObservabilityBuilder { .with_line_number(true) .with_timer(timer) .json(), - ) - .init(); + ); + if let Some(opts) = self.opentelemetry_options { + let subscriber = Self::add_opentelemetry_layer( + opts.opentelemetry_level, + opts.otlp_endpoint, + opts.service_name, + subscriber, + ); + subscriber.init() + } else { + subscriber.init() + } } }; diff --git a/core/lib/zksync_core/src/basic_witness_input_producer/mod.rs b/core/lib/zksync_core/src/basic_witness_input_producer/mod.rs index 0c44997faf72..549c6941b646 100644 --- a/core/lib/zksync_core/src/basic_witness_input_producer/mod.rs +++ b/core/lib/zksync_core/src/basic_witness_input_producer/mod.rs @@ -142,6 +142,7 @@ impl JobProcessor for BasicWitnessInputProducer { async fn process_job( &self, + _job_id: &Self::JobId, job: Self::Job, started_at: Instant, ) -> JoinHandle> { diff --git a/prover/Cargo.lock b/prover/Cargo.lock index 2b0540b3f201..d1ea842fbcb3 100644 --- a/prover/Cargo.lock +++ b/prover/Cargo.lock @@ -262,6 +262,51 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +[[package]] +name = "axum" +version = "0.6.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b829e4e32b91e643de6eafe82b1d90675f5874230191a4ffbc1b336dec4d6bf" +dependencies = [ + "async-trait", + "axum-core", + "bitflags 1.3.2", + "bytes", + "futures-util", + "http", + "http-body", + "hyper", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "rustversion", + "serde", + "sync_wrapper", + "tower", + "tower-layer", + "tower-service", +] + +[[package]] +name = "axum-core" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "759fa577a247914fd3f7f76d62972792636412fbfd634cd452f6a385a74d2d2c" +dependencies = [ + "async-trait", + "bytes", + "futures-util", + "http", + "http-body", + "mime", + "rustversion", + "tower-layer", + "tower-service", +] + [[package]] name = "backtrace" version = "0.3.69" @@ -2524,6 +2569,18 @@ dependencies = [ "tokio-rustls", ] +[[package]] +name = "hyper-timeout" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" +dependencies = [ + "hyper", + "pin-project-lite", + "tokio", + "tokio-io-timeout", +] + [[package]] name = "hyper-tls" version = "0.5.0" @@ -3003,6 +3060,12 @@ dependencies = [ "regex-automata 0.1.10", ] +[[package]] +name = "matchit" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" + [[package]] name = "maybe-uninit" version = "2.0.0" @@ -3549,6 +3612,110 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "opentelemetry" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9591d937bc0e6d2feb6f71a559540ab300ea49955229c347a517a28d27784c54" +dependencies = [ + "opentelemetry_api", + "opentelemetry_sdk", +] + +[[package]] +name = "opentelemetry-http" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7594ec0e11d8e33faf03530a4c49af7064ebba81c1480e01be67d90b356508b" +dependencies = [ + "async-trait", + "bytes", + "http", + "opentelemetry_api", + "reqwest", +] + +[[package]] +name = "opentelemetry-otlp" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e5e5a5c4135864099f3faafbe939eb4d7f9b80ebf68a8448da961b32a7c1275" +dependencies = [ + "async-trait", + "futures-core", + "http", + "opentelemetry-http", + "opentelemetry-proto", + "opentelemetry-semantic-conventions", + "opentelemetry_api", + "opentelemetry_sdk", + "prost 0.11.9", + "reqwest", + "thiserror", + "tokio", + "tonic", +] + +[[package]] +name = "opentelemetry-proto" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1e3f814aa9f8c905d0ee4bde026afd3b2577a97c10e1699912e3e44f0c4cbeb" +dependencies = [ + "opentelemetry_api", + "opentelemetry_sdk", + "prost 0.11.9", + "tonic", +] + +[[package]] +name = "opentelemetry-semantic-conventions" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73c9f9340ad135068800e7f1b24e9e09ed9e7143f5bf8518ded3d3ec69789269" +dependencies = [ + "opentelemetry", +] + +[[package]] +name = "opentelemetry_api" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a81f725323db1b1206ca3da8bb19874bbd3f57c3bcd59471bfb04525b265b9b" +dependencies = [ + "futures-channel", + "futures-util", + "indexmap 1.9.3", + "js-sys", + "once_cell", + "pin-project-lite", + "thiserror", + "urlencoding", +] + +[[package]] +name = "opentelemetry_sdk" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa8e705a0612d48139799fcbaba0d4a90f06277153e43dd2bdc16c6f0edd8026" +dependencies = [ + "async-trait", + "crossbeam-channel 0.5.11", + "futures-channel", + "futures-executor", + "futures-util", + "once_cell", + "opentelemetry_api", + "ordered-float 3.9.2", + "percent-encoding", + "rand 0.8.5", + "regex", + "serde_json", + "thiserror", + "tokio", + "tokio-stream", +] + [[package]] name = "ordered-float" version = "2.10.1" @@ -3558,6 +3725,15 @@ dependencies = [ "num-traits", ] +[[package]] +name = "ordered-float" +version = "3.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1e1c390732d15f1d48471625cd92d154e66db2c56645e29a9cd26f4699f72dc" +dependencies = [ + "num-traits", +] + [[package]] name = "os_info" version = "3.7.0" @@ -3964,6 +4140,16 @@ dependencies = [ "unarray", ] +[[package]] +name = "prost" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b82eaa1d779e9a4bc1c3217db8ffbeabaae1dca241bf70183242128d48681cd" +dependencies = [ + "bytes", + "prost-derive 0.11.9", +] + [[package]] name = "prost" version = "0.12.3" @@ -3971,7 +4157,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "146c289cda302b98a28d40c8b3b90498d6e526dd24ac2ecea73e4e491685b94a" dependencies = [ "bytes", - "prost-derive", + "prost-derive 0.12.3", ] [[package]] @@ -3988,7 +4174,7 @@ dependencies = [ "once_cell", "petgraph", "prettyplease", - "prost", + "prost 0.12.3", "prost-types", "regex", "syn 2.0.48", @@ -3996,6 +4182,19 @@ dependencies = [ "which", ] +[[package]] +name = "prost-derive" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5d2d8d10f3c6ded6da8b05b5fb3b8a5082514344d56c9f871412d29b4e075b4" +dependencies = [ + "anyhow", + "itertools 0.10.5", + "proc-macro2 1.0.78", + "quote 1.0.35", + "syn 1.0.109", +] + [[package]] name = "prost-derive" version = "0.12.3" @@ -4019,7 +4218,7 @@ dependencies = [ "logos", "miette", "once_cell", - "prost", + "prost 0.12.3", "prost-types", "serde", "serde-value", @@ -4031,7 +4230,7 @@ version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "193898f59edcf43c26227dcd4c8427f00d99d61e95dcde58dabd49fa291d470e" dependencies = [ - "prost", + "prost 0.12.3", ] [[package]] @@ -4042,7 +4241,7 @@ checksum = "00bb76c5f6221de491fe2c8f39b106330bbd9762c6511119c07940e10eb9ff11" dependencies = [ "bytes", "miette", - "prost", + "prost 0.12.3", "prost-reflect", "prost-types", "protox-parse", @@ -4898,7 +5097,7 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f3a1a3341211875ef120e117ea7fd5228530ae7e7036a779fdc9117be6b3282c" dependencies = [ - "ordered-float", + "ordered-float 2.10.1", "serde", ] @@ -5834,6 +6033,16 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "tokio-io-timeout" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30b74022ada614a1b4834de765f9bb43877f910cc8ce4be40e89042c9223a8bf" +dependencies = [ + "pin-project-lite", + "tokio", +] + [[package]] name = "tokio-macros" version = "2.2.0" @@ -5941,6 +6150,34 @@ dependencies = [ "winnow", ] +[[package]] +name = "tonic" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3082666a3a6433f7f511c7192923fa1fe07c69332d3c6a2e6bb040b569199d5a" +dependencies = [ + "async-trait", + "axum", + "base64 0.21.7", + "bytes", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "hyper", + "hyper-timeout", + "percent-encoding", + "pin-project", + "prost 0.11.9", + "tokio", + "tokio-stream", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + [[package]] name = "tools" version = "0.1.0" @@ -5955,6 +6192,32 @@ dependencies = [ "zksync_prover_interface", ] +[[package]] +name = "tower" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" +dependencies = [ + "futures-core", + "futures-util", + "indexmap 1.9.3", + "pin-project", + "pin-project-lite", + "rand 0.8.5", + "slab", + "tokio", + "tokio-util", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-layer" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0" + [[package]] name = "tower-service" version = "0.3.2" @@ -5994,6 +6257,17 @@ dependencies = [ "valuable", ] +[[package]] +name = "tracing-log" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f751112709b4e791d8ce53e32c4ed2d353565a795ce84da2285393f41557bdf2" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + [[package]] name = "tracing-log" version = "0.2.0" @@ -6005,6 +6279,22 @@ dependencies = [ "tracing-core", ] +[[package]] +name = "tracing-opentelemetry" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75327c6b667828ddc28f5e3f169036cb793c3f588d83bf0f262a7f062ffed3c8" +dependencies = [ + "once_cell", + "opentelemetry", + "opentelemetry_sdk", + "smallvec", + "tracing", + "tracing-core", + "tracing-log 0.1.4", + "tracing-subscriber", +] + [[package]] name = "tracing-serde" version = "0.1.3" @@ -6033,7 +6323,7 @@ dependencies = [ "time", "tracing", "tracing-core", - "tracing-log", + "tracing-log 0.2.0", "tracing-serde", ] @@ -6312,9 +6602,13 @@ name = "vlog" version = "0.1.0" dependencies = [ "chrono", + "opentelemetry", + "opentelemetry-otlp", + "opentelemetry-semantic-conventions", "sentry", "serde_json", "tracing", + "tracing-opentelemetry", "tracing-subscriber", ] @@ -7098,7 +7392,7 @@ dependencies = [ "anyhow", "bit-vec", "hex", - "prost", + "prost 0.12.3", "rand 0.8.5", "serde", "thiserror", @@ -7117,7 +7411,7 @@ source = "git+https://github.com/matter-labs/era-consensus.git?rev=5329a809cfc06 dependencies = [ "anyhow", "async-trait", - "prost", + "prost 0.12.3", "rand 0.8.5", "thiserror", "tracing", @@ -7173,7 +7467,7 @@ dependencies = [ "chrono", "hex", "itertools 0.10.5", - "prost", + "prost 0.12.3", "rand 0.8.5", "serde", "serde_json", @@ -7254,7 +7548,7 @@ dependencies = [ "google-cloud-auth", "google-cloud-storage", "http", - "prost", + "prost 0.12.3", "serde_json", "tokio", "tracing", @@ -7305,7 +7599,7 @@ dependencies = [ "anyhow", "bit-vec", "once_cell", - "prost", + "prost 0.12.3", "prost-reflect", "quick-protobuf", "rand 0.8.5", @@ -7490,7 +7784,7 @@ dependencies = [ "num", "num_enum 0.7.2", "once_cell", - "prost", + "prost 0.12.3", "rlp", "secp256k1", "serde", diff --git a/prover/proof_fri_compressor/src/compressor.rs b/prover/proof_fri_compressor/src/compressor.rs index 0e7fced76737..7b1352b6a7c9 100644 --- a/prover/proof_fri_compressor/src/compressor.rs +++ b/prover/proof_fri_compressor/src/compressor.rs @@ -180,12 +180,15 @@ impl JobProcessor for ProofCompressor { async fn process_job( &self, + job_id: &L1BatchNumber, job: ZkSyncRecursionLayerProof, _started_at: Instant, ) -> JoinHandle> { let compression_mode = self.compression_mode; let verify_wrapper_proof = self.verify_wrapper_proof; + let block_number = *job_id; tokio::task::spawn_blocking(move || { + let _span = tracing::info_span!("compress", %block_number).entered(); Self::compress_proof(job, compression_mode, verify_wrapper_proof) }) } diff --git a/prover/proof_fri_compressor/src/main.rs b/prover/proof_fri_compressor/src/main.rs index 7aa61db60297..2ae19423d453 100644 --- a/prover/proof_fri_compressor/src/main.rs +++ b/prover/proof_fri_compressor/src/main.rs @@ -46,6 +46,15 @@ async fn main() -> anyhow::Result<()> { .expect("Invalid Sentry URL") .with_sentry_environment(observability_config.sentry_environment); } + if let Some(opentelemetry) = observability_config.opentelemetry { + builder = builder + .with_opentelemetry( + &opentelemetry.level, + opentelemetry.endpoint, + "zksync-prover-fri-compressor".into(), + ) + .expect("Invalid OpenTelemetry config"); + } let _guard = builder.build(); let opt = Opt::from_args(); diff --git a/prover/prover_fri/src/gpu_prover_job_processor.rs b/prover/prover_fri/src/gpu_prover_job_processor.rs index c2748f005dee..98180e68282c 100644 --- a/prover/prover_fri/src/gpu_prover_job_processor.rs +++ b/prover/prover_fri/src/gpu_prover_job_processor.rs @@ -259,6 +259,7 @@ pub mod gpu_prover { async fn process_job( &self, + _job_id: &Self::JobId, job: Self::Job, _started_at: Instant, ) -> JoinHandle> { @@ -269,6 +270,8 @@ pub mod gpu_prover { .clone(), ); tokio::task::spawn_blocking(move || { + let block_number = job.witness_vector_artifacts.prover_job.block_number; + let _span = tracing::info_span!("gpu_prove", %block_number).entered(); Ok(Self::prove(job, setup_data.context("get_setup_data()")?)) }) } diff --git a/prover/prover_fri/src/main.rs b/prover/prover_fri/src/main.rs index e13c2be5147e..841406c28334 100644 --- a/prover/prover_fri/src/main.rs +++ b/prover/prover_fri/src/main.rs @@ -70,6 +70,16 @@ async fn main() -> anyhow::Result<()> { .expect("Invalid Sentry URL") .with_sentry_environment(observability_config.sentry_environment); } + + if let Some(opentelemetry) = observability_config.opentelemetry { + builder = builder + .with_opentelemetry( + &opentelemetry.level, + opentelemetry.endpoint, + "zksync-prover-fri".into(), + ) + .expect("Invalid OpenTelemetry config"); + } let _guard = builder.build(); // Report whether sentry is running after the logging subsystem was initialized. diff --git a/prover/prover_fri/src/prover_job_processor.rs b/prover/prover_fri/src/prover_job_processor.rs index 8f45aa5621ab..077c6edf0c1b 100644 --- a/prover/prover_fri/src/prover_job_processor.rs +++ b/prover/prover_fri/src/prover_job_processor.rs @@ -257,12 +257,15 @@ impl JobProcessor for Prover { async fn process_job( &self, + _job_id: &Self::JobId, job: Self::Job, _started_at: Instant, ) -> JoinHandle> { let config = Arc::clone(&self.config); let setup_data = self.get_setup_data(job.setup_data_key.clone()); tokio::task::spawn_blocking(move || { + let block_number = job.block_number; + let _span = tracing::info_span!("cpu_prove", %block_number).entered(); Ok(Self::prove( job, config, diff --git a/prover/witness_generator/src/basic_circuits.rs b/prover/witness_generator/src/basic_circuits.rs index 4f7256a4bc02..f960ae5ce97b 100644 --- a/prover/witness_generator/src/basic_circuits.rs +++ b/prover/witness_generator/src/basic_circuits.rs @@ -23,6 +23,7 @@ use prover_dal::{ }; use rand::Rng; use serde::{Deserialize, Serialize}; +use tracing::Instrument; use zkevm_test_harness::{ geometry_config::get_geometry_config, toolset::GeometryConfig, utils::generate_eip4844_circuit_and_witness, @@ -248,6 +249,7 @@ impl JobProcessor for BasicWitnessGenerator { #[allow(clippy::async_yields_async)] async fn process_job( &self, + _job_id: &Self::JobId, job: BasicWitnessGeneratorJob, started_at: Instant, ) -> tokio::task::JoinHandle>> { @@ -256,6 +258,7 @@ impl JobProcessor for BasicWitnessGenerator { let connection_pool = self.connection_pool.clone(); let prover_connection_pool = self.prover_connection_pool.clone(); tokio::spawn(async move { + let block_number = job.block_number; Ok(Self::process_job_impl( object_store, connection_pool, @@ -264,6 +267,7 @@ impl JobProcessor for BasicWitnessGenerator { started_at, config, ) + .instrument(tracing::info_span!("basic_circuit", %block_number)) .await) }) } diff --git a/prover/witness_generator/src/leaf_aggregation.rs b/prover/witness_generator/src/leaf_aggregation.rs index ffc6e5c26a85..4775eb82ad0f 100644 --- a/prover/witness_generator/src/leaf_aggregation.rs +++ b/prover/witness_generator/src/leaf_aggregation.rs @@ -146,10 +146,15 @@ impl JobProcessor for LeafAggregationWitnessGenerator { #[allow(clippy::async_yields_async)] async fn process_job( &self, + _job_id: &Self::JobId, job: LeafAggregationWitnessGeneratorJob, started_at: Instant, ) -> tokio::task::JoinHandle> { - tokio::task::spawn_blocking(move || Ok(Self::process_job_sync(job, started_at))) + tokio::task::spawn_blocking(move || { + let block_number = job.block_number; + let _span = tracing::info_span!("leaf_aggregation", %block_number).entered(); + Ok(Self::process_job_sync(job, started_at)) + }) } async fn save_result( diff --git a/prover/witness_generator/src/main.rs b/prover/witness_generator/src/main.rs index 5d56e60674d2..0c4e992a2406 100644 --- a/prover/witness_generator/src/main.rs +++ b/prover/witness_generator/src/main.rs @@ -77,6 +77,15 @@ async fn main() -> anyhow::Result<()> { .expect("Invalid Sentry URL") .with_sentry_environment(observability_config.sentry_environment); } + if let Some(opentelemetry) = observability_config.opentelemetry { + builder = builder + .with_opentelemetry( + &opentelemetry.level, + opentelemetry.endpoint, + "zksync-witness-generator".into(), + ) + .expect("Invalid OpenTelemetry config"); + } let _guard = builder.build(); // Report whether sentry is running after the logging subsystem was initialized. diff --git a/prover/witness_generator/src/node_aggregation.rs b/prover/witness_generator/src/node_aggregation.rs index 17a9cfd1e581..79d621999d57 100644 --- a/prover/witness_generator/src/node_aggregation.rs +++ b/prover/witness_generator/src/node_aggregation.rs @@ -179,10 +179,15 @@ impl JobProcessor for NodeAggregationWitnessGenerator { #[allow(clippy::async_yields_async)] async fn process_job( &self, + _job_id: &Self::JobId, job: NodeAggregationWitnessGeneratorJob, started_at: Instant, ) -> tokio::task::JoinHandle> { - tokio::task::spawn_blocking(move || Ok(Self::process_job_sync(job, started_at))) + tokio::task::spawn_blocking(move || { + let block_number = job.block_number; + let _span = tracing::info_span!("node_aggregation", %block_number).entered(); + Ok(Self::process_job_sync(job, started_at)) + }) } async fn save_result( diff --git a/prover/witness_generator/src/scheduler.rs b/prover/witness_generator/src/scheduler.rs index 86f039632f1c..681f4fc504a1 100644 --- a/prover/witness_generator/src/scheduler.rs +++ b/prover/witness_generator/src/scheduler.rs @@ -168,10 +168,15 @@ impl JobProcessor for SchedulerWitnessGenerator { #[allow(clippy::async_yields_async)] async fn process_job( &self, + _job_id: &Self::JobId, job: SchedulerWitnessGeneratorJob, started_at: Instant, ) -> tokio::task::JoinHandle> { - tokio::task::spawn_blocking(move || Ok(Self::process_job_sync(job, started_at))) + tokio::task::spawn_blocking(move || { + let block_number = job.block_number; + let _span = tracing::info_span!("scheduler", %block_number).entered(); + Ok(Self::process_job_sync(job, started_at)) + }) } async fn save_result( diff --git a/prover/witness_vector_generator/src/generator.rs b/prover/witness_vector_generator/src/generator.rs index e7cb9cfff95c..d08c65a900d2 100644 --- a/prover/witness_vector_generator/src/generator.rs +++ b/prover/witness_vector_generator/src/generator.rs @@ -123,10 +123,13 @@ impl JobProcessor for WitnessVectorGenerator { async fn process_job( &self, + _job_id: &Self::JobId, job: ProverJob, _started_at: Instant, ) -> JoinHandle> { tokio::task::spawn_blocking(move || { + let block_number = job.block_number; + let _span = tracing::info_span!("witness_vector_generator", %block_number).entered(); Self::generate_witness_vector(job, &Keystore::default()) }) } diff --git a/prover/witness_vector_generator/src/main.rs b/prover/witness_vector_generator/src/main.rs index e57a279435d9..3eaf8eda11e0 100644 --- a/prover/witness_vector_generator/src/main.rs +++ b/prover/witness_vector_generator/src/main.rs @@ -50,6 +50,15 @@ async fn main() -> anyhow::Result<()> { .expect("Invalid Sentry URL") .with_sentry_environment(observability_config.sentry_environment); } + if let Some(opentelemetry) = observability_config.opentelemetry { + builder = builder + .with_opentelemetry( + &opentelemetry.level, + opentelemetry.endpoint, + "zksync-witness-vector-generator".into(), + ) + .expect("Invalid OpenTelemetry config"); + } let _guard = builder.build(); let opt = Opt::from_args();