From c7c6b321a63dbcc7f1af045aa7416e697beab08f Mon Sep 17 00:00:00 2001 From: Igor Aleksanov Date: Tue, 7 Nov 2023 11:36:14 +0400 Subject: [PATCH] feat(core): Split config definitions and deserialization (#414) Instead of a thousand words: image This is the first PR of N preparing zkSync codebase for zkStack. --- In this episode: - `::from_env()` methods were extracted from `zksync_config` to a new crate `zksync_env_config`. - Any usage of `from_env()` has been removed from `core/lib` directory. In other words, configs are now deserialized only in binaries. - Some crates were redesigned somewhat, for example, `ObjectStoreFactory` now lacks `*_from_env` methods and can only be created via config. - Some configs obtained `for_tests()` constructors that allow creating configs without accessing env variables. Values there mostly match localhost defaults. Some tests have been rewritten to enforce values in the configs (thus they are now more resilient than they were before). The focus for this PR has been to remove any `from_env()` usage, so there are some unsolved issues, e.g. - We still access some env variables via `std::env::var`. - We have introduced a temporary ugly structure `TempConfigStore` that _reminds_ the former `ZkSyncConfig` but doesn't have the same level of danger. These (and other) things are to be resolved soon. --- Cargo.lock | 23 +- Cargo.toml | 1 + core/bin/block_reverter/Cargo.toml | 1 + core/bin/block_reverter/src/main.rs | 1 + core/bin/contract-verifier/Cargo.toml | 1 + core/bin/contract-verifier/src/main.rs | 1 + core/bin/contract-verifier/src/verifier.rs | 1 + core/bin/external_node/src/config/mod.rs | 18 +- .../Cargo.toml | 1 + .../src/main.rs | 1 + core/bin/rocksdb_util/Cargo.toml | 1 + core/bin/rocksdb_util/src/main.rs | 1 + core/bin/zksync_server/Cargo.toml | 1 + core/bin/zksync_server/src/main.rs | 54 +++- core/lib/config/Cargo.toml | 7 - core/lib/config/src/configs/alerts.rs | 39 --- core/lib/config/src/configs/api.rs | 186 +++---------- core/lib/config/src/configs/chain.rs | 182 +++---------- .../config/src/configs/circuit_synthesizer.rs | 48 ---- .../config/src/configs/contract_verifier.rs | 36 --- core/lib/config/src/configs/contracts.rs | 134 +++------- core/lib/config/src/configs/database.rs | 95 ------- core/lib/config/src/configs/eth_client.rs | 43 --- core/lib/config/src/configs/eth_sender.rs | 146 +++-------- core/lib/config/src/configs/eth_watch.rs | 44 ---- core/lib/config/src/configs/fetcher.rs | 69 +---- .../src/configs/fri_proof_compressor.rs | 51 ---- core/lib/config/src/configs/fri_prover.rs | 53 ---- .../config/src/configs/fri_prover_gateway.rs | 38 --- .../config/src/configs/fri_prover_group.rs | 240 +---------------- .../src/configs/fri_witness_generator.rs | 45 ---- .../configs/fri_witness_vector_generator.rs | 47 ---- core/lib/config/src/configs/house_keeper.rs | 60 ----- core/lib/config/src/configs/mod.rs | 17 -- core/lib/config/src/configs/object_store.rs | 79 ------ .../config/src/configs/proof_data_handler.rs | 37 --- core/lib/config/src/configs/prover.rs | 197 -------------- core/lib/config/src/configs/prover_group.rs | 146 ----------- core/lib/config/src/configs/utils.rs | 6 - .../config/src/configs/witness_generator.rs | 46 ---- core/lib/env_config/Cargo.toml | 18 ++ core/lib/env_config/src/alerts.rs | 37 +++ core/lib/env_config/src/api.rs | 151 +++++++++++ core/lib/env_config/src/chain.rs | 151 +++++++++++ .../lib/env_config/src/circuit_synthesizer.rs | 51 ++++ core/lib/env_config/src/contract_verifier.rs | 39 +++ core/lib/env_config/src/contracts.rs | 108 ++++++++ core/lib/env_config/src/database.rs | 100 +++++++ core/lib/env_config/src/eth_client.rs | 44 ++++ core/lib/env_config/src/eth_sender.rs | 111 ++++++++ core/lib/env_config/src/eth_watch.rs | 37 +++ core/lib/env_config/src/fetcher.rs | 68 +++++ .../env_config/src/fri_proof_compressor.rs | 54 ++++ core/lib/env_config/src/fri_prover.rs | 59 +++++ core/lib/env_config/src/fri_prover_gateway.rs | 42 +++ core/lib/env_config/src/fri_prover_group.rs | 247 ++++++++++++++++++ .../env_config/src/fri_witness_generator.rs | 46 ++++ .../src/fri_witness_vector_generator.rs | 50 ++++ core/lib/env_config/src/house_keeper.rs | 61 +++++ core/lib/env_config/src/lib.rs | 42 +++ core/lib/env_config/src/object_store.rs | 96 +++++++ core/lib/env_config/src/proof_data_handler.rs | 42 +++ core/lib/env_config/src/prover.rs | 197 ++++++++++++++ core/lib/env_config/src/prover_group.rs | 149 +++++++++++ .../configs => env_config/src}/test_utils.rs | 0 core/lib/env_config/src/utils.rs | 8 + core/lib/env_config/src/witness_generator.rs | 50 ++++ core/lib/object_store/src/raw.rs | 26 +- core/lib/prover_utils/src/region_fetcher.rs | 63 ++++- .../src/api_server/tx_sender/mod.rs | 4 - .../zksync_core/src/api_server/web3/tests.rs | 8 +- .../src/data_fetchers/token_list/one_inch.rs | 16 -- .../data_fetchers/token_price/coingecko.rs | 29 -- core/lib/zksync_core/src/eth_sender/tests.rs | 6 +- core/lib/zksync_core/src/eth_watch/mod.rs | 7 +- .../src/house_keeper/prover_queue_monitor.rs | 14 +- .../zksync_core/src/l1_gas_price/singleton.rs | 35 +-- core/lib/zksync_core/src/lib.rs | 235 ++++++++++++----- .../src/metadata_calculator/mod.rs | 16 +- .../src/metadata_calculator/tests.rs | 23 +- .../zksync_core/src/proof_data_handler/mod.rs | 18 +- .../batch_executor/tests/tester.rs | 12 +- .../src/state_keeper/io/mempool.rs | 13 +- .../src/state_keeper/io/tests/tester.rs | 6 +- core/lib/zksync_core/src/state_keeper/mod.rs | 3 + .../seal_criteria/criteria/gas.rs | 9 +- .../seal_criteria/criteria/pubdata_bytes.rs | 8 +- .../seal_criteria/criteria/slots.rs | 7 +- .../criteria/tx_encoding_size.rs | 8 +- core/lib/zksync_core/src/sync_layer/tests.rs | 2 +- core/lib/zksync_core/src/temp_config_store.rs | 47 ++++ .../src/witness_generator/basic_circuits.rs | 11 +- .../src/witness_generator/node_aggregation.rs | 7 +- etc/env/base/chain.toml | 3 - prover/circuit_synthesizer/Cargo.toml | 1 + .../src/circuit_synthesizer.rs | 5 +- prover/circuit_synthesizer/src/main.rs | 9 +- prover/proof_fri_compressor/Cargo.toml | 1 + prover/proof_fri_compressor/src/main.rs | 6 +- prover/prover/Cargo.toml | 1 + prover/prover/src/run.rs | 30 ++- prover/prover_fri/Cargo.toml | 1 + .../src/gpu_prover_job_processor.rs | 2 + prover/prover_fri/src/main.rs | 35 ++- prover/prover_fri/src/prover_job_processor.rs | 2 + prover/prover_fri/tests/basic_test.rs | 1 + prover/prover_fri_gateway/Cargo.toml | 1 + prover/prover_fri_gateway/src/main.rs | 6 +- .../setup_key_generator_and_server/Cargo.toml | 1 + .../setup_key_generator_and_server/src/lib.rs | 1 + .../Cargo.toml | 1 + .../src/lib.rs | 1 + prover/witness_generator/Cargo.toml | 1 + prover/witness_generator/src/main.rs | 21 +- prover/witness_generator/tests/basic_test.rs | 1 + prover/witness_vector_generator/Cargo.toml | 1 + prover/witness_vector_generator/src/main.rs | 13 +- 117 files changed, 2763 insertions(+), 2270 deletions(-) create mode 100644 core/lib/env_config/Cargo.toml create mode 100644 core/lib/env_config/src/alerts.rs create mode 100644 core/lib/env_config/src/api.rs create mode 100644 core/lib/env_config/src/chain.rs create mode 100644 core/lib/env_config/src/circuit_synthesizer.rs create mode 100644 core/lib/env_config/src/contract_verifier.rs create mode 100644 core/lib/env_config/src/contracts.rs create mode 100644 core/lib/env_config/src/database.rs create mode 100644 core/lib/env_config/src/eth_client.rs create mode 100644 core/lib/env_config/src/eth_sender.rs create mode 100644 core/lib/env_config/src/eth_watch.rs create mode 100644 core/lib/env_config/src/fetcher.rs create mode 100644 core/lib/env_config/src/fri_proof_compressor.rs create mode 100644 core/lib/env_config/src/fri_prover.rs create mode 100644 core/lib/env_config/src/fri_prover_gateway.rs create mode 100644 core/lib/env_config/src/fri_prover_group.rs create mode 100644 core/lib/env_config/src/fri_witness_generator.rs create mode 100644 core/lib/env_config/src/fri_witness_vector_generator.rs create mode 100644 core/lib/env_config/src/house_keeper.rs create mode 100644 core/lib/env_config/src/lib.rs create mode 100644 core/lib/env_config/src/object_store.rs create mode 100644 core/lib/env_config/src/proof_data_handler.rs create mode 100644 core/lib/env_config/src/prover.rs create mode 100644 core/lib/env_config/src/prover_group.rs rename core/lib/{config/src/configs => env_config/src}/test_utils.rs (100%) create mode 100644 core/lib/env_config/src/utils.rs create mode 100644 core/lib/env_config/src/witness_generator.rs create mode 100644 core/lib/zksync_core/src/temp_config_store.rs diff --git a/Cargo.lock b/Cargo.lock index 29e9b3be3b9f..953001c820a4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -924,6 +924,7 @@ dependencies = [ "zksync_config", "zksync_core", "zksync_dal", + "zksync_env_config", "zksync_types", ] @@ -3811,6 +3812,7 @@ dependencies = [ "tracing", "vlog", "zksync_config", + "zksync_env_config", "zksync_merkle_tree", "zksync_storage", "zksync_types", @@ -5471,6 +5473,7 @@ dependencies = [ "clap 4.4.6", "tempfile", "zksync_config", + "zksync_env_config", "zksync_storage", ] @@ -7935,17 +7938,10 @@ name = "zksync_config" version = "0.1.0" dependencies = [ "anyhow", - "bigdecimal", "envy", - "hex", - "num 0.3.1", - "once_cell", "serde", - "serde_json", - "url", "zksync_basic_types", "zksync_contracts", - "zksync_utils", ] [[package]] @@ -7973,6 +7969,7 @@ dependencies = [ "zksync_config", "zksync_contracts", "zksync_dal", + "zksync_env_config", "zksync_queued_job_processor", "zksync_types", "zksync_utils", @@ -8101,6 +8098,17 @@ dependencies = [ "zksync_utils", ] +[[package]] +name = "zksync_env_config" +version = "0.1.0" +dependencies = [ + "anyhow", + "envy", + "serde", + "zksync_basic_types", + "zksync_config", +] + [[package]] name = "zksync_eth_client" version = "0.1.0" @@ -8287,6 +8295,7 @@ dependencies = [ "vlog", "zksync_config", "zksync_core", + "zksync_env_config", "zksync_storage", "zksync_types", "zksync_utils", diff --git a/Cargo.toml b/Cargo.toml index a02652c3f617..75a4c7237d2b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,6 +21,7 @@ members = [ "core/lib/circuit_breaker", "core/lib/commitment_utils", "core/lib/dal", + "core/lib/env_config", "core/lib/eth_client", "core/lib/eth_signer", "core/lib/mempool", diff --git a/core/bin/block_reverter/Cargo.toml b/core/bin/block_reverter/Cargo.toml index 33147ad1cac1..634132f33058 100644 --- a/core/bin/block_reverter/Cargo.toml +++ b/core/bin/block_reverter/Cargo.toml @@ -12,6 +12,7 @@ publish = false # We don't want to publish our binaries. [dependencies] zksync_config = { path = "../../lib/config" } +zksync_env_config = { path = "../../lib/env_config" } zksync_dal = { path = "../../lib/dal" } zksync_types = { path = "../../lib/types" } zksync_core = { path = "../../lib/zksync_core" } diff --git a/core/bin/block_reverter/src/main.rs b/core/bin/block_reverter/src/main.rs index 971dfbeb8d15..443f6fe634b5 100644 --- a/core/bin/block_reverter/src/main.rs +++ b/core/bin/block_reverter/src/main.rs @@ -4,6 +4,7 @@ use tokio::io::{self, AsyncReadExt}; use zksync_config::{ContractsConfig, DBConfig, ETHClientConfig, ETHSenderConfig}; use zksync_dal::{connection::DbVariant, ConnectionPool}; +use zksync_env_config::FromEnv; use zksync_types::{L1BatchNumber, U256}; use zksync_core::block_reverter::{ diff --git a/core/bin/contract-verifier/Cargo.toml b/core/bin/contract-verifier/Cargo.toml index e86d24931bc8..1a22c1825b51 100644 --- a/core/bin/contract-verifier/Cargo.toml +++ b/core/bin/contract-verifier/Cargo.toml @@ -13,6 +13,7 @@ publish = false # We don't want to publish our binaries. [dependencies] zksync_types = { path = "../../lib/types" } zksync_dal = { path = "../../lib/dal" } +zksync_env_config = { path = "../../lib/env_config" } zksync_config = { path = "../../lib/config" } zksync_contracts = { path = "../../lib/contracts" } zksync_queued_job_processor = { path = "../../lib/queued_job_processor" } diff --git a/core/bin/contract-verifier/src/main.rs b/core/bin/contract-verifier/src/main.rs index 29696e58896d..a5c24450f6b6 100644 --- a/core/bin/contract-verifier/src/main.rs +++ b/core/bin/contract-verifier/src/main.rs @@ -4,6 +4,7 @@ use anyhow::Context as _; use prometheus_exporter::PrometheusExporterConfig; use zksync_config::{configs::PrometheusConfig, ApiConfig, ContractVerifierConfig}; use zksync_dal::ConnectionPool; +use zksync_env_config::FromEnv; use zksync_queued_job_processor::JobProcessor; use zksync_utils::wait_for_tasks::wait_for_tasks; diff --git a/core/bin/contract-verifier/src/verifier.rs b/core/bin/contract-verifier/src/verifier.rs index 97c42899254b..03e94dde75ce 100644 --- a/core/bin/contract-verifier/src/verifier.rs +++ b/core/bin/contract-verifier/src/verifier.rs @@ -12,6 +12,7 @@ use tokio::time; use zksync_config::ContractVerifierConfig; use zksync_dal::{ConnectionPool, StorageProcessor}; +use zksync_env_config::FromEnv; use zksync_queued_job_processor::{async_trait, JobProcessor}; use zksync_types::{ contract_verification_api::{ diff --git a/core/bin/external_node/src/config/mod.rs b/core/bin/external_node/src/config/mod.rs index 652ddd6e33ca..2c5666ee0e03 100644 --- a/core/bin/external_node/src/config/mod.rs +++ b/core/bin/external_node/src/config/mod.rs @@ -3,8 +3,7 @@ use serde::Deserialize; use std::{env, time::Duration}; use url::Url; -use zksync_basic_types::{Address, L1ChainId, L2ChainId, MiniblockNumber, H256}; -use zksync_contracts::BaseSystemContractsHashes; +use zksync_basic_types::{Address, L1ChainId, L2ChainId, MiniblockNumber}; use zksync_core::api_server::{ tx_sender::TxSenderConfig, web3::state::InternalApiConfig, web3::Namespace, }; @@ -32,9 +31,6 @@ pub struct RemoteENConfig { pub l2_chain_id: L2ChainId, pub l1_chain_id: L1ChainId, - pub default_aa_hash: H256, - pub bootloader_hash: H256, - pub fair_l2_gas_price: u64, } @@ -76,7 +72,6 @@ impl RemoteENConfig { .await .context("Failed to fetch last miniblock header")? .expect("Block is known to exist"); - let base_system_contract_hashes = block_header.base_system_contracts_hashes; Ok(Self { diamond_proxy_addr, @@ -87,18 +82,9 @@ impl RemoteENConfig { l2_weth_bridge_addr: bridges.l2_weth_bridge, l2_chain_id, l1_chain_id, - default_aa_hash: base_system_contract_hashes.default_aa, - bootloader_hash: base_system_contract_hashes.bootloader, fair_l2_gas_price: block_header.l2_fair_gas_price, }) } - - pub fn base_system_contracts_hashes(&self) -> BaseSystemContractsHashes { - BaseSystemContractsHashes { - default_aa: self.default_aa_hash, - bootloader: self.bootloader_hash, - } - } } /// This part of the external node config is completely optional to provide. @@ -509,8 +495,6 @@ impl From for TxSenderConfig { max_nonce_ahead: config.optional.max_nonce_ahead, fair_l2_gas_price: config.remote.fair_l2_gas_price, vm_execution_cache_misses_limit: config.optional.vm_execution_cache_misses_limit, - default_aa: config.remote.default_aa_hash, - bootloader: config.remote.bootloader_hash, // We set these values to the maximum since we don't know the actual values // and they will be enforced by the main node anyway. max_allowed_l2_tx_gas_limit: u32::MAX, diff --git a/core/bin/merkle_tree_consistency_checker/Cargo.toml b/core/bin/merkle_tree_consistency_checker/Cargo.toml index 344170c956af..6bd30586d22d 100644 --- a/core/bin/merkle_tree_consistency_checker/Cargo.toml +++ b/core/bin/merkle_tree_consistency_checker/Cargo.toml @@ -12,6 +12,7 @@ publish = false # We don't want to publish our binaries. [dependencies] zksync_config = { path = "../../lib/config" } +zksync_env_config = { path = "../../lib/env_config" } zksync_merkle_tree = { path = "../../lib/merkle_tree" } zksync_types = { path = "../../lib/types" } zksync_storage = { path = "../../lib/storage" } diff --git a/core/bin/merkle_tree_consistency_checker/src/main.rs b/core/bin/merkle_tree_consistency_checker/src/main.rs index a4a4aeec0bf6..b132bda87fa0 100644 --- a/core/bin/merkle_tree_consistency_checker/src/main.rs +++ b/core/bin/merkle_tree_consistency_checker/src/main.rs @@ -4,6 +4,7 @@ use clap::Parser; use std::{path::Path, time::Instant}; use zksync_config::DBConfig; +use zksync_env_config::FromEnv; use zksync_merkle_tree::domain::ZkSyncTree; use zksync_storage::RocksDB; use zksync_types::L1BatchNumber; diff --git a/core/bin/rocksdb_util/Cargo.toml b/core/bin/rocksdb_util/Cargo.toml index 5418169609f0..75d1afe19797 100644 --- a/core/bin/rocksdb_util/Cargo.toml +++ b/core/bin/rocksdb_util/Cargo.toml @@ -12,6 +12,7 @@ publish = false # We don't want to publish our binaries. [dependencies] zksync_config = { path = "../../lib/config" } +zksync_env_config = { path = "../../lib/env_config" } zksync_storage = { path = "../../lib/storage" } anyhow = "1.0" diff --git a/core/bin/rocksdb_util/src/main.rs b/core/bin/rocksdb_util/src/main.rs index a579dfdbfb24..30d3d42e771c 100644 --- a/core/bin/rocksdb_util/src/main.rs +++ b/core/bin/rocksdb_util/src/main.rs @@ -2,6 +2,7 @@ use anyhow::Context as _; use clap::{Parser, Subcommand}; use zksync_config::DBConfig; +use zksync_env_config::FromEnv; use zksync_storage::rocksdb::{ backup::{BackupEngine, BackupEngineOptions, RestoreOptions}, Env, Error, Options, DB, diff --git a/core/bin/zksync_server/Cargo.toml b/core/bin/zksync_server/Cargo.toml index 674736fe98bc..a1de2ef057ad 100644 --- a/core/bin/zksync_server/Cargo.toml +++ b/core/bin/zksync_server/Cargo.toml @@ -12,6 +12,7 @@ publish = false # We don't want to publish our binaries. [dependencies] zksync_config = { path = "../../lib/config" } +zksync_env_config = { path = "../../lib/env_config" } zksync_storage = { path = "../../lib/storage" } zksync_utils = { path = "../../lib/utils" } zksync_types = { path = "../../lib/types" } diff --git a/core/bin/zksync_server/src/main.rs b/core/bin/zksync_server/src/main.rs index 4a0bff0cd442..23794f5dafd4 100644 --- a/core/bin/zksync_server/src/main.rs +++ b/core/bin/zksync_server/src/main.rs @@ -2,13 +2,28 @@ use anyhow::Context as _; use clap::Parser; use std::{str::FromStr, time::Duration}; -use zksync_config::configs::chain::NetworkConfig; -use zksync_config::{ContractsConfig, ETHClientConfig, ETHSenderConfig}; +use zksync_config::{ + configs::{ + api::{HealthCheckConfig, MerkleTreeApiConfig, Web3JsonRpcConfig}, + chain::{ + CircuitBreakerConfig, MempoolConfig, NetworkConfig, OperationsManagerConfig, + StateKeeperConfig, + }, + house_keeper::HouseKeeperConfig, + FriProofCompressorConfig, FriProverConfig, FriWitnessGeneratorConfig, PrometheusConfig, + ProofDataHandlerConfig, ProverGroupConfig, WitnessGeneratorConfig, + }, + ApiConfig, ContractsConfig, DBConfig, ETHClientConfig, ETHSenderConfig, ETHWatchConfig, + FetcherConfig, GasAdjusterConfig, ObjectStoreConfig, ProverConfigs, +}; + +use zksync_core::temp_config_store::TempConfigStore; use zksync_core::{ genesis_init, initialize_components, is_genesis_needed, setup_sigint_handler, Component, Components, }; +use zksync_env_config::FromEnv; use zksync_storage::RocksDB; use zksync_utils::wait_for_tasks::wait_for_tasks; @@ -102,9 +117,42 @@ async fn main() -> anyhow::Result<()> { [Component::WitnessGenerator(Some(_), _)] ); + // TODO (QIT-22): Only deserialize configs on demand. + // Right now, we are trying to deserialize all the configs that may be needed by `zksync_core`. + // "May" is the key word here, since some configs are only used by certain component configuration, + // hence we are using `Option`s. + let configs = TempConfigStore { + health_check_config: HealthCheckConfig::from_env().ok(), + merkle_tree_api_config: MerkleTreeApiConfig::from_env().ok(), + web3_json_rpc_config: Web3JsonRpcConfig::from_env().ok(), + circuit_breaker_config: CircuitBreakerConfig::from_env().ok(), + mempool_config: MempoolConfig::from_env().ok(), + network_config: NetworkConfig::from_env().ok(), + operations_manager_config: OperationsManagerConfig::from_env().ok(), + state_keeper_config: StateKeeperConfig::from_env().ok(), + house_keeper_config: HouseKeeperConfig::from_env().ok(), + fri_proof_compressor_config: FriProofCompressorConfig::from_env().ok(), + fri_prover_config: FriProverConfig::from_env().ok(), + fri_witness_generator_config: FriWitnessGeneratorConfig::from_env().ok(), + prometheus_config: PrometheusConfig::from_env().ok(), + proof_data_handler_config: ProofDataHandlerConfig::from_env().ok(), + prover_group_config: ProverGroupConfig::from_env().ok(), + witness_generator_config: WitnessGeneratorConfig::from_env().ok(), + api_config: ApiConfig::from_env().ok(), + contracts_config: ContractsConfig::from_env().ok(), + db_config: DBConfig::from_env().ok(), + eth_client_config: ETHClientConfig::from_env().ok(), + eth_sender_config: ETHSenderConfig::from_env().ok(), + eth_watch_config: ETHWatchConfig::from_env().ok(), + fetcher_config: FetcherConfig::from_env().ok(), + gas_adjuster_config: GasAdjusterConfig::from_env().ok(), + prover_configs: ProverConfigs::from_env().ok(), + object_store_config: ObjectStoreConfig::from_env().ok(), + }; + // Run core actors. let (core_task_handles, stop_sender, cb_receiver, health_check_handle) = - initialize_components(components, is_only_oneshot_witness_generator_task) + initialize_components(&configs, components, is_only_oneshot_witness_generator_task) .await .context("Unable to start Core actors")?; diff --git a/core/lib/config/Cargo.toml b/core/lib/config/Cargo.toml index 948ec9aba56e..6ff3363f895a 100644 --- a/core/lib/config/Cargo.toml +++ b/core/lib/config/Cargo.toml @@ -11,15 +11,8 @@ categories = ["cryptography"] [dependencies] zksync_basic_types = { path = "../../lib/basic_types" } -zksync_utils = { path = "../../lib/utils" } zksync_contracts = { path = "../../lib/contracts" } anyhow = "1.0" -url = "2.1" -num = "0.3.1" serde = { version = "1.0", features = ["derive"] } -serde_json = "1.0" envy = "0.4" -once_cell = "1.13.0" -bigdecimal = "0.2.2" -hex = "0.4" diff --git a/core/lib/config/src/configs/alerts.rs b/core/lib/config/src/configs/alerts.rs index ffdd0dea0441..070ebb970fe8 100644 --- a/core/lib/config/src/configs/alerts.rs +++ b/core/lib/config/src/configs/alerts.rs @@ -1,47 +1,8 @@ -// Built-in uses -// External uses use serde::Deserialize; -use super::envy_load; - #[derive(Debug, Deserialize, Clone, PartialEq)] pub struct AlertsConfig { /// List of panics' messages from external crypto code, /// that are sporadic and needed to be handled separately pub sporadic_crypto_errors_substrs: Vec, } - -impl AlertsConfig { - pub fn from_env() -> anyhow::Result { - envy_load("sporadic_crypto_errors_substrs", "ALERTS_") - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::configs::test_utils::EnvMutex; - - static MUTEX: EnvMutex = EnvMutex::new(); - - fn expected_config() -> AlertsConfig { - AlertsConfig { - sporadic_crypto_errors_substrs: vec![ - "EventDestroyErr".to_string(), - "Can't free memory of DeviceBuf".to_string(), - "value: PoisonError".to_string(), - ], - } - } - - #[test] - fn test_from_env() { - let mut lock = MUTEX.lock(); - let config = r#" - ALERTS_SPORADIC_CRYPTO_ERRORS_SUBSTRS="EventDestroyErr,Can't free memory of DeviceBuf,value: PoisonError" - "#; - lock.set_env(config); - - assert_eq!(AlertsConfig::from_env().unwrap(), expected_config()); - } -} diff --git a/core/lib/config/src/configs/api.rs b/core/lib/config/src/configs/api.rs index e1f79d052260..dc14514aec45 100644 --- a/core/lib/config/src/configs/api.rs +++ b/core/lib/config/src/configs/api.rs @@ -1,9 +1,7 @@ -use anyhow::Context as _; use serde::Deserialize; use std::{net::SocketAddr, time::Duration}; -use super::envy_load; pub use crate::configs::PrometheusConfig; use zksync_basic_types::H256; @@ -22,19 +20,6 @@ pub struct ApiConfig { pub merkle_tree: MerkleTreeApiConfig, } -impl ApiConfig { - pub fn from_env() -> anyhow::Result { - Ok(Self { - web3_json_rpc: Web3JsonRpcConfig::from_env().context("Web3JsonRpcConfig")?, - contract_verification: ContractVerificationApiConfig::from_env() - .context("ContractVerificationApiConfig")?, - prometheus: PrometheusConfig::from_env().context("PrometheusConfig")?, - healthcheck: HealthCheckConfig::from_env().context("HealthCheckConfig")?, - merkle_tree: MerkleTreeApiConfig::from_env().context("MerkleTreeApiConfig")?, - }) - } -} - #[derive(Debug, Deserialize, Clone, PartialEq)] pub struct Web3JsonRpcConfig { /// Port to which the HTTP RPC server is listening. @@ -105,8 +90,40 @@ pub struct Web3JsonRpcConfig { } impl Web3JsonRpcConfig { - pub fn from_env() -> anyhow::Result { - envy_load("web3_json_rpc", "API_WEB3_JSON_RPC_") + /// Creates a mock instance of `Web3JsonRpcConfig` to be used in tests. + /// Ports and some fields that may affect execution are set to the same values used by default in + /// the localhost environment. Other fields are set to default values. + pub fn for_tests() -> Self { + Self { + http_port: 3050, + http_url: "http://localhost:3050".into(), + ws_port: 3051, + ws_url: "ws://localhost:3051".into(), + req_entities_limit: Some(10000), + filters_limit: Some(10000), + subscriptions_limit: Some(10000), + pubsub_polling_interval: Some(200), + threads_per_server: 1, + max_nonce_ahead: 50, + gas_price_scale_factor: 1.2, + transactions_per_sec_limit: Default::default(), + request_timeout: Default::default(), + account_pks: Default::default(), + estimate_gas_scale_factor: 1.2, + estimate_gas_acceptable_overestimation: 1000, + max_tx_size: 1000000, + vm_execution_cache_misses_limit: Default::default(), + vm_concurrency_limit: Default::default(), + factory_deps_cache_size_mb: Default::default(), + initial_writes_cache_size_mb: Default::default(), + latest_values_cache_size_mb: Default::default(), + http_threads: Default::default(), + ws_threads: Default::default(), + fee_history_limit: Default::default(), + max_batch_request_size: Default::default(), + max_response_body_size_mb: Default::default(), + websocket_requests_per_minute_limit: Default::default(), + } } pub fn http_bind_addr(&self) -> SocketAddr { @@ -197,10 +214,6 @@ pub struct HealthCheckConfig { } impl HealthCheckConfig { - pub fn from_env() -> anyhow::Result { - envy_load("healthcheck", "API_HEALTHCHECK_") - } - pub fn bind_addr(&self) -> SocketAddr { SocketAddr::new("0.0.0.0".parse().unwrap(), self.port) } @@ -220,10 +233,6 @@ impl ContractVerificationApiConfig { pub fn bind_addr(&self) -> SocketAddr { SocketAddr::new("0.0.0.0".parse().unwrap(), self.port) } - - pub fn from_env() -> anyhow::Result { - envy_load("contract_verification", "API_CONTRACT_VERIFICATION_") - } } /// Configuration for the Merkle tree API. @@ -238,131 +247,4 @@ impl MerkleTreeApiConfig { const fn default_port() -> u16 { 3_072 } - - /// Loads configuration from env variables. - pub fn from_env() -> anyhow::Result { - envy_load("merkle_tree_api", "API_MERKLE_TREE_") - } -} - -#[cfg(test)] -mod tests { - use std::net::IpAddr; - - use super::*; - use crate::configs::test_utils::{hash, EnvMutex}; - - static MUTEX: EnvMutex = EnvMutex::new(); - - fn expected_config() -> ApiConfig { - ApiConfig { - web3_json_rpc: Web3JsonRpcConfig { - http_port: 3050, - http_url: "http://127.0.0.1:3050".into(), - ws_port: 3051, - ws_url: "ws://127.0.0.1:3051".into(), - req_entities_limit: Some(10000), - filters_limit: Some(10000), - subscriptions_limit: Some(10000), - pubsub_polling_interval: Some(200), - threads_per_server: 128, - max_nonce_ahead: 5, - transactions_per_sec_limit: Some(1000), - request_timeout: Some(10), - account_pks: Some(vec![ - hash("0x0000000000000000000000000000000000000000000000000000000000000001"), - hash("0x0000000000000000000000000000000000000000000000000000000000000002"), - ]), - estimate_gas_scale_factor: 1.0f64, - gas_price_scale_factor: 1.2, - estimate_gas_acceptable_overestimation: 1000, - max_tx_size: 1000000, - vm_execution_cache_misses_limit: None, - vm_concurrency_limit: Some(512), - factory_deps_cache_size_mb: Some(128), - initial_writes_cache_size_mb: Some(32), - latest_values_cache_size_mb: Some(256), - http_threads: Some(128), - ws_threads: Some(256), - fee_history_limit: Some(100), - max_batch_request_size: Some(200), - max_response_body_size_mb: Some(10), - websocket_requests_per_minute_limit: Some(10), - }, - contract_verification: ContractVerificationApiConfig { - port: 3070, - url: "http://127.0.0.1:3070".into(), - threads_per_server: 128, - }, - prometheus: PrometheusConfig { - listener_port: 3312, - pushgateway_url: "http://127.0.0.1:9091".into(), - push_interval_ms: Some(100), - }, - healthcheck: HealthCheckConfig { port: 8081 }, - merkle_tree: MerkleTreeApiConfig { port: 8082 }, - } - } - - #[test] - fn from_env() { - let mut lock = MUTEX.lock(); - let config = r#" - API_WEB3_JSON_RPC_HTTP_PORT="3050" - API_WEB3_JSON_RPC_HTTP_URL="http://127.0.0.1:3050" - API_WEB3_JSON_RPC_WS_PORT="3051" - API_WEB3_JSON_RPC_WS_URL="ws://127.0.0.1:3051" - API_WEB3_JSON_RPC_REQ_ENTITIES_LIMIT=10000 - API_WEB3_JSON_RPC_FILTERS_LIMIT=10000 - API_WEB3_JSON_RPC_SUBSCRIPTIONS_LIMIT=10000 - API_WEB3_JSON_RPC_PUBSUB_POLLING_INTERVAL=200 - API_WEB3_JSON_RPC_THREADS_PER_SERVER=128 - API_WEB3_JSON_RPC_MAX_NONCE_AHEAD=5 - API_WEB3_JSON_RPC_GAS_PRICE_SCALE_FACTOR=1.2 - API_WEB3_JSON_RPC_TRANSACTIONS_PER_SEC_LIMIT=1000 - API_WEB3_JSON_RPC_REQUEST_TIMEOUT=10 - API_WEB3_JSON_RPC_ACCOUNT_PKS="0x0000000000000000000000000000000000000000000000000000000000000001,0x0000000000000000000000000000000000000000000000000000000000000002" - API_WEB3_JSON_RPC_ESTIMATE_GAS_SCALE_FACTOR=1.0 - API_WEB3_JSON_RPC_ESTIMATE_GAS_ACCEPTABLE_OVERESTIMATION=1000 - API_WEB3_JSON_RPC_MAX_TX_SIZE=1000000 - API_WEB3_JSON_RPC_VM_CONCURRENCY_LIMIT=512 - API_WEB3_JSON_RPC_FACTORY_DEPS_CACHE_SIZE_MB=128 - API_WEB3_JSON_RPC_INITIAL_WRITES_CACHE_SIZE_MB=32 - API_WEB3_JSON_RPC_LATEST_VALUES_CACHE_SIZE_MB=256 - API_WEB3_JSON_RPC_HTTP_THREADS=128 - API_WEB3_JSON_RPC_WS_THREADS=256 - API_WEB3_JSON_RPC_FEE_HISTORY_LIMIT=100 - API_WEB3_JSON_RPC_MAX_BATCH_REQUEST_SIZE=200 - API_WEB3_JSON_RPC_WEBSOCKET_REQUESTS_PER_MINUTE_LIMIT=10 - API_CONTRACT_VERIFICATION_PORT="3070" - API_CONTRACT_VERIFICATION_URL="http://127.0.0.1:3070" - API_CONTRACT_VERIFICATION_THREADS_PER_SERVER=128 - API_WEB3_JSON_RPC_MAX_RESPONSE_BODY_SIZE_MB=10 - API_PROMETHEUS_LISTENER_PORT="3312" - API_PROMETHEUS_PUSHGATEWAY_URL="http://127.0.0.1:9091" - API_PROMETHEUS_PUSH_INTERVAL_MS=100 - API_HEALTHCHECK_PORT=8081 - API_MERKLE_TREE_PORT=8082 - "#; - lock.set_env(config); - - let actual = ApiConfig::from_env().unwrap(); - assert_eq!(actual, expected_config()); - } - - /// Checks the correctness of the config helper methods. - #[test] - fn methods() { - let config = expected_config(); - let bind_broadcast_addr: IpAddr = "0.0.0.0".parse().unwrap(); - - assert_eq!( - config.web3_json_rpc.pubsub_interval(), - Duration::from_millis(200) - ); - assert_eq!( - config.contract_verification.bind_addr(), - SocketAddr::new(bind_broadcast_addr, config.contract_verification.port) - ); - } } diff --git a/core/lib/config/src/configs/chain.rs b/core/lib/config/src/configs/chain.rs index ddf0c85e63e5..95392c8df839 100644 --- a/core/lib/config/src/configs/chain.rs +++ b/core/lib/config/src/configs/chain.rs @@ -1,14 +1,11 @@ /// External uses -use anyhow::Context as _; use serde::Deserialize; +use std::str::FromStr; /// Built-in uses use std::time::Duration; // Local uses use zksync_basic_types::network::Network; -use zksync_basic_types::{Address, L2ChainId, H256}; -use zksync_contracts::BaseSystemContractsHashes; - -use super::envy_load; +use zksync_basic_types::{Address, L2ChainId}; #[derive(Debug, Deserialize, Clone, PartialEq)] pub struct ChainConfig { @@ -24,20 +21,6 @@ pub struct ChainConfig { pub circuit_breaker: CircuitBreakerConfig, } -impl ChainConfig { - pub fn from_env() -> anyhow::Result { - Ok(Self { - // TODO rename `eth` to `network` - network: NetworkConfig::from_env().context("NetworkConfig")?, - state_keeper: StateKeeperConfig::from_env().context("StateKeeperConfig")?, - operations_manager: OperationsManagerConfig::from_env() - .context("OperationsManagerConfig")?, - mempool: MempoolConfig::from_env().context("MempoolConfig")?, - circuit_breaker: CircuitBreakerConfig::from_env().context("CircuitBreakerConfig")?, - }) - } -} - #[derive(Debug, Deserialize, Clone, PartialEq)] pub struct NetworkConfig { /// Name of the used Ethereum network, e.g. `localhost` or `rinkeby`. @@ -51,8 +34,13 @@ pub struct NetworkConfig { } impl NetworkConfig { - pub fn from_env() -> anyhow::Result { - envy_load("network", "CHAIN_ETH_") + /// Creates a config object suitable for use in unit tests. + pub fn for_tests() -> NetworkConfig { + Self { + network: Network::Localhost, + zksync_network: "localhost".into(), + zksync_network_id: L2ChainId::default(), + } } } @@ -96,9 +84,6 @@ pub struct StateKeeperConfig { /// The price the operator spends on 1 gas of computation in wei. pub fair_l2_gas_price: u64, - pub bootloader_hash: H256, - pub default_aa_hash: H256, - /// Max number of computational gas that validation step is allowed to take. pub validation_computational_gas_limit: u32, pub save_call_traces: bool, @@ -115,14 +100,31 @@ pub struct StateKeeperConfig { } impl StateKeeperConfig { - pub fn from_env() -> anyhow::Result { - envy_load("state_keeper", "CHAIN_STATE_KEEPER_") - } - - pub fn base_system_contracts_hashes(&self) -> BaseSystemContractsHashes { - BaseSystemContractsHashes { - bootloader: self.bootloader_hash, - default_aa: self.default_aa_hash, + /// Creates a config object suitable for use in unit tests. + /// Values mostly repeat the values used in the localhost environment. + pub fn for_tests() -> Self { + Self { + transaction_slots: 250, + block_commit_deadline_ms: 2500, + miniblock_commit_deadline_ms: 1000, + miniblock_seal_queue_capacity: 10, + max_single_tx_gas: 6000000, + max_allowed_l2_tx_gas_limit: 4000000000, + reject_tx_at_geometry_percentage: 0.95, + reject_tx_at_eth_params_percentage: 0.95, + reject_tx_at_gas_percentage: 0.95, + close_block_at_geometry_percentage: 0.95, + close_block_at_eth_params_percentage: 0.95, + close_block_at_gas_percentage: 0.95, + fee_account_addr: Address::from_str("0xde03a0B5963f75f1C8485B355fF6D30f3093BDE7") + .unwrap(), + fair_l2_gas_price: 250000000, + validation_computational_gas_limit: 300000, + save_call_traces: true, + virtual_blocks_interval: 1, + virtual_blocks_per_miniblock: 1, + upload_witness_inputs_to_gcs: false, + enum_index_migration_chunk_size: None, } } @@ -138,10 +140,6 @@ pub struct OperationsManagerConfig { } impl OperationsManagerConfig { - pub fn from_env() -> anyhow::Result { - envy_load("operations_manager", "CHAIN_OPERATIONS_MANAGER_") - } - pub fn delay_interval(&self) -> Duration { Duration::from_millis(self.delay_interval) } @@ -156,10 +154,6 @@ pub struct CircuitBreakerConfig { } impl CircuitBreakerConfig { - pub fn from_env() -> anyhow::Result { - envy_load("circuit_breaker", "CHAIN_CIRCUIT_BREAKER_") - } - pub fn sync_interval(&self) -> Duration { Duration::from_millis(self.sync_interval_ms) } @@ -191,112 +185,4 @@ impl MempoolConfig { pub fn delay_interval(&self) -> Duration { Duration::from_millis(self.delay_interval) } - - pub fn from_env() -> anyhow::Result { - envy_load("mempool", "CHAIN_MEMPOOL_") - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::configs::test_utils::{addr, EnvMutex}; - - static MUTEX: EnvMutex = EnvMutex::new(); - - fn expected_config() -> ChainConfig { - ChainConfig { - network: NetworkConfig { - network: "localhost".parse().unwrap(), - zksync_network: "localhost".to_string(), - zksync_network_id: L2ChainId::from(270), - }, - state_keeper: StateKeeperConfig { - transaction_slots: 50, - block_commit_deadline_ms: 2500, - miniblock_commit_deadline_ms: 1000, - miniblock_seal_queue_capacity: 10, - max_single_tx_gas: 1_000_000, - max_allowed_l2_tx_gas_limit: 2_000_000_000, - close_block_at_eth_params_percentage: 0.2, - close_block_at_gas_percentage: 0.8, - close_block_at_geometry_percentage: 0.5, - reject_tx_at_eth_params_percentage: 0.8, - reject_tx_at_geometry_percentage: 0.3, - fee_account_addr: addr("de03a0B5963f75f1C8485B355fF6D30f3093BDE7"), - reject_tx_at_gas_percentage: 0.5, - fair_l2_gas_price: 250000000, - bootloader_hash: H256::from(&[254; 32]), - default_aa_hash: H256::from(&[254; 32]), - validation_computational_gas_limit: 10_000_000, - save_call_traces: false, - virtual_blocks_interval: 1, - virtual_blocks_per_miniblock: 1, - upload_witness_inputs_to_gcs: false, - enum_index_migration_chunk_size: Some(2_000), - }, - operations_manager: OperationsManagerConfig { - delay_interval: 100, - }, - mempool: MempoolConfig { - sync_interval_ms: 10, - sync_batch_size: 1000, - capacity: 1_000_000, - stuck_tx_timeout: 10, - remove_stuck_txs: true, - delay_interval: 100, - }, - circuit_breaker: CircuitBreakerConfig { - sync_interval_ms: 1000, - http_req_max_retry_number: 5, - http_req_retry_interval_sec: 2, - replication_lag_limit_sec: Some(10), - }, - } - } - - #[test] - fn from_env() { - let mut lock = MUTEX.lock(); - let config = r#" - CHAIN_ETH_NETWORK="localhost" - CHAIN_ETH_ZKSYNC_NETWORK="localhost" - CHAIN_ETH_ZKSYNC_NETWORK_ID=270 - CHAIN_STATE_KEEPER_TRANSACTION_SLOTS="50" - CHAIN_STATE_KEEPER_FEE_ACCOUNT_ADDR="0xde03a0B5963f75f1C8485B355fF6D30f3093BDE7" - CHAIN_STATE_KEEPER_MAX_SINGLE_TX_GAS="1000000" - CHAIN_STATE_KEEPER_MAX_ALLOWED_L2_TX_GAS_LIMIT="2000000000" - CHAIN_STATE_KEEPER_CLOSE_BLOCK_AT_GEOMETRY_PERCENTAGE="0.5" - CHAIN_STATE_KEEPER_CLOSE_BLOCK_AT_GAS_PERCENTAGE="0.8" - CHAIN_STATE_KEEPER_CLOSE_BLOCK_AT_ETH_PARAMS_PERCENTAGE="0.2" - CHAIN_STATE_KEEPER_REJECT_TX_AT_GEOMETRY_PERCENTAGE="0.3" - CHAIN_STATE_KEEPER_REJECT_TX_AT_ETH_PARAMS_PERCENTAGE="0.8" - CHAIN_STATE_KEEPER_REJECT_TX_AT_GAS_PERCENTAGE="0.5" - CHAIN_STATE_KEEPER_BLOCK_COMMIT_DEADLINE_MS="2500" - CHAIN_STATE_KEEPER_MINIBLOCK_COMMIT_DEADLINE_MS="1000" - CHAIN_STATE_KEEPER_MINIBLOCK_SEAL_QUEUE_CAPACITY="10" - CHAIN_STATE_KEEPER_FAIR_L2_GAS_PRICE="250000000" - CHAIN_STATE_KEEPER_BOOTLOADER_HASH="0xfefefefefefefefefefefefefefefefefefefefefefefefefefefefefefefefe" - CHAIN_STATE_KEEPER_DEFAULT_AA_HASH="0xfefefefefefefefefefefefefefefefefefefefefefefefefefefefefefefefe" - CHAIN_STATE_KEEPER_VALIDATION_COMPUTATIONAL_GAS_LIMIT="10000000" - CHAIN_STATE_KEEPER_SAVE_CALL_TRACES="false" - CHAIN_STATE_KEEPER_UPLOAD_WITNESS_INPUTS_TO_GCS="false" - CHAIN_STATE_KEEPER_ENUM_INDEX_MIGRATION_CHUNK_SIZE="2000" - CHAIN_OPERATIONS_MANAGER_DELAY_INTERVAL="100" - CHAIN_MEMPOOL_SYNC_INTERVAL_MS="10" - CHAIN_MEMPOOL_SYNC_BATCH_SIZE="1000" - CHAIN_MEMPOOL_STUCK_TX_TIMEOUT="10" - CHAIN_MEMPOOL_REMOVE_STUCK_TXS="true" - CHAIN_MEMPOOL_DELAY_INTERVAL="100" - CHAIN_MEMPOOL_CAPACITY="1000000" - CHAIN_CIRCUIT_BREAKER_SYNC_INTERVAL_MS="1000" - CHAIN_CIRCUIT_BREAKER_HTTP_REQ_MAX_RETRY_NUMBER="5" - CHAIN_CIRCUIT_BREAKER_HTTP_REQ_RETRY_INTERVAL_SEC="2" - CHAIN_CIRCUIT_BREAKER_REPLICATION_LAG_LIMIT_SEC="10" - "#; - lock.set_env(config); - - let actual = ChainConfig::from_env().unwrap(); - assert_eq!(actual, expected_config()); - } } diff --git a/core/lib/config/src/configs/circuit_synthesizer.rs b/core/lib/config/src/configs/circuit_synthesizer.rs index d336b4f5b18c..8df0f4bbd1a1 100644 --- a/core/lib/config/src/configs/circuit_synthesizer.rs +++ b/core/lib/config/src/configs/circuit_synthesizer.rs @@ -2,8 +2,6 @@ use std::time::Duration; use serde::Deserialize; -use super::envy_load; - /// Configuration for the witness generation #[derive(Debug, Deserialize, Clone, PartialEq)] pub struct CircuitSynthesizerConfig { @@ -26,10 +24,6 @@ pub struct CircuitSynthesizerConfig { } impl CircuitSynthesizerConfig { - pub fn from_env() -> anyhow::Result { - envy_load("circuit_synthesizer", "CIRCUIT_SYNTHESIZER_") - } - pub fn generation_timeout(&self) -> Duration { Duration::from_secs(self.generation_timeout_in_secs as u64) } @@ -46,45 +40,3 @@ impl CircuitSynthesizerConfig { Duration::from_millis(self.prover_instance_poll_time_in_milli_secs as u64) } } - -#[cfg(test)] -mod tests { - use super::*; - use crate::configs::test_utils::EnvMutex; - - static MUTEX: EnvMutex = EnvMutex::new(); - - fn expected_config() -> CircuitSynthesizerConfig { - CircuitSynthesizerConfig { - generation_timeout_in_secs: 1000u16, - max_attempts: 2, - gpu_prover_queue_timeout_in_secs: 1000u16, - prover_instance_wait_timeout_in_secs: 1000u16, - prover_instance_poll_time_in_milli_secs: 250u16, - prometheus_listener_port: 3314, - prometheus_pushgateway_url: "http://127.0.0.1:9091".to_string(), - prometheus_push_interval_ms: Some(100), - prover_group_id: 0, - } - } - - #[test] - fn from_env() { - let mut lock = MUTEX.lock(); - let config = r#" - CIRCUIT_SYNTHESIZER_GENERATION_TIMEOUT_IN_SECS=1000 - CIRCUIT_SYNTHESIZER_MAX_ATTEMPTS=2 - CIRCUIT_SYNTHESIZER_GPU_PROVER_QUEUE_TIMEOUT_IN_SECS=1000 - CIRCUIT_SYNTHESIZER_PROVER_INSTANCE_WAIT_TIMEOUT_IN_SECS=1000 - CIRCUIT_SYNTHESIZER_PROVER_INSTANCE_POLL_TIME_IN_MILLI_SECS=250 - CIRCUIT_SYNTHESIZER_PROMETHEUS_LISTENER_PORT=3314 - CIRCUIT_SYNTHESIZER_PROMETHEUS_PUSHGATEWAY_URL="http://127.0.0.1:9091" - CIRCUIT_SYNTHESIZER_PROMETHEUS_PUSH_INTERVAL_MS=100 - CIRCUIT_SYNTHESIZER_PROVER_GROUP_ID=0 - "#; - lock.set_env(config); - - let actual = CircuitSynthesizerConfig::from_env().unwrap(); - assert_eq!(actual, expected_config()); - } -} diff --git a/core/lib/config/src/configs/contract_verifier.rs b/core/lib/config/src/configs/contract_verifier.rs index 08f05e8f264d..5c2a1608c8f5 100644 --- a/core/lib/config/src/configs/contract_verifier.rs +++ b/core/lib/config/src/configs/contract_verifier.rs @@ -2,8 +2,6 @@ use std::time::Duration; // External uses use serde::Deserialize; -// Local uses -use super::envy_load; #[derive(Debug, Deserialize, Clone, PartialEq)] pub struct ContractVerifierConfig { @@ -16,10 +14,6 @@ pub struct ContractVerifierConfig { } impl ContractVerifierConfig { - pub fn from_env() -> anyhow::Result { - envy_load("contract_verifier", "CONTRACT_VERIFIER_") - } - pub fn compilation_timeout(&self) -> Duration { Duration::from_secs(self.compilation_timeout) } @@ -28,33 +22,3 @@ impl ContractVerifierConfig { Duration::from_millis(self.polling_interval.unwrap_or(1000)) } } - -#[cfg(test)] -mod tests { - use super::*; - use crate::configs::test_utils::EnvMutex; - - static MUTEX: EnvMutex = EnvMutex::new(); - - fn expected_config() -> ContractVerifierConfig { - ContractVerifierConfig { - compilation_timeout: 30, - polling_interval: Some(1000), - prometheus_port: 3314, - } - } - - #[test] - fn from_env() { - let mut lock = MUTEX.lock(); - let config = r#" - CONTRACT_VERIFIER_COMPILATION_TIMEOUT=30 - CONTRACT_VERIFIER_POLLING_INTERVAL=1000 - CONTRACT_VERIFIER_PROMETHEUS_PORT=3314 - "#; - lock.set_env(config); - - let actual = ContractVerifierConfig::from_env().unwrap(); - assert_eq!(actual, expected_config()); - } -} diff --git a/core/lib/config/src/configs/contracts.rs b/core/lib/config/src/configs/contracts.rs index 5980a165cfed..a05027e0bf0d 100644 --- a/core/lib/config/src/configs/contracts.rs +++ b/core/lib/config/src/configs/contracts.rs @@ -2,9 +2,6 @@ use serde::Deserialize; // Workspace uses use zksync_basic_types::{Address, H256}; -// Local uses -use super::envy_load; - #[derive(Debug, Deserialize, Clone, PartialEq)] #[serde(rename_all = "lowercase")] pub enum ProverAtGenesis { @@ -46,105 +43,40 @@ pub struct ContractsConfig { } impl ContractsConfig { - pub fn from_env() -> anyhow::Result { - envy_load("contracts", "CONTRACTS_") - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::configs::test_utils::{addr, hash, EnvMutex}; - - static MUTEX: EnvMutex = EnvMutex::new(); - - fn expected_config() -> ContractsConfig { - ContractsConfig { - governance_addr: addr("d8dA6BF26964aF9D7eEd9e03E53415D37aA96045"), - mailbox_facet_addr: addr("0f6Fa881EF414Fc6E818180657c2d5CD7Ac6cCAd"), - executor_facet_addr: addr("18B631537801963A964211C0E86645c1aBfbB2d3"), - admin_facet_addr: addr("1e12b20BE86bEc3A0aC95aA52ade345cB9AE7a32"), - getters_facet_addr: addr("8656770FA78c830456B00B4fFCeE6b1De0e1b888"), - verifier_addr: addr("34782eE00206EAB6478F2692caa800e4A581687b"), - diamond_init_addr: addr("FFC35A5e767BE36057c34586303498e3de7C62Ba"), - diamond_upgrade_init_addr: addr("FFC35A5e767BE36057c34586303498e3de7C62Ba"), - diamond_proxy_addr: addr("F00B988a98Ca742e7958DeF9F7823b5908715f4a"), - validator_timelock_addr: addr("F00B988a98Ca742e7958DeF9F7823b5908715f4a"), - genesis_tx_hash: hash( - "b99ebfea46cbe05a21cd80fe5597d97b204befc52a16303f579c607dc1ac2e2e", - ), - l1_erc20_bridge_proxy_addr: addr("8656770FA78c830456B00B4fFCeE6b1De0e1b888"), - l1_erc20_bridge_impl_addr: addr("8656770FA78c830456B00B4fFCeE6b1De0e1b888"), - l2_erc20_bridge_addr: addr("8656770FA78c830456B00B4fFCeE6b1De0e1b888"), - l1_allow_list_addr: addr("8656770FA78c830456B00B4fFCeE6b1De0e1b888"), - l1_weth_bridge_proxy_addr: Some(addr("8656770FA78c830456B00B4fFCeE6b1De0e1b888")), - l2_weth_bridge_addr: Some(addr("8656770FA78c830456B00B4fFCeE6b1De0e1b888")), - l2_testnet_paymaster_addr: Some(addr("FC073319977e314F251EAE6ae6bE76B0B3BAeeCF")), - recursion_scheduler_level_vk_hash: hash( - "0x1186ec268d49f1905f8d9c1e9d39fc33e98c74f91d91a21b8f7ef78bd09a8db8", - ), - recursion_node_level_vk_hash: hash( - "0x1186ec268d49f1905f8d9c1e9d39fc33e98c74f91d91a21b8f7ef78bd09a8db8", - ), - recursion_leaf_level_vk_hash: hash( - "0x101e08b00193e529145ee09823378ef51a3bc8966504064f1f6ba3f1ba863210", - ), - recursion_circuits_set_vks_hash: hash( - "0x142a364ef2073132eaf07aa7f3d8495065be5b92a2dc14fda09b4216affed9c0", - ), - l1_multicall3_addr: addr("0xcA11bde05977b3631167028862bE2a173976CA11"), - fri_recursion_scheduler_level_vk_hash: hash( - "0x201d4c7d8e781d51a3bbd451a43a8f45240bb765b565ae6ce69192d918c3563d", - ), - fri_recursion_node_level_vk_hash: hash( - "0x5a3ef282b21e12fe1f4438e5bb158fc5060b160559c5158c6389d62d9fe3d080", - ), - fri_recursion_leaf_level_vk_hash: hash( - "0x72167c43a46cf38875b267d67716edc4563861364a3c03ab7aee73498421e828", - ), + /// Creates a mock instance of `ContractsConfig` to be used in tests. + /// No data in the created object is valid. + /// Every contract address is set to a random but unique non-zero value. + /// Same goes for hashes. + pub fn for_tests() -> Self { + Self { + mailbox_facet_addr: Address::repeat_byte(0x01), + executor_facet_addr: Address::repeat_byte(0x02), + admin_facet_addr: Address::repeat_byte(0x03), + getters_facet_addr: Address::repeat_byte(0x05), + verifier_addr: Address::repeat_byte(0x06), + diamond_init_addr: Address::repeat_byte(0x07), + diamond_upgrade_init_addr: Address::repeat_byte(0x08), + diamond_proxy_addr: Address::repeat_byte(0x09), + validator_timelock_addr: Address::repeat_byte(0x0a), + genesis_tx_hash: H256::repeat_byte(0x01), + l1_erc20_bridge_proxy_addr: Address::repeat_byte(0x0b), + l1_erc20_bridge_impl_addr: Address::repeat_byte(0x0c), + l2_erc20_bridge_addr: Address::repeat_byte(0x0d), + l1_weth_bridge_proxy_addr: Some(Address::repeat_byte(0x0e)), + l2_weth_bridge_addr: Some(Address::repeat_byte(0x0f)), + l1_allow_list_addr: Address::repeat_byte(0x10), + l2_testnet_paymaster_addr: Some(Address::repeat_byte(0x11)), + recursion_scheduler_level_vk_hash: H256::repeat_byte(0x02), + recursion_node_level_vk_hash: H256::repeat_byte(0x03), + recursion_leaf_level_vk_hash: H256::repeat_byte(0x04), + recursion_circuits_set_vks_hash: H256::repeat_byte(0x05), + l1_multicall3_addr: Address::repeat_byte(0x12), + fri_recursion_scheduler_level_vk_hash: H256::repeat_byte(0x06), + fri_recursion_node_level_vk_hash: H256::repeat_byte(0x07), + fri_recursion_leaf_level_vk_hash: H256::repeat_byte(0x08), + governance_addr: Address::repeat_byte(0x13), prover_at_genesis: ProverAtGenesis::Fri, - snark_wrapper_vk_hash: hash( - "0x4be443afd605a782b6e56d199df2460a025c81b3dea144e135bece83612563f2", - ), + snark_wrapper_vk_hash: H256::repeat_byte(0x09), } } - - #[test] - fn from_env() { - let mut lock = MUTEX.lock(); - let config = r#" -CONTRACTS_GOVERNANCE_ADDR="0xd8dA6BF26964aF9D7eEd9e03E53415D37aA96045" -CONTRACTS_MAILBOX_FACET_ADDR="0x0f6Fa881EF414Fc6E818180657c2d5CD7Ac6cCAd" -CONTRACTS_EXECUTOR_FACET_ADDR="0x18B631537801963A964211C0E86645c1aBfbB2d3" -CONTRACTS_ADMIN_FACET_ADDR="0x1e12b20BE86bEc3A0aC95aA52ade345cB9AE7a32" -CONTRACTS_GETTERS_FACET_ADDR="0x8656770FA78c830456B00B4fFCeE6b1De0e1b888" -CONTRACTS_VERIFIER_ADDR="0x34782eE00206EAB6478F2692caa800e4A581687b" -CONTRACTS_DIAMOND_INIT_ADDR="0xFFC35A5e767BE36057c34586303498e3de7C62Ba" -CONTRACTS_DIAMOND_UPGRADE_INIT_ADDR="0xFFC35A5e767BE36057c34586303498e3de7C62Ba" -CONTRACTS_DIAMOND_PROXY_ADDR="0xF00B988a98Ca742e7958DeF9F7823b5908715f4a" -CONTRACTS_VALIDATOR_TIMELOCK_ADDR="0xF00B988a98Ca742e7958DeF9F7823b5908715f4a" -CONTRACTS_GENESIS_TX_HASH="0xb99ebfea46cbe05a21cd80fe5597d97b204befc52a16303f579c607dc1ac2e2e" -CONTRACTS_L1_ERC20_BRIDGE_PROXY_ADDR="0x8656770FA78c830456B00B4fFCeE6b1De0e1b888" -CONTRACTS_L1_ALLOW_LIST_ADDR="0x8656770FA78c830456B00B4fFCeE6b1De0e1b888" -CONTRACTS_L1_ERC20_BRIDGE_IMPL_ADDR="0x8656770FA78c830456B00B4fFCeE6b1De0e1b888" -CONTRACTS_L2_ERC20_BRIDGE_ADDR="0x8656770FA78c830456B00B4fFCeE6b1De0e1b888" -CONTRACTS_L1_WETH_BRIDGE_PROXY_ADDR="0x8656770FA78c830456B00B4fFCeE6b1De0e1b888" -CONTRACTS_L2_WETH_BRIDGE_ADDR="0x8656770FA78c830456B00B4fFCeE6b1De0e1b888" -CONTRACTS_L2_TESTNET_PAYMASTER_ADDR="FC073319977e314F251EAE6ae6bE76B0B3BAeeCF" -CONTRACTS_RECURSION_SCHEDULER_LEVEL_VK_HASH="0x1186ec268d49f1905f8d9c1e9d39fc33e98c74f91d91a21b8f7ef78bd09a8db8" -CONTRACTS_RECURSION_NODE_LEVEL_VK_HASH="0x1186ec268d49f1905f8d9c1e9d39fc33e98c74f91d91a21b8f7ef78bd09a8db8" -CONTRACTS_RECURSION_LEAF_LEVEL_VK_HASH="0x101e08b00193e529145ee09823378ef51a3bc8966504064f1f6ba3f1ba863210" -CONTRACTS_RECURSION_CIRCUITS_SET_VKS_HASH="0x142a364ef2073132eaf07aa7f3d8495065be5b92a2dc14fda09b4216affed9c0" -CONTRACTS_L1_MULTICALL3_ADDR="0xcA11bde05977b3631167028862bE2a173976CA11" -CONTRACTS_FRI_RECURSION_SCHEDULER_LEVEL_VK_HASH="0x201d4c7d8e781d51a3bbd451a43a8f45240bb765b565ae6ce69192d918c3563d" -CONTRACTS_FRI_RECURSION_NODE_LEVEL_VK_HASH="0x5a3ef282b21e12fe1f4438e5bb158fc5060b160559c5158c6389d62d9fe3d080" -CONTRACTS_FRI_RECURSION_LEAF_LEVEL_VK_HASH="0x72167c43a46cf38875b267d67716edc4563861364a3c03ab7aee73498421e828" -CONTRACTS_PROVER_AT_GENESIS="fri" -CONTRACTS_SNARK_WRAPPER_VK_HASH="0x4be443afd605a782b6e56d199df2460a025c81b3dea144e135bece83612563f2" - "#; - lock.set_env(config); - - let actual = ContractsConfig::from_env().unwrap(); - assert_eq!(actual, expected_config()); - } } diff --git a/core/lib/config/src/configs/database.rs b/core/lib/config/src/configs/database.rs index 0628d8d45653..e694401aa939 100644 --- a/core/lib/config/src/configs/database.rs +++ b/core/lib/config/src/configs/database.rs @@ -2,8 +2,6 @@ use serde::{Deserialize, Serialize}; use std::time::Duration; -use super::envy_load; - /// Mode of operation for the Merkle tree. /// /// The mode does not influence how tree data is stored; i.e., a mode can be switched on the fly. @@ -145,13 +143,6 @@ impl DBConfig { 60_000 } - pub fn from_env() -> anyhow::Result { - Ok(Self { - merkle_tree: envy_load("database_merkle_tree", "DATABASE_MERKLE_TREE_")?, - ..envy_load("database", "DATABASE_")? - }) - } - /// Returns the Postgres statement timeout. pub fn statement_timeout(&self) -> Option { self.statement_timeout_sec.map(Duration::from_secs) @@ -161,89 +152,3 @@ impl DBConfig { Duration::from_millis(self.backup_interval_ms) } } - -#[cfg(test)] -mod tests { - use super::*; - use crate::configs::test_utils::EnvMutex; - - static MUTEX: EnvMutex = EnvMutex::new(); - - #[test] - fn from_env() { - let mut lock = MUTEX.lock(); - let config = r#" - DATABASE_STATE_KEEPER_DB_PATH="/db/state_keeper" - DATABASE_MERKLE_TREE_BACKUP_PATH="/db/backups" - DATABASE_MERKLE_TREE_PATH="/db/tree" - DATABASE_MERKLE_TREE_MODE=lightweight - DATABASE_MERKLE_TREE_MULTI_GET_CHUNK_SIZE=250 - DATABASE_MERKLE_TREE_MEMTABLE_CAPACITY_MB=512 - DATABASE_MERKLE_TREE_STALLED_WRITES_TIMEOUT_SEC=60 - DATABASE_MERKLE_TREE_MAX_L1_BATCHES_PER_ITER=50 - DATABASE_BACKUP_COUNT=5 - DATABASE_BACKUP_INTERVAL_MS=60000 - "#; - lock.set_env(config); - - let db_config = DBConfig::from_env().unwrap(); - assert_eq!(db_config.state_keeper_db_path, "/db/state_keeper"); - assert_eq!(db_config.merkle_tree.path, "/db/tree"); - assert_eq!(db_config.merkle_tree.backup_path, "/db/backups"); - assert_eq!(db_config.merkle_tree.mode, MerkleTreeMode::Lightweight); - assert_eq!(db_config.merkle_tree.multi_get_chunk_size, 250); - assert_eq!(db_config.merkle_tree.max_l1_batches_per_iter, 50); - assert_eq!(db_config.merkle_tree.memtable_capacity_mb, 512); - assert_eq!(db_config.merkle_tree.stalled_writes_timeout_sec, 60); - assert_eq!(db_config.backup_count, 5); - assert_eq!(db_config.backup_interval().as_secs(), 60); - } - - #[test] - fn from_empty_env() { - let mut lock = MUTEX.lock(); - lock.remove_env(&[ - "DATABASE_STATE_KEEPER_DB_PATH", - "DATABASE_MERKLE_TREE_BACKUP_PATH", - "DATABASE_MERKLE_TREE_PATH", - "DATABASE_MERKLE_TREE_MODE", - "DATABASE_MERKLE_TREE_MULTI_GET_CHUNK_SIZE", - "DATABASE_MERKLE_TREE_BLOCK_CACHE_SIZE_MB", - "DATABASE_MERKLE_TREE_MEMTABLE_CAPACITY_MB", - "DATABASE_MERKLE_TREE_STALLED_WRITES_TIMEOUT_SEC", - "DATABASE_MERKLE_TREE_MAX_L1_BATCHES_PER_ITER", - "DATABASE_BACKUP_COUNT", - "DATABASE_BACKUP_INTERVAL_MS", - ]); - - let db_config = DBConfig::from_env().unwrap(); - assert_eq!(db_config.state_keeper_db_path, "./db/state_keeper"); - assert_eq!(db_config.merkle_tree.path, "./db/lightweight-new"); - assert_eq!(db_config.merkle_tree.backup_path, "./db/backups"); - assert_eq!(db_config.merkle_tree.mode, MerkleTreeMode::Full); - assert_eq!(db_config.merkle_tree.multi_get_chunk_size, 500); - assert_eq!(db_config.merkle_tree.max_l1_batches_per_iter, 20); - assert_eq!(db_config.merkle_tree.block_cache_size_mb, 128); - assert_eq!(db_config.merkle_tree.memtable_capacity_mb, 256); - assert_eq!(db_config.merkle_tree.stalled_writes_timeout_sec, 30); - assert_eq!(db_config.backup_count, 5); - assert_eq!(db_config.backup_interval().as_secs(), 60); - - // Check that new env variable for Merkle tree path is supported - lock.set_env("DATABASE_MERKLE_TREE_PATH=/db/tree/main"); - let db_config = DBConfig::from_env().unwrap(); - assert_eq!(db_config.merkle_tree.path, "/db/tree/main"); - - lock.set_env("DATABASE_MERKLE_TREE_MULTI_GET_CHUNK_SIZE=200"); - let db_config = DBConfig::from_env().unwrap(); - assert_eq!(db_config.merkle_tree.multi_get_chunk_size, 200); - - lock.set_env("DATABASE_MERKLE_TREE_BLOCK_CACHE_SIZE_MB=256"); - let db_config = DBConfig::from_env().unwrap(); - assert_eq!(db_config.merkle_tree.block_cache_size_mb, 256); - - lock.set_env("DATABASE_MERKLE_TREE_MAX_L1_BATCHES_PER_ITER=50"); - let db_config = DBConfig::from_env().unwrap(); - assert_eq!(db_config.merkle_tree.max_l1_batches_per_iter, 50); - } -} diff --git a/core/lib/config/src/configs/eth_client.rs b/core/lib/config/src/configs/eth_client.rs index 4cebdacc7fcd..69b722f1f704 100644 --- a/core/lib/config/src/configs/eth_client.rs +++ b/core/lib/config/src/configs/eth_client.rs @@ -1,7 +1,5 @@ // External uses use serde::Deserialize; -// Local uses -use super::envy_load; /// Configuration for the Ethereum gateways. #[derive(Debug, Deserialize, Clone, PartialEq)] @@ -11,44 +9,3 @@ pub struct ETHClientConfig { /// Address of the Ethereum node API. pub web3_url: String, } - -impl ETHClientConfig { - pub fn from_env() -> anyhow::Result { - let config: Self = envy_load("eth_client", "ETH_CLIENT_")?; - if config.web3_url.find(',').is_some() { - anyhow::bail!( - "Multiple web3 URLs aren't supported anymore. Provided invalid value: {}", - config.web3_url - ); - } - Ok(config) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::configs::test_utils::EnvMutex; - - static MUTEX: EnvMutex = EnvMutex::new(); - - fn expected_config() -> ETHClientConfig { - ETHClientConfig { - chain_id: 9, - web3_url: "http://127.0.0.1:8545".into(), - } - } - - #[test] - fn from_env() { - let mut lock = MUTEX.lock(); - let config = r#" - ETH_CLIENT_CHAIN_ID="9" - ETH_CLIENT_WEB3_URL="http://127.0.0.1:8545" - "#; - lock.set_env(config); - - let actual = ETHClientConfig::from_env().unwrap(); - assert_eq!(actual, expected_config()); - } -} diff --git a/core/lib/config/src/configs/eth_sender.rs b/core/lib/config/src/configs/eth_sender.rs index 5516340035f9..3d036483347f 100644 --- a/core/lib/config/src/configs/eth_sender.rs +++ b/core/lib/config/src/configs/eth_sender.rs @@ -1,12 +1,9 @@ // Built-in uses use std::time::Duration; // External uses -use anyhow::Context as _; use serde::Deserialize; // Workspace uses use zksync_basic_types::H256; -// Local uses -use super::envy_load; /// Configuration for the Ethereum sender crate. #[derive(Debug, Deserialize, Clone, PartialEq)] @@ -18,11 +15,40 @@ pub struct ETHSenderConfig { } impl ETHSenderConfig { - pub fn from_env() -> anyhow::Result { - Ok(Self { - sender: SenderConfig::from_env().context("SenderConfig")?, - gas_adjuster: GasAdjusterConfig::from_env().context("GasAdjusterConfig")?, - }) + /// Creates a mock configuration object suitable for unit tests. + /// Values inside match the config used for localhost development. + pub fn for_tests() -> Self { + Self { + sender: SenderConfig { + aggregated_proof_sizes: vec![1, 4], + wait_confirmations: Some(1), + tx_poll_period: 1, + aggregate_tx_poll_period: 1, + max_txs_in_flight: 30, + proof_sending_mode: ProofSendingMode::SkipEveryProof, + max_aggregated_tx_gas: 4000000, + max_eth_tx_data_size: 6000000, + max_aggregated_blocks_to_commit: 10, + max_aggregated_blocks_to_execute: 10, + aggregated_block_commit_deadline: 1, + aggregated_block_prove_deadline: 10, + aggregated_block_execute_deadline: 10, + timestamp_criteria_max_allowed_lag: 30, + l1_batch_min_age_before_execute_seconds: None, + max_acceptable_priority_fee_in_gwei: 100000000000, + proof_loading_mode: ProofLoadingMode::OldProofFromDb, + }, + gas_adjuster: GasAdjusterConfig { + default_priority_fee_per_gas: 1000000000, + max_base_fee_samples: 10000, + pricing_formula_parameter_a: 1.5, + pricing_formula_parameter_b: 1.0005, + internal_l1_pricing_multiplier: 0.8, + internal_enforced_l1_gas_price: None, + poll_period: 5, + max_l1_gas_price: None, + }, + } } } @@ -79,6 +105,7 @@ impl SenderConfig { pub fn tx_poll_period(&self) -> Duration { Duration::from_secs(self.tx_poll_period) } + /// Converts `self.aggregate_tx_poll_period` into `Duration`. pub fn aggregate_tx_poll_period(&self) -> Duration { Duration::from_secs(self.aggregate_tx_poll_period) @@ -90,10 +117,6 @@ impl SenderConfig { .ok() .map(|pk| pk.parse().unwrap()) } - - pub fn from_env() -> anyhow::Result { - envy_load("eth_sender", "ETH_SENDER_SENDER_") - } } #[derive(Debug, Deserialize, Copy, Clone, PartialEq)] @@ -125,103 +148,4 @@ impl GasAdjusterConfig { pub fn max_l1_gas_price(&self) -> u64 { self.max_l1_gas_price.unwrap_or(u64::MAX) } - - pub fn from_env() -> anyhow::Result { - envy_load("eth_sender.gas_adjuster", "ETH_SENDER_GAS_ADJUSTER_") - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::configs::test_utils::{hash, EnvMutex}; - - static MUTEX: EnvMutex = EnvMutex::new(); - - fn expected_config() -> ETHSenderConfig { - ETHSenderConfig { - sender: SenderConfig { - aggregated_proof_sizes: vec![1, 5], - aggregated_block_commit_deadline: 30, - aggregated_block_prove_deadline: 3_000, - aggregated_block_execute_deadline: 4_000, - max_aggregated_tx_gas: 4_000_000, - max_eth_tx_data_size: 120_000, - - timestamp_criteria_max_allowed_lag: 30, - max_aggregated_blocks_to_commit: 3, - max_aggregated_blocks_to_execute: 4, - wait_confirmations: Some(1), - tx_poll_period: 3, - aggregate_tx_poll_period: 3, - max_txs_in_flight: 3, - proof_sending_mode: ProofSendingMode::SkipEveryProof, - l1_batch_min_age_before_execute_seconds: Some(1000), - max_acceptable_priority_fee_in_gwei: 100_000_000_000, - proof_loading_mode: ProofLoadingMode::OldProofFromDb, - }, - gas_adjuster: GasAdjusterConfig { - default_priority_fee_per_gas: 20000000000, - max_base_fee_samples: 10000, - pricing_formula_parameter_a: 1.5, - pricing_formula_parameter_b: 1.0005, - internal_l1_pricing_multiplier: 0.8, - internal_enforced_l1_gas_price: None, - poll_period: 15, - max_l1_gas_price: Some(100000000), - }, - } - } - - #[test] - fn from_env() { - let mut lock = MUTEX.lock(); - let config = r#" - ETH_SENDER_SENDER_WAIT_CONFIRMATIONS="1" - ETH_SENDER_SENDER_TX_POLL_PERIOD="3" - ETH_SENDER_SENDER_AGGREGATE_TX_POLL_PERIOD="3" - ETH_SENDER_SENDER_MAX_TXS_IN_FLIGHT="3" - ETH_SENDER_SENDER_OPERATOR_PRIVATE_KEY="0x27593fea79697e947890ecbecce7901b0008345e5d7259710d0dd5e500d040be" - ETH_SENDER_SENDER_PROOF_SENDING_MODE="SkipEveryProof" - ETH_SENDER_GAS_ADJUSTER_DEFAULT_PRIORITY_FEE_PER_GAS="20000000000" - ETH_SENDER_GAS_ADJUSTER_MAX_BASE_FEE_SAMPLES="10000" - ETH_SENDER_GAS_ADJUSTER_PRICING_FORMULA_PARAMETER_A="1.5" - ETH_SENDER_GAS_ADJUSTER_PRICING_FORMULA_PARAMETER_B="1.0005" - ETH_SENDER_GAS_ADJUSTER_INTERNAL_L1_PRICING_MULTIPLIER="0.8" - ETH_SENDER_GAS_ADJUSTER_POLL_PERIOD="15" - ETH_SENDER_GAS_ADJUSTER_MAX_L1_GAS_PRICE="100000000" - ETH_SENDER_WAIT_FOR_PROOFS="false" - ETH_SENDER_SENDER_AGGREGATED_PROOF_SIZES="1,5" - ETH_SENDER_SENDER_MAX_AGGREGATED_BLOCKS_TO_COMMIT="3" - ETH_SENDER_SENDER_MAX_AGGREGATED_BLOCKS_TO_EXECUTE="4" - ETH_SENDER_SENDER_AGGREGATED_BLOCK_COMMIT_DEADLINE="30" - ETH_SENDER_SENDER_AGGREGATED_BLOCK_PROVE_DEADLINE="3000" - ETH_SENDER_SENDER_AGGREGATED_BLOCK_EXECUTE_DEADLINE="4000" - ETH_SENDER_SENDER_TIMESTAMP_CRITERIA_MAX_ALLOWED_LAG="30" - ETH_SENDER_SENDER_MAX_AGGREGATED_TX_GAS="4000000" - ETH_SENDER_SENDER_MAX_ETH_TX_DATA_SIZE="120000" - ETH_SENDER_SENDER_L1_BATCH_MIN_AGE_BEFORE_EXECUTE_SECONDS="1000" - ETH_SENDER_SENDER_MAX_ACCEPTABLE_PRIORITY_FEE_IN_GWEI="100000000000" - ETH_SENDER_SENDER_PROOF_LOADING_MODE="OldProofFromDb" - "#; - lock.set_env(config); - - let actual = ETHSenderConfig::from_env().unwrap(); - assert_eq!(actual, expected_config()); - assert_eq!( - actual.sender.private_key().unwrap(), - hash("27593fea79697e947890ecbecce7901b0008345e5d7259710d0dd5e500d040be") - ); - } - - /// Checks the correctness of the config helper methods. - #[test] - fn methods() { - let config = expected_config(); - - assert_eq!( - config.sender.tx_poll_period(), - Duration::from_secs(config.sender.tx_poll_period) - ); - } } diff --git a/core/lib/config/src/configs/eth_watch.rs b/core/lib/config/src/configs/eth_watch.rs index a8662ddecb95..93d73ddf6bfd 100644 --- a/core/lib/config/src/configs/eth_watch.rs +++ b/core/lib/config/src/configs/eth_watch.rs @@ -2,8 +2,6 @@ use std::time::Duration; // External uses use serde::Deserialize; -// Local uses -use super::envy_load; /// Configuration for the Ethereum sender crate. #[derive(Debug, Deserialize, Clone, PartialEq)] @@ -17,50 +15,8 @@ pub struct ETHWatchConfig { } impl ETHWatchConfig { - pub fn from_env() -> anyhow::Result { - envy_load("eth_watch", "ETH_WATCH_") - } - /// Converts `self.eth_node_poll_interval` into `Duration`. pub fn poll_interval(&self) -> Duration { Duration::from_millis(self.eth_node_poll_interval) } } - -#[cfg(test)] -mod tests { - use super::*; - use crate::configs::test_utils::EnvMutex; - - static MUTEX: EnvMutex = EnvMutex::new(); - - fn expected_config() -> ETHWatchConfig { - ETHWatchConfig { - confirmations_for_eth_event: Some(0), - eth_node_poll_interval: 300, - } - } - - #[test] - fn from_env() { - let mut lock = MUTEX.lock(); - let config = r#" - ETH_WATCH_CONFIRMATIONS_FOR_ETH_EVENT="0" - ETH_WATCH_ETH_NODE_POLL_INTERVAL="300" - "#; - lock.set_env(config); - - let actual = ETHWatchConfig::from_env().unwrap(); - assert_eq!(actual, expected_config()); - } - - /// Checks the correctness of the config helper methods. - #[test] - fn methods() { - let config = expected_config(); - assert_eq!( - config.poll_interval(), - Duration::from_millis(config.eth_node_poll_interval) - ); - } -} diff --git a/core/lib/config/src/configs/fetcher.rs b/core/lib/config/src/configs/fetcher.rs index ac3cff36b40b..b1a5fca4b24a 100644 --- a/core/lib/config/src/configs/fetcher.rs +++ b/core/lib/config/src/configs/fetcher.rs @@ -1,11 +1,5 @@ -use std::time::Duration; - -// Built-in uses -// External uses use serde::Deserialize; -// Workspace uses -// Local uses -use super::envy_load; +use std::time::Duration; #[derive(Debug, Deserialize, Clone, Copy, PartialEq)] pub enum TokenListSource { @@ -49,64 +43,3 @@ pub struct FetcherConfig { pub token_price: SingleFetcherConfig, pub token_trading_volume: SingleFetcherConfig, } - -impl FetcherConfig { - pub fn from_env() -> anyhow::Result { - Ok(Self { - token_list: envy_load("token_list", "FETCHER_TOKEN_LIST_")?, - token_price: envy_load("token_price", "FETCHER_TOKEN_PRICE_")?, - token_trading_volume: envy_load( - "token_trading_volume", - "FETCHER_TOKEN_TRADING_VOLUME_", - )?, - }) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::configs::test_utils::EnvMutex; - - static MUTEX: EnvMutex = EnvMutex::new(); - - fn expected_config() -> FetcherConfig { - FetcherConfig { - token_list: SingleFetcherConfig { - source: TokenListSource::OneInch, - url: "http://127.0.0.1:1020".into(), - fetching_interval: 10, - }, - token_price: SingleFetcherConfig { - source: TokenPriceSource::CoinGecko, - url: "http://127.0.0.1:9876".into(), - fetching_interval: 7, - }, - token_trading_volume: SingleFetcherConfig { - source: TokenTradingVolumeSource::Uniswap, - url: "http://127.0.0.1:9975/graphql".to_string(), - fetching_interval: 5, - }, - } - } - - #[test] - fn from_env() { - let mut lock = MUTEX.lock(); - let config = r#" - FETCHER_TOKEN_LIST_SOURCE="OneInch" - FETCHER_TOKEN_LIST_URL="http://127.0.0.1:1020" - FETCHER_TOKEN_LIST_FETCHING_INTERVAL="10" - FETCHER_TOKEN_PRICE_SOURCE="CoinGecko" - FETCHER_TOKEN_PRICE_URL="http://127.0.0.1:9876" - FETCHER_TOKEN_PRICE_FETCHING_INTERVAL="7" - FETCHER_TOKEN_TRADING_VOLUME_SOURCE="Uniswap" - FETCHER_TOKEN_TRADING_VOLUME_URL="http://127.0.0.1:9975/graphql" - FETCHER_TOKEN_TRADING_VOLUME_FETCHING_INTERVAL="5" - "#; - lock.set_env(config); - - let actual = FetcherConfig::from_env().unwrap(); - assert_eq!(actual, expected_config()); - } -} diff --git a/core/lib/config/src/configs/fri_proof_compressor.rs b/core/lib/config/src/configs/fri_proof_compressor.rs index e544b4f52064..bbf58f2d1c69 100644 --- a/core/lib/config/src/configs/fri_proof_compressor.rs +++ b/core/lib/config/src/configs/fri_proof_compressor.rs @@ -1,8 +1,6 @@ use serde::Deserialize; use std::time::Duration; -use super::envy_load; - /// Configuration for the fri proof compressor #[derive(Debug, Deserialize, Clone, PartialEq)] pub struct FriProofCompressorConfig { @@ -29,56 +27,7 @@ pub struct FriProofCompressorConfig { } impl FriProofCompressorConfig { - pub fn from_env() -> anyhow::Result { - envy_load("fri_proof_compressor", "FRI_PROOF_COMPRESSOR_") - } - pub fn generation_timeout(&self) -> Duration { Duration::from_secs(self.generation_timeout_in_secs as u64) } } - -#[cfg(test)] -mod tests { - use crate::configs::test_utils::EnvMutex; - - use super::*; - - static MUTEX: EnvMutex = EnvMutex::new(); - - fn expected_config() -> FriProofCompressorConfig { - FriProofCompressorConfig { - compression_mode: 1, - prometheus_listener_port: 3316, - prometheus_pushgateway_url: "http://127.0.0.1:9091".to_string(), - prometheus_push_interval_ms: Some(100), - generation_timeout_in_secs: 3000, - max_attempts: 5, - universal_setup_path: "keys/setup/setup_2^26.key".to_string(), - universal_setup_download_url: - "https://storage.googleapis.com/matterlabs-setup-keys-us/setup-keys/setup_2^26.key" - .to_string(), - verify_wrapper_proof: false, - } - } - - #[test] - fn from_env() { - let mut lock = MUTEX.lock(); - let config = r#" - FRI_PROOF_COMPRESSOR_COMPRESSION_MODE=1 - FRI_PROOF_COMPRESSOR_PROMETHEUS_LISTENER_PORT=3316 - FRI_PROOF_COMPRESSOR_PROMETHEUS_PUSHGATEWAY_URL="http://127.0.0.1:9091" - FRI_PROOF_COMPRESSOR_PROMETHEUS_PUSH_INTERVAL_MS=100 - FRI_PROOF_COMPRESSOR_GENERATION_TIMEOUT_IN_SECS=3000 - FRI_PROOF_COMPRESSOR_MAX_ATTEMPTS=5 - FRI_PROOF_COMPRESSOR_UNIVERSAL_SETUP_PATH="keys/setup/setup_2^26.key" - FRI_PROOF_COMPRESSOR_UNIVERSAL_SETUP_DOWNLOAD_URL="https://storage.googleapis.com/matterlabs-setup-keys-us/setup-keys/setup_2^26.key" - FRI_PROOF_COMPRESSOR_VERIFY_WRAPPER_PROOF=false - "#; - lock.set_env(config); - - let actual = FriProofCompressorConfig::from_env().unwrap(); - assert_eq!(actual, expected_config()); - } -} diff --git a/core/lib/config/src/configs/fri_prover.rs b/core/lib/config/src/configs/fri_prover.rs index 205f13e114c8..aab358a4adaf 100644 --- a/core/lib/config/src/configs/fri_prover.rs +++ b/core/lib/config/src/configs/fri_prover.rs @@ -1,4 +1,3 @@ -use super::envy_load; use serde::Deserialize; use std::time::Duration; @@ -28,59 +27,7 @@ pub struct FriProverConfig { } impl FriProverConfig { - pub fn from_env() -> anyhow::Result { - envy_load("fri_prover", "FRI_PROVER_") - } - pub fn proof_generation_timeout(&self) -> Duration { Duration::from_secs(self.generation_timeout_in_secs as u64) } } - -#[cfg(test)] -mod tests { - use super::*; - use crate::configs::test_utils::EnvMutex; - - static MUTEX: EnvMutex = EnvMutex::new(); - - fn expected_config() -> FriProverConfig { - FriProverConfig { - setup_data_path: "/usr/src/setup-data".to_string(), - prometheus_port: 3315, - max_attempts: 10, - generation_timeout_in_secs: 300, - base_layer_circuit_ids_to_be_verified: vec![1, 5], - recursive_layer_circuit_ids_to_be_verified: vec![1, 2, 3], - setup_load_mode: SetupLoadMode::FromDisk, - specialized_group_id: 10, - witness_vector_generator_thread_count: Some(5), - queue_capacity: 10, - witness_vector_receiver_port: 3316, - shall_save_to_public_bucket: true, - } - } - - #[test] - fn from_env() { - let mut lock = MUTEX.lock(); - let config = r#" - FRI_PROVER_SETUP_DATA_PATH="/usr/src/setup-data" - FRI_PROVER_PROMETHEUS_PORT="3315" - FRI_PROVER_MAX_ATTEMPTS="10" - FRI_PROVER_GENERATION_TIMEOUT_IN_SECS="300" - FRI_PROVER_BASE_LAYER_CIRCUIT_IDS_TO_BE_VERIFIED="1,5" - FRI_PROVER_RECURSIVE_LAYER_CIRCUIT_IDS_TO_BE_VERIFIED="1,2,3" - FRI_PROVER_SETUP_LOAD_MODE="FromDisk" - FRI_PROVER_SPECIALIZED_GROUP_ID="10" - FRI_PROVER_WITNESS_VECTOR_GENERATOR_THREAD_COUNT="5" - FRI_PROVER_QUEUE_CAPACITY="10" - FRI_PROVER_WITNESS_VECTOR_RECEIVER_PORT="3316" - FRI_PROVER_SHALL_SAVE_TO_PUBLIC_BUCKET=true - "#; - lock.set_env(config); - - let actual = FriProverConfig::from_env().unwrap(); - assert_eq!(actual, expected_config()); - } -} diff --git a/core/lib/config/src/configs/fri_prover_gateway.rs b/core/lib/config/src/configs/fri_prover_gateway.rs index f8eb7fbd2ca7..652c7d1bc0f5 100644 --- a/core/lib/config/src/configs/fri_prover_gateway.rs +++ b/core/lib/config/src/configs/fri_prover_gateway.rs @@ -1,4 +1,3 @@ -use super::envy_load; use serde::Deserialize; use std::time::Duration; @@ -14,44 +13,7 @@ pub struct FriProverGatewayConfig { } impl FriProverGatewayConfig { - pub fn from_env() -> anyhow::Result { - envy_load("fri_prover_gateway", "FRI_PROVER_GATEWAY_") - } - pub fn api_poll_duration(&self) -> Duration { Duration::from_secs(self.api_poll_duration_secs as u64) } } - -#[cfg(test)] -mod tests { - use super::*; - use crate::configs::test_utils::EnvMutex; - - static MUTEX: EnvMutex = EnvMutex::new(); - - fn expected_config() -> FriProverGatewayConfig { - FriProverGatewayConfig { - api_url: "http://private-dns-for-server".to_string(), - api_poll_duration_secs: 100, - prometheus_listener_port: 3316, - prometheus_pushgateway_url: "http://127.0.0.1:9091".to_string(), - prometheus_push_interval_ms: Some(100), - } - } - - #[test] - fn from_env() { - let config = r#" - FRI_PROVER_GATEWAY_API_URL="http://private-dns-for-server" - FRI_PROVER_GATEWAY_API_POLL_DURATION_SECS="100" - FRI_PROVER_GATEWAY_PROMETHEUS_LISTENER_PORT=3316 - FRI_PROVER_GATEWAY_PROMETHEUS_PUSHGATEWAY_URL="http://127.0.0.1:9091" - FRI_PROVER_GATEWAY_PROMETHEUS_PUSH_INTERVAL_MS=100 - "#; - let mut lock = MUTEX.lock(); - lock.set_env(config); - let actual = FriProverGatewayConfig::from_env().unwrap(); - assert_eq!(actual, expected_config()); - } -} diff --git a/core/lib/config/src/configs/fri_prover_group.rs b/core/lib/config/src/configs/fri_prover_group.rs index bce956cfffc0..966a4b324db6 100644 --- a/core/lib/config/src/configs/fri_prover_group.rs +++ b/core/lib/config/src/configs/fri_prover_group.rs @@ -1,51 +1,8 @@ use serde::Deserialize; -use std::collections::{HashMap, HashSet}; -use std::env; +use std::collections::HashSet; use zksync_basic_types::basic_fri_types::CircuitIdRoundTuple; -fn load_from_env_variable() -> HashMap> { - // Prepare a hash map to store the mapping of group to a vector of tuples - let mut groups: HashMap> = (0..=12) - .map(|i| (format!("group_{}", i), HashSet::new())) - .collect(); - - // Separate environment variables into Circuit Id and Aggregation Round - let mut circuit_ids = HashMap::new(); - let mut aggregation_rounds = HashMap::new(); - for (key, value) in env::vars() { - if key.contains("_CIRCUIT_ID") { - circuit_ids.insert(key, value); - } else if key.contains("_AGGREGATION_ROUND") { - aggregation_rounds.insert(key, value); - } - } - - // Iterate over all circuit id variables - for (key, value_str) in circuit_ids { - let key_parts: Vec<&str> = key.split('_').collect(); - if let (Some(group_key), Some(value), Some(index_str)) = ( - key_parts.get(4), - value_str.parse::().ok(), - key_parts.get(5), - ) { - let round_key = format!( - "FRI_PROVER_GROUP_GROUP_{}_{}_AGGREGATION_ROUND", - group_key, index_str - ); - if let Some(round_str) = aggregation_rounds.get(&round_key) { - if let Ok(round) = round_str.parse::() { - let tuple = CircuitIdRoundTuple::new(value, round); - if let Some(group) = groups.get_mut(&format!("group_{}", group_key)) { - group.insert(tuple); - } - } - } - } - } - groups -} - /// Configuration for the grouping of specialized provers. #[derive(Debug, Deserialize, Clone, PartialEq)] pub struct FriProverGroupConfig { @@ -63,29 +20,7 @@ pub struct FriProverGroupConfig { pub group_11: HashSet, pub group_12: HashSet, } - impl FriProverGroupConfig { - pub fn from_env() -> Self { - let mut groups = load_from_env_variable(); - let config = FriProverGroupConfig { - group_0: groups.remove("group_0").unwrap_or_default(), - group_1: groups.remove("group_1").unwrap_or_default(), - group_2: groups.remove("group_2").unwrap_or_default(), - group_3: groups.remove("group_3").unwrap_or_default(), - group_4: groups.remove("group_4").unwrap_or_default(), - group_5: groups.remove("group_5").unwrap_or_default(), - group_6: groups.remove("group_6").unwrap_or_default(), - group_7: groups.remove("group_7").unwrap_or_default(), - group_8: groups.remove("group_8").unwrap_or_default(), - group_9: groups.remove("group_9").unwrap_or_default(), - group_10: groups.remove("group_10").unwrap_or_default(), - group_11: groups.remove("group_11").unwrap_or_default(), - group_12: groups.remove("group_12").unwrap_or_default(), - }; - config.validate(); - config - } - pub fn get_circuit_ids_for_group_id(&self, group_id: u8) -> Option> { match group_id { 0 => Some(self.group_0.clone().into_iter().collect()), @@ -117,7 +52,7 @@ impl FriProverGroupConfig { /// In aggregation round 1, the circuit ids should be 3 to 15. /// In aggregation round 2, the circuit ids should be 2. /// In aggregation round 3, the circuit ids should be 1. - fn validate(&self) { + pub fn validate(&self) { let mut rounds: Vec> = vec![Vec::new(); 4]; let groups = [ &self.group_0, @@ -257,174 +192,3 @@ impl FriProverGroupConfig { } } } - -#[cfg(test)] -mod tests { - use super::*; - - fn expected_config() -> FriProverGroupConfig { - FriProverGroupConfig { - group_0: vec![ - CircuitIdRoundTuple::new(1, 3), - CircuitIdRoundTuple::new(2, 2), - ] - .into_iter() - .collect::>(), - group_1: vec![CircuitIdRoundTuple::new(1, 0)] - .into_iter() - .collect::>(), - group_2: vec![ - CircuitIdRoundTuple::new(2, 0), - CircuitIdRoundTuple::new(4, 0), - CircuitIdRoundTuple::new(6, 0), - CircuitIdRoundTuple::new(9, 0), - ] - .into_iter() - .collect::>(), - group_3: vec![CircuitIdRoundTuple::new(3, 0)] - .into_iter() - .collect::>(), - group_4: vec![ - CircuitIdRoundTuple::new(11, 0), - CircuitIdRoundTuple::new(12, 0), - CircuitIdRoundTuple::new(13, 0), - ] - .into_iter() - .collect::>(), - group_5: vec![CircuitIdRoundTuple::new(5, 0)] - .into_iter() - .collect::>(), - group_6: vec![CircuitIdRoundTuple::new(3, 1)] - .into_iter() - .collect::>(), - group_7: vec![CircuitIdRoundTuple::new(7, 0)] - .into_iter() - .collect::>(), - group_8: vec![CircuitIdRoundTuple::new(8, 0)] - .into_iter() - .collect::>(), - group_9: vec![ - CircuitIdRoundTuple::new(12, 1), - CircuitIdRoundTuple::new(13, 1), - CircuitIdRoundTuple::new(14, 1), - CircuitIdRoundTuple::new(15, 1), - ] - .into_iter() - .collect::>(), - group_10: vec![CircuitIdRoundTuple::new(10, 0)] - .into_iter() - .collect::>(), - group_11: vec![ - CircuitIdRoundTuple::new(7, 1), - CircuitIdRoundTuple::new(8, 1), - CircuitIdRoundTuple::new(10, 1), - CircuitIdRoundTuple::new(11, 1), - ] - .into_iter() - .collect::>(), - group_12: vec![ - CircuitIdRoundTuple::new(4, 1), - CircuitIdRoundTuple::new(5, 1), - CircuitIdRoundTuple::new(6, 1), - CircuitIdRoundTuple::new(9, 1), - ] - .into_iter() - .collect::>(), - } - } - - #[test] - fn from_env() { - let groups = [ - ("FRI_PROVER_GROUP_GROUP_0_0", CircuitIdRoundTuple::new(1, 3)), - ("FRI_PROVER_GROUP_GROUP_0_1", CircuitIdRoundTuple::new(2, 2)), - ("FRI_PROVER_GROUP_GROUP_1_0", CircuitIdRoundTuple::new(1, 0)), - ("FRI_PROVER_GROUP_GROUP_2_0", CircuitIdRoundTuple::new(2, 0)), - ("FRI_PROVER_GROUP_GROUP_2_1", CircuitIdRoundTuple::new(4, 0)), - ("FRI_PROVER_GROUP_GROUP_2_2", CircuitIdRoundTuple::new(6, 0)), - ("FRI_PROVER_GROUP_GROUP_2_3", CircuitIdRoundTuple::new(9, 0)), - ("FRI_PROVER_GROUP_GROUP_3_0", CircuitIdRoundTuple::new(3, 0)), - ( - "FRI_PROVER_GROUP_GROUP_4_0", - CircuitIdRoundTuple::new(11, 0), - ), - ( - "FRI_PROVER_GROUP_GROUP_4_1", - CircuitIdRoundTuple::new(12, 0), - ), - ( - "FRI_PROVER_GROUP_GROUP_4_2", - CircuitIdRoundTuple::new(13, 0), - ), - ("FRI_PROVER_GROUP_GROUP_5_0", CircuitIdRoundTuple::new(5, 0)), - ("FRI_PROVER_GROUP_GROUP_6_0", CircuitIdRoundTuple::new(3, 1)), - ("FRI_PROVER_GROUP_GROUP_7_0", CircuitIdRoundTuple::new(7, 0)), - ("FRI_PROVER_GROUP_GROUP_8_0", CircuitIdRoundTuple::new(8, 0)), - ( - "FRI_PROVER_GROUP_GROUP_9_0", - CircuitIdRoundTuple::new(12, 1), - ), - ( - "FRI_PROVER_GROUP_GROUP_9_1", - CircuitIdRoundTuple::new(13, 1), - ), - ( - "FRI_PROVER_GROUP_GROUP_9_2", - CircuitIdRoundTuple::new(14, 1), - ), - ( - "FRI_PROVER_GROUP_GROUP_9_3", - CircuitIdRoundTuple::new(15, 1), - ), - ( - "FRI_PROVER_GROUP_GROUP_10_0", - CircuitIdRoundTuple::new(10, 0), - ), - ( - "FRI_PROVER_GROUP_GROUP_11_0", - CircuitIdRoundTuple::new(7, 1), - ), - ( - "FRI_PROVER_GROUP_GROUP_11_1", - CircuitIdRoundTuple::new(8, 1), - ), - ( - "FRI_PROVER_GROUP_GROUP_11_2", - CircuitIdRoundTuple::new(10, 1), - ), - ( - "FRI_PROVER_GROUP_GROUP_11_3", - CircuitIdRoundTuple::new(11, 1), - ), - ( - "FRI_PROVER_GROUP_GROUP_12_0", - CircuitIdRoundTuple::new(4, 1), - ), - ( - "FRI_PROVER_GROUP_GROUP_12_1", - CircuitIdRoundTuple::new(5, 1), - ), - ( - "FRI_PROVER_GROUP_GROUP_12_2", - CircuitIdRoundTuple::new(6, 1), - ), - ( - "FRI_PROVER_GROUP_GROUP_12_3", - CircuitIdRoundTuple::new(9, 1), - ), - ]; - - for (key_base, circuit_round_tuple) in &groups { - let circuit_id_key = format!("{}_CIRCUIT_ID", key_base); - let aggregation_round_key = format!("{}_AGGREGATION_ROUND", key_base); - env::set_var(&circuit_id_key, circuit_round_tuple.circuit_id.to_string()); - env::set_var( - &aggregation_round_key, - circuit_round_tuple.aggregation_round.to_string(), - ); - } - - let actual = FriProverGroupConfig::from_env(); - assert_eq!(actual, expected_config()); - } -} diff --git a/core/lib/config/src/configs/fri_witness_generator.rs b/core/lib/config/src/configs/fri_witness_generator.rs index 4abc839635af..17b6d773c6fe 100644 --- a/core/lib/config/src/configs/fri_witness_generator.rs +++ b/core/lib/config/src/configs/fri_witness_generator.rs @@ -4,9 +4,6 @@ use std::time::Duration; // External uses use serde::Deserialize; -// Local uses -use super::envy_load; - /// Configuration for the fri witness generation #[derive(Debug, Deserialize, Clone, PartialEq)] pub struct FriWitnessGeneratorConfig { @@ -28,12 +25,7 @@ pub struct FriWitnessGeneratorConfig { // whether to write to public GCS bucket for https://github.com/matter-labs/era-boojum-validator-cli pub shall_save_to_public_bucket: bool, } - impl FriWitnessGeneratorConfig { - pub fn from_env() -> anyhow::Result { - envy_load("fri_witness", "FRI_WITNESS_") - } - pub fn witness_generation_timeout(&self) -> Duration { Duration::from_secs(self.generation_timeout_in_secs as u64) } @@ -42,40 +34,3 @@ impl FriWitnessGeneratorConfig { self.last_l1_batch_to_process.unwrap_or(u32::MAX) } } - -#[cfg(test)] -mod tests { - use super::*; - use crate::configs::test_utils::EnvMutex; - - static MUTEX: EnvMutex = EnvMutex::new(); - - fn expected_config() -> FriWitnessGeneratorConfig { - FriWitnessGeneratorConfig { - generation_timeout_in_secs: 900u16, - max_attempts: 4, - blocks_proving_percentage: Some(30), - dump_arguments_for_blocks: vec![2, 3], - last_l1_batch_to_process: None, - force_process_block: Some(1), - shall_save_to_public_bucket: true, - } - } - - #[test] - fn from_env() { - let mut lock = MUTEX.lock(); - let config = r#" - FRI_WITNESS_GENERATION_TIMEOUT_IN_SECS=900 - FRI_WITNESS_MAX_ATTEMPTS=4 - FRI_WITNESS_DUMP_ARGUMENTS_FOR_BLOCKS="2,3" - FRI_WITNESS_BLOCKS_PROVING_PERCENTAGE="30" - FRI_WITNESS_FORCE_PROCESS_BLOCK="1" - FRI_WITNESS_SHALL_SAVE_TO_PUBLIC_BUCKET=true - "#; - lock.set_env(config); - - let actual = FriWitnessGeneratorConfig::from_env().unwrap(); - assert_eq!(actual, expected_config()); - } -} diff --git a/core/lib/config/src/configs/fri_witness_vector_generator.rs b/core/lib/config/src/configs/fri_witness_vector_generator.rs index 0bb2ddac318d..98ce7ea9390b 100644 --- a/core/lib/config/src/configs/fri_witness_vector_generator.rs +++ b/core/lib/config/src/configs/fri_witness_vector_generator.rs @@ -2,8 +2,6 @@ use std::time::Duration; use serde::Deserialize; -use super::envy_load; - /// Configuration for the witness vector generator #[derive(Debug, Deserialize, Clone, PartialEq)] pub struct FriWitnessVectorGeneratorConfig { @@ -25,13 +23,6 @@ pub struct FriWitnessVectorGeneratorConfig { } impl FriWitnessVectorGeneratorConfig { - pub fn from_env() -> anyhow::Result { - envy_load( - "fri_witness_vector_generator", - "FRI_WITNESS_VECTOR_GENERATOR_", - ) - } - pub fn prover_instance_wait_timeout(&self) -> Duration { Duration::from_secs(self.prover_instance_wait_timeout_in_secs as u64) } @@ -44,41 +35,3 @@ impl FriWitnessVectorGeneratorConfig { Duration::from_secs(self.max_prover_reservation_duration_in_secs as u64) } } - -#[cfg(test)] -mod tests { - use super::*; - use crate::configs::test_utils::EnvMutex; - - static MUTEX: EnvMutex = EnvMutex::new(); - - fn expected_config() -> FriWitnessVectorGeneratorConfig { - FriWitnessVectorGeneratorConfig { - max_prover_reservation_duration_in_secs: 1000u16, - prover_instance_wait_timeout_in_secs: 1000u16, - prover_instance_poll_time_in_milli_secs: 250u16, - prometheus_listener_port: 3316, - prometheus_pushgateway_url: "http://127.0.0.1:9091".to_string(), - prometheus_push_interval_ms: Some(100), - specialized_group_id: 1, - } - } - - #[test] - fn from_env() { - let mut lock = MUTEX.lock(); - let config = r#" - FRI_WITNESS_VECTOR_GENERATOR_MAX_PROVER_RESERVATION_DURATION_IN_SECS=1000 - FRI_WITNESS_VECTOR_GENERATOR_PROVER_INSTANCE_WAIT_TIMEOUT_IN_SECS=1000 - FRI_WITNESS_VECTOR_GENERATOR_PROVER_INSTANCE_POLL_TIME_IN_MILLI_SECS=250 - FRI_WITNESS_VECTOR_GENERATOR_PROMETHEUS_LISTENER_PORT=3316 - FRI_WITNESS_VECTOR_GENERATOR_PROMETHEUS_PUSHGATEWAY_URL="http://127.0.0.1:9091" - FRI_WITNESS_VECTOR_GENERATOR_PROMETHEUS_PUSH_INTERVAL_MS=100 - FRI_WITNESS_VECTOR_GENERATOR_SPECIALIZED_GROUP_ID=1 - "#; - lock.set_env(config); - - let actual = FriWitnessVectorGeneratorConfig::from_env().unwrap(); - assert_eq!(actual, expected_config()); - } -} diff --git a/core/lib/config/src/configs/house_keeper.rs b/core/lib/config/src/configs/house_keeper.rs index 0215f6676404..f7eb35ac3708 100644 --- a/core/lib/config/src/configs/house_keeper.rs +++ b/core/lib/config/src/configs/house_keeper.rs @@ -1,7 +1,5 @@ use serde::Deserialize; -use super::envy_load; - /// Configuration for the house keeper. #[derive(Debug, Deserialize, Clone, PartialEq)] pub struct HouseKeeperConfig { @@ -20,61 +18,3 @@ pub struct HouseKeeperConfig { pub fri_proof_compressor_job_retrying_interval_ms: u64, pub fri_proof_compressor_stats_reporting_interval_ms: u64, } - -impl HouseKeeperConfig { - pub fn from_env() -> anyhow::Result { - envy_load("house_keeper", "HOUSE_KEEPER_") - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::configs::test_utils::EnvMutex; - - static MUTEX: EnvMutex = EnvMutex::new(); - - fn expected_config() -> HouseKeeperConfig { - HouseKeeperConfig { - l1_batch_metrics_reporting_interval_ms: 10_000, - blob_cleaning_interval_ms: 60_000, - gpu_prover_queue_reporting_interval_ms: 10_000, - prover_job_retrying_interval_ms: 300_000, - prover_stats_reporting_interval_ms: 5_000, - witness_job_moving_interval_ms: 30_000, - witness_generator_stats_reporting_interval_ms: 10_000, - fri_witness_job_moving_interval_ms: 40_000, - fri_prover_job_retrying_interval_ms: 30_000, - fri_witness_generator_job_retrying_interval_ms: 30_000, - prover_db_pool_size: 2, - fri_prover_stats_reporting_interval_ms: 30_000, - fri_proof_compressor_job_retrying_interval_ms: 30_000, - fri_proof_compressor_stats_reporting_interval_ms: 30_000, - } - } - - #[test] - fn from_env() { - let mut lock = MUTEX.lock(); - let config = r#" - HOUSE_KEEPER_L1_BATCH_METRICS_REPORTING_INTERVAL_MS="10000" - HOUSE_KEEPER_BLOB_CLEANING_INTERVAL_MS="60000" - HOUSE_KEEPER_GPU_PROVER_QUEUE_REPORTING_INTERVAL_MS="10000" - HOUSE_KEEPER_PROVER_JOB_RETRYING_INTERVAL_MS="300000" - HOUSE_KEEPER_PROVER_STATS_REPORTING_INTERVAL_MS="5000" - HOUSE_KEEPER_WITNESS_JOB_MOVING_INTERVAL_MS="30000" - HOUSE_KEEPER_WITNESS_GENERATOR_STATS_REPORTING_INTERVAL_MS="10000" - HOUSE_KEEPER_FRI_WITNESS_JOB_MOVING_INTERVAL_MS="40000" - HOUSE_KEEPER_FRI_PROVER_JOB_RETRYING_INTERVAL_MS="30000" - HOUSE_KEEPER_FRI_WITNESS_GENERATOR_JOB_RETRYING_INTERVAL_MS="30000" - HOUSE_KEEPER_PROVER_DB_POOL_SIZE="2" - HOUSE_KEEPER_FRI_PROVER_STATS_REPORTING_INTERVAL_MS="30000" - HOUSE_KEEPER_FRI_PROOF_COMPRESSOR_STATS_REPORTING_INTERVAL_MS="30000" - HOUSE_KEEPER_FRI_PROOF_COMPRESSOR_JOB_RETRYING_INTERVAL_MS="30000" - "#; - lock.set_env(config); - - let actual = HouseKeeperConfig::from_env().unwrap(); - assert_eq!(actual, expected_config()); - } -} diff --git a/core/lib/config/src/configs/mod.rs b/core/lib/config/src/configs/mod.rs index 4fac42c18ef0..f23b6d414e33 100644 --- a/core/lib/config/src/configs/mod.rs +++ b/core/lib/config/src/configs/mod.rs @@ -13,9 +13,6 @@ pub use self::{ witness_generator::WitnessGeneratorConfig, }; -use anyhow::Context as _; -use serde::de::DeserializeOwned; - pub mod alerts; pub mod api; pub mod chain; @@ -41,18 +38,4 @@ pub mod prover_group; pub mod utils; pub mod witness_generator; -#[cfg(test)] -pub(crate) mod test_utils; - const BYTES_IN_MEGABYTE: usize = 1_024 * 1_024; - -/// Convenience function that loads the structure from the environment variable given the prefix. -/// Panics if the config cannot be loaded from the environment variables. -pub fn envy_load(name: &str, prefix: &str) -> anyhow::Result { - envy_try_load(prefix).with_context(|| format!("Cannot load config <{name}>")) -} - -/// Convenience function that loads the structure from the environment variable given the prefix. -pub fn envy_try_load(prefix: &str) -> Result { - envy::prefixed(prefix).from_env() -} diff --git a/core/lib/config/src/configs/object_store.rs b/core/lib/config/src/configs/object_store.rs index 379bb7d24f53..5524b6ade253 100644 --- a/core/lib/config/src/configs/object_store.rs +++ b/core/lib/config/src/configs/object_store.rs @@ -1,7 +1,5 @@ use serde::Deserialize; -use super::envy_load; - #[derive(Debug, Deserialize, Eq, PartialEq, Clone, Copy)] pub enum ObjectStoreMode { GCS, @@ -18,80 +16,3 @@ pub struct ObjectStoreConfig { pub gcs_credential_file_path: String, pub max_retries: u16, } - -impl ObjectStoreConfig { - pub fn from_env() -> anyhow::Result { - envy_load("object_store", "OBJECT_STORE_") - } - - pub fn public_from_env() -> anyhow::Result { - envy_load("public_object_store", "PUBLIC_OBJECT_STORE_") - } - - pub fn prover_from_env() -> anyhow::Result { - envy_load("prover_object_store", "PROVER_OBJECT_STORE_") - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::configs::test_utils::EnvMutex; - - static MUTEX: EnvMutex = EnvMutex::new(); - - fn expected_config(bucket_base_url: &str) -> ObjectStoreConfig { - ObjectStoreConfig { - bucket_base_url: bucket_base_url.to_string(), - mode: ObjectStoreMode::FileBacked, - file_backed_base_path: "artifacts".to_string(), - gcs_credential_file_path: "/path/to/credentials.json".to_string(), - max_retries: 5, - } - } - - #[test] - fn from_env() { - let mut lock = MUTEX.lock(); - let config = r#" - OBJECT_STORE_BUCKET_BASE_URL="/base/url" - OBJECT_STORE_MODE="FileBacked" - OBJECT_STORE_FILE_BACKED_BASE_PATH="artifacts" - OBJECT_STORE_GCS_CREDENTIAL_FILE_PATH="/path/to/credentials.json" - OBJECT_STORE_MAX_RETRIES="5" - "#; - lock.set_env(config); - let actual = ObjectStoreConfig::from_env().unwrap(); - assert_eq!(actual, expected_config("/base/url")); - } - - #[test] - fn public_bucket_config_from_env() { - let mut lock = MUTEX.lock(); - let config = r#" - PUBLIC_OBJECT_STORE_BUCKET_BASE_URL="/public_base_url" - PUBLIC_OBJECT_STORE_MODE="FileBacked" - PUBLIC_OBJECT_STORE_FILE_BACKED_BASE_PATH="artifacts" - PUBLIC_OBJECT_STORE_GCS_CREDENTIAL_FILE_PATH="/path/to/credentials.json" - PUBLIC_OBJECT_STORE_MAX_RETRIES="5" - "#; - lock.set_env(config); - let actual = ObjectStoreConfig::public_from_env().unwrap(); - assert_eq!(actual, expected_config("/public_base_url")); - } - - #[test] - fn prover_bucket_config_from_env() { - let mut lock = MUTEX.lock(); - let config = r#" - PROVER_OBJECT_STORE_BUCKET_BASE_URL="/prover_base_url" - PROVER_OBJECT_STORE_MODE="FileBacked" - PROVER_OBJECT_STORE_FILE_BACKED_BASE_PATH="artifacts" - PROVER_OBJECT_STORE_GCS_CREDENTIAL_FILE_PATH="/path/to/credentials.json" - PROVER_OBJECT_STORE_MAX_RETRIES="5" - "#; - lock.set_env(config); - let actual = ObjectStoreConfig::prover_from_env().unwrap(); - assert_eq!(actual, expected_config("/prover_base_url")); - } -} diff --git a/core/lib/config/src/configs/proof_data_handler.rs b/core/lib/config/src/configs/proof_data_handler.rs index 577a01c055f5..e3efd6b7a4d7 100644 --- a/core/lib/config/src/configs/proof_data_handler.rs +++ b/core/lib/config/src/configs/proof_data_handler.rs @@ -1,4 +1,3 @@ -use super::envy_load; use serde::Deserialize; use std::time::Duration; @@ -15,44 +14,8 @@ pub struct ProofDataHandlerConfig { pub protocol_version_loading_mode: ProtocolVersionLoadingMode, pub fri_protocol_version_id: u16, } - impl ProofDataHandlerConfig { - pub fn from_env() -> anyhow::Result { - envy_load("proof_data_handler", "PROOF_DATA_HANDLER_") - } - pub fn proof_generation_timeout(&self) -> Duration { Duration::from_secs(self.proof_generation_timeout_in_secs as u64) } } - -#[cfg(test)] -mod tests { - use super::*; - use crate::configs::test_utils::EnvMutex; - - static MUTEX: EnvMutex = EnvMutex::new(); - - fn expected_config() -> ProofDataHandlerConfig { - ProofDataHandlerConfig { - http_port: 3320, - proof_generation_timeout_in_secs: 18000, - protocol_version_loading_mode: ProtocolVersionLoadingMode::FromEnvVar, - fri_protocol_version_id: 2, - } - } - - #[test] - fn from_env() { - let config = r#" - PROOF_DATA_HANDLER_PROOF_GENERATION_TIMEOUT_IN_SECS="18000" - PROOF_DATA_HANDLER_HTTP_PORT="3320" - PROOF_DATA_HANDLER_PROTOCOL_VERSION_LOADING_MODE="FromEnvVar" - PROOF_DATA_HANDLER_FRI_PROTOCOL_VERSION_ID="2" - "#; - let mut lock = MUTEX.lock(); - lock.set_env(config); - let actual = ProofDataHandlerConfig::from_env().unwrap(); - assert_eq!(actual, expected_config()); - } -} diff --git a/core/lib/config/src/configs/prover.rs b/core/lib/config/src/configs/prover.rs index 483706b8244c..45ed7100f9f4 100644 --- a/core/lib/config/src/configs/prover.rs +++ b/core/lib/config/src/configs/prover.rs @@ -1,12 +1,7 @@ use std::time::Duration; -// Built-in uses -// External uses use serde::Deserialize; -// Local uses -use super::envy_load; - /// Configuration for the prover application #[derive(Debug, Deserialize, Clone, PartialEq)] pub struct ProverConfig { @@ -64,195 +59,3 @@ impl ProverConfig { Duration::from_secs(self.generation_timeout_in_secs as u64) } } - -impl ProverConfigs { - pub fn from_env() -> anyhow::Result { - Ok(Self { - non_gpu: envy_load("non_gpu", "PROVER_NON_GPU_")?, - two_gpu_forty_gb_mem: envy_load( - "two_gpu_forty_gb_mem", - "PROVER_TWO_GPU_FORTY_GB_MEM_", - )?, - one_gpu_eighty_gb_mem: envy_load( - "one_gpu_eighty_gb_mem", - "PROVER_ONE_GPU_EIGHTY_GB_MEM_", - )?, - two_gpu_eighty_gb_mem: envy_load( - "two_gpu_eighty_gb_mem", - "PROVER_TWO_GPU_EIGHTY_GB_MEM_", - )?, - four_gpu_eighty_gb_mem: envy_load( - "four_gpu_eighty_gb_mem", - "PROVER_FOUR_GPU_EIGHTY_GB_MEM_", - )?, - }) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::configs::test_utils::EnvMutex; - - static MUTEX: EnvMutex = EnvMutex::new(); - - fn expected_config() -> ProverConfigs { - ProverConfigs { - non_gpu: ProverConfig { - prometheus_port: 3313, - initial_setup_key_path: "key".to_owned(), - key_download_url: "value".to_owned(), - generation_timeout_in_secs: 2700u16, - number_of_threads: 2, - max_attempts: 4, - polling_duration_in_millis: 5, - setup_keys_path: "/usr/src/setup-keys".to_string(), - specialized_prover_group_id: 0, - number_of_setup_slots: 2, - assembly_receiver_port: 17791, - assembly_receiver_poll_time_in_millis: 250, - assembly_queue_capacity: 5, - }, - two_gpu_forty_gb_mem: ProverConfig { - prometheus_port: 3313, - initial_setup_key_path: "key".to_owned(), - key_download_url: "value".to_owned(), - generation_timeout_in_secs: 2700u16, - number_of_threads: 2, - max_attempts: 4, - polling_duration_in_millis: 5, - setup_keys_path: "/usr/src/setup-keys".to_string(), - specialized_prover_group_id: 1, - number_of_setup_slots: 5, - assembly_receiver_port: 17791, - assembly_receiver_poll_time_in_millis: 250, - assembly_queue_capacity: 5, - }, - one_gpu_eighty_gb_mem: ProverConfig { - prometheus_port: 3313, - initial_setup_key_path: "key".to_owned(), - key_download_url: "value".to_owned(), - generation_timeout_in_secs: 2700u16, - number_of_threads: 4, - max_attempts: 4, - polling_duration_in_millis: 5, - setup_keys_path: "/usr/src/setup-keys".to_string(), - specialized_prover_group_id: 2, - number_of_setup_slots: 5, - assembly_receiver_port: 17791, - assembly_receiver_poll_time_in_millis: 250, - assembly_queue_capacity: 5, - }, - two_gpu_eighty_gb_mem: ProverConfig { - prometheus_port: 3313, - initial_setup_key_path: "key".to_owned(), - key_download_url: "value".to_owned(), - generation_timeout_in_secs: 2700u16, - number_of_threads: 9, - max_attempts: 4, - polling_duration_in_millis: 5, - setup_keys_path: "/usr/src/setup-keys".to_string(), - specialized_prover_group_id: 3, - number_of_setup_slots: 9, - assembly_receiver_port: 17791, - assembly_receiver_poll_time_in_millis: 250, - assembly_queue_capacity: 5, - }, - four_gpu_eighty_gb_mem: ProverConfig { - prometheus_port: 3313, - initial_setup_key_path: "key".to_owned(), - key_download_url: "value".to_owned(), - generation_timeout_in_secs: 2700u16, - number_of_threads: 18, - max_attempts: 4, - polling_duration_in_millis: 5, - setup_keys_path: "/usr/src/setup-keys".to_string(), - specialized_prover_group_id: 4, - number_of_setup_slots: 18, - assembly_receiver_port: 17791, - assembly_receiver_poll_time_in_millis: 250, - assembly_queue_capacity: 5, - }, - } - } - - const CONFIG: &str = r#" - PROVER_NON_GPU_PROMETHEUS_PORT="3313" - PROVER_NON_GPU_INITIAL_SETUP_KEY_PATH="key" - PROVER_NON_GPU_KEY_DOWNLOAD_URL="value" - PROVER_NON_GPU_GENERATION_TIMEOUT_IN_SECS=2700 - PROVER_NON_GPU_NUMBER_OF_THREADS="2" - PROVER_NON_GPU_MAX_ATTEMPTS="4" - PROVER_NON_GPU_POLLING_DURATION_IN_MILLIS=5 - PROVER_NON_GPU_SETUP_KEYS_PATH="/usr/src/setup-keys" - PROVER_NON_GPU_NUMBER_OF_SETUP_SLOTS=2 - PROVER_NON_GPU_ASSEMBLY_RECEIVER_PORT=17791 - PROVER_NON_GPU_ASSEMBLY_RECEIVER_POLL_TIME_IN_MILLIS=250 - PROVER_NON_GPU_ASSEMBLY_QUEUE_CAPACITY=5 - PROVER_NON_GPU_SPECIALIZED_PROVER_GROUP_ID=0 - - PROVER_TWO_GPU_FORTY_GB_MEM_PROMETHEUS_PORT="3313" - PROVER_TWO_GPU_FORTY_GB_MEM_INITIAL_SETUP_KEY_PATH="key" - PROVER_TWO_GPU_FORTY_GB_MEM_KEY_DOWNLOAD_URL="value" - PROVER_TWO_GPU_FORTY_GB_MEM_GENERATION_TIMEOUT_IN_SECS=2700 - PROVER_TWO_GPU_FORTY_GB_MEM_NUMBER_OF_THREADS="2" - PROVER_TWO_GPU_FORTY_GB_MEM_MAX_ATTEMPTS="4" - PROVER_TWO_GPU_FORTY_GB_MEM_POLLING_DURATION_IN_MILLIS=5 - PROVER_TWO_GPU_FORTY_GB_MEM_SETUP_KEYS_PATH="/usr/src/setup-keys" - PROVER_TWO_GPU_FORTY_GB_MEM_NUMBER_OF_SETUP_SLOTS=5 - PROVER_TWO_GPU_FORTY_GB_MEM_ASSEMBLY_RECEIVER_PORT=17791 - PROVER_TWO_GPU_FORTY_GB_MEM_ASSEMBLY_RECEIVER_POLL_TIME_IN_MILLIS=250 - PROVER_TWO_GPU_FORTY_GB_MEM_ASSEMBLY_QUEUE_CAPACITY=5 - PROVER_TWO_GPU_FORTY_GB_MEM_SPECIALIZED_PROVER_GROUP_ID=1 - - PROVER_ONE_GPU_EIGHTY_GB_MEM_PROMETHEUS_PORT="3313" - PROVER_ONE_GPU_EIGHTY_GB_MEM_INITIAL_SETUP_KEY_PATH="key" - PROVER_ONE_GPU_EIGHTY_GB_MEM_KEY_DOWNLOAD_URL="value" - PROVER_ONE_GPU_EIGHTY_GB_MEM_GENERATION_TIMEOUT_IN_SECS=2700 - PROVER_ONE_GPU_EIGHTY_GB_MEM_NUMBER_OF_THREADS="4" - PROVER_ONE_GPU_EIGHTY_GB_MEM_MAX_ATTEMPTS="4" - PROVER_ONE_GPU_EIGHTY_GB_MEM_POLLING_DURATION_IN_MILLIS=5 - PROVER_ONE_GPU_EIGHTY_GB_MEM_SETUP_KEYS_PATH="/usr/src/setup-keys" - PROVER_ONE_GPU_EIGHTY_GB_MEM_NUMBER_OF_SETUP_SLOTS=5 - PROVER_ONE_GPU_EIGHTY_GB_MEM_ASSEMBLY_RECEIVER_PORT=17791 - PROVER_ONE_GPU_EIGHTY_GB_MEM_ASSEMBLY_RECEIVER_POLL_TIME_IN_MILLIS=250 - PROVER_ONE_GPU_EIGHTY_GB_MEM_ASSEMBLY_QUEUE_CAPACITY=5 - PROVER_ONE_GPU_EIGHTY_GB_MEM_SPECIALIZED_PROVER_GROUP_ID=2 - - PROVER_TWO_GPU_EIGHTY_GB_MEM_PROMETHEUS_PORT="3313" - PROVER_TWO_GPU_EIGHTY_GB_MEM_INITIAL_SETUP_KEY_PATH="key" - PROVER_TWO_GPU_EIGHTY_GB_MEM_KEY_DOWNLOAD_URL="value" - PROVER_TWO_GPU_EIGHTY_GB_MEM_GENERATION_TIMEOUT_IN_SECS=2700 - PROVER_TWO_GPU_EIGHTY_GB_MEM_NUMBER_OF_THREADS="9" - PROVER_TWO_GPU_EIGHTY_GB_MEM_MAX_ATTEMPTS="4" - PROVER_TWO_GPU_EIGHTY_GB_MEM_POLLING_DURATION_IN_MILLIS=5 - PROVER_TWO_GPU_EIGHTY_GB_MEM_SETUP_KEYS_PATH="/usr/src/setup-keys" - PROVER_TWO_GPU_EIGHTY_GB_MEM_NUMBER_OF_SETUP_SLOTS=9 - PROVER_TWO_GPU_EIGHTY_GB_MEM_ASSEMBLY_RECEIVER_PORT=17791 - PROVER_TWO_GPU_EIGHTY_GB_MEM_ASSEMBLY_RECEIVER_POLL_TIME_IN_MILLIS=250 - PROVER_TWO_GPU_EIGHTY_GB_MEM_ASSEMBLY_QUEUE_CAPACITY=5 - PROVER_TWO_GPU_EIGHTY_GB_MEM_SPECIALIZED_PROVER_GROUP_ID=3 - - PROVER_FOUR_GPU_EIGHTY_GB_MEM_PROMETHEUS_PORT="3313" - PROVER_FOUR_GPU_EIGHTY_GB_MEM_INITIAL_SETUP_KEY_PATH="key" - PROVER_FOUR_GPU_EIGHTY_GB_MEM_KEY_DOWNLOAD_URL="value" - PROVER_FOUR_GPU_EIGHTY_GB_MEM_GENERATION_TIMEOUT_IN_SECS=2700 - PROVER_FOUR_GPU_EIGHTY_GB_MEM_NUMBER_OF_THREADS="18" - PROVER_FOUR_GPU_EIGHTY_GB_MEM_MAX_ATTEMPTS="4" - PROVER_FOUR_GPU_EIGHTY_GB_MEM_POLLING_DURATION_IN_MILLIS=5 - PROVER_FOUR_GPU_EIGHTY_GB_MEM_SETUP_KEYS_PATH="/usr/src/setup-keys" - PROVER_FOUR_GPU_EIGHTY_GB_MEM_NUMBER_OF_SETUP_SLOTS=18 - PROVER_FOUR_GPU_EIGHTY_GB_MEM_ASSEMBLY_RECEIVER_PORT=17791 - PROVER_FOUR_GPU_EIGHTY_GB_MEM_ASSEMBLY_RECEIVER_POLL_TIME_IN_MILLIS=250 - PROVER_FOUR_GPU_EIGHTY_GB_MEM_ASSEMBLY_QUEUE_CAPACITY=5 - PROVER_FOUR_GPU_EIGHTY_GB_MEM_SPECIALIZED_PROVER_GROUP_ID=4 - "#; - - #[test] - fn from_env() { - let mut lock = MUTEX.lock(); - lock.set_env(CONFIG); - let actual = ProverConfigs::from_env().unwrap(); - assert_eq!(actual, expected_config()); - } -} diff --git a/core/lib/config/src/configs/prover_group.rs b/core/lib/config/src/configs/prover_group.rs index d4dfee82a203..2d40d47ba8c1 100644 --- a/core/lib/config/src/configs/prover_group.rs +++ b/core/lib/config/src/configs/prover_group.rs @@ -1,7 +1,5 @@ use serde::Deserialize; -use super::envy_load; - /// Configuration for the grouping of specialized provers. /// This config would be used by circuit-synthesizer and provers. #[derive(Debug, Deserialize, Clone, PartialEq)] @@ -26,10 +24,6 @@ pub struct ProverGroupConfig { } impl ProverGroupConfig { - pub fn from_env() -> anyhow::Result { - envy_load("prover_group", "PROVER_GROUP_") - } - pub fn get_circuit_ids_for_group_id(&self, group_id: u8) -> Option> { match group_id { 0 => Some(self.group_0_circuit_ids.clone()), @@ -70,143 +64,3 @@ impl ProverGroupConfig { .map(|(group_id, _)| group_id as u8) } } - -#[cfg(test)] -mod tests { - use super::*; - use crate::configs::test_utils::EnvMutex; - - static MUTEX: EnvMutex = EnvMutex::new(); - - fn expected_config() -> ProverGroupConfig { - ProverGroupConfig { - group_0_circuit_ids: vec![0, 18], - group_1_circuit_ids: vec![1, 4], - group_2_circuit_ids: vec![2, 5], - group_3_circuit_ids: vec![6, 7], - group_4_circuit_ids: vec![8, 9], - group_5_circuit_ids: vec![10, 11], - group_6_circuit_ids: vec![12, 13], - group_7_circuit_ids: vec![14, 15], - group_8_circuit_ids: vec![16, 17], - group_9_circuit_ids: vec![3], - region_read_url: "http://metadata.google.internal/computeMetadata/v1/instance/attributes/cluster-location".to_string(), - region_override: Some("us-central-1".to_string()), - zone_read_url: "http://metadata.google.internal/computeMetadata/v1/instance/zone".to_string(), - zone_override: Some("us-central-1-b".to_string()), - synthesizer_per_gpu: 10, - } - } - - const CONFIG: &str = r#" - PROVER_GROUP_GROUP_0_CIRCUIT_IDS="0,18" - PROVER_GROUP_GROUP_1_CIRCUIT_IDS="1,4" - PROVER_GROUP_GROUP_2_CIRCUIT_IDS="2,5" - PROVER_GROUP_GROUP_3_CIRCUIT_IDS="6,7" - PROVER_GROUP_GROUP_4_CIRCUIT_IDS="8,9" - PROVER_GROUP_GROUP_5_CIRCUIT_IDS="10,11" - PROVER_GROUP_GROUP_6_CIRCUIT_IDS="12,13" - PROVER_GROUP_GROUP_7_CIRCUIT_IDS="14,15" - PROVER_GROUP_GROUP_8_CIRCUIT_IDS="16,17" - PROVER_GROUP_GROUP_9_CIRCUIT_IDS="3" - PROVER_GROUP_REGION_READ_URL="http://metadata.google.internal/computeMetadata/v1/instance/attributes/cluster-location" - PROVER_GROUP_REGION_OVERRIDE="us-central-1" - PROVER_GROUP_ZONE_READ_URL="http://metadata.google.internal/computeMetadata/v1/instance/zone" - PROVER_GROUP_ZONE_OVERRIDE="us-central-1-b" - PROVER_GROUP_SYNTHESIZER_PER_GPU="10" - "#; - - #[test] - fn from_env() { - let mut lock = MUTEX.lock(); - lock.set_env(CONFIG); - let actual = ProverGroupConfig::from_env().unwrap(); - assert_eq!(actual, expected_config()); - } - - #[test] - fn get_group_id_for_circuit_id() { - let prover_group_config = expected_config(); - - assert_eq!(Some(0), prover_group_config.get_group_id_for_circuit_id(0)); - assert_eq!(Some(0), prover_group_config.get_group_id_for_circuit_id(18)); - - assert_eq!(Some(1), prover_group_config.get_group_id_for_circuit_id(1)); - assert_eq!(Some(1), prover_group_config.get_group_id_for_circuit_id(4)); - - assert_eq!(Some(2), prover_group_config.get_group_id_for_circuit_id(2)); - assert_eq!(Some(2), prover_group_config.get_group_id_for_circuit_id(5)); - - assert_eq!(Some(3), prover_group_config.get_group_id_for_circuit_id(6)); - assert_eq!(Some(3), prover_group_config.get_group_id_for_circuit_id(7)); - - assert_eq!(Some(4), prover_group_config.get_group_id_for_circuit_id(8)); - assert_eq!(Some(4), prover_group_config.get_group_id_for_circuit_id(9)); - - assert_eq!(Some(5), prover_group_config.get_group_id_for_circuit_id(10)); - assert_eq!(Some(5), prover_group_config.get_group_id_for_circuit_id(11)); - - assert_eq!(Some(6), prover_group_config.get_group_id_for_circuit_id(12)); - assert_eq!(Some(6), prover_group_config.get_group_id_for_circuit_id(13)); - - assert_eq!(Some(7), prover_group_config.get_group_id_for_circuit_id(14)); - assert_eq!(Some(7), prover_group_config.get_group_id_for_circuit_id(15)); - - assert_eq!(Some(8), prover_group_config.get_group_id_for_circuit_id(16)); - assert_eq!(Some(8), prover_group_config.get_group_id_for_circuit_id(17)); - - assert_eq!(Some(9), prover_group_config.get_group_id_for_circuit_id(3)); - assert!(prover_group_config - .get_group_id_for_circuit_id(19) - .is_none()); - } - - #[test] - fn get_circuit_ids_for_group_id() { - let prover_group_config = expected_config(); - - assert_eq!( - Some(vec![0, 18]), - prover_group_config.get_circuit_ids_for_group_id(0) - ); - assert_eq!( - Some(vec![1, 4]), - prover_group_config.get_circuit_ids_for_group_id(1) - ); - assert_eq!( - Some(vec![2, 5]), - prover_group_config.get_circuit_ids_for_group_id(2) - ); - assert_eq!( - Some(vec![6, 7]), - prover_group_config.get_circuit_ids_for_group_id(3) - ); - assert_eq!( - Some(vec![8, 9]), - prover_group_config.get_circuit_ids_for_group_id(4) - ); - assert_eq!( - Some(vec![10, 11]), - prover_group_config.get_circuit_ids_for_group_id(5) - ); - assert_eq!( - Some(vec![12, 13]), - prover_group_config.get_circuit_ids_for_group_id(6) - ); - assert_eq!( - Some(vec![14, 15]), - prover_group_config.get_circuit_ids_for_group_id(7) - ); - assert_eq!( - Some(vec![16, 17]), - prover_group_config.get_circuit_ids_for_group_id(8) - ); - assert_eq!( - Some(vec![3]), - prover_group_config.get_circuit_ids_for_group_id(9) - ); - assert!(prover_group_config - .get_circuit_ids_for_group_id(10) - .is_none()); - } -} diff --git a/core/lib/config/src/configs/utils.rs b/core/lib/config/src/configs/utils.rs index 5de6abd97185..bfa9e7e7f3e6 100644 --- a/core/lib/config/src/configs/utils.rs +++ b/core/lib/config/src/configs/utils.rs @@ -2,8 +2,6 @@ use serde::Deserialize; use std::{env, time::Duration}; -use crate::configs::envy_load; - #[derive(Debug, Deserialize, Clone, PartialEq)] pub struct PrometheusConfig { /// Port to which the Prometheus exporter server is listening. @@ -15,10 +13,6 @@ pub struct PrometheusConfig { } impl PrometheusConfig { - pub fn from_env() -> anyhow::Result { - envy_load("prometheus", "API_PROMETHEUS_") - } - pub fn push_interval(&self) -> Duration { Duration::from_millis(self.push_interval_ms.unwrap_or(100)) } diff --git a/core/lib/config/src/configs/witness_generator.rs b/core/lib/config/src/configs/witness_generator.rs index de3bb29d08e9..010742e4b047 100644 --- a/core/lib/config/src/configs/witness_generator.rs +++ b/core/lib/config/src/configs/witness_generator.rs @@ -4,9 +4,6 @@ use std::time::Duration; // External uses use serde::Deserialize; -// Local uses -use super::envy_load; - #[derive(Debug, Deserialize, Clone, PartialEq)] pub enum BasicWitnessGeneratorDataSource { FromPostgres, @@ -38,10 +35,6 @@ pub struct WitnessGeneratorConfig { } impl WitnessGeneratorConfig { - pub fn from_env() -> anyhow::Result { - envy_load("witness", "WITNESS_") - } - pub fn witness_generation_timeout(&self) -> Duration { Duration::from_secs(self.generation_timeout_in_secs as u64) } @@ -50,42 +43,3 @@ impl WitnessGeneratorConfig { self.last_l1_batch_to_process.unwrap_or(u32::MAX) } } - -#[cfg(test)] -mod tests { - use super::*; - use crate::configs::test_utils::EnvMutex; - - static MUTEX: EnvMutex = EnvMutex::new(); - - fn expected_config() -> WitnessGeneratorConfig { - WitnessGeneratorConfig { - generation_timeout_in_secs: 900_u16, - initial_setup_key_path: "key".to_owned(), - key_download_url: "value".to_owned(), - max_attempts: 4, - blocks_proving_percentage: Some(30), - dump_arguments_for_blocks: vec![2, 3], - last_l1_batch_to_process: None, - data_source: BasicWitnessGeneratorDataSource::FromBlob, - } - } - - #[test] - fn from_env() { - let mut lock = MUTEX.lock(); - let config = r#" - WITNESS_GENERATION_TIMEOUT_IN_SECS=900 - WITNESS_INITIAL_SETUP_KEY_PATH="key" - WITNESS_KEY_DOWNLOAD_URL="value" - WITNESS_MAX_ATTEMPTS=4 - WITNESS_DUMP_ARGUMENTS_FOR_BLOCKS="2,3" - WITNESS_BLOCKS_PROVING_PERCENTAGE="30" - WITNESS_DATA_SOURCE="FromBlob" - "#; - lock.set_env(config); - - let actual = WitnessGeneratorConfig::from_env().unwrap(); - assert_eq!(actual, expected_config()); - } -} diff --git a/core/lib/env_config/Cargo.toml b/core/lib/env_config/Cargo.toml new file mode 100644 index 000000000000..a084a3768bfd --- /dev/null +++ b/core/lib/env_config/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "zksync_env_config" +version = "0.1.0" +edition = "2018" +authors = ["The Matter Labs Team "] +homepage = "https://zksync.io/" +repository = "https://github.com/matter-labs/zksync-era" +license = "MIT OR Apache-2.0" +keywords = ["blockchain", "zksync"] +categories = ["cryptography"] + +[dependencies] +zksync_basic_types = { path = "../../lib/basic_types" } +zksync_config = { path = "../../lib/config" } + +anyhow = "1.0" +serde = "1.0" +envy = "0.4" diff --git a/core/lib/env_config/src/alerts.rs b/core/lib/env_config/src/alerts.rs new file mode 100644 index 000000000000..c72b23bbd9fa --- /dev/null +++ b/core/lib/env_config/src/alerts.rs @@ -0,0 +1,37 @@ +use crate::{envy_load, FromEnv}; +use zksync_config::configs::AlertsConfig; + +impl FromEnv for AlertsConfig { + fn from_env() -> anyhow::Result { + envy_load("sporadic_crypto_errors_substrs", "ALERTS_") + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_utils::EnvMutex; + + static MUTEX: EnvMutex = EnvMutex::new(); + + fn expected_config() -> AlertsConfig { + AlertsConfig { + sporadic_crypto_errors_substrs: vec![ + "EventDestroyErr".to_string(), + "Can't free memory of DeviceBuf".to_string(), + "value: PoisonError".to_string(), + ], + } + } + + #[test] + fn test_from_env() { + let mut lock = MUTEX.lock(); + let config = r#" + ALERTS_SPORADIC_CRYPTO_ERRORS_SUBSTRS="EventDestroyErr,Can't free memory of DeviceBuf,value: PoisonError" + "#; + lock.set_env(config); + + assert_eq!(AlertsConfig::from_env().unwrap(), expected_config()); + } +} diff --git a/core/lib/env_config/src/api.rs b/core/lib/env_config/src/api.rs new file mode 100644 index 000000000000..92428fe45f1a --- /dev/null +++ b/core/lib/env_config/src/api.rs @@ -0,0 +1,151 @@ +use anyhow::Context as _; + +use crate::{envy_load, FromEnv}; +use zksync_config::configs::{ + api::{ + ContractVerificationApiConfig, HealthCheckConfig, MerkleTreeApiConfig, Web3JsonRpcConfig, + }, + ApiConfig, PrometheusConfig, +}; + +impl FromEnv for ApiConfig { + fn from_env() -> anyhow::Result { + Ok(Self { + web3_json_rpc: Web3JsonRpcConfig::from_env().context("Web3JsonRpcConfig")?, + contract_verification: ContractVerificationApiConfig::from_env() + .context("ContractVerificationApiConfig")?, + prometheus: PrometheusConfig::from_env().context("PrometheusConfig")?, + healthcheck: HealthCheckConfig::from_env().context("HealthCheckConfig")?, + merkle_tree: MerkleTreeApiConfig::from_env().context("MerkleTreeApiConfig")?, + }) + } +} + +impl FromEnv for Web3JsonRpcConfig { + fn from_env() -> anyhow::Result { + envy_load("web3_json_rpc", "API_WEB3_JSON_RPC_") + } +} + +impl FromEnv for HealthCheckConfig { + fn from_env() -> anyhow::Result { + envy_load("healthcheck", "API_HEALTHCHECK_") + } +} + +impl FromEnv for ContractVerificationApiConfig { + fn from_env() -> anyhow::Result { + envy_load("contract_verification", "API_CONTRACT_VERIFICATION_") + } +} + +impl FromEnv for MerkleTreeApiConfig { + /// Loads configuration from env variables. + fn from_env() -> anyhow::Result { + envy_load("merkle_tree_api", "API_MERKLE_TREE_") + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_utils::{hash, EnvMutex}; + + static MUTEX: EnvMutex = EnvMutex::new(); + + fn expected_config() -> ApiConfig { + ApiConfig { + web3_json_rpc: Web3JsonRpcConfig { + http_port: 3050, + http_url: "http://127.0.0.1:3050".into(), + ws_port: 3051, + ws_url: "ws://127.0.0.1:3051".into(), + req_entities_limit: Some(10000), + filters_limit: Some(10000), + subscriptions_limit: Some(10000), + pubsub_polling_interval: Some(200), + threads_per_server: 128, + max_nonce_ahead: 5, + transactions_per_sec_limit: Some(1000), + request_timeout: Some(10), + account_pks: Some(vec![ + hash("0x0000000000000000000000000000000000000000000000000000000000000001"), + hash("0x0000000000000000000000000000000000000000000000000000000000000002"), + ]), + estimate_gas_scale_factor: 1.0f64, + gas_price_scale_factor: 1.2, + estimate_gas_acceptable_overestimation: 1000, + max_tx_size: 1000000, + vm_execution_cache_misses_limit: None, + vm_concurrency_limit: Some(512), + factory_deps_cache_size_mb: Some(128), + initial_writes_cache_size_mb: Some(32), + latest_values_cache_size_mb: Some(256), + http_threads: Some(128), + ws_threads: Some(256), + fee_history_limit: Some(100), + max_batch_request_size: Some(200), + max_response_body_size_mb: Some(10), + websocket_requests_per_minute_limit: Some(10), + }, + contract_verification: ContractVerificationApiConfig { + port: 3070, + url: "http://127.0.0.1:3070".into(), + threads_per_server: 128, + }, + prometheus: PrometheusConfig { + listener_port: 3312, + pushgateway_url: "http://127.0.0.1:9091".into(), + push_interval_ms: Some(100), + }, + healthcheck: HealthCheckConfig { port: 8081 }, + merkle_tree: MerkleTreeApiConfig { port: 8082 }, + } + } + + #[test] + fn from_env() { + let mut lock = MUTEX.lock(); + let config = r#" + API_WEB3_JSON_RPC_HTTP_PORT="3050" + API_WEB3_JSON_RPC_HTTP_URL="http://127.0.0.1:3050" + API_WEB3_JSON_RPC_WS_PORT="3051" + API_WEB3_JSON_RPC_WS_URL="ws://127.0.0.1:3051" + API_WEB3_JSON_RPC_REQ_ENTITIES_LIMIT=10000 + API_WEB3_JSON_RPC_FILTERS_LIMIT=10000 + API_WEB3_JSON_RPC_SUBSCRIPTIONS_LIMIT=10000 + API_WEB3_JSON_RPC_PUBSUB_POLLING_INTERVAL=200 + API_WEB3_JSON_RPC_THREADS_PER_SERVER=128 + API_WEB3_JSON_RPC_MAX_NONCE_AHEAD=5 + API_WEB3_JSON_RPC_GAS_PRICE_SCALE_FACTOR=1.2 + API_WEB3_JSON_RPC_TRANSACTIONS_PER_SEC_LIMIT=1000 + API_WEB3_JSON_RPC_REQUEST_TIMEOUT=10 + API_WEB3_JSON_RPC_ACCOUNT_PKS="0x0000000000000000000000000000000000000000000000000000000000000001,0x0000000000000000000000000000000000000000000000000000000000000002" + API_WEB3_JSON_RPC_ESTIMATE_GAS_SCALE_FACTOR=1.0 + API_WEB3_JSON_RPC_ESTIMATE_GAS_ACCEPTABLE_OVERESTIMATION=1000 + API_WEB3_JSON_RPC_MAX_TX_SIZE=1000000 + API_WEB3_JSON_RPC_VM_CONCURRENCY_LIMIT=512 + API_WEB3_JSON_RPC_FACTORY_DEPS_CACHE_SIZE_MB=128 + API_WEB3_JSON_RPC_INITIAL_WRITES_CACHE_SIZE_MB=32 + API_WEB3_JSON_RPC_LATEST_VALUES_CACHE_SIZE_MB=256 + API_WEB3_JSON_RPC_HTTP_THREADS=128 + API_WEB3_JSON_RPC_WS_THREADS=256 + API_WEB3_JSON_RPC_FEE_HISTORY_LIMIT=100 + API_WEB3_JSON_RPC_MAX_BATCH_REQUEST_SIZE=200 + API_WEB3_JSON_RPC_WEBSOCKET_REQUESTS_PER_MINUTE_LIMIT=10 + API_CONTRACT_VERIFICATION_PORT="3070" + API_CONTRACT_VERIFICATION_URL="http://127.0.0.1:3070" + API_CONTRACT_VERIFICATION_THREADS_PER_SERVER=128 + API_WEB3_JSON_RPC_MAX_RESPONSE_BODY_SIZE_MB=10 + API_PROMETHEUS_LISTENER_PORT="3312" + API_PROMETHEUS_PUSHGATEWAY_URL="http://127.0.0.1:9091" + API_PROMETHEUS_PUSH_INTERVAL_MS=100 + API_HEALTHCHECK_PORT=8081 + API_MERKLE_TREE_PORT=8082 + "#; + lock.set_env(config); + + let actual = ApiConfig::from_env().unwrap(); + assert_eq!(actual, expected_config()); + } +} diff --git a/core/lib/env_config/src/chain.rs b/core/lib/env_config/src/chain.rs new file mode 100644 index 000000000000..e64ba3c36b89 --- /dev/null +++ b/core/lib/env_config/src/chain.rs @@ -0,0 +1,151 @@ +use crate::{envy_load, FromEnv}; +use anyhow::Context as _; +use zksync_config::configs::chain::{ + ChainConfig, CircuitBreakerConfig, MempoolConfig, NetworkConfig, OperationsManagerConfig, + StateKeeperConfig, +}; + +impl FromEnv for ChainConfig { + fn from_env() -> anyhow::Result { + Ok(Self { + network: NetworkConfig::from_env().context("NetworkConfig")?, + state_keeper: StateKeeperConfig::from_env().context("StateKeeperConfig")?, + operations_manager: OperationsManagerConfig::from_env() + .context("OperationsManagerConfig")?, + mempool: MempoolConfig::from_env().context("MempoolConfig")?, + circuit_breaker: CircuitBreakerConfig::from_env().context("CircuitBreakerConfig")?, + }) + } +} + +impl FromEnv for NetworkConfig { + fn from_env() -> anyhow::Result { + envy_load("network", "CHAIN_ETH_") + } +} + +impl FromEnv for StateKeeperConfig { + fn from_env() -> anyhow::Result { + envy_load("state_keeper", "CHAIN_STATE_KEEPER_") + } +} + +impl FromEnv for OperationsManagerConfig { + fn from_env() -> anyhow::Result { + envy_load("operations_manager", "CHAIN_OPERATIONS_MANAGER_") + } +} + +impl FromEnv for CircuitBreakerConfig { + fn from_env() -> anyhow::Result { + envy_load("circuit_breaker", "CHAIN_CIRCUIT_BREAKER_") + } +} + +impl FromEnv for MempoolConfig { + fn from_env() -> anyhow::Result { + envy_load("mempool", "CHAIN_MEMPOOL_") + } +} + +#[cfg(test)] +mod tests { + use zksync_basic_types::L2ChainId; + + use super::*; + use crate::test_utils::{addr, EnvMutex}; + + static MUTEX: EnvMutex = EnvMutex::new(); + + fn expected_config() -> ChainConfig { + ChainConfig { + network: NetworkConfig { + network: "localhost".parse().unwrap(), + zksync_network: "localhost".to_string(), + zksync_network_id: L2ChainId::from(270), + }, + state_keeper: StateKeeperConfig { + transaction_slots: 50, + block_commit_deadline_ms: 2500, + miniblock_commit_deadline_ms: 1000, + miniblock_seal_queue_capacity: 10, + max_single_tx_gas: 1_000_000, + max_allowed_l2_tx_gas_limit: 2_000_000_000, + close_block_at_eth_params_percentage: 0.2, + close_block_at_gas_percentage: 0.8, + close_block_at_geometry_percentage: 0.5, + reject_tx_at_eth_params_percentage: 0.8, + reject_tx_at_geometry_percentage: 0.3, + fee_account_addr: addr("de03a0B5963f75f1C8485B355fF6D30f3093BDE7"), + reject_tx_at_gas_percentage: 0.5, + fair_l2_gas_price: 250000000, + validation_computational_gas_limit: 10_000_000, + save_call_traces: false, + virtual_blocks_interval: 1, + virtual_blocks_per_miniblock: 1, + upload_witness_inputs_to_gcs: false, + enum_index_migration_chunk_size: Some(2_000), + }, + operations_manager: OperationsManagerConfig { + delay_interval: 100, + }, + mempool: MempoolConfig { + sync_interval_ms: 10, + sync_batch_size: 1000, + capacity: 1_000_000, + stuck_tx_timeout: 10, + remove_stuck_txs: true, + delay_interval: 100, + }, + circuit_breaker: CircuitBreakerConfig { + sync_interval_ms: 1000, + http_req_max_retry_number: 5, + http_req_retry_interval_sec: 2, + replication_lag_limit_sec: Some(10), + }, + } + } + + #[test] + fn from_env() { + let mut lock = MUTEX.lock(); + let config = r#" + CHAIN_ETH_NETWORK="localhost" + CHAIN_ETH_ZKSYNC_NETWORK="localhost" + CHAIN_ETH_ZKSYNC_NETWORK_ID=270 + CHAIN_STATE_KEEPER_TRANSACTION_SLOTS="50" + CHAIN_STATE_KEEPER_FEE_ACCOUNT_ADDR="0xde03a0B5963f75f1C8485B355fF6D30f3093BDE7" + CHAIN_STATE_KEEPER_MAX_SINGLE_TX_GAS="1000000" + CHAIN_STATE_KEEPER_MAX_ALLOWED_L2_TX_GAS_LIMIT="2000000000" + CHAIN_STATE_KEEPER_CLOSE_BLOCK_AT_GEOMETRY_PERCENTAGE="0.5" + CHAIN_STATE_KEEPER_CLOSE_BLOCK_AT_GAS_PERCENTAGE="0.8" + CHAIN_STATE_KEEPER_CLOSE_BLOCK_AT_ETH_PARAMS_PERCENTAGE="0.2" + CHAIN_STATE_KEEPER_REJECT_TX_AT_GEOMETRY_PERCENTAGE="0.3" + CHAIN_STATE_KEEPER_REJECT_TX_AT_ETH_PARAMS_PERCENTAGE="0.8" + CHAIN_STATE_KEEPER_REJECT_TX_AT_GAS_PERCENTAGE="0.5" + CHAIN_STATE_KEEPER_BLOCK_COMMIT_DEADLINE_MS="2500" + CHAIN_STATE_KEEPER_MINIBLOCK_COMMIT_DEADLINE_MS="1000" + CHAIN_STATE_KEEPER_MINIBLOCK_SEAL_QUEUE_CAPACITY="10" + CHAIN_STATE_KEEPER_FAIR_L2_GAS_PRICE="250000000" + CHAIN_STATE_KEEPER_VALIDATION_COMPUTATIONAL_GAS_LIMIT="10000000" + CHAIN_STATE_KEEPER_SAVE_CALL_TRACES="false" + CHAIN_STATE_KEEPER_UPLOAD_WITNESS_INPUTS_TO_GCS="false" + CHAIN_STATE_KEEPER_ENUM_INDEX_MIGRATION_CHUNK_SIZE="2000" + CHAIN_OPERATIONS_MANAGER_DELAY_INTERVAL="100" + CHAIN_MEMPOOL_SYNC_INTERVAL_MS="10" + CHAIN_MEMPOOL_SYNC_BATCH_SIZE="1000" + CHAIN_MEMPOOL_STUCK_TX_TIMEOUT="10" + CHAIN_MEMPOOL_REMOVE_STUCK_TXS="true" + CHAIN_MEMPOOL_DELAY_INTERVAL="100" + CHAIN_MEMPOOL_CAPACITY="1000000" + CHAIN_CIRCUIT_BREAKER_SYNC_INTERVAL_MS="1000" + CHAIN_CIRCUIT_BREAKER_HTTP_REQ_MAX_RETRY_NUMBER="5" + CHAIN_CIRCUIT_BREAKER_HTTP_REQ_RETRY_INTERVAL_SEC="2" + CHAIN_CIRCUIT_BREAKER_REPLICATION_LAG_LIMIT_SEC="10" + "#; + lock.set_env(config); + + let actual = ChainConfig::from_env().unwrap(); + assert_eq!(actual, expected_config()); + } +} diff --git a/core/lib/env_config/src/circuit_synthesizer.rs b/core/lib/env_config/src/circuit_synthesizer.rs new file mode 100644 index 000000000000..a59e1adcd656 --- /dev/null +++ b/core/lib/env_config/src/circuit_synthesizer.rs @@ -0,0 +1,51 @@ +use zksync_config::configs::CircuitSynthesizerConfig; + +use crate::{envy_load, FromEnv}; + +impl FromEnv for CircuitSynthesizerConfig { + fn from_env() -> anyhow::Result { + envy_load("circuit_synthesizer", "CIRCUIT_SYNTHESIZER_") + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_utils::EnvMutex; + + static MUTEX: EnvMutex = EnvMutex::new(); + + fn expected_config() -> CircuitSynthesizerConfig { + CircuitSynthesizerConfig { + generation_timeout_in_secs: 1000u16, + max_attempts: 2, + gpu_prover_queue_timeout_in_secs: 1000u16, + prover_instance_wait_timeout_in_secs: 1000u16, + prover_instance_poll_time_in_milli_secs: 250u16, + prometheus_listener_port: 3314, + prometheus_pushgateway_url: "http://127.0.0.1:9091".to_string(), + prometheus_push_interval_ms: Some(100), + prover_group_id: 0, + } + } + + #[test] + fn from_env() { + let mut lock = MUTEX.lock(); + let config = r#" + CIRCUIT_SYNTHESIZER_GENERATION_TIMEOUT_IN_SECS=1000 + CIRCUIT_SYNTHESIZER_MAX_ATTEMPTS=2 + CIRCUIT_SYNTHESIZER_GPU_PROVER_QUEUE_TIMEOUT_IN_SECS=1000 + CIRCUIT_SYNTHESIZER_PROVER_INSTANCE_WAIT_TIMEOUT_IN_SECS=1000 + CIRCUIT_SYNTHESIZER_PROVER_INSTANCE_POLL_TIME_IN_MILLI_SECS=250 + CIRCUIT_SYNTHESIZER_PROMETHEUS_LISTENER_PORT=3314 + CIRCUIT_SYNTHESIZER_PROMETHEUS_PUSHGATEWAY_URL="http://127.0.0.1:9091" + CIRCUIT_SYNTHESIZER_PROMETHEUS_PUSH_INTERVAL_MS=100 + CIRCUIT_SYNTHESIZER_PROVER_GROUP_ID=0 + "#; + lock.set_env(config); + + let actual = CircuitSynthesizerConfig::from_env().unwrap(); + assert_eq!(actual, expected_config()); + } +} diff --git a/core/lib/env_config/src/contract_verifier.rs b/core/lib/env_config/src/contract_verifier.rs new file mode 100644 index 000000000000..88c462cef2df --- /dev/null +++ b/core/lib/env_config/src/contract_verifier.rs @@ -0,0 +1,39 @@ +use zksync_config::ContractVerifierConfig; + +use crate::{envy_load, FromEnv}; + +impl FromEnv for ContractVerifierConfig { + fn from_env() -> anyhow::Result { + envy_load("contract_verifier", "CONTRACT_VERIFIER_") + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_utils::EnvMutex; + + static MUTEX: EnvMutex = EnvMutex::new(); + + fn expected_config() -> ContractVerifierConfig { + ContractVerifierConfig { + compilation_timeout: 30, + polling_interval: Some(1000), + prometheus_port: 3314, + } + } + + #[test] + fn from_env() { + let mut lock = MUTEX.lock(); + let config = r#" + CONTRACT_VERIFIER_COMPILATION_TIMEOUT=30 + CONTRACT_VERIFIER_POLLING_INTERVAL=1000 + CONTRACT_VERIFIER_PROMETHEUS_PORT=3314 + "#; + lock.set_env(config); + + let actual = ContractVerifierConfig::from_env().unwrap(); + assert_eq!(actual, expected_config()); + } +} diff --git a/core/lib/env_config/src/contracts.rs b/core/lib/env_config/src/contracts.rs new file mode 100644 index 000000000000..8c58483db064 --- /dev/null +++ b/core/lib/env_config/src/contracts.rs @@ -0,0 +1,108 @@ +use zksync_config::ContractsConfig; + +use crate::{envy_load, FromEnv}; + +impl FromEnv for ContractsConfig { + fn from_env() -> anyhow::Result { + envy_load("contracts", "CONTRACTS_") + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_utils::{addr, hash, EnvMutex}; + use zksync_config::configs::contracts::ProverAtGenesis; + + static MUTEX: EnvMutex = EnvMutex::new(); + + fn expected_config() -> ContractsConfig { + ContractsConfig { + governance_addr: addr("d8dA6BF26964aF9D7eEd9e03E53415D37aA96045"), + mailbox_facet_addr: addr("0f6Fa881EF414Fc6E818180657c2d5CD7Ac6cCAd"), + executor_facet_addr: addr("18B631537801963A964211C0E86645c1aBfbB2d3"), + admin_facet_addr: addr("1e12b20BE86bEc3A0aC95aA52ade345cB9AE7a32"), + getters_facet_addr: addr("8656770FA78c830456B00B4fFCeE6b1De0e1b888"), + verifier_addr: addr("34782eE00206EAB6478F2692caa800e4A581687b"), + diamond_init_addr: addr("FFC35A5e767BE36057c34586303498e3de7C62Ba"), + diamond_upgrade_init_addr: addr("FFC35A5e767BE36057c34586303498e3de7C62Ba"), + diamond_proxy_addr: addr("F00B988a98Ca742e7958DeF9F7823b5908715f4a"), + validator_timelock_addr: addr("F00B988a98Ca742e7958DeF9F7823b5908715f4a"), + genesis_tx_hash: hash( + "b99ebfea46cbe05a21cd80fe5597d97b204befc52a16303f579c607dc1ac2e2e", + ), + l1_erc20_bridge_proxy_addr: addr("8656770FA78c830456B00B4fFCeE6b1De0e1b888"), + l1_erc20_bridge_impl_addr: addr("8656770FA78c830456B00B4fFCeE6b1De0e1b888"), + l2_erc20_bridge_addr: addr("8656770FA78c830456B00B4fFCeE6b1De0e1b888"), + l1_allow_list_addr: addr("8656770FA78c830456B00B4fFCeE6b1De0e1b888"), + l1_weth_bridge_proxy_addr: Some(addr("8656770FA78c830456B00B4fFCeE6b1De0e1b888")), + l2_weth_bridge_addr: Some(addr("8656770FA78c830456B00B4fFCeE6b1De0e1b888")), + l2_testnet_paymaster_addr: Some(addr("FC073319977e314F251EAE6ae6bE76B0B3BAeeCF")), + recursion_scheduler_level_vk_hash: hash( + "0x1186ec268d49f1905f8d9c1e9d39fc33e98c74f91d91a21b8f7ef78bd09a8db8", + ), + recursion_node_level_vk_hash: hash( + "0x1186ec268d49f1905f8d9c1e9d39fc33e98c74f91d91a21b8f7ef78bd09a8db8", + ), + recursion_leaf_level_vk_hash: hash( + "0x101e08b00193e529145ee09823378ef51a3bc8966504064f1f6ba3f1ba863210", + ), + recursion_circuits_set_vks_hash: hash( + "0x142a364ef2073132eaf07aa7f3d8495065be5b92a2dc14fda09b4216affed9c0", + ), + l1_multicall3_addr: addr("0xcA11bde05977b3631167028862bE2a173976CA11"), + fri_recursion_scheduler_level_vk_hash: hash( + "0x201d4c7d8e781d51a3bbd451a43a8f45240bb765b565ae6ce69192d918c3563d", + ), + fri_recursion_node_level_vk_hash: hash( + "0x5a3ef282b21e12fe1f4438e5bb158fc5060b160559c5158c6389d62d9fe3d080", + ), + fri_recursion_leaf_level_vk_hash: hash( + "0x72167c43a46cf38875b267d67716edc4563861364a3c03ab7aee73498421e828", + ), + prover_at_genesis: ProverAtGenesis::Fri, + snark_wrapper_vk_hash: hash( + "0x4be443afd605a782b6e56d199df2460a025c81b3dea144e135bece83612563f2", + ), + } + } + + #[test] + fn from_env() { + let mut lock = MUTEX.lock(); + let config = r#" +CONTRACTS_GOVERNANCE_ADDR="0xd8dA6BF26964aF9D7eEd9e03E53415D37aA96045" +CONTRACTS_MAILBOX_FACET_ADDR="0x0f6Fa881EF414Fc6E818180657c2d5CD7Ac6cCAd" +CONTRACTS_EXECUTOR_FACET_ADDR="0x18B631537801963A964211C0E86645c1aBfbB2d3" +CONTRACTS_ADMIN_FACET_ADDR="0x1e12b20BE86bEc3A0aC95aA52ade345cB9AE7a32" +CONTRACTS_GETTERS_FACET_ADDR="0x8656770FA78c830456B00B4fFCeE6b1De0e1b888" +CONTRACTS_VERIFIER_ADDR="0x34782eE00206EAB6478F2692caa800e4A581687b" +CONTRACTS_DIAMOND_INIT_ADDR="0xFFC35A5e767BE36057c34586303498e3de7C62Ba" +CONTRACTS_DIAMOND_UPGRADE_INIT_ADDR="0xFFC35A5e767BE36057c34586303498e3de7C62Ba" +CONTRACTS_DIAMOND_PROXY_ADDR="0xF00B988a98Ca742e7958DeF9F7823b5908715f4a" +CONTRACTS_VALIDATOR_TIMELOCK_ADDR="0xF00B988a98Ca742e7958DeF9F7823b5908715f4a" +CONTRACTS_GENESIS_TX_HASH="0xb99ebfea46cbe05a21cd80fe5597d97b204befc52a16303f579c607dc1ac2e2e" +CONTRACTS_L1_ERC20_BRIDGE_PROXY_ADDR="0x8656770FA78c830456B00B4fFCeE6b1De0e1b888" +CONTRACTS_L1_ALLOW_LIST_ADDR="0x8656770FA78c830456B00B4fFCeE6b1De0e1b888" +CONTRACTS_L1_ERC20_BRIDGE_IMPL_ADDR="0x8656770FA78c830456B00B4fFCeE6b1De0e1b888" +CONTRACTS_L2_ERC20_BRIDGE_ADDR="0x8656770FA78c830456B00B4fFCeE6b1De0e1b888" +CONTRACTS_L1_WETH_BRIDGE_PROXY_ADDR="0x8656770FA78c830456B00B4fFCeE6b1De0e1b888" +CONTRACTS_L2_WETH_BRIDGE_ADDR="0x8656770FA78c830456B00B4fFCeE6b1De0e1b888" +CONTRACTS_L2_TESTNET_PAYMASTER_ADDR="FC073319977e314F251EAE6ae6bE76B0B3BAeeCF" +CONTRACTS_RECURSION_SCHEDULER_LEVEL_VK_HASH="0x1186ec268d49f1905f8d9c1e9d39fc33e98c74f91d91a21b8f7ef78bd09a8db8" +CONTRACTS_RECURSION_NODE_LEVEL_VK_HASH="0x1186ec268d49f1905f8d9c1e9d39fc33e98c74f91d91a21b8f7ef78bd09a8db8" +CONTRACTS_RECURSION_LEAF_LEVEL_VK_HASH="0x101e08b00193e529145ee09823378ef51a3bc8966504064f1f6ba3f1ba863210" +CONTRACTS_RECURSION_CIRCUITS_SET_VKS_HASH="0x142a364ef2073132eaf07aa7f3d8495065be5b92a2dc14fda09b4216affed9c0" +CONTRACTS_L1_MULTICALL3_ADDR="0xcA11bde05977b3631167028862bE2a173976CA11" +CONTRACTS_FRI_RECURSION_SCHEDULER_LEVEL_VK_HASH="0x201d4c7d8e781d51a3bbd451a43a8f45240bb765b565ae6ce69192d918c3563d" +CONTRACTS_FRI_RECURSION_NODE_LEVEL_VK_HASH="0x5a3ef282b21e12fe1f4438e5bb158fc5060b160559c5158c6389d62d9fe3d080" +CONTRACTS_FRI_RECURSION_LEAF_LEVEL_VK_HASH="0x72167c43a46cf38875b267d67716edc4563861364a3c03ab7aee73498421e828" +CONTRACTS_PROVER_AT_GENESIS="fri" +CONTRACTS_SNARK_WRAPPER_VK_HASH="0x4be443afd605a782b6e56d199df2460a025c81b3dea144e135bece83612563f2" + "#; + lock.set_env(config); + + let actual = ContractsConfig::from_env().unwrap(); + assert_eq!(actual, expected_config()); + } +} diff --git a/core/lib/env_config/src/database.rs b/core/lib/env_config/src/database.rs new file mode 100644 index 000000000000..eeb67205d4de --- /dev/null +++ b/core/lib/env_config/src/database.rs @@ -0,0 +1,100 @@ +use zksync_config::DBConfig; + +use crate::{envy_load, FromEnv}; + +impl FromEnv for DBConfig { + fn from_env() -> anyhow::Result { + Ok(Self { + merkle_tree: envy_load("database_merkle_tree", "DATABASE_MERKLE_TREE_")?, + ..envy_load("database", "DATABASE_")? + }) + } +} + +#[cfg(test)] +mod tests { + use zksync_config::configs::database::MerkleTreeMode; + + use super::*; + use crate::test_utils::EnvMutex; + + static MUTEX: EnvMutex = EnvMutex::new(); + + #[test] + fn from_env() { + let mut lock = MUTEX.lock(); + let config = r#" + DATABASE_STATE_KEEPER_DB_PATH="/db/state_keeper" + DATABASE_MERKLE_TREE_BACKUP_PATH="/db/backups" + DATABASE_MERKLE_TREE_PATH="/db/tree" + DATABASE_MERKLE_TREE_MODE=lightweight + DATABASE_MERKLE_TREE_MULTI_GET_CHUNK_SIZE=250 + DATABASE_MERKLE_TREE_MEMTABLE_CAPACITY_MB=512 + DATABASE_MERKLE_TREE_STALLED_WRITES_TIMEOUT_SEC=60 + DATABASE_MERKLE_TREE_MAX_L1_BATCHES_PER_ITER=50 + DATABASE_BACKUP_COUNT=5 + DATABASE_BACKUP_INTERVAL_MS=60000 + "#; + lock.set_env(config); + + let db_config = DBConfig::from_env().unwrap(); + assert_eq!(db_config.state_keeper_db_path, "/db/state_keeper"); + assert_eq!(db_config.merkle_tree.path, "/db/tree"); + assert_eq!(db_config.merkle_tree.backup_path, "/db/backups"); + assert_eq!(db_config.merkle_tree.mode, MerkleTreeMode::Lightweight); + assert_eq!(db_config.merkle_tree.multi_get_chunk_size, 250); + assert_eq!(db_config.merkle_tree.max_l1_batches_per_iter, 50); + assert_eq!(db_config.merkle_tree.memtable_capacity_mb, 512); + assert_eq!(db_config.merkle_tree.stalled_writes_timeout_sec, 60); + assert_eq!(db_config.backup_count, 5); + assert_eq!(db_config.backup_interval().as_secs(), 60); + } + + #[test] + fn from_empty_env() { + let mut lock = MUTEX.lock(); + lock.remove_env(&[ + "DATABASE_STATE_KEEPER_DB_PATH", + "DATABASE_MERKLE_TREE_BACKUP_PATH", + "DATABASE_MERKLE_TREE_PATH", + "DATABASE_MERKLE_TREE_MODE", + "DATABASE_MERKLE_TREE_MULTI_GET_CHUNK_SIZE", + "DATABASE_MERKLE_TREE_BLOCK_CACHE_SIZE_MB", + "DATABASE_MERKLE_TREE_MEMTABLE_CAPACITY_MB", + "DATABASE_MERKLE_TREE_STALLED_WRITES_TIMEOUT_SEC", + "DATABASE_MERKLE_TREE_MAX_L1_BATCHES_PER_ITER", + "DATABASE_BACKUP_COUNT", + "DATABASE_BACKUP_INTERVAL_MS", + ]); + + let db_config = DBConfig::from_env().unwrap(); + assert_eq!(db_config.state_keeper_db_path, "./db/state_keeper"); + assert_eq!(db_config.merkle_tree.path, "./db/lightweight-new"); + assert_eq!(db_config.merkle_tree.backup_path, "./db/backups"); + assert_eq!(db_config.merkle_tree.mode, MerkleTreeMode::Full); + assert_eq!(db_config.merkle_tree.multi_get_chunk_size, 500); + assert_eq!(db_config.merkle_tree.max_l1_batches_per_iter, 20); + assert_eq!(db_config.merkle_tree.block_cache_size_mb, 128); + assert_eq!(db_config.merkle_tree.memtable_capacity_mb, 256); + assert_eq!(db_config.merkle_tree.stalled_writes_timeout_sec, 30); + assert_eq!(db_config.backup_count, 5); + assert_eq!(db_config.backup_interval().as_secs(), 60); + + // Check that new env variable for Merkle tree path is supported + lock.set_env("DATABASE_MERKLE_TREE_PATH=/db/tree/main"); + let db_config = DBConfig::from_env().unwrap(); + assert_eq!(db_config.merkle_tree.path, "/db/tree/main"); + + lock.set_env("DATABASE_MERKLE_TREE_MULTI_GET_CHUNK_SIZE=200"); + let db_config = DBConfig::from_env().unwrap(); + assert_eq!(db_config.merkle_tree.multi_get_chunk_size, 200); + + lock.set_env("DATABASE_MERKLE_TREE_BLOCK_CACHE_SIZE_MB=256"); + let db_config = DBConfig::from_env().unwrap(); + assert_eq!(db_config.merkle_tree.block_cache_size_mb, 256); + + lock.set_env("DATABASE_MERKLE_TREE_MAX_L1_BATCHES_PER_ITER=50"); + let db_config = DBConfig::from_env().unwrap(); + assert_eq!(db_config.merkle_tree.max_l1_batches_per_iter, 50); + } +} diff --git a/core/lib/env_config/src/eth_client.rs b/core/lib/env_config/src/eth_client.rs new file mode 100644 index 000000000000..378ca84b0e2d --- /dev/null +++ b/core/lib/env_config/src/eth_client.rs @@ -0,0 +1,44 @@ +use zksync_config::ETHClientConfig; + +use crate::{envy_load, FromEnv}; + +impl FromEnv for ETHClientConfig { + fn from_env() -> anyhow::Result { + let config: Self = envy_load("eth_client", "ETH_CLIENT_")?; + if config.web3_url.find(',').is_some() { + anyhow::bail!( + "Multiple web3 URLs aren't supported anymore. Provided invalid value: {}", + config.web3_url + ); + } + Ok(config) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_utils::EnvMutex; + + static MUTEX: EnvMutex = EnvMutex::new(); + + fn expected_config() -> ETHClientConfig { + ETHClientConfig { + chain_id: 9, + web3_url: "http://127.0.0.1:8545".into(), + } + } + + #[test] + fn from_env() { + let mut lock = MUTEX.lock(); + let config = r#" + ETH_CLIENT_CHAIN_ID="9" + ETH_CLIENT_WEB3_URL="http://127.0.0.1:8545" + "#; + lock.set_env(config); + + let actual = ETHClientConfig::from_env().unwrap(); + assert_eq!(actual, expected_config()); + } +} diff --git a/core/lib/env_config/src/eth_sender.rs b/core/lib/env_config/src/eth_sender.rs new file mode 100644 index 000000000000..f0ee6030b029 --- /dev/null +++ b/core/lib/env_config/src/eth_sender.rs @@ -0,0 +1,111 @@ +use anyhow::Context as _; +use zksync_config::{configs::eth_sender::SenderConfig, ETHSenderConfig, GasAdjusterConfig}; + +use crate::{envy_load, FromEnv}; + +impl FromEnv for ETHSenderConfig { + fn from_env() -> anyhow::Result { + Ok(Self { + sender: SenderConfig::from_env().context("SenderConfig")?, + gas_adjuster: GasAdjusterConfig::from_env().context("GasAdjusterConfig")?, + }) + } +} + +impl FromEnv for SenderConfig { + fn from_env() -> anyhow::Result { + envy_load("eth_sender", "ETH_SENDER_SENDER_") + } +} + +impl FromEnv for GasAdjusterConfig { + fn from_env() -> anyhow::Result { + envy_load("eth_sender.gas_adjuster", "ETH_SENDER_GAS_ADJUSTER_") + } +} + +#[cfg(test)] +mod tests { + use zksync_config::configs::eth_sender::{ProofLoadingMode, ProofSendingMode}; + + use super::*; + use crate::test_utils::{hash, EnvMutex}; + + static MUTEX: EnvMutex = EnvMutex::new(); + + fn expected_config() -> ETHSenderConfig { + ETHSenderConfig { + sender: SenderConfig { + aggregated_proof_sizes: vec![1, 5], + aggregated_block_commit_deadline: 30, + aggregated_block_prove_deadline: 3_000, + aggregated_block_execute_deadline: 4_000, + max_aggregated_tx_gas: 4_000_000, + max_eth_tx_data_size: 120_000, + + timestamp_criteria_max_allowed_lag: 30, + max_aggregated_blocks_to_commit: 3, + max_aggregated_blocks_to_execute: 4, + wait_confirmations: Some(1), + tx_poll_period: 3, + aggregate_tx_poll_period: 3, + max_txs_in_flight: 3, + proof_sending_mode: ProofSendingMode::SkipEveryProof, + l1_batch_min_age_before_execute_seconds: Some(1000), + max_acceptable_priority_fee_in_gwei: 100_000_000_000, + proof_loading_mode: ProofLoadingMode::OldProofFromDb, + }, + gas_adjuster: GasAdjusterConfig { + default_priority_fee_per_gas: 20000000000, + max_base_fee_samples: 10000, + pricing_formula_parameter_a: 1.5, + pricing_formula_parameter_b: 1.0005, + internal_l1_pricing_multiplier: 0.8, + internal_enforced_l1_gas_price: None, + poll_period: 15, + max_l1_gas_price: Some(100000000), + }, + } + } + + #[test] + fn from_env() { + let mut lock = MUTEX.lock(); + let config = r#" + ETH_SENDER_SENDER_WAIT_CONFIRMATIONS="1" + ETH_SENDER_SENDER_TX_POLL_PERIOD="3" + ETH_SENDER_SENDER_AGGREGATE_TX_POLL_PERIOD="3" + ETH_SENDER_SENDER_MAX_TXS_IN_FLIGHT="3" + ETH_SENDER_SENDER_OPERATOR_PRIVATE_KEY="0x27593fea79697e947890ecbecce7901b0008345e5d7259710d0dd5e500d040be" + ETH_SENDER_SENDER_PROOF_SENDING_MODE="SkipEveryProof" + ETH_SENDER_GAS_ADJUSTER_DEFAULT_PRIORITY_FEE_PER_GAS="20000000000" + ETH_SENDER_GAS_ADJUSTER_MAX_BASE_FEE_SAMPLES="10000" + ETH_SENDER_GAS_ADJUSTER_PRICING_FORMULA_PARAMETER_A="1.5" + ETH_SENDER_GAS_ADJUSTER_PRICING_FORMULA_PARAMETER_B="1.0005" + ETH_SENDER_GAS_ADJUSTER_INTERNAL_L1_PRICING_MULTIPLIER="0.8" + ETH_SENDER_GAS_ADJUSTER_POLL_PERIOD="15" + ETH_SENDER_GAS_ADJUSTER_MAX_L1_GAS_PRICE="100000000" + ETH_SENDER_WAIT_FOR_PROOFS="false" + ETH_SENDER_SENDER_AGGREGATED_PROOF_SIZES="1,5" + ETH_SENDER_SENDER_MAX_AGGREGATED_BLOCKS_TO_COMMIT="3" + ETH_SENDER_SENDER_MAX_AGGREGATED_BLOCKS_TO_EXECUTE="4" + ETH_SENDER_SENDER_AGGREGATED_BLOCK_COMMIT_DEADLINE="30" + ETH_SENDER_SENDER_AGGREGATED_BLOCK_PROVE_DEADLINE="3000" + ETH_SENDER_SENDER_AGGREGATED_BLOCK_EXECUTE_DEADLINE="4000" + ETH_SENDER_SENDER_TIMESTAMP_CRITERIA_MAX_ALLOWED_LAG="30" + ETH_SENDER_SENDER_MAX_AGGREGATED_TX_GAS="4000000" + ETH_SENDER_SENDER_MAX_ETH_TX_DATA_SIZE="120000" + ETH_SENDER_SENDER_L1_BATCH_MIN_AGE_BEFORE_EXECUTE_SECONDS="1000" + ETH_SENDER_SENDER_MAX_ACCEPTABLE_PRIORITY_FEE_IN_GWEI="100000000000" + ETH_SENDER_SENDER_PROOF_LOADING_MODE="OldProofFromDb" + "#; + lock.set_env(config); + + let actual = ETHSenderConfig::from_env().unwrap(); + assert_eq!(actual, expected_config()); + assert_eq!( + actual.sender.private_key().unwrap(), + hash("27593fea79697e947890ecbecce7901b0008345e5d7259710d0dd5e500d040be") + ); + } +} diff --git a/core/lib/env_config/src/eth_watch.rs b/core/lib/env_config/src/eth_watch.rs new file mode 100644 index 000000000000..ef90b7e9c46c --- /dev/null +++ b/core/lib/env_config/src/eth_watch.rs @@ -0,0 +1,37 @@ +use zksync_config::ETHWatchConfig; + +use crate::{envy_load, FromEnv}; + +impl FromEnv for ETHWatchConfig { + fn from_env() -> anyhow::Result { + envy_load("eth_watch", "ETH_WATCH_") + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_utils::EnvMutex; + + static MUTEX: EnvMutex = EnvMutex::new(); + + fn expected_config() -> ETHWatchConfig { + ETHWatchConfig { + confirmations_for_eth_event: Some(0), + eth_node_poll_interval: 300, + } + } + + #[test] + fn from_env() { + let mut lock = MUTEX.lock(); + let config = r#" + ETH_WATCH_CONFIRMATIONS_FOR_ETH_EVENT="0" + ETH_WATCH_ETH_NODE_POLL_INTERVAL="300" + "#; + lock.set_env(config); + + let actual = ETHWatchConfig::from_env().unwrap(); + assert_eq!(actual, expected_config()); + } +} diff --git a/core/lib/env_config/src/fetcher.rs b/core/lib/env_config/src/fetcher.rs new file mode 100644 index 000000000000..4f86488b2c41 --- /dev/null +++ b/core/lib/env_config/src/fetcher.rs @@ -0,0 +1,68 @@ +use zksync_config::FetcherConfig; + +use crate::{envy_load, FromEnv}; + +impl FromEnv for FetcherConfig { + fn from_env() -> anyhow::Result { + Ok(Self { + token_list: envy_load("token_list", "FETCHER_TOKEN_LIST_")?, + token_price: envy_load("token_price", "FETCHER_TOKEN_PRICE_")?, + token_trading_volume: envy_load( + "token_trading_volume", + "FETCHER_TOKEN_TRADING_VOLUME_", + )?, + }) + } +} + +#[cfg(test)] +mod tests { + use zksync_config::configs::fetcher::{ + SingleFetcherConfig, TokenListSource, TokenPriceSource, TokenTradingVolumeSource, + }; + + use super::*; + use crate::test_utils::EnvMutex; + + static MUTEX: EnvMutex = EnvMutex::new(); + + fn expected_config() -> FetcherConfig { + FetcherConfig { + token_list: SingleFetcherConfig { + source: TokenListSource::OneInch, + url: "http://127.0.0.1:1020".into(), + fetching_interval: 10, + }, + token_price: SingleFetcherConfig { + source: TokenPriceSource::CoinGecko, + url: "http://127.0.0.1:9876".into(), + fetching_interval: 7, + }, + token_trading_volume: SingleFetcherConfig { + source: TokenTradingVolumeSource::Uniswap, + url: "http://127.0.0.1:9975/graphql".to_string(), + fetching_interval: 5, + }, + } + } + + #[test] + fn from_env() { + let mut lock = MUTEX.lock(); + let config = r#" + FETCHER_TOKEN_LIST_SOURCE="OneInch" + FETCHER_TOKEN_LIST_URL="http://127.0.0.1:1020" + FETCHER_TOKEN_LIST_FETCHING_INTERVAL="10" + FETCHER_TOKEN_PRICE_SOURCE="CoinGecko" + FETCHER_TOKEN_PRICE_URL="http://127.0.0.1:9876" + FETCHER_TOKEN_PRICE_FETCHING_INTERVAL="7" + FETCHER_TOKEN_TRADING_VOLUME_SOURCE="Uniswap" + FETCHER_TOKEN_TRADING_VOLUME_URL="http://127.0.0.1:9975/graphql" + FETCHER_TOKEN_TRADING_VOLUME_FETCHING_INTERVAL="5" + "#; + lock.set_env(config); + + let actual = FetcherConfig::from_env().unwrap(); + assert_eq!(actual, expected_config()); + } +} diff --git a/core/lib/env_config/src/fri_proof_compressor.rs b/core/lib/env_config/src/fri_proof_compressor.rs new file mode 100644 index 000000000000..2594433025e4 --- /dev/null +++ b/core/lib/env_config/src/fri_proof_compressor.rs @@ -0,0 +1,54 @@ +use zksync_config::configs::FriProofCompressorConfig; + +use crate::{envy_load, FromEnv}; + +impl FromEnv for FriProofCompressorConfig { + fn from_env() -> anyhow::Result { + envy_load("fri_proof_compressor", "FRI_PROOF_COMPRESSOR_") + } +} + +#[cfg(test)] +mod tests { + use crate::test_utils::EnvMutex; + + use super::*; + + static MUTEX: EnvMutex = EnvMutex::new(); + + fn expected_config() -> FriProofCompressorConfig { + FriProofCompressorConfig { + compression_mode: 1, + prometheus_listener_port: 3316, + prometheus_pushgateway_url: "http://127.0.0.1:9091".to_string(), + prometheus_push_interval_ms: Some(100), + generation_timeout_in_secs: 3000, + max_attempts: 5, + universal_setup_path: "keys/setup/setup_2^26.key".to_string(), + universal_setup_download_url: + "https://storage.googleapis.com/matterlabs-setup-keys-us/setup-keys/setup_2^26.key" + .to_string(), + verify_wrapper_proof: false, + } + } + + #[test] + fn from_env() { + let mut lock = MUTEX.lock(); + let config = r#" + FRI_PROOF_COMPRESSOR_COMPRESSION_MODE=1 + FRI_PROOF_COMPRESSOR_PROMETHEUS_LISTENER_PORT=3316 + FRI_PROOF_COMPRESSOR_PROMETHEUS_PUSHGATEWAY_URL="http://127.0.0.1:9091" + FRI_PROOF_COMPRESSOR_PROMETHEUS_PUSH_INTERVAL_MS=100 + FRI_PROOF_COMPRESSOR_GENERATION_TIMEOUT_IN_SECS=3000 + FRI_PROOF_COMPRESSOR_MAX_ATTEMPTS=5 + FRI_PROOF_COMPRESSOR_UNIVERSAL_SETUP_PATH="keys/setup/setup_2^26.key" + FRI_PROOF_COMPRESSOR_UNIVERSAL_SETUP_DOWNLOAD_URL="https://storage.googleapis.com/matterlabs-setup-keys-us/setup-keys/setup_2^26.key" + FRI_PROOF_COMPRESSOR_VERIFY_WRAPPER_PROOF=false + "#; + lock.set_env(config); + + let actual = FriProofCompressorConfig::from_env().unwrap(); + assert_eq!(actual, expected_config()); + } +} diff --git a/core/lib/env_config/src/fri_prover.rs b/core/lib/env_config/src/fri_prover.rs new file mode 100644 index 000000000000..200f22d89b7e --- /dev/null +++ b/core/lib/env_config/src/fri_prover.rs @@ -0,0 +1,59 @@ +use zksync_config::configs::FriProverConfig; + +use crate::{envy_load, FromEnv}; + +impl FromEnv for FriProverConfig { + fn from_env() -> anyhow::Result { + envy_load("fri_prover", "FRI_PROVER_") + } +} + +#[cfg(test)] +mod tests { + use zksync_config::configs::fri_prover::SetupLoadMode; + + use super::*; + use crate::test_utils::EnvMutex; + + static MUTEX: EnvMutex = EnvMutex::new(); + + fn expected_config() -> FriProverConfig { + FriProverConfig { + setup_data_path: "/usr/src/setup-data".to_string(), + prometheus_port: 3315, + max_attempts: 10, + generation_timeout_in_secs: 300, + base_layer_circuit_ids_to_be_verified: vec![1, 5], + recursive_layer_circuit_ids_to_be_verified: vec![1, 2, 3], + setup_load_mode: SetupLoadMode::FromDisk, + specialized_group_id: 10, + witness_vector_generator_thread_count: Some(5), + queue_capacity: 10, + witness_vector_receiver_port: 3316, + shall_save_to_public_bucket: true, + } + } + + #[test] + fn from_env() { + let mut lock = MUTEX.lock(); + let config = r#" + FRI_PROVER_SETUP_DATA_PATH="/usr/src/setup-data" + FRI_PROVER_PROMETHEUS_PORT="3315" + FRI_PROVER_MAX_ATTEMPTS="10" + FRI_PROVER_GENERATION_TIMEOUT_IN_SECS="300" + FRI_PROVER_BASE_LAYER_CIRCUIT_IDS_TO_BE_VERIFIED="1,5" + FRI_PROVER_RECURSIVE_LAYER_CIRCUIT_IDS_TO_BE_VERIFIED="1,2,3" + FRI_PROVER_SETUP_LOAD_MODE="FromDisk" + FRI_PROVER_SPECIALIZED_GROUP_ID="10" + FRI_PROVER_WITNESS_VECTOR_GENERATOR_THREAD_COUNT="5" + FRI_PROVER_QUEUE_CAPACITY="10" + FRI_PROVER_WITNESS_VECTOR_RECEIVER_PORT="3316" + FRI_PROVER_SHALL_SAVE_TO_PUBLIC_BUCKET=true + "#; + lock.set_env(config); + + let actual = FriProverConfig::from_env().unwrap(); + assert_eq!(actual, expected_config()); + } +} diff --git a/core/lib/env_config/src/fri_prover_gateway.rs b/core/lib/env_config/src/fri_prover_gateway.rs new file mode 100644 index 000000000000..cfb2622bb12a --- /dev/null +++ b/core/lib/env_config/src/fri_prover_gateway.rs @@ -0,0 +1,42 @@ +use zksync_config::configs::FriProverGatewayConfig; + +use crate::{envy_load, FromEnv}; + +impl FromEnv for FriProverGatewayConfig { + fn from_env() -> anyhow::Result { + envy_load("fri_prover_gateway", "FRI_PROVER_GATEWAY_") + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_utils::EnvMutex; + + static MUTEX: EnvMutex = EnvMutex::new(); + + fn expected_config() -> FriProverGatewayConfig { + FriProverGatewayConfig { + api_url: "http://private-dns-for-server".to_string(), + api_poll_duration_secs: 100, + prometheus_listener_port: 3316, + prometheus_pushgateway_url: "http://127.0.0.1:9091".to_string(), + prometheus_push_interval_ms: Some(100), + } + } + + #[test] + fn from_env() { + let config = r#" + FRI_PROVER_GATEWAY_API_URL="http://private-dns-for-server" + FRI_PROVER_GATEWAY_API_POLL_DURATION_SECS="100" + FRI_PROVER_GATEWAY_PROMETHEUS_LISTENER_PORT=3316 + FRI_PROVER_GATEWAY_PROMETHEUS_PUSHGATEWAY_URL="http://127.0.0.1:9091" + FRI_PROVER_GATEWAY_PROMETHEUS_PUSH_INTERVAL_MS=100 + "#; + let mut lock = MUTEX.lock(); + lock.set_env(config); + let actual = FriProverGatewayConfig::from_env().unwrap(); + assert_eq!(actual, expected_config()); + } +} diff --git a/core/lib/env_config/src/fri_prover_group.rs b/core/lib/env_config/src/fri_prover_group.rs new file mode 100644 index 000000000000..373febb6abb2 --- /dev/null +++ b/core/lib/env_config/src/fri_prover_group.rs @@ -0,0 +1,247 @@ +use std::{ + collections::{HashMap, HashSet}, + env, +}; + +use zksync_basic_types::basic_fri_types::CircuitIdRoundTuple; +use zksync_config::configs::fri_prover_group::FriProverGroupConfig; + +use crate::FromEnv; + +fn load_from_env_variable() -> HashMap> { + // Prepare a hash map to store the mapping of group to a vector of tuples + let mut groups: HashMap> = (0..=12) + .map(|i| (format!("group_{}", i), HashSet::new())) + .collect(); + + // Separate environment variables into Circuit Id and Aggregation Round + let mut circuit_ids = HashMap::new(); + let mut aggregation_rounds = HashMap::new(); + for (key, value) in env::vars() { + if key.contains("_CIRCUIT_ID") { + circuit_ids.insert(key, value); + } else if key.contains("_AGGREGATION_ROUND") { + aggregation_rounds.insert(key, value); + } + } + + // Iterate over all circuit id variables + for (key, value_str) in circuit_ids { + let key_parts: Vec<&str> = key.split('_').collect(); + if let (Some(group_key), Some(value), Some(index_str)) = ( + key_parts.get(4), + value_str.parse::().ok(), + key_parts.get(5), + ) { + let round_key = format!( + "FRI_PROVER_GROUP_GROUP_{}_{}_AGGREGATION_ROUND", + group_key, index_str + ); + if let Some(round_str) = aggregation_rounds.get(&round_key) { + if let Ok(round) = round_str.parse::() { + let tuple = CircuitIdRoundTuple::new(value, round); + if let Some(group) = groups.get_mut(&format!("group_{}", group_key)) { + group.insert(tuple); + } + } + } + } + } + groups +} + +impl FromEnv for FriProverGroupConfig { + fn from_env() -> anyhow::Result { + let mut groups = load_from_env_variable(); + let config = FriProverGroupConfig { + group_0: groups.remove("group_0").unwrap_or_default(), + group_1: groups.remove("group_1").unwrap_or_default(), + group_2: groups.remove("group_2").unwrap_or_default(), + group_3: groups.remove("group_3").unwrap_or_default(), + group_4: groups.remove("group_4").unwrap_or_default(), + group_5: groups.remove("group_5").unwrap_or_default(), + group_6: groups.remove("group_6").unwrap_or_default(), + group_7: groups.remove("group_7").unwrap_or_default(), + group_8: groups.remove("group_8").unwrap_or_default(), + group_9: groups.remove("group_9").unwrap_or_default(), + group_10: groups.remove("group_10").unwrap_or_default(), + group_11: groups.remove("group_11").unwrap_or_default(), + group_12: groups.remove("group_12").unwrap_or_default(), + }; + config.validate(); + Ok(config) + } +} + +#[cfg(test)] +mod tests { + use std::collections::HashSet; + + use super::*; + + fn expected_config() -> FriProverGroupConfig { + FriProverGroupConfig { + group_0: vec![ + CircuitIdRoundTuple::new(1, 3), + CircuitIdRoundTuple::new(2, 2), + ] + .into_iter() + .collect::>(), + group_1: vec![CircuitIdRoundTuple::new(1, 0)] + .into_iter() + .collect::>(), + group_2: vec![ + CircuitIdRoundTuple::new(2, 0), + CircuitIdRoundTuple::new(4, 0), + CircuitIdRoundTuple::new(6, 0), + CircuitIdRoundTuple::new(9, 0), + ] + .into_iter() + .collect::>(), + group_3: vec![CircuitIdRoundTuple::new(3, 0)] + .into_iter() + .collect::>(), + group_4: vec![ + CircuitIdRoundTuple::new(11, 0), + CircuitIdRoundTuple::new(12, 0), + CircuitIdRoundTuple::new(13, 0), + ] + .into_iter() + .collect::>(), + group_5: vec![CircuitIdRoundTuple::new(5, 0)] + .into_iter() + .collect::>(), + group_6: vec![CircuitIdRoundTuple::new(3, 1)] + .into_iter() + .collect::>(), + group_7: vec![CircuitIdRoundTuple::new(7, 0)] + .into_iter() + .collect::>(), + group_8: vec![CircuitIdRoundTuple::new(8, 0)] + .into_iter() + .collect::>(), + group_9: vec![ + CircuitIdRoundTuple::new(12, 1), + CircuitIdRoundTuple::new(13, 1), + CircuitIdRoundTuple::new(14, 1), + CircuitIdRoundTuple::new(15, 1), + ] + .into_iter() + .collect::>(), + group_10: vec![CircuitIdRoundTuple::new(10, 0)] + .into_iter() + .collect::>(), + group_11: vec![ + CircuitIdRoundTuple::new(7, 1), + CircuitIdRoundTuple::new(8, 1), + CircuitIdRoundTuple::new(10, 1), + CircuitIdRoundTuple::new(11, 1), + ] + .into_iter() + .collect::>(), + group_12: vec![ + CircuitIdRoundTuple::new(4, 1), + CircuitIdRoundTuple::new(5, 1), + CircuitIdRoundTuple::new(6, 1), + CircuitIdRoundTuple::new(9, 1), + ] + .into_iter() + .collect::>(), + } + } + + #[test] + fn from_env() { + let groups = [ + ("FRI_PROVER_GROUP_GROUP_0_0", CircuitIdRoundTuple::new(1, 3)), + ("FRI_PROVER_GROUP_GROUP_0_1", CircuitIdRoundTuple::new(2, 2)), + ("FRI_PROVER_GROUP_GROUP_1_0", CircuitIdRoundTuple::new(1, 0)), + ("FRI_PROVER_GROUP_GROUP_2_0", CircuitIdRoundTuple::new(2, 0)), + ("FRI_PROVER_GROUP_GROUP_2_1", CircuitIdRoundTuple::new(4, 0)), + ("FRI_PROVER_GROUP_GROUP_2_2", CircuitIdRoundTuple::new(6, 0)), + ("FRI_PROVER_GROUP_GROUP_2_3", CircuitIdRoundTuple::new(9, 0)), + ("FRI_PROVER_GROUP_GROUP_3_0", CircuitIdRoundTuple::new(3, 0)), + ( + "FRI_PROVER_GROUP_GROUP_4_0", + CircuitIdRoundTuple::new(11, 0), + ), + ( + "FRI_PROVER_GROUP_GROUP_4_1", + CircuitIdRoundTuple::new(12, 0), + ), + ( + "FRI_PROVER_GROUP_GROUP_4_2", + CircuitIdRoundTuple::new(13, 0), + ), + ("FRI_PROVER_GROUP_GROUP_5_0", CircuitIdRoundTuple::new(5, 0)), + ("FRI_PROVER_GROUP_GROUP_6_0", CircuitIdRoundTuple::new(3, 1)), + ("FRI_PROVER_GROUP_GROUP_7_0", CircuitIdRoundTuple::new(7, 0)), + ("FRI_PROVER_GROUP_GROUP_8_0", CircuitIdRoundTuple::new(8, 0)), + ( + "FRI_PROVER_GROUP_GROUP_9_0", + CircuitIdRoundTuple::new(12, 1), + ), + ( + "FRI_PROVER_GROUP_GROUP_9_1", + CircuitIdRoundTuple::new(13, 1), + ), + ( + "FRI_PROVER_GROUP_GROUP_9_2", + CircuitIdRoundTuple::new(14, 1), + ), + ( + "FRI_PROVER_GROUP_GROUP_9_3", + CircuitIdRoundTuple::new(15, 1), + ), + ( + "FRI_PROVER_GROUP_GROUP_10_0", + CircuitIdRoundTuple::new(10, 0), + ), + ( + "FRI_PROVER_GROUP_GROUP_11_0", + CircuitIdRoundTuple::new(7, 1), + ), + ( + "FRI_PROVER_GROUP_GROUP_11_1", + CircuitIdRoundTuple::new(8, 1), + ), + ( + "FRI_PROVER_GROUP_GROUP_11_2", + CircuitIdRoundTuple::new(10, 1), + ), + ( + "FRI_PROVER_GROUP_GROUP_11_3", + CircuitIdRoundTuple::new(11, 1), + ), + ( + "FRI_PROVER_GROUP_GROUP_12_0", + CircuitIdRoundTuple::new(4, 1), + ), + ( + "FRI_PROVER_GROUP_GROUP_12_1", + CircuitIdRoundTuple::new(5, 1), + ), + ( + "FRI_PROVER_GROUP_GROUP_12_2", + CircuitIdRoundTuple::new(6, 1), + ), + ( + "FRI_PROVER_GROUP_GROUP_12_3", + CircuitIdRoundTuple::new(9, 1), + ), + ]; + + for (key_base, circuit_round_tuple) in &groups { + let circuit_id_key = format!("{}_CIRCUIT_ID", key_base); + let aggregation_round_key = format!("{}_AGGREGATION_ROUND", key_base); + env::set_var(&circuit_id_key, circuit_round_tuple.circuit_id.to_string()); + env::set_var( + &aggregation_round_key, + circuit_round_tuple.aggregation_round.to_string(), + ); + } + + let actual = FriProverGroupConfig::from_env().unwrap(); + assert_eq!(actual, expected_config()); + } +} diff --git a/core/lib/env_config/src/fri_witness_generator.rs b/core/lib/env_config/src/fri_witness_generator.rs new file mode 100644 index 000000000000..ab3d5dc4d133 --- /dev/null +++ b/core/lib/env_config/src/fri_witness_generator.rs @@ -0,0 +1,46 @@ +use zksync_config::configs::FriWitnessGeneratorConfig; + +use crate::{envy_load, FromEnv}; + +impl FromEnv for FriWitnessGeneratorConfig { + fn from_env() -> anyhow::Result { + envy_load("fri_witness", "FRI_WITNESS_") + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_utils::EnvMutex; + + static MUTEX: EnvMutex = EnvMutex::new(); + + fn expected_config() -> FriWitnessGeneratorConfig { + FriWitnessGeneratorConfig { + generation_timeout_in_secs: 900u16, + max_attempts: 4, + blocks_proving_percentage: Some(30), + dump_arguments_for_blocks: vec![2, 3], + last_l1_batch_to_process: None, + force_process_block: Some(1), + shall_save_to_public_bucket: true, + } + } + + #[test] + fn from_env() { + let mut lock = MUTEX.lock(); + let config = r#" + FRI_WITNESS_GENERATION_TIMEOUT_IN_SECS=900 + FRI_WITNESS_MAX_ATTEMPTS=4 + FRI_WITNESS_DUMP_ARGUMENTS_FOR_BLOCKS="2,3" + FRI_WITNESS_BLOCKS_PROVING_PERCENTAGE="30" + FRI_WITNESS_FORCE_PROCESS_BLOCK="1" + FRI_WITNESS_SHALL_SAVE_TO_PUBLIC_BUCKET=true + "#; + lock.set_env(config); + + let actual = FriWitnessGeneratorConfig::from_env().unwrap(); + assert_eq!(actual, expected_config()); + } +} diff --git a/core/lib/env_config/src/fri_witness_vector_generator.rs b/core/lib/env_config/src/fri_witness_vector_generator.rs new file mode 100644 index 000000000000..f564aaf13cec --- /dev/null +++ b/core/lib/env_config/src/fri_witness_vector_generator.rs @@ -0,0 +1,50 @@ +use zksync_config::configs::FriWitnessVectorGeneratorConfig; + +use crate::{envy_load, FromEnv}; + +impl FromEnv for FriWitnessVectorGeneratorConfig { + fn from_env() -> anyhow::Result { + envy_load( + "fri_witness_vector_generator", + "FRI_WITNESS_VECTOR_GENERATOR_", + ) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_utils::EnvMutex; + + static MUTEX: EnvMutex = EnvMutex::new(); + + fn expected_config() -> FriWitnessVectorGeneratorConfig { + FriWitnessVectorGeneratorConfig { + max_prover_reservation_duration_in_secs: 1000u16, + prover_instance_wait_timeout_in_secs: 1000u16, + prover_instance_poll_time_in_milli_secs: 250u16, + prometheus_listener_port: 3316, + prometheus_pushgateway_url: "http://127.0.0.1:9091".to_string(), + prometheus_push_interval_ms: Some(100), + specialized_group_id: 1, + } + } + + #[test] + fn from_env() { + let mut lock = MUTEX.lock(); + let config = r#" + FRI_WITNESS_VECTOR_GENERATOR_MAX_PROVER_RESERVATION_DURATION_IN_SECS=1000 + FRI_WITNESS_VECTOR_GENERATOR_PROVER_INSTANCE_WAIT_TIMEOUT_IN_SECS=1000 + FRI_WITNESS_VECTOR_GENERATOR_PROVER_INSTANCE_POLL_TIME_IN_MILLI_SECS=250 + FRI_WITNESS_VECTOR_GENERATOR_PROMETHEUS_LISTENER_PORT=3316 + FRI_WITNESS_VECTOR_GENERATOR_PROMETHEUS_PUSHGATEWAY_URL="http://127.0.0.1:9091" + FRI_WITNESS_VECTOR_GENERATOR_PROMETHEUS_PUSH_INTERVAL_MS=100 + FRI_WITNESS_VECTOR_GENERATOR_SPECIALIZED_GROUP_ID=1 + "#; + lock.set_env(config); + + let actual = FriWitnessVectorGeneratorConfig::from_env().unwrap(); + assert_eq!(actual, expected_config()); + } +} diff --git a/core/lib/env_config/src/house_keeper.rs b/core/lib/env_config/src/house_keeper.rs new file mode 100644 index 000000000000..d6b2906a7a35 --- /dev/null +++ b/core/lib/env_config/src/house_keeper.rs @@ -0,0 +1,61 @@ +use zksync_config::configs::house_keeper::HouseKeeperConfig; + +use crate::{envy_load, FromEnv}; + +impl FromEnv for HouseKeeperConfig { + fn from_env() -> anyhow::Result { + envy_load("house_keeper", "HOUSE_KEEPER_") + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_utils::EnvMutex; + + static MUTEX: EnvMutex = EnvMutex::new(); + + fn expected_config() -> HouseKeeperConfig { + HouseKeeperConfig { + l1_batch_metrics_reporting_interval_ms: 10_000, + blob_cleaning_interval_ms: 60_000, + gpu_prover_queue_reporting_interval_ms: 10_000, + prover_job_retrying_interval_ms: 300_000, + prover_stats_reporting_interval_ms: 5_000, + witness_job_moving_interval_ms: 30_000, + witness_generator_stats_reporting_interval_ms: 10_000, + fri_witness_job_moving_interval_ms: 40_000, + fri_prover_job_retrying_interval_ms: 30_000, + fri_witness_generator_job_retrying_interval_ms: 30_000, + prover_db_pool_size: 2, + fri_prover_stats_reporting_interval_ms: 30_000, + fri_proof_compressor_job_retrying_interval_ms: 30_000, + fri_proof_compressor_stats_reporting_interval_ms: 30_000, + } + } + + #[test] + fn from_env() { + let mut lock = MUTEX.lock(); + let config = r#" + HOUSE_KEEPER_L1_BATCH_METRICS_REPORTING_INTERVAL_MS="10000" + HOUSE_KEEPER_BLOB_CLEANING_INTERVAL_MS="60000" + HOUSE_KEEPER_GPU_PROVER_QUEUE_REPORTING_INTERVAL_MS="10000" + HOUSE_KEEPER_PROVER_JOB_RETRYING_INTERVAL_MS="300000" + HOUSE_KEEPER_PROVER_STATS_REPORTING_INTERVAL_MS="5000" + HOUSE_KEEPER_WITNESS_JOB_MOVING_INTERVAL_MS="30000" + HOUSE_KEEPER_WITNESS_GENERATOR_STATS_REPORTING_INTERVAL_MS="10000" + HOUSE_KEEPER_FRI_WITNESS_JOB_MOVING_INTERVAL_MS="40000" + HOUSE_KEEPER_FRI_PROVER_JOB_RETRYING_INTERVAL_MS="30000" + HOUSE_KEEPER_FRI_WITNESS_GENERATOR_JOB_RETRYING_INTERVAL_MS="30000" + HOUSE_KEEPER_PROVER_DB_POOL_SIZE="2" + HOUSE_KEEPER_FRI_PROVER_STATS_REPORTING_INTERVAL_MS="30000" + HOUSE_KEEPER_FRI_PROOF_COMPRESSOR_STATS_REPORTING_INTERVAL_MS="30000" + HOUSE_KEEPER_FRI_PROOF_COMPRESSOR_JOB_RETRYING_INTERVAL_MS="30000" + "#; + lock.set_env(config); + + let actual = HouseKeeperConfig::from_env().unwrap(); + assert_eq!(actual, expected_config()); + } +} diff --git a/core/lib/env_config/src/lib.rs b/core/lib/env_config/src/lib.rs new file mode 100644 index 000000000000..a4a4af3f1ec0 --- /dev/null +++ b/core/lib/env_config/src/lib.rs @@ -0,0 +1,42 @@ +use anyhow::Context as _; +use serde::de::DeserializeOwned; + +mod alerts; +mod api; +mod chain; +mod circuit_synthesizer; +mod contract_verifier; +mod contracts; +mod database; +mod eth_client; +mod eth_sender; +mod eth_watch; +mod fetcher; +mod fri_proof_compressor; +mod fri_prover; +mod fri_prover_gateway; +mod fri_prover_group; +mod fri_witness_generator; +mod fri_witness_vector_generator; +mod house_keeper; +pub mod object_store; +mod proof_data_handler; +mod prover; +mod prover_group; +mod utils; +mod witness_generator; + +#[cfg(test)] +mod test_utils; + +pub trait FromEnv: Sized { + fn from_env() -> anyhow::Result; +} + +/// Convenience function that loads the structure from the environment variable given the prefix. +/// Panics if the config cannot be loaded from the environment variables. +pub(crate) fn envy_load(name: &str, prefix: &str) -> anyhow::Result { + envy::prefixed(prefix) + .from_env() + .with_context(|| format!("Cannot load config <{name}>")) +} diff --git a/core/lib/env_config/src/object_store.rs b/core/lib/env_config/src/object_store.rs new file mode 100644 index 000000000000..3b4afe86b522 --- /dev/null +++ b/core/lib/env_config/src/object_store.rs @@ -0,0 +1,96 @@ +use zksync_config::ObjectStoreConfig; + +use crate::{envy_load, FromEnv}; + +impl FromEnv for ObjectStoreConfig { + fn from_env() -> anyhow::Result { + envy_load("object_store", "OBJECT_STORE_") + } +} + +/// Wrapper for `ObjectStoreConfig` that allows loading object store config using `PUBLIC_` prefix. +#[derive(Debug)] +pub struct PublicObjectStoreConfig(pub ObjectStoreConfig); + +impl FromEnv for PublicObjectStoreConfig { + fn from_env() -> anyhow::Result { + let config = envy_load("public_object_store", "PUBLIC_OBJECT_STORE_")?; + Ok(Self(config)) + } +} + +/// Wrapper for `ObjectStoreConfig` that allows loading object store config using `PROVER_` prefix. +#[derive(Debug)] +pub struct ProverObjectStoreConfig(pub ObjectStoreConfig); + +impl FromEnv for ProverObjectStoreConfig { + fn from_env() -> anyhow::Result { + let config = envy_load("prover_object_store", "PROVER_OBJECT_STORE_")?; + Ok(Self(config)) + } +} + +#[cfg(test)] +mod tests { + use zksync_config::{configs::object_store::ObjectStoreMode, ObjectStoreConfig}; + + use super::*; + use crate::test_utils::EnvMutex; + + static MUTEX: EnvMutex = EnvMutex::new(); + + fn expected_config(bucket_base_url: &str) -> ObjectStoreConfig { + ObjectStoreConfig { + bucket_base_url: bucket_base_url.to_string(), + mode: ObjectStoreMode::FileBacked, + file_backed_base_path: "artifacts".to_string(), + gcs_credential_file_path: "/path/to/credentials.json".to_string(), + max_retries: 5, + } + } + + #[test] + fn from_env() { + let mut lock = MUTEX.lock(); + let config = r#" + OBJECT_STORE_BUCKET_BASE_URL="/base/url" + OBJECT_STORE_MODE="FileBacked" + OBJECT_STORE_FILE_BACKED_BASE_PATH="artifacts" + OBJECT_STORE_GCS_CREDENTIAL_FILE_PATH="/path/to/credentials.json" + OBJECT_STORE_MAX_RETRIES="5" + "#; + lock.set_env(config); + let actual = ObjectStoreConfig::from_env().unwrap(); + assert_eq!(actual, expected_config("/base/url")); + } + + #[test] + fn public_bucket_config_from_env() { + let mut lock = MUTEX.lock(); + let config = r#" + PUBLIC_OBJECT_STORE_BUCKET_BASE_URL="/public_base_url" + PUBLIC_OBJECT_STORE_MODE="FileBacked" + PUBLIC_OBJECT_STORE_FILE_BACKED_BASE_PATH="artifacts" + PUBLIC_OBJECT_STORE_GCS_CREDENTIAL_FILE_PATH="/path/to/credentials.json" + PUBLIC_OBJECT_STORE_MAX_RETRIES="5" + "#; + lock.set_env(config); + let actual = PublicObjectStoreConfig::from_env().unwrap().0; + assert_eq!(actual, expected_config("/public_base_url")); + } + + #[test] + fn prover_bucket_config_from_env() { + let mut lock = MUTEX.lock(); + let config = r#" + PROVER_OBJECT_STORE_BUCKET_BASE_URL="/prover_base_url" + PROVER_OBJECT_STORE_MODE="FileBacked" + PROVER_OBJECT_STORE_FILE_BACKED_BASE_PATH="artifacts" + PROVER_OBJECT_STORE_GCS_CREDENTIAL_FILE_PATH="/path/to/credentials.json" + PROVER_OBJECT_STORE_MAX_RETRIES="5" + "#; + lock.set_env(config); + let actual = ProverObjectStoreConfig::from_env().unwrap().0; + assert_eq!(actual, expected_config("/prover_base_url")); + } +} diff --git a/core/lib/env_config/src/proof_data_handler.rs b/core/lib/env_config/src/proof_data_handler.rs new file mode 100644 index 000000000000..67aad80fbe7f --- /dev/null +++ b/core/lib/env_config/src/proof_data_handler.rs @@ -0,0 +1,42 @@ +use zksync_config::configs::ProofDataHandlerConfig; + +use crate::{envy_load, FromEnv}; + +impl FromEnv for ProofDataHandlerConfig { + fn from_env() -> anyhow::Result { + envy_load("proof_data_handler", "PROOF_DATA_HANDLER_") + } +} + +#[cfg(test)] +mod tests { + use zksync_config::configs::proof_data_handler::ProtocolVersionLoadingMode; + + use super::*; + use crate::test_utils::EnvMutex; + + static MUTEX: EnvMutex = EnvMutex::new(); + + fn expected_config() -> ProofDataHandlerConfig { + ProofDataHandlerConfig { + http_port: 3320, + proof_generation_timeout_in_secs: 18000, + protocol_version_loading_mode: ProtocolVersionLoadingMode::FromEnvVar, + fri_protocol_version_id: 2, + } + } + + #[test] + fn from_env() { + let config = r#" + PROOF_DATA_HANDLER_PROOF_GENERATION_TIMEOUT_IN_SECS="18000" + PROOF_DATA_HANDLER_HTTP_PORT="3320" + PROOF_DATA_HANDLER_PROTOCOL_VERSION_LOADING_MODE="FromEnvVar" + PROOF_DATA_HANDLER_FRI_PROTOCOL_VERSION_ID="2" + "#; + let mut lock = MUTEX.lock(); + lock.set_env(config); + let actual = ProofDataHandlerConfig::from_env().unwrap(); + assert_eq!(actual, expected_config()); + } +} diff --git a/core/lib/env_config/src/prover.rs b/core/lib/env_config/src/prover.rs new file mode 100644 index 000000000000..700f0fffb96a --- /dev/null +++ b/core/lib/env_config/src/prover.rs @@ -0,0 +1,197 @@ +use zksync_config::ProverConfigs; + +use crate::{envy_load, FromEnv}; + +impl FromEnv for ProverConfigs { + fn from_env() -> anyhow::Result { + Ok(Self { + non_gpu: envy_load("non_gpu", "PROVER_NON_GPU_")?, + two_gpu_forty_gb_mem: envy_load( + "two_gpu_forty_gb_mem", + "PROVER_TWO_GPU_FORTY_GB_MEM_", + )?, + one_gpu_eighty_gb_mem: envy_load( + "one_gpu_eighty_gb_mem", + "PROVER_ONE_GPU_EIGHTY_GB_MEM_", + )?, + two_gpu_eighty_gb_mem: envy_load( + "two_gpu_eighty_gb_mem", + "PROVER_TWO_GPU_EIGHTY_GB_MEM_", + )?, + four_gpu_eighty_gb_mem: envy_load( + "four_gpu_eighty_gb_mem", + "PROVER_FOUR_GPU_EIGHTY_GB_MEM_", + )?, + }) + } +} + +#[cfg(test)] +mod tests { + use zksync_config::ProverConfig; + + use super::*; + use crate::test_utils::EnvMutex; + + static MUTEX: EnvMutex = EnvMutex::new(); + + fn expected_config() -> ProverConfigs { + ProverConfigs { + non_gpu: ProverConfig { + prometheus_port: 3313, + initial_setup_key_path: "key".to_owned(), + key_download_url: "value".to_owned(), + generation_timeout_in_secs: 2700u16, + number_of_threads: 2, + max_attempts: 4, + polling_duration_in_millis: 5, + setup_keys_path: "/usr/src/setup-keys".to_string(), + specialized_prover_group_id: 0, + number_of_setup_slots: 2, + assembly_receiver_port: 17791, + assembly_receiver_poll_time_in_millis: 250, + assembly_queue_capacity: 5, + }, + two_gpu_forty_gb_mem: ProverConfig { + prometheus_port: 3313, + initial_setup_key_path: "key".to_owned(), + key_download_url: "value".to_owned(), + generation_timeout_in_secs: 2700u16, + number_of_threads: 2, + max_attempts: 4, + polling_duration_in_millis: 5, + setup_keys_path: "/usr/src/setup-keys".to_string(), + specialized_prover_group_id: 1, + number_of_setup_slots: 5, + assembly_receiver_port: 17791, + assembly_receiver_poll_time_in_millis: 250, + assembly_queue_capacity: 5, + }, + one_gpu_eighty_gb_mem: ProverConfig { + prometheus_port: 3313, + initial_setup_key_path: "key".to_owned(), + key_download_url: "value".to_owned(), + generation_timeout_in_secs: 2700u16, + number_of_threads: 4, + max_attempts: 4, + polling_duration_in_millis: 5, + setup_keys_path: "/usr/src/setup-keys".to_string(), + specialized_prover_group_id: 2, + number_of_setup_slots: 5, + assembly_receiver_port: 17791, + assembly_receiver_poll_time_in_millis: 250, + assembly_queue_capacity: 5, + }, + two_gpu_eighty_gb_mem: ProverConfig { + prometheus_port: 3313, + initial_setup_key_path: "key".to_owned(), + key_download_url: "value".to_owned(), + generation_timeout_in_secs: 2700u16, + number_of_threads: 9, + max_attempts: 4, + polling_duration_in_millis: 5, + setup_keys_path: "/usr/src/setup-keys".to_string(), + specialized_prover_group_id: 3, + number_of_setup_slots: 9, + assembly_receiver_port: 17791, + assembly_receiver_poll_time_in_millis: 250, + assembly_queue_capacity: 5, + }, + four_gpu_eighty_gb_mem: ProverConfig { + prometheus_port: 3313, + initial_setup_key_path: "key".to_owned(), + key_download_url: "value".to_owned(), + generation_timeout_in_secs: 2700u16, + number_of_threads: 18, + max_attempts: 4, + polling_duration_in_millis: 5, + setup_keys_path: "/usr/src/setup-keys".to_string(), + specialized_prover_group_id: 4, + number_of_setup_slots: 18, + assembly_receiver_port: 17791, + assembly_receiver_poll_time_in_millis: 250, + assembly_queue_capacity: 5, + }, + } + } + + const CONFIG: &str = r#" + PROVER_NON_GPU_PROMETHEUS_PORT="3313" + PROVER_NON_GPU_INITIAL_SETUP_KEY_PATH="key" + PROVER_NON_GPU_KEY_DOWNLOAD_URL="value" + PROVER_NON_GPU_GENERATION_TIMEOUT_IN_SECS=2700 + PROVER_NON_GPU_NUMBER_OF_THREADS="2" + PROVER_NON_GPU_MAX_ATTEMPTS="4" + PROVER_NON_GPU_POLLING_DURATION_IN_MILLIS=5 + PROVER_NON_GPU_SETUP_KEYS_PATH="/usr/src/setup-keys" + PROVER_NON_GPU_NUMBER_OF_SETUP_SLOTS=2 + PROVER_NON_GPU_ASSEMBLY_RECEIVER_PORT=17791 + PROVER_NON_GPU_ASSEMBLY_RECEIVER_POLL_TIME_IN_MILLIS=250 + PROVER_NON_GPU_ASSEMBLY_QUEUE_CAPACITY=5 + PROVER_NON_GPU_SPECIALIZED_PROVER_GROUP_ID=0 + + PROVER_TWO_GPU_FORTY_GB_MEM_PROMETHEUS_PORT="3313" + PROVER_TWO_GPU_FORTY_GB_MEM_INITIAL_SETUP_KEY_PATH="key" + PROVER_TWO_GPU_FORTY_GB_MEM_KEY_DOWNLOAD_URL="value" + PROVER_TWO_GPU_FORTY_GB_MEM_GENERATION_TIMEOUT_IN_SECS=2700 + PROVER_TWO_GPU_FORTY_GB_MEM_NUMBER_OF_THREADS="2" + PROVER_TWO_GPU_FORTY_GB_MEM_MAX_ATTEMPTS="4" + PROVER_TWO_GPU_FORTY_GB_MEM_POLLING_DURATION_IN_MILLIS=5 + PROVER_TWO_GPU_FORTY_GB_MEM_SETUP_KEYS_PATH="/usr/src/setup-keys" + PROVER_TWO_GPU_FORTY_GB_MEM_NUMBER_OF_SETUP_SLOTS=5 + PROVER_TWO_GPU_FORTY_GB_MEM_ASSEMBLY_RECEIVER_PORT=17791 + PROVER_TWO_GPU_FORTY_GB_MEM_ASSEMBLY_RECEIVER_POLL_TIME_IN_MILLIS=250 + PROVER_TWO_GPU_FORTY_GB_MEM_ASSEMBLY_QUEUE_CAPACITY=5 + PROVER_TWO_GPU_FORTY_GB_MEM_SPECIALIZED_PROVER_GROUP_ID=1 + + PROVER_ONE_GPU_EIGHTY_GB_MEM_PROMETHEUS_PORT="3313" + PROVER_ONE_GPU_EIGHTY_GB_MEM_INITIAL_SETUP_KEY_PATH="key" + PROVER_ONE_GPU_EIGHTY_GB_MEM_KEY_DOWNLOAD_URL="value" + PROVER_ONE_GPU_EIGHTY_GB_MEM_GENERATION_TIMEOUT_IN_SECS=2700 + PROVER_ONE_GPU_EIGHTY_GB_MEM_NUMBER_OF_THREADS="4" + PROVER_ONE_GPU_EIGHTY_GB_MEM_MAX_ATTEMPTS="4" + PROVER_ONE_GPU_EIGHTY_GB_MEM_POLLING_DURATION_IN_MILLIS=5 + PROVER_ONE_GPU_EIGHTY_GB_MEM_SETUP_KEYS_PATH="/usr/src/setup-keys" + PROVER_ONE_GPU_EIGHTY_GB_MEM_NUMBER_OF_SETUP_SLOTS=5 + PROVER_ONE_GPU_EIGHTY_GB_MEM_ASSEMBLY_RECEIVER_PORT=17791 + PROVER_ONE_GPU_EIGHTY_GB_MEM_ASSEMBLY_RECEIVER_POLL_TIME_IN_MILLIS=250 + PROVER_ONE_GPU_EIGHTY_GB_MEM_ASSEMBLY_QUEUE_CAPACITY=5 + PROVER_ONE_GPU_EIGHTY_GB_MEM_SPECIALIZED_PROVER_GROUP_ID=2 + + PROVER_TWO_GPU_EIGHTY_GB_MEM_PROMETHEUS_PORT="3313" + PROVER_TWO_GPU_EIGHTY_GB_MEM_INITIAL_SETUP_KEY_PATH="key" + PROVER_TWO_GPU_EIGHTY_GB_MEM_KEY_DOWNLOAD_URL="value" + PROVER_TWO_GPU_EIGHTY_GB_MEM_GENERATION_TIMEOUT_IN_SECS=2700 + PROVER_TWO_GPU_EIGHTY_GB_MEM_NUMBER_OF_THREADS="9" + PROVER_TWO_GPU_EIGHTY_GB_MEM_MAX_ATTEMPTS="4" + PROVER_TWO_GPU_EIGHTY_GB_MEM_POLLING_DURATION_IN_MILLIS=5 + PROVER_TWO_GPU_EIGHTY_GB_MEM_SETUP_KEYS_PATH="/usr/src/setup-keys" + PROVER_TWO_GPU_EIGHTY_GB_MEM_NUMBER_OF_SETUP_SLOTS=9 + PROVER_TWO_GPU_EIGHTY_GB_MEM_ASSEMBLY_RECEIVER_PORT=17791 + PROVER_TWO_GPU_EIGHTY_GB_MEM_ASSEMBLY_RECEIVER_POLL_TIME_IN_MILLIS=250 + PROVER_TWO_GPU_EIGHTY_GB_MEM_ASSEMBLY_QUEUE_CAPACITY=5 + PROVER_TWO_GPU_EIGHTY_GB_MEM_SPECIALIZED_PROVER_GROUP_ID=3 + + PROVER_FOUR_GPU_EIGHTY_GB_MEM_PROMETHEUS_PORT="3313" + PROVER_FOUR_GPU_EIGHTY_GB_MEM_INITIAL_SETUP_KEY_PATH="key" + PROVER_FOUR_GPU_EIGHTY_GB_MEM_KEY_DOWNLOAD_URL="value" + PROVER_FOUR_GPU_EIGHTY_GB_MEM_GENERATION_TIMEOUT_IN_SECS=2700 + PROVER_FOUR_GPU_EIGHTY_GB_MEM_NUMBER_OF_THREADS="18" + PROVER_FOUR_GPU_EIGHTY_GB_MEM_MAX_ATTEMPTS="4" + PROVER_FOUR_GPU_EIGHTY_GB_MEM_POLLING_DURATION_IN_MILLIS=5 + PROVER_FOUR_GPU_EIGHTY_GB_MEM_SETUP_KEYS_PATH="/usr/src/setup-keys" + PROVER_FOUR_GPU_EIGHTY_GB_MEM_NUMBER_OF_SETUP_SLOTS=18 + PROVER_FOUR_GPU_EIGHTY_GB_MEM_ASSEMBLY_RECEIVER_PORT=17791 + PROVER_FOUR_GPU_EIGHTY_GB_MEM_ASSEMBLY_RECEIVER_POLL_TIME_IN_MILLIS=250 + PROVER_FOUR_GPU_EIGHTY_GB_MEM_ASSEMBLY_QUEUE_CAPACITY=5 + PROVER_FOUR_GPU_EIGHTY_GB_MEM_SPECIALIZED_PROVER_GROUP_ID=4 + "#; + + #[test] + fn from_env() { + let mut lock = MUTEX.lock(); + lock.set_env(CONFIG); + let actual = ProverConfigs::from_env().unwrap(); + assert_eq!(actual, expected_config()); + } +} diff --git a/core/lib/env_config/src/prover_group.rs b/core/lib/env_config/src/prover_group.rs new file mode 100644 index 000000000000..bdac82cbb9cc --- /dev/null +++ b/core/lib/env_config/src/prover_group.rs @@ -0,0 +1,149 @@ +use zksync_config::configs::ProverGroupConfig; + +use crate::{envy_load, FromEnv}; + +impl FromEnv for ProverGroupConfig { + fn from_env() -> anyhow::Result { + envy_load("prover_group", "PROVER_GROUP_") + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_utils::EnvMutex; + + static MUTEX: EnvMutex = EnvMutex::new(); + + fn expected_config() -> ProverGroupConfig { + ProverGroupConfig { + group_0_circuit_ids: vec![0, 18], + group_1_circuit_ids: vec![1, 4], + group_2_circuit_ids: vec![2, 5], + group_3_circuit_ids: vec![6, 7], + group_4_circuit_ids: vec![8, 9], + group_5_circuit_ids: vec![10, 11], + group_6_circuit_ids: vec![12, 13], + group_7_circuit_ids: vec![14, 15], + group_8_circuit_ids: vec![16, 17], + group_9_circuit_ids: vec![3], + region_read_url: "http://metadata.google.internal/computeMetadata/v1/instance/attributes/cluster-location".to_string(), + region_override: Some("us-central-1".to_string()), + zone_read_url: "http://metadata.google.internal/computeMetadata/v1/instance/zone".to_string(), + zone_override: Some("us-central-1-b".to_string()), + synthesizer_per_gpu: 10, + } + } + + const CONFIG: &str = r#" + PROVER_GROUP_GROUP_0_CIRCUIT_IDS="0,18" + PROVER_GROUP_GROUP_1_CIRCUIT_IDS="1,4" + PROVER_GROUP_GROUP_2_CIRCUIT_IDS="2,5" + PROVER_GROUP_GROUP_3_CIRCUIT_IDS="6,7" + PROVER_GROUP_GROUP_4_CIRCUIT_IDS="8,9" + PROVER_GROUP_GROUP_5_CIRCUIT_IDS="10,11" + PROVER_GROUP_GROUP_6_CIRCUIT_IDS="12,13" + PROVER_GROUP_GROUP_7_CIRCUIT_IDS="14,15" + PROVER_GROUP_GROUP_8_CIRCUIT_IDS="16,17" + PROVER_GROUP_GROUP_9_CIRCUIT_IDS="3" + PROVER_GROUP_REGION_READ_URL="http://metadata.google.internal/computeMetadata/v1/instance/attributes/cluster-location" + PROVER_GROUP_REGION_OVERRIDE="us-central-1" + PROVER_GROUP_ZONE_READ_URL="http://metadata.google.internal/computeMetadata/v1/instance/zone" + PROVER_GROUP_ZONE_OVERRIDE="us-central-1-b" + PROVER_GROUP_SYNTHESIZER_PER_GPU="10" + "#; + + #[test] + fn from_env() { + let mut lock = MUTEX.lock(); + lock.set_env(CONFIG); + let actual = ProverGroupConfig::from_env().unwrap(); + assert_eq!(actual, expected_config()); + } + + #[test] + fn get_group_id_for_circuit_id() { + let prover_group_config = expected_config(); + + assert_eq!(Some(0), prover_group_config.get_group_id_for_circuit_id(0)); + assert_eq!(Some(0), prover_group_config.get_group_id_for_circuit_id(18)); + + assert_eq!(Some(1), prover_group_config.get_group_id_for_circuit_id(1)); + assert_eq!(Some(1), prover_group_config.get_group_id_for_circuit_id(4)); + + assert_eq!(Some(2), prover_group_config.get_group_id_for_circuit_id(2)); + assert_eq!(Some(2), prover_group_config.get_group_id_for_circuit_id(5)); + + assert_eq!(Some(3), prover_group_config.get_group_id_for_circuit_id(6)); + assert_eq!(Some(3), prover_group_config.get_group_id_for_circuit_id(7)); + + assert_eq!(Some(4), prover_group_config.get_group_id_for_circuit_id(8)); + assert_eq!(Some(4), prover_group_config.get_group_id_for_circuit_id(9)); + + assert_eq!(Some(5), prover_group_config.get_group_id_for_circuit_id(10)); + assert_eq!(Some(5), prover_group_config.get_group_id_for_circuit_id(11)); + + assert_eq!(Some(6), prover_group_config.get_group_id_for_circuit_id(12)); + assert_eq!(Some(6), prover_group_config.get_group_id_for_circuit_id(13)); + + assert_eq!(Some(7), prover_group_config.get_group_id_for_circuit_id(14)); + assert_eq!(Some(7), prover_group_config.get_group_id_for_circuit_id(15)); + + assert_eq!(Some(8), prover_group_config.get_group_id_for_circuit_id(16)); + assert_eq!(Some(8), prover_group_config.get_group_id_for_circuit_id(17)); + + assert_eq!(Some(9), prover_group_config.get_group_id_for_circuit_id(3)); + assert!(prover_group_config + .get_group_id_for_circuit_id(19) + .is_none()); + } + + #[test] + fn get_circuit_ids_for_group_id() { + let prover_group_config = expected_config(); + + assert_eq!( + Some(vec![0, 18]), + prover_group_config.get_circuit_ids_for_group_id(0) + ); + assert_eq!( + Some(vec![1, 4]), + prover_group_config.get_circuit_ids_for_group_id(1) + ); + assert_eq!( + Some(vec![2, 5]), + prover_group_config.get_circuit_ids_for_group_id(2) + ); + assert_eq!( + Some(vec![6, 7]), + prover_group_config.get_circuit_ids_for_group_id(3) + ); + assert_eq!( + Some(vec![8, 9]), + prover_group_config.get_circuit_ids_for_group_id(4) + ); + assert_eq!( + Some(vec![10, 11]), + prover_group_config.get_circuit_ids_for_group_id(5) + ); + assert_eq!( + Some(vec![12, 13]), + prover_group_config.get_circuit_ids_for_group_id(6) + ); + assert_eq!( + Some(vec![14, 15]), + prover_group_config.get_circuit_ids_for_group_id(7) + ); + assert_eq!( + Some(vec![16, 17]), + prover_group_config.get_circuit_ids_for_group_id(8) + ); + assert_eq!( + Some(vec![3]), + prover_group_config.get_circuit_ids_for_group_id(9) + ); + assert!(prover_group_config + .get_circuit_ids_for_group_id(10) + .is_none()); + } +} diff --git a/core/lib/config/src/configs/test_utils.rs b/core/lib/env_config/src/test_utils.rs similarity index 100% rename from core/lib/config/src/configs/test_utils.rs rename to core/lib/env_config/src/test_utils.rs diff --git a/core/lib/env_config/src/utils.rs b/core/lib/env_config/src/utils.rs new file mode 100644 index 000000000000..655d3b2e6d56 --- /dev/null +++ b/core/lib/env_config/src/utils.rs @@ -0,0 +1,8 @@ +use crate::{envy_load, FromEnv}; +use zksync_config::configs::PrometheusConfig; + +impl FromEnv for PrometheusConfig { + fn from_env() -> anyhow::Result { + envy_load("prometheus", "API_PROMETHEUS_") + } +} diff --git a/core/lib/env_config/src/witness_generator.rs b/core/lib/env_config/src/witness_generator.rs new file mode 100644 index 000000000000..28d7a9bbbd9d --- /dev/null +++ b/core/lib/env_config/src/witness_generator.rs @@ -0,0 +1,50 @@ +use zksync_config::configs::WitnessGeneratorConfig; + +use crate::{envy_load, FromEnv}; + +impl FromEnv for WitnessGeneratorConfig { + fn from_env() -> anyhow::Result { + envy_load("witness", "WITNESS_") + } +} + +#[cfg(test)] +mod tests { + use zksync_config::configs::witness_generator::BasicWitnessGeneratorDataSource; + + use super::*; + use crate::test_utils::EnvMutex; + + static MUTEX: EnvMutex = EnvMutex::new(); + + fn expected_config() -> WitnessGeneratorConfig { + WitnessGeneratorConfig { + generation_timeout_in_secs: 900_u16, + initial_setup_key_path: "key".to_owned(), + key_download_url: "value".to_owned(), + max_attempts: 4, + blocks_proving_percentage: Some(30), + dump_arguments_for_blocks: vec![2, 3], + last_l1_batch_to_process: None, + data_source: BasicWitnessGeneratorDataSource::FromBlob, + } + } + + #[test] + fn from_env() { + let mut lock = MUTEX.lock(); + let config = r#" + WITNESS_GENERATION_TIMEOUT_IN_SECS=900 + WITNESS_INITIAL_SETUP_KEY_PATH="key" + WITNESS_KEY_DOWNLOAD_URL="value" + WITNESS_MAX_ATTEMPTS=4 + WITNESS_DUMP_ARGUMENTS_FOR_BLOCKS="2,3" + WITNESS_BLOCKS_PROVING_PERCENTAGE="30" + WITNESS_DATA_SOURCE="FromBlob" + "#; + lock.set_env(config); + + let actual = WitnessGeneratorConfig::from_env().unwrap(); + assert_eq!(actual, expected_config()); + } +} diff --git a/core/lib/object_store/src/raw.rs b/core/lib/object_store/src/raw.rs index f369eacc2caf..bf318a61610f 100644 --- a/core/lib/object_store/src/raw.rs +++ b/core/lib/object_store/src/raw.rs @@ -1,11 +1,9 @@ -use anyhow::Context as _; use async_trait::async_trait; use std::{error, fmt, sync::Arc}; use crate::{file::FileBackedObjectStore, gcs::GoogleCloudStorage, mock::MockStore}; -use zksync_config::configs::object_store::ObjectStoreMode; -use zksync_config::ObjectStoreConfig; +use zksync_config::configs::object_store::{ObjectStoreConfig, ObjectStoreMode}; /// Bucket for [`ObjectStore`] in which objects can be placed. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] @@ -162,28 +160,6 @@ impl ObjectStoreFactory { } } - /// Creates an object store factory with the configuration taken from the environment. - /// - /// # Errors - /// - /// Invalid or missing configuration. - pub fn from_env() -> anyhow::Result { - Ok(Self::new( - ObjectStoreConfig::from_env().context("ObjectStoreConfig::from_env()")?, - )) - } - - /// Creates an object store factory with the prover configuration taken from the environment. - /// - /// # Errors - /// - /// Invalid or missing configuration. - pub fn prover_from_env() -> anyhow::Result { - Ok(Self::new( - ObjectStoreConfig::prover_from_env().context("ObjectStoreConfig::prover_from_env()")?, - )) - } - /// Creates an object store factory with a mock in-memory store. /// All calls to [`Self::create_store()`] will return the same store; thus, the testing code /// can use [`ObjectStore`] methods for assertions. diff --git a/core/lib/prover_utils/src/region_fetcher.rs b/core/lib/prover_utils/src/region_fetcher.rs index c2b17f143328..22a0cedce494 100644 --- a/core/lib/prover_utils/src/region_fetcher.rs +++ b/core/lib/prover_utils/src/region_fetcher.rs @@ -6,29 +6,27 @@ use reqwest::Method; use zksync_config::configs::ProverGroupConfig; use zksync_utils::http_with_retries::send_request_with_retries; -pub async fn get_region() -> anyhow::Result { - let prover_group_config = ProverGroupConfig::from_env().context("ProverGroupConfig")?; - if let Some(region) = prover_group_config.region_override { - return Ok(region); +pub async fn get_region(prover_group_config: &ProverGroupConfig) -> anyhow::Result { + if let Some(region) = &prover_group_config.region_override { + return Ok(region.clone()); } - let url = prover_group_config.region_read_url; + let url = &prover_group_config.region_read_url; fetch_from_url(url).await.context("fetch_from_url()") } -pub async fn get_zone() -> anyhow::Result { - let prover_group_config = ProverGroupConfig::from_env().context("ProverGroupConfig")?; - if let Some(zone) = prover_group_config.zone_override { - return Ok(zone); +pub async fn get_zone(prover_group_config: &ProverGroupConfig) -> anyhow::Result { + if let Some(zone) = &prover_group_config.zone_override { + return Ok(zone.clone()); } - let url = prover_group_config.zone_read_url; + let url = &prover_group_config.zone_read_url; let data = fetch_from_url(url).await.context("fetch_from_url()")?; parse_zone(&data).context("parse_zone") } -async fn fetch_from_url(url: String) -> anyhow::Result { +async fn fetch_from_url(url: &str) -> anyhow::Result { let mut headers = HeaderMap::new(); headers.insert("Metadata-Flavor", HeaderValue::from_static("Google")); - let response = send_request_with_retries(&url, 5, Method::GET, Some(headers), None).await; + let response = send_request_with_retries(url, 5, Method::GET, Some(headers), None).await; response .map_err(|err| anyhow::anyhow!("Failed fetching response from url: {url}: {err:?}"))? .text() @@ -48,6 +46,8 @@ fn parse_zone(data: &str) -> anyhow::Result { #[cfg(test)] mod tests { + use zksync_config::configs::ProverGroupConfig; + use crate::region_fetcher::{get_region, get_zone, parse_zone}; #[test] @@ -65,11 +65,46 @@ mod tests { #[tokio::test] async fn test_get_region_with_override() { - assert_eq!("us-central-1", get_region().await.unwrap()); + let config = ProverGroupConfig { + group_0_circuit_ids: vec![], + group_1_circuit_ids: vec![], + group_2_circuit_ids: vec![], + group_3_circuit_ids: vec![], + group_4_circuit_ids: vec![], + group_5_circuit_ids: vec![], + group_6_circuit_ids: vec![], + group_7_circuit_ids: vec![], + group_8_circuit_ids: vec![], + group_9_circuit_ids: vec![], + region_override: Some("us-central-1".to_string()), + region_read_url: "".to_string(), + zone_override: Some("us-central-1-b".to_string()), + zone_read_url: "".to_string(), + synthesizer_per_gpu: 0, + }; + + assert_eq!("us-central-1", get_region(&config).await.unwrap()); } #[tokio::test] async fn test_get_zone_with_override() { - assert_eq!("us-central-1-b", get_zone().await.unwrap()); + let config = ProverGroupConfig { + group_0_circuit_ids: vec![], + group_1_circuit_ids: vec![], + group_2_circuit_ids: vec![], + group_3_circuit_ids: vec![], + group_4_circuit_ids: vec![], + group_5_circuit_ids: vec![], + group_6_circuit_ids: vec![], + group_7_circuit_ids: vec![], + group_8_circuit_ids: vec![], + group_9_circuit_ids: vec![], + region_override: Some("us-central-1".to_string()), + region_read_url: "".to_string(), + zone_override: Some("us-central-1-b".to_string()), + zone_read_url: "".to_string(), + synthesizer_per_gpu: 0, + }; + assert_eq!("us-central-1-b", get_zone(&config).await.unwrap()); } } diff --git a/core/lib/zksync_core/src/api_server/tx_sender/mod.rs b/core/lib/zksync_core/src/api_server/tx_sender/mod.rs index fd9c2cad99af..b994fbff8824 100644 --- a/core/lib/zksync_core/src/api_server/tx_sender/mod.rs +++ b/core/lib/zksync_core/src/api_server/tx_sender/mod.rs @@ -235,8 +235,6 @@ pub struct TxSenderConfig { pub fair_l2_gas_price: u64, pub vm_execution_cache_misses_limit: Option, pub validation_computational_gas_limit: u32, - pub default_aa: H256, - pub bootloader: H256, pub chain_id: L2ChainId, } @@ -255,8 +253,6 @@ impl TxSenderConfig { vm_execution_cache_misses_limit: web3_json_config.vm_execution_cache_misses_limit, validation_computational_gas_limit: state_keeper_config .validation_computational_gas_limit, - default_aa: state_keeper_config.default_aa_hash, - bootloader: state_keeper_config.bootloader_hash, chain_id, } } diff --git a/core/lib/zksync_core/src/api_server/web3/tests.rs b/core/lib/zksync_core/src/api_server/web3/tests.rs index 5fde6cc3aebf..0d595830d34f 100644 --- a/core/lib/zksync_core/src/api_server/web3/tests.rs +++ b/core/lib/zksync_core/src/api_server/web3/tests.rs @@ -70,9 +70,9 @@ pub(crate) async fn spawn_http_server( pool: ConnectionPool, stop_receiver: watch::Receiver, ) -> ApiServerHandles { - let contracts_config = ContractsConfig::from_env().unwrap(); - let web3_config = Web3JsonRpcConfig::from_env().unwrap(); - let state_keeper_config = StateKeeperConfig::from_env().unwrap(); + let contracts_config = ContractsConfig::for_tests(); + let web3_config = Web3JsonRpcConfig::for_tests(); + let state_keeper_config = StateKeeperConfig::for_tests(); let api_config = InternalApiConfig::new(network_config, &web3_config, &contracts_config); let tx_sender_config = TxSenderConfig::new(&state_keeper_config, &web3_config, api_config.l2_chain_id); @@ -103,7 +103,7 @@ pub(crate) async fn spawn_http_server( #[tokio::test] async fn http_server_can_start() { let pool = ConnectionPool::test_pool().await; - let network_config = NetworkConfig::from_env().unwrap(); + let network_config = NetworkConfig::for_tests(); let mut storage = pool.access_storage().await.unwrap(); if storage.blocks_dal().is_genesis_needed().await.unwrap() { ensure_genesis_state( diff --git a/core/lib/zksync_core/src/data_fetchers/token_list/one_inch.rs b/core/lib/zksync_core/src/data_fetchers/token_list/one_inch.rs index 0052b091f2a2..1d022e4700d4 100644 --- a/core/lib/zksync_core/src/data_fetchers/token_list/one_inch.rs +++ b/core/lib/zksync_core/src/data_fetchers/token_list/one_inch.rs @@ -55,19 +55,3 @@ impl FetcherImpl for OneInchTokenListFetcher { pub(super) struct OneInchTokensResponse { pub tokens: HashMap, } - -#[tokio::test] -#[ignore] -// We can't rely on 1inch API in unit tests, so we ignore this test. -async fn test_fetch_one_inch_token_list() { - let mut config = FetcherConfig::from_env().unwrap(); - config.token_list.url = "https://api.1inch.exchange".to_string(); - - let fetcher = OneInchTokenListFetcher::new(&config); - - let token_list = fetcher - .fetch_token_list() - .await - .expect("failed get token list"); - assert!(!token_list.is_empty(), "Token list is empty"); -} diff --git a/core/lib/zksync_core/src/data_fetchers/token_price/coingecko.rs b/core/lib/zksync_core/src/data_fetchers/token_price/coingecko.rs index 3b7125f8c632..a046c23ea2d2 100644 --- a/core/lib/zksync_core/src/data_fetchers/token_price/coingecko.rs +++ b/core/lib/zksync_core/src/data_fetchers/token_price/coingecko.rs @@ -165,32 +165,3 @@ pub struct CoinGeckoTokenPrice { #[serde(with = "UnsignedRatioSerializeAsDecimal")] pub usd: Ratio, } - -#[tokio::test] -#[ignore] -async fn test_fetch_coingecko_prices() { - let mut config = FetcherConfig::from_env().unwrap(); - config.token_price.url = "https://api.coingecko.com".to_string(); - - let fetcher = CoinGeckoFetcher::new(&config); - - let tokens = vec![ - ETHEREUM_ADDRESS, - Address::from_str("6b175474e89094c44da98b954eedeac495271d0f").expect("DAI"), - Address::from_str("1f9840a85d5af5bf1d1762f925bdaddc4201f984").expect("UNI"), - Address::from_str("514910771af9ca656af840dff83e8264ecf986ca").expect("LINK"), - ]; - - let token_prices = fetcher - .fetch_token_price(&tokens) - .await - .expect("failed get tokens price"); - assert_eq!( - token_prices.len(), - tokens.len(), - "not all data was received" - ); - for token_address in tokens { - assert!(token_prices.get(&token_address).is_some()); - } -} diff --git a/core/lib/zksync_core/src/eth_sender/tests.rs b/core/lib/zksync_core/src/eth_sender/tests.rs index b2c1fa77e608..08069df6414c 100644 --- a/core/lib/zksync_core/src/eth_sender/tests.rs +++ b/core/lib/zksync_core/src/eth_sender/tests.rs @@ -65,8 +65,8 @@ impl EthSenderTester { history: Vec, non_ordering_confirmations: bool, ) -> Self { - let eth_sender_config = ETHSenderConfig::from_env().unwrap(); - let contracts_config = ContractsConfig::from_env().unwrap(); + let eth_sender_config = ETHSenderConfig::for_tests(); + let contracts_config = ContractsConfig::for_tests(); let aggregator_config = SenderConfig { aggregated_proof_sizes: vec![1], ..eth_sender_config.sender.clone() @@ -100,7 +100,7 @@ impl EthSenderTester { .await .unwrap(), ); - let store_factory = ObjectStoreFactory::from_env().unwrap(); + let store_factory = ObjectStoreFactory::mock(); let aggregator = EthTxAggregator::new( SenderConfig { diff --git a/core/lib/zksync_core/src/eth_watch/mod.rs b/core/lib/zksync_core/src/eth_watch/mod.rs index 640f96707e4a..fdb629bce28a 100644 --- a/core/lib/zksync_core/src/eth_watch/mod.rs +++ b/core/lib/zksync_core/src/eth_watch/mod.rs @@ -4,7 +4,6 @@ //! Poll interval is configured using the `ETH_POLL_INTERVAL` constant. //! Number of confirmations is configured using the `CONFIRMATIONS_FOR_ETH_EVENT` environment variable. -use anyhow::Context as _; use tokio::{sync::watch, task::JoinHandle}; use std::time::Duration; @@ -188,18 +187,18 @@ impl EthWatch { } pub async fn start_eth_watch( + config: ETHWatchConfig, pool: ConnectionPool, eth_gateway: E, diamond_proxy_addr: Address, governance: (Contract, Address), stop_receiver: watch::Receiver, ) -> anyhow::Result>> { - let eth_watch = ETHWatchConfig::from_env().context("ETHWatchConfig::from_env()")?; let eth_client = EthHttpQueryClient::new( eth_gateway, diamond_proxy_addr, Some(governance.1), - eth_watch.confirmations_for_eth_event, + config.confirmations_for_eth_event, ); let mut eth_watch = EthWatch::new( @@ -207,7 +206,7 @@ pub async fn start_eth_watch( Some(governance.0), eth_client, &pool, - eth_watch.poll_interval(), + config.poll_interval(), ) .await; diff --git a/core/lib/zksync_core/src/house_keeper/prover_queue_monitor.rs b/core/lib/zksync_core/src/house_keeper/prover_queue_monitor.rs index dcd209860205..5b41ee74ac9a 100644 --- a/core/lib/zksync_core/src/house_keeper/prover_queue_monitor.rs +++ b/core/lib/zksync_core/src/house_keeper/prover_queue_monitor.rs @@ -1,4 +1,3 @@ -use anyhow::Context as _; use async_trait::async_trait; use zksync_config::configs::ProverGroupConfig; use zksync_dal::ConnectionPool; @@ -10,13 +9,19 @@ use zksync_prover_utils::periodic_job::PeriodicJob; pub struct ProverStatsReporter { reporting_interval_ms: u64, prover_connection_pool: ConnectionPool, + config: ProverGroupConfig, } impl ProverStatsReporter { - pub fn new(reporting_interval_ms: u64, prover_connection_pool: ConnectionPool) -> Self { + pub fn new( + reporting_interval_ms: u64, + prover_connection_pool: ConnectionPool, + config: ProverGroupConfig, + ) -> Self { Self { reporting_interval_ms, prover_connection_pool, + config, } } } @@ -28,13 +33,12 @@ impl PeriodicJob for ProverStatsReporter { const SERVICE_NAME: &'static str = "ProverStatsReporter"; async fn run_routine_task(&mut self) -> anyhow::Result<()> { - let prover_group_config = - ProverGroupConfig::from_env().context("ProverGroupConfig::from_env()")?; let mut conn = self.prover_connection_pool.access_storage().await.unwrap(); let stats = conn.prover_dal().get_prover_jobs_stats_per_circuit().await; for (circuit_name, stats) in stats.into_iter() { - let group_id = prover_group_config + let group_id = self + .config .get_group_id_for_circuit_id(circuit_name_to_numeric_index(&circuit_name).unwrap()) .unwrap(); diff --git a/core/lib/zksync_core/src/l1_gas_price/singleton.rs b/core/lib/zksync_core/src/l1_gas_price/singleton.rs index 766dc246a949..4808dee548ba 100644 --- a/core/lib/zksync_core/src/l1_gas_price/singleton.rs +++ b/core/lib/zksync_core/src/l1_gas_price/singleton.rs @@ -3,13 +3,17 @@ use anyhow::Context as _; use std::sync::Arc; use tokio::sync::{watch, OnceCell}; use tokio::task::JoinHandle; -use zksync_config::{ETHClientConfig, GasAdjusterConfig}; +use zksync_config::GasAdjusterConfig; use zksync_eth_client::clients::http::QueryClient; /// Special struct for creating a singleton of `GasAdjuster`. -/// This is needed only for running the server. This struct uses all configs from env. -#[derive(Debug, Default)] -pub struct GasAdjusterSingleton(OnceCell>, Error>>); +/// This is needed only for running the server. +#[derive(Debug)] +pub struct GasAdjusterSingleton { + web3_url: String, + gas_adjuster_config: GasAdjusterConfig, + singleton: OnceCell>, Error>>, +} #[derive(thiserror::Error, Debug, Clone)] #[error(transparent)] @@ -22,21 +26,21 @@ impl From for Error { } impl GasAdjusterSingleton { - pub fn new() -> Self { - Default::default() + pub fn new(web3_url: String, gas_adjuster_config: GasAdjusterConfig) -> Self { + Self { + web3_url, + gas_adjuster_config, + singleton: OnceCell::new(), + } } pub async fn get_or_init(&mut self) -> Result>, Error> { let adjuster = self - .0 + .singleton .get_or_init(|| async { - let eth_client_config = - ETHClientConfig::from_env().context("ETHClientConfig::from_env()")?; let query_client = - QueryClient::new(ð_client_config.web3_url).context("QueryClient::new()")?; - let gas_adjuster_config = - GasAdjusterConfig::from_env().context("GasAdjusterConfig::from_env()")?; - let adjuster = GasAdjuster::new(query_client.clone(), gas_adjuster_config) + QueryClient::new(&self.web3_url).context("QueryClient::new()")?; + let adjuster = GasAdjuster::new(query_client.clone(), self.gas_adjuster_config) .await .context("GasAdjuster::new()")?; Ok(Arc::new(adjuster)) @@ -48,10 +52,9 @@ impl GasAdjusterSingleton { pub async fn get_or_init_bounded( &mut self, ) -> anyhow::Result>>> { - let config = GasAdjusterConfig::from_env().context("GasAdjusterConfig::from_env()")?; let adjuster = self.get_or_init().await.context("get_or_init()")?; Ok(Arc::new(BoundedGasAdjuster::new( - config.max_l1_gas_price(), + self.gas_adjuster_config.max_l1_gas_price(), adjuster, ))) } @@ -60,7 +63,7 @@ impl GasAdjusterSingleton { self, stop_signal: watch::Receiver, ) -> Option>> { - let gas_adjuster = self.0.get()?.clone(); + let gas_adjuster = self.singleton.get()?.clone(); Some(tokio::spawn( async move { gas_adjuster?.run(stop_signal).await }, )) diff --git a/core/lib/zksync_core/src/lib.rs b/core/lib/zksync_core/src/lib.rs index 2e35cf573a94..9294a3e65def 100644 --- a/core/lib/zksync_core/src/lib.rs +++ b/core/lib/zksync_core/src/lib.rs @@ -5,28 +5,24 @@ use std::{net::Ipv4Addr, str::FromStr, sync::Arc, time::Instant}; use anyhow::Context as _; use futures::channel::oneshot; use prometheus_exporter::PrometheusExporterConfig; +use temp_config_store::TempConfigStore; use tokio::{sync::watch, task::JoinHandle}; use zksync_circuit_breaker::{ l1_txs::FailedL1TransactionChecker, replication_lag::ReplicationLagChecker, CircuitBreaker, CircuitBreakerChecker, CircuitBreakerError, }; -use zksync_config::configs::api::MerkleTreeApiConfig; -use zksync_config::configs::contracts::ProverAtGenesis; -use zksync_config::configs::{ - api::{HealthCheckConfig, Web3JsonRpcConfig}, - chain::{ - self, CircuitBreakerConfig, MempoolConfig, NetworkConfig, OperationsManagerConfig, - StateKeeperConfig, - }, - database::MerkleTreeMode, - house_keeper::HouseKeeperConfig, - FriProofCompressorConfig, FriProverConfig, FriWitnessGeneratorConfig, PrometheusConfig, - ProofDataHandlerConfig, ProverGroupConfig, WitnessGeneratorConfig, -}; use zksync_config::{ - ApiConfig, ContractsConfig, DBConfig, ETHClientConfig, ETHSenderConfig, FetcherConfig, - ProverConfigs, + configs::{ + api::{MerkleTreeApiConfig, Web3JsonRpcConfig}, + chain::{ + CircuitBreakerConfig, MempoolConfig, NetworkConfig, OperationsManagerConfig, + StateKeeperConfig, + }, + contracts::ProverAtGenesis, + database::MerkleTreeMode, + }, + ApiConfig, ContractsConfig, DBConfig, ETHSenderConfig, }; use zksync_contracts::{governance_contract, BaseSystemContracts}; use zksync_dal::{ @@ -36,7 +32,7 @@ use zksync_eth_client::clients::http::QueryClient; use zksync_eth_client::EthInterface; use zksync_eth_client::{clients::http::PKSigningClient, BoundEthInterface}; use zksync_health_check::{CheckHealth, HealthStatus, ReactiveHealthCheck}; -use zksync_object_store::ObjectStoreFactory; +use zksync_object_store::{ObjectStore, ObjectStoreFactory}; use zksync_prover_utils::periodic_job::PeriodicJob; use zksync_queued_job_processor::JobProcessor; use zksync_state::PostgresStorageCaches; @@ -65,6 +61,7 @@ pub mod proof_data_handler; pub mod reorg_detector; pub mod state_keeper; pub mod sync_layer; +pub mod temp_config_store; pub mod witness_generator; use crate::api_server::healthcheck::HealthCheckHandle; @@ -327,6 +324,7 @@ impl FromStr for Components { } pub async fn initialize_components( + configs: &TempConfigStore, components: Vec, use_prometheus_push_gateway: bool, ) -> anyhow::Result<( @@ -337,7 +335,10 @@ pub async fn initialize_components( )> { tracing::info!("Starting the components: {components:?}"); - let db_config = DBConfig::from_env().context("DbConfig::from_env()")?; + let db_config = configs + .db_config + .clone() + .ok_or_else(|| anyhow::anyhow!("db_config"))?; let connection_pool = ConnectionPool::builder(DbVariant::Master) .build() .await @@ -353,10 +354,18 @@ pub async fn initialize_components( .context("failed to build replica_connection_pool")?; let mut healthchecks: Vec> = Vec::new(); - let contracts_config = ContractsConfig::from_env().context("ContractsConfig::from_env()")?; - let eth_client_config = ETHClientConfig::from_env().context("ETHClientConfig::from_env()")?; - let circuit_breaker_config = - CircuitBreakerConfig::from_env().context("CircuitBreakerConfig::from_env()")?; + let contracts_config = configs + .contracts_config + .clone() + .ok_or_else(|| anyhow::anyhow!("contracts_config"))?; + let eth_client_config = configs + .eth_client_config + .clone() + .ok_or_else(|| anyhow::anyhow!("eth_client_config"))?; + let circuit_breaker_config = configs + .circuit_breaker_config + .clone() + .ok_or_else(|| anyhow::anyhow!("circuit_breaker_config"))?; let circuit_breaker_checker = CircuitBreakerChecker::new( circuit_breakers_for_components(&components, &circuit_breaker_config) @@ -369,13 +378,20 @@ pub async fn initialize_components( }); let query_client = QueryClient::new(ð_client_config.web3_url).unwrap(); - let mut gas_adjuster = GasAdjusterSingleton::new(); + let gas_adjuster_config = configs + .gas_adjuster_config + .ok_or_else(|| anyhow::anyhow!("gas_adjuster_config"))?; + let mut gas_adjuster = + GasAdjusterSingleton::new(eth_client_config.web3_url.clone(), gas_adjuster_config); let (stop_sender, stop_receiver) = watch::channel(false); let (cb_sender, cb_receiver) = oneshot::channel(); // Prometheus exporter and circuit breaker checker should run for every component configuration. - let prom_config = PrometheusConfig::from_env().context("PrometheusConfig::from_env()")?; + let prom_config = configs + .prometheus_config + .clone() + .ok_or_else(|| anyhow::anyhow!("prometheus_config"))?; let prom_config = if use_prometheus_push_gateway { PrometheusExporterConfig::push(prom_config.gateway_endpoint(), prom_config.push_interval()) } else { @@ -403,10 +419,18 @@ pub async fn initialize_components( || components.contains(&Component::ContractVerificationApi) || components.contains(&Component::ApiTranslator) { - let api_config = ApiConfig::from_env().context("ApiConfig::from_env()")?; - let state_keeper_config = - StateKeeperConfig::from_env().context("StateKeeperConfig::from_env()")?; - let network_config = NetworkConfig::from_env().context("NetworkConfig::from_env()")?; + let api_config = configs + .api_config + .clone() + .ok_or_else(|| anyhow::anyhow!("api_config"))?; + let state_keeper_config = configs + .state_keeper_config + .clone() + .ok_or_else(|| anyhow::anyhow!("state_keeper_config"))?; + let network_config = configs + .network_config + .clone() + .ok_or_else(|| anyhow::anyhow!("network_config"))?; let tx_sender_config = TxSenderConfig::new( &state_keeper_config, &api_config.web3_json_rpc, @@ -426,7 +450,7 @@ pub async fn initialize_components( if components.contains(&Component::HttpApi) { storage_caches = Some( - build_storage_caches(&replica_connection_pool, &mut task_futures) + build_storage_caches(configs, &replica_connection_pool, &mut task_futures) .context("build_storage_caches()")?, ); @@ -465,7 +489,7 @@ pub async fn initialize_components( if components.contains(&Component::WsApi) { let storage_caches = match storage_caches { Some(storage_caches) => storage_caches, - None => build_storage_caches(&replica_connection_pool, &mut task_futures) + None => build_storage_caches(configs, &replica_connection_pool, &mut task_futures) .context("build_storage_caches()")?, }; @@ -515,6 +539,12 @@ pub async fn initialize_components( } } + let object_store_config = configs + .object_store_config + .clone() + .ok_or_else(|| anyhow::anyhow!("object_store_config"))?; + let store_factory = ObjectStoreFactory::new(object_store_config); + if components.contains(&Component::StateKeeper) { let started_at = Instant::now(); tracing::info!("initializing State Keeper"); @@ -525,11 +555,21 @@ pub async fn initialize_components( add_state_keeper_to_task_futures( &mut task_futures, &contracts_config, - StateKeeperConfig::from_env().context("StateKeeperConfig::from_env()")?, - &NetworkConfig::from_env().context("NetworkConfig::from_env()")?, + configs + .state_keeper_config + .clone() + .ok_or_else(|| anyhow::anyhow!("state_keeper_config"))?, + &configs + .network_config + .clone() + .ok_or_else(|| anyhow::anyhow!("network_config"))?, &db_config, - &MempoolConfig::from_env().context("MempoolConfig::from_env()")?, + &configs + .mempool_config + .clone() + .ok_or_else(|| anyhow::anyhow!("mempool_config"))?, bounded_gas_adjuster, + store_factory.create_store().await, stop_receiver.clone(), ) .await @@ -549,8 +589,13 @@ pub async fn initialize_components( .await .context("failed to build eth_watch_pool")?; let governance = (governance_contract(), contracts_config.governance_addr); + let eth_watch_config = configs + .eth_watch_config + .clone() + .ok_or_else(|| anyhow::anyhow!("eth_watch_config"))?; task_futures.push( start_eth_watch( + eth_watch_config, eth_watch_pool, query_client.clone(), main_zksync_contract_address, @@ -565,8 +610,6 @@ pub async fn initialize_components( tracing::info!("initialized ETH-Watcher in {elapsed:?}"); } - let store_factory = ObjectStoreFactory::from_env().context("ObjectStoreFactor::from_env()")?; - if components.contains(&Component::EthTxAggregator) { let started_at = Instant::now(); tracing::info!("initializing ETH-TxAggregator"); @@ -579,7 +622,10 @@ pub async fn initialize_components( .await .context("failed to build eth_sender_prover_pool")?; - let eth_sender = ETHSenderConfig::from_env().context("ETHSenderConfig::from_env()")?; + let eth_sender = configs + .eth_sender_config + .clone() + .ok_or_else(|| anyhow::anyhow!("eth_sender_config"))?; let eth_client = PKSigningClient::from_config(ð_sender, &contracts_config, ð_client_config); let nonce = eth_client.pending_nonce("eth_sender").await.unwrap(); @@ -612,7 +658,10 @@ pub async fn initialize_components( .build() .await .context("failed to build eth_manager_pool")?; - let eth_sender = ETHSenderConfig::from_env().context("ETHSenderConfig::from_env()")?; + let eth_sender = configs + .eth_sender_config + .clone() + .ok_or_else(|| anyhow::anyhow!("eth_sender_config"))?; let eth_client = PKSigningClient::from_config(ð_sender, &contracts_config, ð_client_config); let eth_tx_manager_actor = EthTxManager::new( @@ -633,8 +682,14 @@ pub async fn initialize_components( if components.contains(&Component::DataFetcher) { let started_at = Instant::now(); - let fetcher_config = FetcherConfig::from_env().context("FetcherConfig::from_env()")?; - let eth_network = chain::NetworkConfig::from_env().context("NetworkConfig::from_env()")?; + let fetcher_config = configs + .fetcher_config + .clone() + .ok_or_else(|| anyhow::anyhow!("fetcher_config"))?; + let eth_network = configs + .network_config + .clone() + .ok_or_else(|| anyhow::anyhow!("network_config"))?; tracing::info!("initializing data fetchers"); task_futures.extend(run_data_fetchers( &fetcher_config, @@ -648,6 +703,7 @@ pub async fn initialize_components( } add_trees_to_task_futures( + configs, &mut task_futures, &mut healthchecks, &components, @@ -657,6 +713,7 @@ pub async fn initialize_components( .await .context("add_trees_to_task_futures()")?; add_witness_generator_to_task_futures( + configs, &mut task_futures, &components, &connection_pool, @@ -672,13 +729,15 @@ pub async fn initialize_components( .build() .await .context("failed to build singleton connection_pool")?; + let network_config = configs + .network_config + .clone() + .ok_or_else(|| anyhow::anyhow!("network_config"))?; add_basic_witness_input_producer_to_task_futures( &mut task_futures, &singleton_connection_pool, &store_factory, - NetworkConfig::from_env() - .context("NetworkConfig::from_env()")? - .zksync_network_id, + network_config.zksync_network_id, stop_receiver.clone(), ) .await @@ -686,14 +745,21 @@ pub async fn initialize_components( } if components.contains(&Component::Housekeeper) { - add_house_keeper_to_task_futures(&mut task_futures, &store_factory) + add_house_keeper_to_task_futures(configs, &mut task_futures, &store_factory) .await .context("add_house_keeper_to_task_futures()")?; } if components.contains(&Component::ProofDataHandler) { task_futures.push(tokio::spawn(proof_data_handler::run_server( - ProofDataHandlerConfig::from_env().context("ProofDataHandlerConfig::from_env()")?, + configs + .proof_data_handler_config + .clone() + .ok_or_else(|| anyhow::anyhow!("proof_data_handler_config"))?, + configs + .contracts_config + .clone() + .ok_or_else(|| anyhow::anyhow!("contracts_config"))?, store_factory.create_store().await, connection_pool.clone(), stop_receiver.clone(), @@ -705,8 +771,10 @@ pub async fn initialize_components( replica_connection_pool, ))); - let healtcheck_api_config = - HealthCheckConfig::from_env().context("HealthCheckConfig::from_env()")?; + let healtcheck_api_config = configs + .health_check_config + .clone() + .ok_or_else(|| anyhow::anyhow!("health_check_config"))?; let health_check_handle = HealthCheckHandle::spawn_server(healtcheck_api_config.bind_addr(), healthchecks); @@ -725,6 +793,7 @@ async fn add_state_keeper_to_task_futures, + object_store: Box, stop_receiver: watch::Receiver, ) -> anyhow::Result<()> { let fair_l2_gas_price = state_keeper_config.fair_l2_gas_price; @@ -763,6 +832,7 @@ async fn add_state_keeper_to_task_futures>>, healthchecks: &mut Vec>, components: &[Component], @@ -795,11 +866,18 @@ async fn add_trees_to_task_futures( anyhow::bail!("Tree backup mode is disabled"); } - let db_config = DBConfig::from_env().context("DBConfig::from_env()")?; - let operation_config = - OperationsManagerConfig::from_env().context("OperationManagerConfig::from_env()")?; - let api_config = ApiConfig::from_env() - .context("ApiConfig::from_env()")? + let db_config = configs + .db_config + .clone() + .ok_or_else(|| anyhow::anyhow!("db_config"))?; + let operation_config = configs + .operations_manager_config + .clone() + .ok_or_else(|| anyhow::anyhow!("operations_manager_config"))?; + let api_config = configs + .api_config + .clone() + .ok_or_else(|| anyhow::anyhow!("api_config"))? .merkle_tree; let api_config = components .contains(&Component::TreeApi) @@ -859,7 +937,8 @@ async fn run_tree( }; tracing::info!("Initializing Merkle tree in {mode_str} mode"); - let config = MetadataCalculatorConfig::for_main_node(db_config, operation_manager, mode); + let config = + MetadataCalculatorConfig::for_main_node(&db_config.merkle_tree, operation_manager, mode); let metadata_calculator = MetadataCalculator::new(&config).await; if let Some(api_config) = api_config { let address = (Ipv4Addr::UNSPECIFIED, api_config.port).into(); @@ -915,6 +994,7 @@ async fn add_basic_witness_input_producer_to_task_futures( } async fn add_witness_generator_to_task_futures( + configs: &TempConfigStore, task_futures: &mut Vec>>, components: &[Component], connection_pool: &ConnectionPool, @@ -949,8 +1029,10 @@ async fn add_witness_generator_to_task_futures( .protocol_versions_dal() .protocol_version_for(&vk_commitments) .await; - let config = - WitnessGeneratorConfig::from_env().context("WitnessGeneratorConfig::from_env()")?; + let config = configs + .witness_generator_config + .clone() + .ok_or_else(|| anyhow::anyhow!("witness_generator_config"))?; let task = match component_type { AggregationRound::BasicCircuits => { let witness_generator = BasicWitnessGenerator::new( @@ -1007,11 +1089,14 @@ async fn add_witness_generator_to_task_futures( } async fn add_house_keeper_to_task_futures( + configs: &TempConfigStore, task_futures: &mut Vec>>, store_factory: &ObjectStoreFactory, ) -> anyhow::Result<()> { - let house_keeper_config = - HouseKeeperConfig::from_env().context("HouseKeeperConfig::from_env()")?; + let house_keeper_config = configs + .house_keeper_config + .clone() + .ok_or_else(|| anyhow::anyhow!("house_keeper_config"))?; let connection_pool = ConnectionPool::singleton(DbVariant::Replica) .build() .await @@ -1026,16 +1111,20 @@ async fn add_house_keeper_to_task_futures( .build() .await .context("failed to build a prover_connection_pool")?; + let prover_group_config = configs + .prover_group_config + .clone() + .ok_or_else(|| anyhow::anyhow!("prover_group_config"))?; + let prover_configs = configs + .prover_configs + .clone() + .ok_or_else(|| anyhow::anyhow!("prover_configs"))?; let gpu_prover_queue = GpuProverQueueMonitor::new( - ProverGroupConfig::from_env() - .context("ProverGroupConfig::from_env()")? - .synthesizer_per_gpu, + prover_group_config.synthesizer_per_gpu, house_keeper_config.gpu_prover_queue_reporting_interval_ms, prover_connection_pool.clone(), ); - let config = ProverConfigs::from_env() - .context("ProverCOnfigs::from_env()")? - .non_gpu; + let config = prover_configs.non_gpu.clone(); let prover_job_retry_manager = ProverJobRetryManager::new( config.max_attempts, config.proof_generation_timeout(), @@ -1045,6 +1134,7 @@ async fn add_house_keeper_to_task_futures( let prover_stats_reporter = ProverStatsReporter::new( house_keeper_config.prover_stats_reporting_interval_ms, prover_connection_pool.clone(), + prover_group_config.clone(), ); let waiting_to_queued_witness_job_mover = WaitingToQueuedWitnessJobMover::new( house_keeper_config.witness_job_moving_interval_ms, @@ -1070,7 +1160,10 @@ async fn add_house_keeper_to_task_futures( task_futures.push(tokio::spawn(prover_job_retry_manager.run())); // All FRI Prover related components are configured below. - let fri_prover_config = FriProverConfig::from_env().context("FriProverConfig::from_env()")?; + let fri_prover_config = configs + .fri_prover_config + .clone() + .ok_or_else(|| anyhow::anyhow!("fri_prover_config"))?; let fri_prover_job_retry_manager = FriProverJobRetryManager::new( fri_prover_config.max_attempts, fri_prover_config.proof_generation_timeout(), @@ -1079,8 +1172,10 @@ async fn add_house_keeper_to_task_futures( ); task_futures.push(tokio::spawn(fri_prover_job_retry_manager.run())); - let fri_witness_gen_config = - FriWitnessGeneratorConfig::from_env().context("FriWitnessGeneratorConfig::from_env")?; + let fri_witness_gen_config = configs + .fri_witness_generator_config + .clone() + .ok_or_else(|| anyhow::anyhow!("fri_witness_generator_config"))?; let fri_witness_gen_job_retry_manager = FriWitnessGeneratorJobRetryManager::new( fri_witness_gen_config.max_attempts, fri_witness_gen_config.witness_generation_timeout(), @@ -1113,8 +1208,10 @@ async fn add_house_keeper_to_task_futures( ); task_futures.push(tokio::spawn(fri_prover_stats_reporter.run())); - let proof_compressor_config = - FriProofCompressorConfig::from_env().context("FriProofCompressorConfig")?; + let proof_compressor_config = configs + .fri_proof_compressor_config + .clone() + .ok_or_else(|| anyhow::anyhow!("fri_proof_compressor_config"))?; let fri_proof_compressor_stats_reporter = FriProofCompressorStatsReporter::new( house_keeper_config.fri_proof_compressor_stats_reporting_interval_ms, prover_connection_pool.clone(), @@ -1132,10 +1229,14 @@ async fn add_house_keeper_to_task_futures( } fn build_storage_caches( + configs: &TempConfigStore, replica_connection_pool: &ConnectionPool, task_futures: &mut Vec>>, ) -> anyhow::Result { - let rpc_config = Web3JsonRpcConfig::from_env().context("Web3JsonRpcConfig::from_env()")?; + let rpc_config = configs + .web3_json_rpc_config + .clone() + .ok_or_else(|| anyhow::anyhow!("web3_json_rpc_config"))?; let factory_deps_capacity = rpc_config.factory_deps_cache_size() as u64; let initial_writes_capacity = rpc_config.initial_writes_cache_size() as u64; let values_capacity = rpc_config.latest_values_cache_size() as u64; diff --git a/core/lib/zksync_core/src/metadata_calculator/mod.rs b/core/lib/zksync_core/src/metadata_calculator/mod.rs index 3602a79fa829..0f50b2bb09b7 100644 --- a/core/lib/zksync_core/src/metadata_calculator/mod.rs +++ b/core/lib/zksync_core/src/metadata_calculator/mod.rs @@ -7,7 +7,7 @@ use std::time::Duration; use zksync_config::configs::{ chain::OperationsManagerConfig, - database::{DBConfig, MerkleTreeMode}, + database::{MerkleTreeConfig, MerkleTreeMode}, }; use zksync_dal::{ConnectionPool, StorageProcessor}; use zksync_health_check::{HealthUpdater, ReactiveHealthCheck}; @@ -81,19 +81,19 @@ pub struct MetadataCalculatorConfig<'a> { impl<'a> MetadataCalculatorConfig<'a> { pub(crate) fn for_main_node( - db_config: &'a DBConfig, + merkle_tree_config: &'a MerkleTreeConfig, operation_config: &'a OperationsManagerConfig, mode: MetadataCalculatorModeConfig<'a>, ) -> Self { Self { - db_path: &db_config.merkle_tree.path, + db_path: &merkle_tree_config.path, mode, delay_interval: operation_config.delay_interval(), - max_l1_batches_per_iter: db_config.merkle_tree.max_l1_batches_per_iter, - multi_get_chunk_size: db_config.merkle_tree.multi_get_chunk_size, - block_cache_capacity: db_config.merkle_tree.block_cache_size(), - memtable_capacity: db_config.merkle_tree.memtable_capacity(), - stalled_writes_timeout: db_config.merkle_tree.stalled_writes_timeout(), + max_l1_batches_per_iter: merkle_tree_config.max_l1_batches_per_iter, + multi_get_chunk_size: merkle_tree_config.multi_get_chunk_size, + block_cache_capacity: merkle_tree_config.block_cache_size(), + memtable_capacity: merkle_tree_config.memtable_capacity(), + stalled_writes_timeout: merkle_tree_config.stalled_writes_timeout(), } } } diff --git a/core/lib/zksync_core/src/metadata_calculator/tests.rs b/core/lib/zksync_core/src/metadata_calculator/tests.rs index dca85e1f7d28..dd72410f950d 100644 --- a/core/lib/zksync_core/src/metadata_calculator/tests.rs +++ b/core/lib/zksync_core/src/metadata_calculator/tests.rs @@ -5,7 +5,7 @@ use tokio::sync::{mpsc, watch}; use std::{future::Future, ops, panic, path::Path, time::Duration}; -use zksync_config::{configs::chain::OperationsManagerConfig, DBConfig}; +use zksync_config::configs::{chain::OperationsManagerConfig, database::MerkleTreeConfig}; use zksync_contracts::BaseSystemContracts; use zksync_dal::{ConnectionPool, StorageProcessor}; use zksync_health_check::{CheckHealth, HealthStatus}; @@ -243,11 +243,11 @@ async fn shutting_down_calculator() { let pool = ConnectionPool::test_pool().await; let prover_pool = ConnectionPool::test_pool().await; let temp_dir = TempDir::new().expect("failed get temporary directory for RocksDB"); - let (db_config, mut operation_config) = create_config(temp_dir.path()); + let (merkle_tree_config, mut operation_config) = create_config(temp_dir.path()); operation_config.delay_interval = 30_000; // ms; chosen to be larger than `RUN_TIMEOUT` let calculator = setup_calculator_with_options( - &db_config, + &merkle_tree_config, &operation_config, &pool, MetadataCalculatorModeConfig::Lightweight, @@ -371,12 +371,12 @@ pub(crate) async fn setup_calculator( pool: &ConnectionPool, ) -> (MetadataCalculator, Box) { let store_factory = &ObjectStoreFactory::mock(); - let (db_config, operation_manager) = create_config(db_path); + let (merkle_tree_config, operation_manager) = create_config(db_path); let mode = MetadataCalculatorModeConfig::Full { store_factory: Some(store_factory), }; let calculator = - setup_calculator_with_options(&db_config, &operation_manager, pool, mode).await; + setup_calculator_with_options(&merkle_tree_config, &operation_manager, pool, mode).await; (calculator, store_factory.create_store().await) } @@ -386,10 +386,11 @@ async fn setup_lightweight_calculator(db_path: &Path, pool: &ConnectionPool) -> setup_calculator_with_options(&db_config, &operation_config, pool, mode).await } -fn create_config(db_path: &Path) -> (DBConfig, OperationsManagerConfig) { - let mut db_config = DBConfig::from_env().unwrap(); - db_config.merkle_tree.path = path_to_string(&db_path.join("new")); - db_config.backup_interval_ms = 0; +fn create_config(db_path: &Path) -> (MerkleTreeConfig, OperationsManagerConfig) { + let db_config = MerkleTreeConfig { + path: path_to_string(&db_path.join("new")), + ..Default::default() + }; let operation_config = OperationsManagerConfig { delay_interval: 50, // ms @@ -398,13 +399,13 @@ fn create_config(db_path: &Path) -> (DBConfig, OperationsManagerConfig) { } async fn setup_calculator_with_options( - db_config: &DBConfig, + merkle_tree_config: &MerkleTreeConfig, operation_config: &OperationsManagerConfig, pool: &ConnectionPool, mode: MetadataCalculatorModeConfig<'_>, ) -> MetadataCalculator { let calculator_config = - MetadataCalculatorConfig::for_main_node(db_config, operation_config, mode); + MetadataCalculatorConfig::for_main_node(merkle_tree_config, operation_config, mode); let metadata_calculator = MetadataCalculator::new(&calculator_config).await; let mut storage = pool.access_storage().await.unwrap(); diff --git a/core/lib/zksync_core/src/proof_data_handler/mod.rs b/core/lib/zksync_core/src/proof_data_handler/mod.rs index 2c3454662324..898ac4652ba2 100644 --- a/core/lib/zksync_core/src/proof_data_handler/mod.rs +++ b/core/lib/zksync_core/src/proof_data_handler/mod.rs @@ -18,21 +18,21 @@ use zksync_types::{ mod request_processor; -fn fri_l1_verifier_config_from_env() -> anyhow::Result { - let config = ContractsConfig::from_env().context("ContractsConfig::from_env()")?; - Ok(L1VerifierConfig { +fn fri_l1_verifier_config(contracts_config: &ContractsConfig) -> L1VerifierConfig { + L1VerifierConfig { params: VerifierParams { - recursion_node_level_vk_hash: config.fri_recursion_node_level_vk_hash, - recursion_leaf_level_vk_hash: config.fri_recursion_leaf_level_vk_hash, + recursion_node_level_vk_hash: contracts_config.fri_recursion_node_level_vk_hash, + recursion_leaf_level_vk_hash: contracts_config.fri_recursion_leaf_level_vk_hash, // The base layer commitment is not used in the FRI prover verification. recursion_circuits_set_vks_hash: H256::zero(), }, - recursion_scheduler_level_vk_hash: config.snark_wrapper_vk_hash, - }) + recursion_scheduler_level_vk_hash: contracts_config.snark_wrapper_vk_hash, + } } pub(crate) async fn run_server( config: ProofDataHandlerConfig, + contracts_config: ContractsConfig, blob_store: Box, pool: ConnectionPool, mut stop_receiver: watch::Receiver, @@ -41,9 +41,7 @@ pub(crate) async fn run_server( tracing::debug!("Starting proof data handler server on {bind_address}"); let l1_verifier_config: Option = match config.protocol_version_loading_mode { ProtocolVersionLoadingMode::FromDb => None, - ProtocolVersionLoadingMode::FromEnvVar => { - Some(fri_l1_verifier_config_from_env().context("fri_l1_verified_config_from_env()")?) - } + ProtocolVersionLoadingMode::FromEnvVar => Some(fri_l1_verifier_config(&contracts_config)), }; let get_proof_gen_processor = RequestProcessor::new(blob_store, pool, config, l1_verifier_config); diff --git a/core/lib/zksync_core/src/state_keeper/batch_executor/tests/tester.rs b/core/lib/zksync_core/src/state_keeper/batch_executor/tests/tester.rs index 84ee799fe12f..c3f03fd86321 100644 --- a/core/lib/zksync_core/src/state_keeper/batch_executor/tests/tester.rs +++ b/core/lib/zksync_core/src/state_keeper/batch_executor/tests/tester.rs @@ -6,7 +6,6 @@ use tempfile::TempDir; use multivm::interface::{L1BatchEnv, SystemEnv}; use multivm::vm_latest::constants::INITIAL_STORAGE_WRITE_PUBDATA_BYTES; -use zksync_config::configs::chain::StateKeeperConfig; use zksync_contracts::{get_loadnext_contract, test_contracts::LoadnextContractExecutionParams}; use zksync_dal::ConnectionPool; use zksync_state::RocksdbStorage; @@ -41,14 +40,17 @@ pub(super) struct TestConfig { impl TestConfig { pub(super) fn new() -> Self { - // It's OK to use env config here, since we would load the postgres URL from there anyway. - let config = StateKeeperConfig::from_env().unwrap(); + // Values are taken from the actual env config used in local development. + // These values remain static for a long time, so it's a better alternative than to introduce dependency + // on env config. + const MAX_ALLOWED_TX_GAS_LIMIT: u32 = 4000000000; + const VALIDATION_COMPUTATIONAL_GAS_LIMIT: u32 = 300000; Self { vm_gas_limit: None, save_call_traces: false, - max_allowed_tx_gas_limit: config.max_allowed_l2_tx_gas_limit, - validation_computational_gas_limit: config.validation_computational_gas_limit, + max_allowed_tx_gas_limit: MAX_ALLOWED_TX_GAS_LIMIT, + validation_computational_gas_limit: VALIDATION_COMPUTATIONAL_GAS_LIMIT, upload_witness_inputs_to_gcs: false, } } diff --git a/core/lib/zksync_core/src/state_keeper/io/mempool.rs b/core/lib/zksync_core/src/state_keeper/io/mempool.rs index fced87479283..f10ad87580c0 100644 --- a/core/lib/zksync_core/src/state_keeper/io/mempool.rs +++ b/core/lib/zksync_core/src/state_keeper/io/mempool.rs @@ -1,4 +1,3 @@ -use anyhow::Context as _; use async_trait::async_trait; use std::{ @@ -14,7 +13,7 @@ use multivm::vm_latest::utils::fee::derive_base_fee_and_gas_per_pubdata; use zksync_config::configs::chain::StateKeeperConfig; use zksync_dal::ConnectionPool; use zksync_mempool::L2TxFilter; -use zksync_object_store::ObjectStoreFactory; +use zksync_object_store::ObjectStore; use zksync_types::{ block::MiniblockHeader, protocol_version::ProtocolUpgradeTx, witness_block_state::WitnessBlockState, Address, L1BatchNumber, L2ChainId, MiniblockNumber, @@ -47,6 +46,7 @@ use crate::{ pub(crate) struct MempoolIO { mempool: MempoolGuard, pool: ConnectionPool, + object_store: Box, timeout_sealer: TimeoutSealer, filter: L2TxFilter, current_miniblock_number: MiniblockNumber, @@ -297,11 +297,8 @@ where self.miniblock_sealer_handle.wait_for_all_commands().await; if let Some(witness_witness_block_state) = witness_block_state { - let object_store = ObjectStoreFactory::from_env() - .context("ObjectsStoreFactor::from_env()")? - .create_store() - .await; - match object_store + match self + .object_store .put(self.current_l1_batch_number(), &witness_witness_block_state) .await { @@ -405,6 +402,7 @@ impl MempoolIO { #[allow(clippy::too_many_arguments)] pub(in crate::state_keeper) async fn new( mempool: MempoolGuard, + object_store: Box, miniblock_sealer_handle: MiniblockSealerHandle, l1_gas_price_provider: Arc, pool: ConnectionPool, @@ -439,6 +437,7 @@ impl MempoolIO { Self { mempool, + object_store, pool, timeout_sealer: TimeoutSealer::new(config), filter: L2TxFilter::default(), diff --git a/core/lib/zksync_core/src/state_keeper/io/tests/tester.rs b/core/lib/zksync_core/src/state_keeper/io/tests/tester.rs index d685e1fc81d9..875bf89e0481 100644 --- a/core/lib/zksync_core/src/state_keeper/io/tests/tester.rs +++ b/core/lib/zksync_core/src/state_keeper/io/tests/tester.rs @@ -2,6 +2,7 @@ use multivm::vm_latest::constants::BLOCK_GAS_LIMIT; use std::{sync::Arc, time::Duration}; +use zksync_object_store::ObjectStoreFactory; use zksync_config::configs::chain::StateKeeperConfig; use zksync_config::GasAdjusterConfig; @@ -72,18 +73,17 @@ impl Tester { MiniblockSealer::new(pool.clone(), miniblock_sealer_capacity); tokio::spawn(miniblock_sealer.run()); - let base_contract_hashes = self.base_system_contracts.hashes(); let config = StateKeeperConfig { fair_l2_gas_price: self.fair_l2_gas_price(), - bootloader_hash: base_contract_hashes.bootloader, - default_aa_hash: base_contract_hashes.default_aa, virtual_blocks_interval: 1, virtual_blocks_per_miniblock: 1, ..StateKeeperConfig::default() }; + let object_store = ObjectStoreFactory::mock().create_store().await; let l2_erc20_bridge_addr = Address::repeat_byte(0x5a); // Isn't relevant. let io = MempoolIO::new( mempool.clone(), + object_store, miniblock_sealer_handle, gas_adjuster, pool, diff --git a/core/lib/zksync_core/src/state_keeper/mod.rs b/core/lib/zksync_core/src/state_keeper/mod.rs index 18401001b4de..bdc1f90e2066 100644 --- a/core/lib/zksync_core/src/state_keeper/mod.rs +++ b/core/lib/zksync_core/src/state_keeper/mod.rs @@ -1,4 +1,5 @@ use tokio::sync::watch; +use zksync_object_store::ObjectStore; use std::sync::Arc; @@ -44,6 +45,7 @@ pub(crate) async fn create_state_keeper( mempool: MempoolGuard, l1_gas_price_provider: Arc, miniblock_sealer_handle: MiniblockSealerHandle, + object_store: Box, stop_receiver: watch::Receiver, ) -> ZkSyncStateKeeper where @@ -67,6 +69,7 @@ where let io = MempoolIO::new( mempool, + object_store, miniblock_sealer_handle, l1_gas_price_provider, pool, diff --git a/core/lib/zksync_core/src/state_keeper/seal_criteria/criteria/gas.rs b/core/lib/zksync_core/src/state_keeper/seal_criteria/criteria/gas.rs index 4683d9cf8798..1472e05fad90 100644 --- a/core/lib/zksync_core/src/state_keeper/seal_criteria/criteria/gas.rs +++ b/core/lib/zksync_core/src/state_keeper/seal_criteria/criteria/gas.rs @@ -56,7 +56,14 @@ mod tests { #[test] fn test_gas_seal_criterion() { - let config = StateKeeperConfig::from_env().unwrap(); + // Create an empty config and only setup fields relevant for the test. + let config = StateKeeperConfig { + max_single_tx_gas: 6000000, + reject_tx_at_gas_percentage: 0.95, + close_block_at_gas_percentage: 0.95, + ..Default::default() + }; + let criterion = GasCriterion; // Empty block should fit into gas criterion. diff --git a/core/lib/zksync_core/src/state_keeper/seal_criteria/criteria/pubdata_bytes.rs b/core/lib/zksync_core/src/state_keeper/seal_criteria/criteria/pubdata_bytes.rs index 143104d5ae53..61f30d724a70 100644 --- a/core/lib/zksync_core/src/state_keeper/seal_criteria/criteria/pubdata_bytes.rs +++ b/core/lib/zksync_core/src/state_keeper/seal_criteria/criteria/pubdata_bytes.rs @@ -58,7 +58,13 @@ mod tests { #[test] fn seal_criterion() { - let config = StateKeeperConfig::from_env().unwrap(); + // Create an empty config and only setup fields relevant for the test. + let config = StateKeeperConfig { + reject_tx_at_eth_params_percentage: 0.95, + close_block_at_eth_params_percentage: 0.95, + ..Default::default() + }; + let criterion = PubDataBytesCriterion; let block_execution_metrics = ExecutionMetrics { diff --git a/core/lib/zksync_core/src/state_keeper/seal_criteria/criteria/slots.rs b/core/lib/zksync_core/src/state_keeper/seal_criteria/criteria/slots.rs index b434c3ba42a7..4c21c41e5e40 100644 --- a/core/lib/zksync_core/src/state_keeper/seal_criteria/criteria/slots.rs +++ b/core/lib/zksync_core/src/state_keeper/seal_criteria/criteria/slots.rs @@ -36,7 +36,12 @@ mod tests { #[test] fn test_slots_seal_criterion() { - let config = StateKeeperConfig::from_env().unwrap(); + // Create an empty config and only setup fields relevant for the test. + let config = StateKeeperConfig { + transaction_slots: 2, + ..Default::default() + }; + let criterion = SlotsCriterion; let almost_full_block_resolution = criterion.should_seal( diff --git a/core/lib/zksync_core/src/state_keeper/seal_criteria/criteria/tx_encoding_size.rs b/core/lib/zksync_core/src/state_keeper/seal_criteria/criteria/tx_encoding_size.rs index 4672ac36e557..ed24e3719338 100644 --- a/core/lib/zksync_core/src/state_keeper/seal_criteria/criteria/tx_encoding_size.rs +++ b/core/lib/zksync_core/src/state_keeper/seal_criteria/criteria/tx_encoding_size.rs @@ -47,7 +47,13 @@ mod tests { #[test] fn seal_criterion() { - let config = StateKeeperConfig::from_env().unwrap(); + // Create an empty config and only setup fields relevant for the test. + let config = StateKeeperConfig { + reject_tx_at_geometry_percentage: 0.95, + close_block_at_geometry_percentage: 0.95, + ..Default::default() + }; + let criterion = TxEncodingSizeCriterion; let empty_block_resolution = criterion.should_seal( diff --git a/core/lib/zksync_core/src/sync_layer/tests.rs b/core/lib/zksync_core/src/sync_layer/tests.rs index 8fe61fe16e6d..e454c0098e12 100644 --- a/core/lib/zksync_core/src/sync_layer/tests.rs +++ b/core/lib/zksync_core/src/sync_layer/tests.rs @@ -578,7 +578,7 @@ async fn fetcher_with_real_server() { let mut tx_hashes = VecDeque::from(tx_hashes); // Start the API server. - let network_config = NetworkConfig::from_env().unwrap(); + let network_config = NetworkConfig::for_tests(); let (stop_sender, stop_receiver) = watch::channel(false); let server_handles = spawn_http_server(&network_config, pool.clone(), stop_receiver.clone()).await; diff --git a/core/lib/zksync_core/src/temp_config_store.rs b/core/lib/zksync_core/src/temp_config_store.rs new file mode 100644 index 000000000000..618f8ac6a274 --- /dev/null +++ b/core/lib/zksync_core/src/temp_config_store.rs @@ -0,0 +1,47 @@ +use zksync_config::{ + configs::{ + api::{HealthCheckConfig, MerkleTreeApiConfig, Web3JsonRpcConfig}, + chain::{ + CircuitBreakerConfig, MempoolConfig, NetworkConfig, OperationsManagerConfig, + StateKeeperConfig, + }, + house_keeper::HouseKeeperConfig, + FriProofCompressorConfig, FriProverConfig, FriWitnessGeneratorConfig, PrometheusConfig, + ProofDataHandlerConfig, ProverGroupConfig, WitnessGeneratorConfig, + }, + ApiConfig, ContractsConfig, DBConfig, ETHClientConfig, ETHSenderConfig, ETHWatchConfig, + FetcherConfig, GasAdjusterConfig, ObjectStoreConfig, ProverConfigs, +}; + +// TODO (QIT-22): This structure is going to be removed when components will be respnsible for their own configs. +/// A temporary config store allowing to pass deserialized configs from `zksync_server` to `zksync_core`. +/// All the configs are optional, since for some component combination it is not needed to pass all the configs. +#[derive(Debug)] +pub struct TempConfigStore { + pub health_check_config: Option, + pub merkle_tree_api_config: Option, + pub web3_json_rpc_config: Option, + pub circuit_breaker_config: Option, + pub mempool_config: Option, + pub network_config: Option, + pub operations_manager_config: Option, + pub state_keeper_config: Option, + pub house_keeper_config: Option, + pub fri_proof_compressor_config: Option, + pub fri_prover_config: Option, + pub fri_witness_generator_config: Option, + pub prometheus_config: Option, + pub proof_data_handler_config: Option, + pub prover_group_config: Option, + pub witness_generator_config: Option, + pub api_config: Option, + pub contracts_config: Option, + pub db_config: Option, + pub eth_client_config: Option, + pub eth_sender_config: Option, + pub eth_watch_config: Option, + pub fetcher_config: Option, + pub gas_adjuster_config: Option, + pub prover_configs: Option, + pub object_store_config: Option, +} diff --git a/core/lib/zksync_core/src/witness_generator/basic_circuits.rs b/core/lib/zksync_core/src/witness_generator/basic_circuits.rs index e48d1f1db6d9..de6ede7e14fd 100644 --- a/core/lib/zksync_core/src/witness_generator/basic_circuits.rs +++ b/core/lib/zksync_core/src/witness_generator/basic_circuits.rs @@ -1,4 +1,3 @@ -use anyhow::Context as _; use async_trait::async_trait; use rand::Rng; use serde::{Deserialize, Serialize}; @@ -13,8 +12,9 @@ use std::{ use multivm::vm_latest::{ constants::MAX_CYCLES_FOR_TX, HistoryDisabled, SimpleMemory, StorageOracle as VmStorageOracle, }; -use zksync_config::configs::witness_generator::BasicWitnessGeneratorDataSource; -use zksync_config::configs::WitnessGeneratorConfig; +use zksync_config::configs::{ + witness_generator::BasicWitnessGeneratorDataSource, WitnessGeneratorConfig, +}; use zksync_dal::ConnectionPool; use zksync_object_store::{Bucket, ObjectStore, ObjectStoreFactory, StoredObject}; use zksync_queued_job_processor::JobProcessor; @@ -88,14 +88,13 @@ impl BasicWitnessGenerator { } async fn process_job_impl( + config: WitnessGeneratorConfig, object_store: Arc, connection_pool: ConnectionPool, prover_connection_pool: ConnectionPool, basic_job: BasicWitnessGeneratorJob, started_at: Instant, ) -> anyhow::Result> { - let config = - WitnessGeneratorConfig::from_env().context("WitnessGeneratorConfig::from_env()")?; let BasicWitnessGeneratorJob { block_number, job } = basic_job; if let Some(blocks_proving_percentage) = config.blocks_proving_percentage { @@ -213,7 +212,9 @@ impl JobProcessor for BasicWitnessGenerator { started_at: Instant, ) -> tokio::task::JoinHandle>> { let object_store = Arc::clone(&self.object_store); + let config = self.config.clone(); tokio::spawn(Self::process_job_impl( + config, object_store, self.connection_pool.clone(), self.prover_connection_pool.clone(), diff --git a/core/lib/zksync_core/src/witness_generator/node_aggregation.rs b/core/lib/zksync_core/src/witness_generator/node_aggregation.rs index 7ba3b746065d..349ab56b1556 100644 --- a/core/lib/zksync_core/src/witness_generator/node_aggregation.rs +++ b/core/lib/zksync_core/src/witness_generator/node_aggregation.rs @@ -1,4 +1,3 @@ -use anyhow::Context as _; use async_trait::async_trait; use std::{collections::HashMap, env, time::Instant}; @@ -76,11 +75,10 @@ impl NodeAggregationWitnessGenerator { } fn process_job_sync( + config: WitnessGeneratorConfig, node_job: NodeAggregationWitnessGeneratorJob, started_at: Instant, ) -> anyhow::Result { - let config = - WitnessGeneratorConfig::from_env().context("WitnessGeneratorConfig::from_env()")?; let NodeAggregationWitnessGeneratorJob { block_number, job } = node_job; tracing::info!( @@ -162,7 +160,8 @@ impl JobProcessor for NodeAggregationWitnessGenerator { job: NodeAggregationWitnessGeneratorJob, started_at: Instant, ) -> tokio::task::JoinHandle> { - tokio::task::spawn_blocking(move || Self::process_job_sync(job, started_at)) + let config = self.config.clone(); + tokio::task::spawn_blocking(move || Self::process_job_sync(config, job, started_at)) } async fn save_result( diff --git a/etc/env/base/chain.toml b/etc/env/base/chain.toml index adf0b7d572c9..db9d87107c41 100644 --- a/etc/env/base/chain.toml +++ b/etc/env/base/chain.toml @@ -45,9 +45,6 @@ reject_tx_at_eth_params_percentage=0.95 # it takes more percentage of the max block gas capacity than this value. reject_tx_at_gas_percentage=0.95 -bootloader_hash="0x0100038581be3d0e201b3cc45d151ef5cc59eb3a0f146ad44f0f72abf00b594c" -default_aa_hash="0x0100038dc66b69be75ec31653c64cb931678299b9b659472772b2550b703f41c" - # The price the operator spends on 1 gas of computation in wei. fair_l2_gas_price=250000000 diff --git a/prover/circuit_synthesizer/Cargo.toml b/prover/circuit_synthesizer/Cargo.toml index 3bee4194af91..d8de49fb7655 100644 --- a/prover/circuit_synthesizer/Cargo.toml +++ b/prover/circuit_synthesizer/Cargo.toml @@ -12,6 +12,7 @@ zksync_dal = { path = "../../core/lib/dal" } zksync_types = { path = "../../core/lib/types" } zksync_queued_job_processor = { path = "../../core/lib/queued_job_processor" } zksync_config = { path = "../../core/lib/config" } +zksync_env_config = { path = "../../core/lib/env_config" } zksync_object_store = { path = "../../core/lib/object_store" } zksync_utils = { path = "../../core/lib/utils" } zksync_prover_fri_utils = { path = "../prover_fri_utils" } diff --git a/prover/circuit_synthesizer/src/circuit_synthesizer.rs b/prover/circuit_synthesizer/src/circuit_synthesizer.rs index 6fbd9d267ab7..ce1a5251d53c 100644 --- a/prover/circuit_synthesizer/src/circuit_synthesizer.rs +++ b/prover/circuit_synthesizer/src/circuit_synthesizer.rs @@ -17,6 +17,7 @@ use zksync_config::configs::prover_group::ProverGroupConfig; use zksync_config::configs::CircuitSynthesizerConfig; use zksync_config::ProverConfigs; use zksync_dal::ConnectionPool; +use zksync_env_config::FromEnv; use zksync_object_store::{CircuitKey, ObjectStore, ObjectStoreError, ObjectStoreFactory}; use zksync_prover_fri_utils::socket_utils::send_assembly; use zksync_prover_utils::numeric_index_to_circuit_name; @@ -88,10 +89,10 @@ impl CircuitSynthesizer { blob_store: store_factory.create_store().await, allowed_circuit_types: allowed_circuit_types .map(|x| x.into_iter().map(|x| x.1).collect()), - region: get_region() + region: get_region(&prover_groups) .await .map_err(CircuitSynthesizerError::GetRegionFailed)?, - zone: get_zone() + zone: get_zone(&prover_groups) .await .map_err(CircuitSynthesizerError::GetZoneFailed)?, vk_commitments, diff --git a/prover/circuit_synthesizer/src/main.rs b/prover/circuit_synthesizer/src/main.rs index 0462c5c78610..5c816d00903e 100644 --- a/prover/circuit_synthesizer/src/main.rs +++ b/prover/circuit_synthesizer/src/main.rs @@ -3,9 +3,12 @@ use prometheus_exporter::PrometheusExporterConfig; use structopt::StructOpt; use tokio::{sync::oneshot, sync::watch}; -use zksync_config::configs::{AlertsConfig, CircuitSynthesizerConfig, ProverGroupConfig}; +use zksync_config::configs::{ + AlertsConfig, CircuitSynthesizerConfig, ObjectStoreConfig, ProverGroupConfig, +}; use zksync_dal::connection::DbVariant; use zksync_dal::ConnectionPool; +use zksync_env_config::FromEnv; use zksync_object_store::ObjectStoreFactory; use zksync_queued_job_processor::JobProcessor; use zksync_utils::wait_for_tasks::wait_for_tasks; @@ -50,10 +53,12 @@ async fn main() -> anyhow::Result<()> { .context("failed to build a connection pool")?; let vk_commitments = get_cached_commitments(); + let object_store_config = + ObjectStoreConfig::from_env().context("ObjectStoreConfig::from_env()")?; let circuit_synthesizer = CircuitSynthesizer::new( config.clone(), ProverGroupConfig::from_env().context("ProverGroupConfig::from_env()")?, - &ObjectStoreFactory::from_env().context("ObjectStoreFactory::from_env()")?, + &ObjectStoreFactory::new(object_store_config), vk_commitments, pool, ) diff --git a/prover/proof_fri_compressor/Cargo.toml b/prover/proof_fri_compressor/Cargo.toml index 02ba39e393c8..45ffe756e196 100644 --- a/prover/proof_fri_compressor/Cargo.toml +++ b/prover/proof_fri_compressor/Cargo.toml @@ -9,6 +9,7 @@ edition = "2021" zksync_types = { path = "../../core/lib/types" } zksync_dal = { path = "../../core/lib/dal" } zksync_config = { path = "../../core/lib/config" } +zksync_env_config = { path = "../../core/lib/env_config" } zksync_object_store = { path = "../../core/lib/object_store" } zksync_utils = { path = "../../core/lib/utils" } prometheus_exporter = { path = "../../core/lib/prometheus_exporter" } diff --git a/prover/proof_fri_compressor/src/main.rs b/prover/proof_fri_compressor/src/main.rs index bb3a8bf3d54c..a805bb3e3216 100644 --- a/prover/proof_fri_compressor/src/main.rs +++ b/prover/proof_fri_compressor/src/main.rs @@ -9,6 +9,7 @@ use prometheus_exporter::PrometheusExporterConfig; use zksync_config::configs::FriProofCompressorConfig; use zksync_dal::connection::DbVariant; use zksync_dal::ConnectionPool; +use zksync_env_config::{object_store::ProverObjectStoreConfig, FromEnv}; use zksync_object_store::ObjectStoreFactory; use zksync_queued_job_processor::JobProcessor; use zksync_utils::wait_for_tasks::wait_for_tasks; @@ -52,8 +53,9 @@ async fn main() -> anyhow::Result<()> { .build() .await .context("failed to build a connection pool")?; - let blob_store = ObjectStoreFactory::prover_from_env() - .context("ObjectSToreFactor::prover_from_env()")? + let object_store_config = + ProverObjectStoreConfig::from_env().context("ProverObjectStoreConfig::from_env()")?; + let blob_store = ObjectStoreFactory::new(object_store_config.0) .create_store() .await; let proof_compressor = ProofCompressor::new( diff --git a/prover/prover/Cargo.toml b/prover/prover/Cargo.toml index 736d42e734e7..d0b66f3e3f7f 100644 --- a/prover/prover/Cargo.toml +++ b/prover/prover/Cargo.toml @@ -13,6 +13,7 @@ publish = false # We don't want to publish our binaries. [dependencies] zksync_dal = { path = "../../core/lib/dal" } zksync_config = { path = "../../core/lib/config" } +zksync_env_config = { path = "../../core/lib/env_config" } zksync_utils = { path = "../../core/lib/utils" } zksync_prover_utils = { path = "../../core/lib/prover_utils" } zksync_circuit_breaker = { path = "../../core/lib/circuit_breaker" } diff --git a/prover/prover/src/run.rs b/prover/prover/src/run.rs index ae66c3a88d82..53a9c821f040 100644 --- a/prover/prover/src/run.rs +++ b/prover/prover/src/run.rs @@ -7,10 +7,13 @@ use queues::Buffer; use prometheus_exporter::PrometheusExporterConfig; use zksync_config::{ - configs::{api::PrometheusConfig, prover_group::ProverGroupConfig, AlertsConfig}, + configs::{ + api::PrometheusConfig, prover_group::ProverGroupConfig, AlertsConfig, ObjectStoreConfig, + }, ApiConfig, ProverConfig, ProverConfigs, }; use zksync_dal::{connection::DbVariant, ConnectionPool}; +use zksync_env_config::FromEnv; use zksync_object_store::ObjectStoreFactory; use zksync_prover_utils::region_fetcher::{get_region, get_zone}; use zksync_types::proofs::{GpuProverInstanceStatus, SocketAddress}; @@ -32,8 +35,12 @@ async fn graceful_shutdown() -> anyhow::Result> { .context("ProverConfigs")? .non_gpu .assembly_receiver_port; - let region = get_region().await.context("get_region()")?; - let zone = get_zone().await.context("get_zone()")?; + let prover_group_config = + ProverGroupConfig::from_env().context("ProverGroupConfig::from_env()")?; + let region = get_region(&prover_group_config) + .await + .context("get_region()")?; + let zone = get_zone(&prover_group_config).await.context("get_zone()")?; let address = SocketAddress { host, port }; Ok(async move { pool.access_storage() @@ -119,8 +126,12 @@ pub async fn run() -> anyhow::Result<()> { .prometheus }; - let region = get_region().await.context("get_regtion()")?; - let zone = get_zone().await.context("get_zone()")?; + let prover_group_config = + ProverGroupConfig::from_env().context("ProverGroupConfig::from_env()")?; + let region = get_region(&prover_group_config) + .await + .context("get_region()")?; + let zone = get_zone(&prover_group_config).await.context("get_zone()")?; let (stop_signal_sender, stop_signal_receiver) = oneshot::channel(); let mut stop_signal_sender = Some(stop_signal_sender); @@ -143,9 +154,8 @@ pub async fn run() -> anyhow::Result<()> { // Though we still need to create a channel because circuit breaker expects `stop_receiver`. let (_stop_sender, stop_receiver) = tokio::sync::watch::channel(false); - let circuit_ids = ProverGroupConfig::from_env() - .context("ProverGroupConfig::from_env()")? - .get_circuit_ids_for_group_id(prover_config.specialized_prover_group_id); + let circuit_ids = + prover_group_config.get_circuit_ids_for_group_id(prover_config.specialized_prover_group_id); tracing::info!( "Starting proof generation for circuits: {circuit_ids:?} \ @@ -183,7 +193,9 @@ pub async fn run() -> anyhow::Result<()> { ))); let params = ProverParams::new(&prover_config); - let store_factory = ObjectStoreFactory::from_env().context("ObjectStoreFactory::from_env()")?; + let object_store_config = + ObjectStoreConfig::from_env().context("ObjectStoreConfig::from_env()")?; + let store_factory = ObjectStoreFactory::new(object_store_config); let circuit_provider_pool = ConnectionPool::singleton(DbVariant::Prover) .build() diff --git a/prover/prover_fri/Cargo.toml b/prover/prover_fri/Cargo.toml index b351b6bf604f..314f034a9a63 100644 --- a/prover/prover_fri/Cargo.toml +++ b/prover/prover_fri/Cargo.toml @@ -9,6 +9,7 @@ edition = "2021" zksync_types = { path = "../../core/lib/types" } zksync_dal = { path = "../../core/lib/dal" } zksync_config = { path = "../../core/lib/config" } +zksync_env_config = { path = "../../core/lib/env_config" } prometheus_exporter = { path = "../../core/lib/prometheus_exporter" } vlog = { path = "../../core/lib/vlog" } zksync_object_store = { path = "../../core/lib/object_store" } diff --git a/prover/prover_fri/src/gpu_prover_job_processor.rs b/prover/prover_fri/src/gpu_prover_job_processor.rs index 79fef4bcc4bc..1919d5888214 100644 --- a/prover/prover_fri/src/gpu_prover_job_processor.rs +++ b/prover/prover_fri/src/gpu_prover_job_processor.rs @@ -19,6 +19,7 @@ pub mod gpu_prover { use zksync_config::configs::fri_prover_group::FriProverGroupConfig; use zksync_config::configs::FriProverConfig; use zksync_dal::ConnectionPool; + use zksync_env_config::FromEnv; use zksync_object_store::ObjectStore; use zksync_prover_fri_types::{CircuitWrapper, FriProofWrapper, ProverServiceDataKey}; use zksync_queued_job_processor::{async_trait, JobProcessor}; @@ -285,6 +286,7 @@ pub mod gpu_prover { &config.specialized_group_id ); let prover_setup_metadata_list = FriProverGroupConfig::from_env() + .context("FriProverGroupConfig::from_env()")? .get_circuit_ids_for_group_id(config.specialized_group_id) .context( "At least one circuit should be configured for group when running in FromMemory mode", diff --git a/prover/prover_fri/src/main.rs b/prover/prover_fri/src/main.rs index f387393c0727..9d2a8e4ff9f1 100644 --- a/prover/prover_fri/src/main.rs +++ b/prover/prover_fri/src/main.rs @@ -7,10 +7,13 @@ use tokio::task::JoinHandle; use prometheus_exporter::PrometheusExporterConfig; use zksync_config::configs::fri_prover_group::FriProverGroupConfig; -use zksync_config::configs::FriProverConfig; -use zksync_config::ObjectStoreConfig; +use zksync_config::configs::{FriProverConfig, ProverGroupConfig}; use zksync_dal::connection::DbVariant; use zksync_dal::ConnectionPool; +use zksync_env_config::{ + object_store::{ProverObjectStoreConfig, PublicObjectStoreConfig}, + FromEnv, +}; use zksync_object_store::{ObjectStore, ObjectStoreFactory}; use zksync_prover_fri_utils::get_all_circuit_id_round_tuples_for; @@ -33,7 +36,9 @@ async fn graceful_shutdown(port: u16) -> anyhow::Result .await .context("failed to build a connection pool")?; let host = local_ip().context("Failed obtaining local IP address")?; - let zone = get_zone().await.context("get_zone()")?; + let prover_group_config = + ProverGroupConfig::from_env().context("ProverGroupConfig::from_env()")?; + let zone = get_zone(&prover_group_config).await.context("get_zone()")?; let address = SocketAddress { host, port }; Ok(async move { pool.access_storage() @@ -83,22 +88,23 @@ async fn main() -> anyhow::Result<()> { .context("Error setting Ctrl+C handler")?; let (stop_sender, stop_receiver) = tokio::sync::watch::channel(false); - let blob_store = - ObjectStoreFactory::prover_from_env().context("ObjectStoreFactory::prover_from_env()")?; + let object_store_config = + ProverObjectStoreConfig::from_env().context("ProverObjectStoreConfig::from_env()")?; + let object_store_factory = ObjectStoreFactory::new(object_store_config.0); + let public_object_store_config = + PublicObjectStoreConfig::from_env().context("PublicObjectStoreConfig::from_env()")?; let public_blob_store = match prover_config.shall_save_to_public_bucket { false => None, true => Some( - ObjectStoreFactory::new( - ObjectStoreConfig::public_from_env() - .context("ObjectStoreConfig::public_from_env()")?, - ) - .create_store() - .await, + ObjectStoreFactory::new(public_object_store_config.0) + .create_store() + .await, ), }; let specialized_group_id = prover_config.specialized_group_id; let circuit_ids_for_round_to_be_proven = FriProverGroupConfig::from_env() + .context("FriProverGroupConfig::from_env()")? .get_circuit_ids_for_group_id(specialized_group_id) .unwrap_or_default(); let circuit_ids_for_round_to_be_proven = @@ -117,7 +123,7 @@ async fn main() -> anyhow::Result<()> { let prover_tasks = get_prover_tasks( prover_config, stop_receiver.clone(), - blob_store, + object_store_factory, public_blob_store, pool, circuit_ids_for_round_to_be_proven, @@ -202,7 +208,10 @@ async fn get_prover_tasks( let witness_vector_queue = FixedSizeQueue::new(prover_config.queue_capacity); let shared_witness_vector_queue = Arc::new(Mutex::new(witness_vector_queue)); let consumer = shared_witness_vector_queue.clone(); - let zone = get_zone().await.context("get_zone()")?; + + let prover_group_config = + ProverGroupConfig::from_env().context("ProverGroupConfig::from_env()")?; + let zone = get_zone(&prover_group_config).await.context("get_zone()")?; let local_ip = local_ip().context("Failed obtaining local IP address")?; let address = SocketAddress { host: local_ip, diff --git a/prover/prover_fri/src/prover_job_processor.rs b/prover/prover_fri/src/prover_job_processor.rs index bedee075b83d..bc53465af91f 100644 --- a/prover/prover_fri/src/prover_job_processor.rs +++ b/prover/prover_fri/src/prover_job_processor.rs @@ -22,6 +22,7 @@ use zkevm_test_harness::prover_utils::{prove_base_layer_circuit, prove_recursion use zksync_config::configs::fri_prover_group::FriProverGroupConfig; use zksync_config::configs::FriProverConfig; use zksync_dal::ConnectionPool; +use zksync_env_config::FromEnv; use zksync_object_store::ObjectStore; use zksync_prover_fri_types::{CircuitWrapper, FriProofWrapper, ProverJob, ProverServiceDataKey}; use zksync_prover_fri_utils::fetch_next_circuit; @@ -271,6 +272,7 @@ pub fn load_setup_data_cache(config: &FriProverConfig) -> anyhow::Result anyhow::Result<()> { .build() .await .context("failed to build a connection pool")?; - let store_factory = - ObjectStoreFactory::prover_from_env().context("ObjectStoreFactory::prover_from_env()")?; + let object_store_config = + ProverObjectStoreConfig::from_env().context("ProverObjectStoreConfig::from_env()")?; + let store_factory = ObjectStoreFactory::new(object_store_config.0); let proof_submitter = PeriodicApiStruct { blob_store: store_factory.create_store().await, diff --git a/prover/setup_key_generator_and_server/Cargo.toml b/prover/setup_key_generator_and_server/Cargo.toml index 2b22a5519437..22b1cef97bd6 100644 --- a/prover/setup_key_generator_and_server/Cargo.toml +++ b/prover/setup_key_generator_and_server/Cargo.toml @@ -21,6 +21,7 @@ path = "src/main.rs" zksync_types = { path = "../../core/lib/types" } vlog = { path = "../../core/lib/vlog" } zksync_config = { path = "../../core/lib/config" } +zksync_env_config = { path = "../../core/lib/env_config" } circuit_testing = { git = "https://github.com/matter-labs/era-circuit_testing.git", branch = "main" } api = { git = "https://github.com/matter-labs/era-heavy-ops-service.git", branch = "v1.3.3", features = [ diff --git a/prover/setup_key_generator_and_server/src/lib.rs b/prover/setup_key_generator_and_server/src/lib.rs index 141a17b7d4f2..a2e4f0998f85 100644 --- a/prover/setup_key_generator_and_server/src/lib.rs +++ b/prover/setup_key_generator_and_server/src/lib.rs @@ -11,6 +11,7 @@ use zkevm_test_harness::witness::vk_set_generator::circuits_for_vk_generation; use zksync_types::circuit::GEOMETRY_CONFIG; use zksync_config::ProverConfigs; +use zksync_env_config::FromEnv; use zksync_types::circuit::{LEAF_SPLITTING_FACTOR, NODE_SPLITTING_FACTOR, SCHEDULER_UPPER_BOUND}; pub fn get_setup_for_circuit_type(circuit_type: u8) -> anyhow::Result> { let filepath = get_setup_key_file_path(circuit_type).context("get_setup_key_file_path()")?; diff --git a/prover/vk_setup_data_generator_server_fri/Cargo.toml b/prover/vk_setup_data_generator_server_fri/Cargo.toml index bae690ed0410..59df5f37cf89 100644 --- a/prover/vk_setup_data_generator_server_fri/Cargo.toml +++ b/prover/vk_setup_data_generator_server_fri/Cargo.toml @@ -32,6 +32,7 @@ circuit_definitions = { git = "https://github.com/matter-labs/era-zkevm_test_har ] } shivini = { git = "https://github.com/matter-labs/era-shivini.git", branch = "main", optional = true } zksync_config = { path = "../../core/lib/config" } +zksync_env_config = { path = "../../core/lib/env_config" } anyhow = "1.0" tracing = "0.1" diff --git a/prover/vk_setup_data_generator_server_fri/src/lib.rs b/prover/vk_setup_data_generator_server_fri/src/lib.rs index ed391fbc86df..75125aa81508 100644 --- a/prover/vk_setup_data_generator_server_fri/src/lib.rs +++ b/prover/vk_setup_data_generator_server_fri/src/lib.rs @@ -43,6 +43,7 @@ use serde::de::DeserializeOwned; use serde::{Deserialize, Serialize}; use zkevm_test_harness::prover_utils::create_base_layer_setup_data; use zksync_config::configs::FriProverConfig; +use zksync_env_config::FromEnv; use zksync_types::proofs::AggregationRound; use zksync_types::zkevm_test_harness::abstract_zksync_circuit::concrete_circuits::ZkSyncCircuit; use zksync_types::zkevm_test_harness::bellman::bn256::Bn256; diff --git a/prover/witness_generator/Cargo.toml b/prover/witness_generator/Cargo.toml index ef2ce86bc579..a8dce0547b23 100644 --- a/prover/witness_generator/Cargo.toml +++ b/prover/witness_generator/Cargo.toml @@ -13,6 +13,7 @@ publish = false # We don't want to publish our binaries. [dependencies] zksync_dal = { path = "../../core/lib/dal" } zksync_config = { path = "../../core/lib/config" } +zksync_env_config = { path = "../../core/lib/env_config" } zksync_system_constants = { path = "../../core/lib/constants" } prometheus_exporter = { path = "../../core/lib/prometheus_exporter" } vlog = { path = "../../core/lib/vlog" } diff --git a/prover/witness_generator/src/main.rs b/prover/witness_generator/src/main.rs index f44afb5909d1..357aff42eb3e 100644 --- a/prover/witness_generator/src/main.rs +++ b/prover/witness_generator/src/main.rs @@ -6,8 +6,11 @@ use std::time::Instant; use structopt::StructOpt; use tokio::sync::watch; use zksync_config::configs::{FriWitnessGeneratorConfig, PrometheusConfig}; -use zksync_config::ObjectStoreConfig; use zksync_dal::{connection::DbVariant, ConnectionPool}; +use zksync_env_config::{ + object_store::{ProverObjectStoreConfig, PublicObjectStoreConfig}, + FromEnv, +}; use zksync_object_store::ObjectStoreFactory; use zksync_prover_utils::get_stop_signal_receiver; use zksync_queued_job_processor::JobProcessor; @@ -71,8 +74,9 @@ async fn main() -> anyhow::Result<()> { let started_at = Instant::now(); let use_push_gateway = opt.batch_size.is_some(); - let store_factory = - ObjectStoreFactory::prover_from_env().context("ObjectStoreFactor::prover_from_env()")?; + let object_store_config = + ProverObjectStoreConfig::from_env().context("ProverObjectStoreConfig::from_env()")?; + let store_factory = ObjectStoreFactory::new(object_store_config.0); let config = FriWitnessGeneratorConfig::from_env().context("FriWitnessGeneratorConfig::from_env()")?; let prometheus_config = PrometheusConfig::from_env().context("PrometheusConfig::from_env()")?; @@ -122,17 +126,16 @@ async fn main() -> anyhow::Result<()> { }; let prometheus_task = prometheus_config.run(stop_receiver.clone()); + let public_object_store_config = + PublicObjectStoreConfig::from_env().context("PublicObjectStoreConfig::from_env()")?; let witness_generator_task = match opt.round { AggregationRound::BasicCircuits => { let public_blob_store = match config.shall_save_to_public_bucket { false => None, true => Some( - ObjectStoreFactory::new( - ObjectStoreConfig::public_from_env() - .context("ObjectStoreConfig::public_from_env()")?, - ) - .create_store() - .await, + ObjectStoreFactory::new(public_object_store_config.0) + .create_store() + .await, ), }; let generator = BasicWitnessGenerator::new( diff --git a/prover/witness_generator/tests/basic_test.rs b/prover/witness_generator/tests/basic_test.rs index 42ab161bd5e3..1c8d00ff35e7 100644 --- a/prover/witness_generator/tests/basic_test.rs +++ b/prover/witness_generator/tests/basic_test.rs @@ -2,6 +2,7 @@ use std::time::Instant; use serde::Serialize; use zksync_config::ObjectStoreConfig; +use zksync_env_config::FromEnv; use zksync_object_store::{AggregationsKey, FriCircuitKey, ObjectStoreFactory}; use zksync_types::proofs::{ AggregationRound, LeafAggregationJobMetadata, NodeAggregationJobMetadata, diff --git a/prover/witness_vector_generator/Cargo.toml b/prover/witness_vector_generator/Cargo.toml index 4360eb7516f2..e8f94849e30e 100644 --- a/prover/witness_vector_generator/Cargo.toml +++ b/prover/witness_vector_generator/Cargo.toml @@ -9,6 +9,7 @@ edition = "2021" zksync_types = { path = "../../core/lib/types" } zksync_dal = { path = "../../core/lib/dal" } zksync_config = { path = "../../core/lib/config" } +zksync_env_config = { path = "../../core/lib/env_config" } zksync_object_store = { path = "../../core/lib/object_store" } zksync_prover_fri_utils = { path = "../prover_fri_utils" } zksync_utils = { path = "../../core/lib/utils" } diff --git a/prover/witness_vector_generator/src/main.rs b/prover/witness_vector_generator/src/main.rs index 48fc372023ab..16eeb660da38 100644 --- a/prover/witness_vector_generator/src/main.rs +++ b/prover/witness_vector_generator/src/main.rs @@ -7,9 +7,10 @@ use tokio::{sync::oneshot, sync::watch}; use crate::generator::WitnessVectorGenerator; use zksync_config::configs::fri_prover_group::FriProverGroupConfig; -use zksync_config::configs::FriWitnessVectorGeneratorConfig; +use zksync_config::configs::{FriWitnessVectorGeneratorConfig, ProverGroupConfig}; use zksync_dal::connection::DbVariant; use zksync_dal::ConnectionPool; +use zksync_env_config::{object_store::ProverObjectStoreConfig, FromEnv}; use zksync_object_store::ObjectStoreFactory; use zksync_prover_fri_utils::get_all_circuit_id_round_tuples_for; use zksync_prover_utils::region_fetcher::get_zone; @@ -58,16 +59,20 @@ async fn main() -> anyhow::Result<()> { .build() .await .context("failed to build a connection pool")?; - let blob_store = ObjectStoreFactory::prover_from_env() - .context("ObjectStoreFactor::prover_from_env()")? + let object_store_config = + ProverObjectStoreConfig::from_env().context("ProverObjectStoreConfig::from_env()")?; + let blob_store = ObjectStoreFactory::new(object_store_config.0) .create_store() .await; let circuit_ids_for_round_to_be_proven = FriProverGroupConfig::from_env() + .context("FriProverGroupConfig::from_env()")? .get_circuit_ids_for_group_id(specialized_group_id) .unwrap_or_default(); let circuit_ids_for_round_to_be_proven = get_all_circuit_id_round_tuples_for(circuit_ids_for_round_to_be_proven); - let zone = get_zone().await.context("get_zone()")?; + let prover_group_config = + ProverGroupConfig::from_env().context("ProverGroupConfig::from_env()")?; + let zone = get_zone(&prover_group_config).await.context("get_zone()")?; let vk_commitments = get_cached_commitments(); let witness_vector_generator = WitnessVectorGenerator::new( blob_store,