From 2ceb9114659f4c4583c87b1bbc8ee230eb1c44db Mon Sep 17 00:00:00 2001 From: EmilLuta Date: Fri, 5 Jan 2024 12:32:27 +0100 Subject: [PATCH] fix(prover): Remove prover-utils from core (#819) Last batch of removing old prover from core. Further improvements will follow up in the core <-> prover integration. These will be tackled as part of prover <-> core separation. ## Checklist - [x] PR title corresponds to the body of PR (we generate changelog entries from PRs). - [ ] Tests for the changes have been added / updated. - [x] Documentation comments have been added / updated. - [x] Code has been formatted via `zk fmt` and `zk lint`. - [x] Spellcheck has been run via `cargo spellcheck --cfg=./spellcheck/era.cfg --code 1`. --- Cargo.lock | 41 ------ Cargo.toml | 1 - core/lib/prover_utils/Cargo.toml | 26 ---- .../lib/prover_utils/src/gcs_proof_fetcher.rs | 22 --- core/lib/prover_utils/src/lib.rs | 126 ------------------ core/lib/zksync_core/Cargo.toml | 1 - .../zksync_core/src/eth_sender/aggregator.rs | 25 +++- .../src/house_keeper/blocks_state_reporter.rs | 6 +- .../fri_proof_compressor_job_retry_manager.rs | 3 +- .../fri_proof_compressor_queue_monitor.rs | 3 +- .../fri_prover_job_retry_manager.rs | 3 +- .../house_keeper/fri_prover_queue_monitor.rs | 3 +- .../fri_scheduler_circuit_queuer.rs | 3 +- ...ri_witness_generator_jobs_retry_manager.rs | 3 +- .../fri_witness_generator_queue_monitor.rs | 3 +- core/lib/zksync_core/src/house_keeper/mod.rs | 1 + .../src/house_keeper}/periodic_job.rs | 2 +- ...waiting_to_queued_fri_witness_job_mover.rs | 3 +- core/lib/zksync_core/src/lib.rs | 2 +- docs/guides/architecture.md | 1 - prover/Cargo.lock | 30 +---- prover/proof_fri_compressor/Cargo.toml | 2 +- .../src/initial_setup_keys.rs | 59 ++++++++ prover/proof_fri_compressor/src/main.rs | 7 +- prover/prover_fri/Cargo.toml | 3 +- prover/prover_fri/src/main.rs | 3 +- prover/prover_fri_utils/Cargo.toml | 4 + prover/prover_fri_utils/src/lib.rs | 1 + .../prover_fri_utils}/src/region_fetcher.rs | 2 +- .../Cargo.toml | 2 +- .../src/commitment_generator.rs | 8 +- .../src/lib.rs | 1 + .../src/vk_commitment_helper.rs | 0 prover/witness_generator/Cargo.toml | 1 - prover/witness_generator/src/main.rs | 8 +- prover/witness_vector_generator/Cargo.toml | 1 - prover/witness_vector_generator/src/main.rs | 3 +- 37 files changed, 139 insertions(+), 274 deletions(-) delete mode 100644 core/lib/prover_utils/Cargo.toml delete mode 100644 core/lib/prover_utils/src/gcs_proof_fetcher.rs delete mode 100644 core/lib/prover_utils/src/lib.rs rename core/lib/{prover_utils/src => zksync_core/src/house_keeper}/periodic_job.rs (97%) create mode 100644 prover/proof_fri_compressor/src/initial_setup_keys.rs rename {core/lib/prover_utils => prover/prover_fri_utils}/src/region_fetcher.rs (98%) rename {core/lib/prover_utils => prover/vk_setup_data_generator_server_fri}/src/vk_commitment_helper.rs (100%) diff --git a/Cargo.lock b/Cargo.lock index 9322e0e2a7b4..461f4df862cc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1369,16 +1369,6 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" -[[package]] -name = "combine" -version = "4.6.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35ed6e9d84f0b51a7f52daf1c7d71dd136fd7a3f41a8462b8cdb8c78d920fad4" -dependencies = [ - "bytes", - "memchr", -] - [[package]] name = "compile-fmt" version = "0.1.0" @@ -7063,17 +7053,6 @@ version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7cda73e2f1397b1262d6dfdcef8aafae14d1de7748d66822d3bfeeb6d03e5e4b" -[[package]] -name = "toml_edit" -version = "0.14.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5376256e44f2443f8896ac012507c19a012df0fe8758b55246ae51a2279db51f" -dependencies = [ - "combine", - "indexmap 1.9.3", - "itertools", -] - [[package]] name = "toml_edit" version = "0.19.15" @@ -8500,7 +8479,6 @@ dependencies = [ "zksync_object_store", "zksync_protobuf", "zksync_protobuf_build", - "zksync_prover_utils", "zksync_queued_job_processor", "zksync_state", "zksync_storage", @@ -8755,25 +8733,6 @@ dependencies = [ "syn 2.0.38", ] -[[package]] -name = "zksync_prover_utils" -version = "0.1.0" -dependencies = [ - "anyhow", - "async-trait", - "ctrlc", - "futures 0.3.28", - "regex", - "reqwest", - "tokio", - "toml_edit 0.14.4", - "tracing", - "zksync_config", - "zksync_object_store", - "zksync_types", - "zksync_utils", -] - [[package]] name = "zksync_queued_job_processor" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 151921aef57e..cd823972a01c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -32,7 +32,6 @@ members = [ "core/lib/state", "core/lib/storage", "core/lib/types", - "core/lib/prover_utils", "core/lib/utils", "core/lib/vlog", "core/lib/multivm", diff --git a/core/lib/prover_utils/Cargo.toml b/core/lib/prover_utils/Cargo.toml deleted file mode 100644 index 3afa050ace07..000000000000 --- a/core/lib/prover_utils/Cargo.toml +++ /dev/null @@ -1,26 +0,0 @@ -[package] -name = "zksync_prover_utils" -version = "0.1.0" -edition = "2018" -authors = ["The Matter Labs Team "] -homepage = "https://zksync.io/" -repository = "https://github.com/matter-labs/zksync-era" -license = "MIT OR Apache-2.0" -keywords = ["blockchain", "zksync"] -categories = ["cryptography"] - -[dependencies] -zksync_config = { path = "../../lib/config" } -zksync_utils = { path = "../../lib/utils" } -zksync_types = { path = "../../lib/types" } -zksync_object_store = { path = "../../lib/object_store" } - -anyhow = "1.0" -reqwest = { version = "0.11", features = ["blocking"] } -regex = "1.7.2" -tokio = "1.27.0" -futures = { version = "0.3", features = ["compat"] } -ctrlc = { version = "3.1", features = ["termination"] } -toml_edit = "0.14.4" -async-trait = "0.1" -tracing = "0.1" diff --git a/core/lib/prover_utils/src/gcs_proof_fetcher.rs b/core/lib/prover_utils/src/gcs_proof_fetcher.rs deleted file mode 100644 index 26872701a1fe..000000000000 --- a/core/lib/prover_utils/src/gcs_proof_fetcher.rs +++ /dev/null @@ -1,22 +0,0 @@ -use zksync_object_store::{ObjectStore, ObjectStoreError}; -use zksync_types::{aggregated_operations::L1BatchProofForL1, L1BatchNumber}; - -pub async fn load_wrapped_fri_proofs_for_range( - from: L1BatchNumber, - to: L1BatchNumber, - blob_store: &dyn ObjectStore, -) -> Vec { - let mut proofs = Vec::new(); - for l1_batch_number in from.0..=to.0 { - let l1_batch_number = L1BatchNumber(l1_batch_number); - match blob_store.get(l1_batch_number).await { - Ok(proof) => proofs.push(proof), - Err(ObjectStoreError::KeyNotFound(_)) => (), // do nothing, proof is not ready yet - Err(err) => panic!( - "Failed to load proof for batch {}: {}", - l1_batch_number.0, err - ), - } - } - proofs -} diff --git a/core/lib/prover_utils/src/lib.rs b/core/lib/prover_utils/src/lib.rs deleted file mode 100644 index 0ee42ffee065..000000000000 --- a/core/lib/prover_utils/src/lib.rs +++ /dev/null @@ -1,126 +0,0 @@ -#![allow(clippy::upper_case_acronyms, clippy::derive_partial_eq_without_eq)] - -extern crate core; - -use std::{fs::create_dir_all, io::Cursor, path::Path, time::Duration}; - -use futures::{channel::mpsc, executor::block_on, SinkExt}; - -pub mod gcs_proof_fetcher; -pub mod periodic_job; -pub mod region_fetcher; -pub mod vk_commitment_helper; - -fn download_bytes(key_download_url: &str) -> reqwest::Result> { - tracing::info!("Downloading initial setup from {:?}", key_download_url); - - const DOWNLOAD_TIMEOUT: Duration = Duration::from_secs(120); - let client = reqwest::blocking::Client::builder() - .timeout(DOWNLOAD_TIMEOUT) - .build() - .unwrap(); - - const DOWNLOAD_RETRIES: usize = 5; - let mut retry_count = 0; - - while retry_count < DOWNLOAD_RETRIES { - let bytes = client - .get(key_download_url) - .send() - .and_then(|response| response.bytes().map(|bytes| bytes.to_vec())); - match bytes { - Ok(bytes) => return Ok(bytes), - Err(_) => retry_count += 1, - } - - tracing::warn!("Failed to download keys. Backing off for 5 second"); - std::thread::sleep(Duration::from_secs(5)); - } - - client - .get(key_download_url) - .send() - .and_then(|response| response.bytes().map(|bytes| bytes.to_vec())) -} - -pub fn ensure_initial_setup_keys_present(initial_setup_key_path: &str, key_download_url: &str) { - if Path::new(initial_setup_key_path).exists() { - tracing::info!( - "Initial setup already present at {:?}", - initial_setup_key_path - ); - return; - } - - let bytes = download_bytes(key_download_url).expect("Failed downloading initial setup"); - let initial_setup_key_dir = Path::new(initial_setup_key_path).parent().unwrap(); - create_dir_all(initial_setup_key_dir).unwrap_or_else(|_| { - panic!( - "Failed creating dirs recursively: {:?}", - initial_setup_key_dir - ) - }); - let mut file = std::fs::File::create(initial_setup_key_path) - .expect("Cannot create file for the initial setup"); - let mut content = Cursor::new(bytes); - std::io::copy(&mut content, &mut file).expect("Cannot write the downloaded key to the file"); -} - -pub fn numeric_index_to_circuit_name(circuit_numeric_index: u8) -> Option<&'static str> { - match circuit_numeric_index { - 0 => Some("Scheduler"), - 1 => Some("Node aggregation"), - 2 => Some("Leaf aggregation"), - 3 => Some("Main VM"), - 4 => Some("Decommitts sorter"), - 5 => Some("Code decommitter"), - 6 => Some("Log demuxer"), - 7 => Some("Keccak"), - 8 => Some("SHA256"), - 9 => Some("ECRecover"), - 10 => Some("RAM permutation"), - 11 => Some("Storage sorter"), - 12 => Some("Storage application"), - 13 => Some("Initial writes pubdata rehasher"), - 14 => Some("Repeated writes pubdata rehasher"), - 15 => Some("Events sorter"), - 16 => Some("L1 messages sorter"), - 17 => Some("L1 messages rehasher"), - 18 => Some("L1 messages merklizer"), - _ => None, - } -} - -pub fn circuit_name_to_numeric_index(circuit_name: &str) -> Option { - match circuit_name { - "Scheduler" => Some(0), - "Node aggregation" => Some(1), - "Leaf aggregation" => Some(2), - "Main VM" => Some(3), - "Decommitts sorter" => Some(4), - "Code decommitter" => Some(5), - "Log demuxer" => Some(6), - "Keccak" => Some(7), - "SHA256" => Some(8), - "ECRecover" => Some(9), - "RAM permutation" => Some(10), - "Storage sorter" => Some(11), - "Storage application" => Some(12), - "Initial writes pubdata rehasher" => Some(13), - "Repeated writes pubdata rehasher" => Some(14), - "Events sorter" => Some(15), - "L1 messages sorter" => Some(16), - "L1 messages rehasher" => Some(17), - "L1 messages merklizer" => Some(18), - _ => None, - } -} - -pub fn get_stop_signal_receiver() -> mpsc::Receiver { - let (mut stop_signal_sender, stop_signal_receiver) = mpsc::channel(256); - ctrlc::set_handler(move || { - block_on(stop_signal_sender.send(true)).expect("Ctrl+C signal send"); - }) - .expect("Error setting Ctrl+C handler"); - stop_signal_receiver -} diff --git a/core/lib/zksync_core/Cargo.toml b/core/lib/zksync_core/Cargo.toml index 64c279ae0cf5..9a86c4e1a1c9 100644 --- a/core/lib/zksync_core/Cargo.toml +++ b/core/lib/zksync_core/Cargo.toml @@ -22,7 +22,6 @@ zksync_commitment_utils = { path = "../commitment_utils" } zksync_eth_client = { path = "../eth_client" } zksync_eth_signer = { path = "../eth_signer" } zksync_mempool = { path = "../mempool" } -zksync_prover_utils = { path = "../prover_utils" } zksync_queued_job_processor = { path = "../queued_job_processor" } zksync_circuit_breaker = { path = "../circuit_breaker" } zksync_storage = { path = "../storage" } diff --git a/core/lib/zksync_core/src/eth_sender/aggregator.rs b/core/lib/zksync_core/src/eth_sender/aggregator.rs index 7368aaa0cd20..ebb11752894f 100644 --- a/core/lib/zksync_core/src/eth_sender/aggregator.rs +++ b/core/lib/zksync_core/src/eth_sender/aggregator.rs @@ -1,12 +1,11 @@ use zksync_config::configs::eth_sender::{ProofLoadingMode, ProofSendingMode, SenderConfig}; use zksync_contracts::BaseSystemContractsHashes; use zksync_dal::StorageProcessor; -use zksync_object_store::ObjectStore; -use zksync_prover_utils::gcs_proof_fetcher::load_wrapped_fri_proofs_for_range; +use zksync_object_store::{ObjectStore, ObjectStoreError}; use zksync_types::{ aggregated_operations::{ AggregatedActionType, AggregatedOperation, L1BatchCommitOperation, L1BatchExecuteOperation, - L1BatchProofOperation, + L1BatchProofForL1, L1BatchProofOperation, }, commitment::L1BatchWithMetadata, helpers::unix_timestamp_ms, @@ -418,3 +417,23 @@ async fn extract_ready_subrange( .collect(), ) } + +pub async fn load_wrapped_fri_proofs_for_range( + from: L1BatchNumber, + to: L1BatchNumber, + blob_store: &dyn ObjectStore, +) -> Vec { + let mut proofs = Vec::new(); + for l1_batch_number in from.0..=to.0 { + let l1_batch_number = L1BatchNumber(l1_batch_number); + match blob_store.get(l1_batch_number).await { + Ok(proof) => proofs.push(proof), + Err(ObjectStoreError::KeyNotFound(_)) => (), // do nothing, proof is not ready yet + Err(err) => panic!( + "Failed to load proof for batch {}: {}", + l1_batch_number.0, err + ), + } + } + proofs +} diff --git a/core/lib/zksync_core/src/house_keeper/blocks_state_reporter.rs b/core/lib/zksync_core/src/house_keeper/blocks_state_reporter.rs index b52edb963758..695c2008e134 100644 --- a/core/lib/zksync_core/src/house_keeper/blocks_state_reporter.rs +++ b/core/lib/zksync_core/src/house_keeper/blocks_state_reporter.rs @@ -1,9 +1,11 @@ use async_trait::async_trait; use zksync_dal::ConnectionPool; -use zksync_prover_utils::periodic_job::PeriodicJob; use zksync_utils::time::seconds_since_epoch; -use crate::metrics::{BlockL1Stage, BlockStage, L1StageLatencyLabel, APP_METRICS}; +use crate::{ + house_keeper::periodic_job::PeriodicJob, + metrics::{BlockL1Stage, BlockStage, L1StageLatencyLabel, APP_METRICS}, +}; #[derive(Debug)] pub struct L1BatchMetricsReporter { diff --git a/core/lib/zksync_core/src/house_keeper/fri_proof_compressor_job_retry_manager.rs b/core/lib/zksync_core/src/house_keeper/fri_proof_compressor_job_retry_manager.rs index fc26524e992e..7cf2c231b67a 100644 --- a/core/lib/zksync_core/src/house_keeper/fri_proof_compressor_job_retry_manager.rs +++ b/core/lib/zksync_core/src/house_keeper/fri_proof_compressor_job_retry_manager.rs @@ -2,7 +2,8 @@ use std::time::Duration; use async_trait::async_trait; use zksync_dal::ConnectionPool; -use zksync_prover_utils::periodic_job::PeriodicJob; + +use crate::house_keeper::periodic_job::PeriodicJob; #[derive(Debug)] pub struct FriProofCompressorJobRetryManager { diff --git a/core/lib/zksync_core/src/house_keeper/fri_proof_compressor_queue_monitor.rs b/core/lib/zksync_core/src/house_keeper/fri_proof_compressor_queue_monitor.rs index 73c752b69558..88e1f0f64654 100644 --- a/core/lib/zksync_core/src/house_keeper/fri_proof_compressor_queue_monitor.rs +++ b/core/lib/zksync_core/src/house_keeper/fri_proof_compressor_queue_monitor.rs @@ -1,8 +1,9 @@ use async_trait::async_trait; use zksync_dal::ConnectionPool; -use zksync_prover_utils::periodic_job::PeriodicJob; use zksync_types::proofs::JobCountStatistics; +use crate::house_keeper::periodic_job::PeriodicJob; + const PROOF_COMPRESSOR_SERVICE_NAME: &str = "proof_compressor"; #[derive(Debug)] diff --git a/core/lib/zksync_core/src/house_keeper/fri_prover_job_retry_manager.rs b/core/lib/zksync_core/src/house_keeper/fri_prover_job_retry_manager.rs index fefb7333a675..8ff847a5ca92 100644 --- a/core/lib/zksync_core/src/house_keeper/fri_prover_job_retry_manager.rs +++ b/core/lib/zksync_core/src/house_keeper/fri_prover_job_retry_manager.rs @@ -2,7 +2,8 @@ use std::time::Duration; use async_trait::async_trait; use zksync_dal::ConnectionPool; -use zksync_prover_utils::periodic_job::PeriodicJob; + +use crate::house_keeper::periodic_job::PeriodicJob; #[derive(Debug)] pub struct FriProverJobRetryManager { diff --git a/core/lib/zksync_core/src/house_keeper/fri_prover_queue_monitor.rs b/core/lib/zksync_core/src/house_keeper/fri_prover_queue_monitor.rs index f0ccf0be7333..3ccf09bdf1e8 100644 --- a/core/lib/zksync_core/src/house_keeper/fri_prover_queue_monitor.rs +++ b/core/lib/zksync_core/src/house_keeper/fri_prover_queue_monitor.rs @@ -1,7 +1,8 @@ use async_trait::async_trait; use zksync_config::configs::fri_prover_group::FriProverGroupConfig; use zksync_dal::ConnectionPool; -use zksync_prover_utils::periodic_job::PeriodicJob; + +use crate::house_keeper::periodic_job::PeriodicJob; #[derive(Debug)] pub struct FriProverStatsReporter { diff --git a/core/lib/zksync_core/src/house_keeper/fri_scheduler_circuit_queuer.rs b/core/lib/zksync_core/src/house_keeper/fri_scheduler_circuit_queuer.rs index 0adfdb470551..70911339a8fd 100644 --- a/core/lib/zksync_core/src/house_keeper/fri_scheduler_circuit_queuer.rs +++ b/core/lib/zksync_core/src/house_keeper/fri_scheduler_circuit_queuer.rs @@ -1,6 +1,7 @@ use async_trait::async_trait; use zksync_dal::ConnectionPool; -use zksync_prover_utils::periodic_job::PeriodicJob; + +use crate::house_keeper::periodic_job::PeriodicJob; #[derive(Debug)] pub struct SchedulerCircuitQueuer { diff --git a/core/lib/zksync_core/src/house_keeper/fri_witness_generator_jobs_retry_manager.rs b/core/lib/zksync_core/src/house_keeper/fri_witness_generator_jobs_retry_manager.rs index f81cb03dc379..3aa21bdd534d 100644 --- a/core/lib/zksync_core/src/house_keeper/fri_witness_generator_jobs_retry_manager.rs +++ b/core/lib/zksync_core/src/house_keeper/fri_witness_generator_jobs_retry_manager.rs @@ -2,7 +2,8 @@ use std::time::Duration; use async_trait::async_trait; use zksync_dal::ConnectionPool; -use zksync_prover_utils::periodic_job::PeriodicJob; + +use crate::house_keeper::periodic_job::PeriodicJob; #[derive(Debug)] pub struct FriWitnessGeneratorJobRetryManager { diff --git a/core/lib/zksync_core/src/house_keeper/fri_witness_generator_queue_monitor.rs b/core/lib/zksync_core/src/house_keeper/fri_witness_generator_queue_monitor.rs index 67f81295b44f..f198d27d97b2 100644 --- a/core/lib/zksync_core/src/house_keeper/fri_witness_generator_queue_monitor.rs +++ b/core/lib/zksync_core/src/house_keeper/fri_witness_generator_queue_monitor.rs @@ -2,9 +2,10 @@ use std::collections::HashMap; use async_trait::async_trait; use zksync_dal::ConnectionPool; -use zksync_prover_utils::periodic_job::PeriodicJob; use zksync_types::proofs::{AggregationRound, JobCountStatistics}; +use crate::house_keeper::periodic_job::PeriodicJob; + const FRI_WITNESS_GENERATOR_SERVICE_NAME: &str = "fri_witness_generator"; #[derive(Debug)] diff --git a/core/lib/zksync_core/src/house_keeper/mod.rs b/core/lib/zksync_core/src/house_keeper/mod.rs index fbf59642d356..2c029d42e779 100644 --- a/core/lib/zksync_core/src/house_keeper/mod.rs +++ b/core/lib/zksync_core/src/house_keeper/mod.rs @@ -6,4 +6,5 @@ pub mod fri_prover_queue_monitor; pub mod fri_scheduler_circuit_queuer; pub mod fri_witness_generator_jobs_retry_manager; pub mod fri_witness_generator_queue_monitor; +pub mod periodic_job; pub mod waiting_to_queued_fri_witness_job_mover; diff --git a/core/lib/prover_utils/src/periodic_job.rs b/core/lib/zksync_core/src/house_keeper/periodic_job.rs similarity index 97% rename from core/lib/prover_utils/src/periodic_job.rs rename to core/lib/zksync_core/src/house_keeper/periodic_job.rs index e58ff33e7890..3f73a01ce200 100644 --- a/core/lib/prover_utils/src/periodic_job.rs +++ b/core/lib/zksync_core/src/house_keeper/periodic_job.rs @@ -1,6 +1,6 @@ use std::time::Duration; -use anyhow::Context as _; +use anyhow::Context; use async_trait::async_trait; use tokio::time::sleep; diff --git a/core/lib/zksync_core/src/house_keeper/waiting_to_queued_fri_witness_job_mover.rs b/core/lib/zksync_core/src/house_keeper/waiting_to_queued_fri_witness_job_mover.rs index 1292ee3f44fa..df9208f1f451 100644 --- a/core/lib/zksync_core/src/house_keeper/waiting_to_queued_fri_witness_job_mover.rs +++ b/core/lib/zksync_core/src/house_keeper/waiting_to_queued_fri_witness_job_mover.rs @@ -1,6 +1,7 @@ use async_trait::async_trait; use zksync_dal::ConnectionPool; -use zksync_prover_utils::periodic_job::PeriodicJob; + +use crate::house_keeper::periodic_job::PeriodicJob; #[derive(Debug)] pub struct WaitingToQueuedFriWitnessJobMover { diff --git a/core/lib/zksync_core/src/lib.rs b/core/lib/zksync_core/src/lib.rs index b3a3d2e580ee..1964651be3ff 100644 --- a/core/lib/zksync_core/src/lib.rs +++ b/core/lib/zksync_core/src/lib.rs @@ -31,7 +31,6 @@ use zksync_eth_client::{ }; use zksync_health_check::{CheckHealth, HealthStatus, ReactiveHealthCheck}; use zksync_object_store::{ObjectStore, ObjectStoreFactory}; -use zksync_prover_utils::periodic_job::PeriodicJob; use zksync_queued_job_processor::JobProcessor; use zksync_state::PostgresStorageCaches; use zksync_types::{ @@ -62,6 +61,7 @@ use crate::{ fri_scheduler_circuit_queuer::SchedulerCircuitQueuer, fri_witness_generator_jobs_retry_manager::FriWitnessGeneratorJobRetryManager, fri_witness_generator_queue_monitor::FriWitnessGeneratorStatsReporter, + periodic_job::PeriodicJob, waiting_to_queued_fri_witness_job_mover::WaitingToQueuedFriWitnessJobMover, }, l1_gas_price::{GasAdjusterSingleton, L1GasPriceProvider}, diff --git a/docs/guides/architecture.md b/docs/guides/architecture.md index dbac73fa09a8..e87f4bca7e55 100644 --- a/docs/guides/architecture.md +++ b/docs/guides/architecture.md @@ -62,7 +62,6 @@ This section provides a physical map of folders & files in this repository. - `/multivm`: A wrapper over several versions of VM that have been used by the main node. - `/object_store`: Abstraction for storing blobs outside the main data store. - `/prometheus_exporter`: Prometheus data exporter. - - `/prover_utils`: Utilities related to the proof generation. - `/queued_job_processor`: An abstraction for async job processing - `/state`: A state keeper responsible for handling transaction execution and creating miniblocks and L1 batches. - `/storage`: An encapsulated database interface. diff --git a/prover/Cargo.lock b/prover/Cargo.lock index c5c7d1bc7be9..27ac29283485 100644 --- a/prover/Cargo.lock +++ b/prover/Cargo.lock @@ -6217,13 +6217,13 @@ dependencies = [ "serde_json", "shivini", "structopt", + "toml_edit 0.14.4", "tracing", "vlog", "zkevm_test_harness 1.4.0", "zksync_config", "zksync_env_config", "zksync_prover_fri_types", - "zksync_prover_utils", "zksync_types", ] @@ -7017,6 +7017,7 @@ dependencies = [ "ctrlc", "futures 0.3.29", "prometheus_exporter", + "reqwest", "structopt", "tokio", "tracing", @@ -7029,7 +7030,6 @@ dependencies = [ "zksync_env_config", "zksync_object_store", "zksync_prover_fri_types", - "zksync_prover_utils", "zksync_queued_job_processor", "zksync_types", "zksync_utils", @@ -7080,6 +7080,8 @@ dependencies = [ "futures 0.3.29", "local-ip-address", "prometheus_exporter", + "regex", + "reqwest", "serde", "shivini", "tokio", @@ -7094,7 +7096,6 @@ dependencies = [ "zksync_object_store", "zksync_prover_fri_types", "zksync_prover_fri_utils", - "zksync_prover_utils", "zksync_queued_job_processor", "zksync_types", "zksync_utils", @@ -7138,6 +7139,9 @@ dependencies = [ name = "zksync_prover_fri_utils" version = "0.1.0" dependencies = [ + "anyhow", + "regex", + "reqwest", "serde", "tracing", "vise", @@ -7146,24 +7150,6 @@ dependencies = [ "zksync_object_store", "zksync_prover_fri_types", "zksync_types", -] - -[[package]] -name = "zksync_prover_utils" -version = "0.1.0" -dependencies = [ - "anyhow", - "async-trait", - "ctrlc", - "futures 0.3.29", - "regex", - "reqwest", - "tokio", - "toml_edit 0.14.4", - "tracing", - "zksync_config", - "zksync_object_store", - "zksync_types", "zksync_utils", ] @@ -7309,7 +7295,6 @@ dependencies = [ "zksync_object_store", "zksync_prover_fri_types", "zksync_prover_fri_utils", - "zksync_prover_utils", "zksync_queued_job_processor", "zksync_state", "zksync_system_constants", @@ -7341,7 +7326,6 @@ dependencies = [ "zksync_object_store", "zksync_prover_fri_types", "zksync_prover_fri_utils", - "zksync_prover_utils", "zksync_queued_job_processor", "zksync_types", "zksync_utils", diff --git a/prover/proof_fri_compressor/Cargo.toml b/prover/proof_fri_compressor/Cargo.toml index ee5bd340686b..659f1fb4ceb4 100644 --- a/prover/proof_fri_compressor/Cargo.toml +++ b/prover/proof_fri_compressor/Cargo.toml @@ -19,7 +19,6 @@ zksync_prover_fri_types = { path = "../prover_fri_types" } zksync_queued_job_processor = { path = "../../core/lib/queued_job_processor" } vk_setup_data_generator_server_fri = { path = "../vk_setup_data_generator_server_fri" } vlog = { path = "../../core/lib/vlog" } -zksync_prover_utils = { path = "../../core/lib/prover_utils" } zkevm_test_harness = { git = "https://github.com/matter-labs/era-zkevm_test_harness.git", branch = "v1.4.0" } @@ -31,3 +30,4 @@ futures = { version = "0.3", features = ["compat"] } ctrlc = { version = "3.1", features = ["termination"] } async-trait = "0.1" bincode = "1.0" +reqwest = { version = "0.11", features = ["blocking"] } diff --git a/prover/proof_fri_compressor/src/initial_setup_keys.rs b/prover/proof_fri_compressor/src/initial_setup_keys.rs new file mode 100644 index 000000000000..222d2ec69cc5 --- /dev/null +++ b/prover/proof_fri_compressor/src/initial_setup_keys.rs @@ -0,0 +1,59 @@ +use std::{fs::create_dir_all, io::Cursor, path::Path, time::Duration}; + +fn download_initial_setup(key_download_url: &str) -> reqwest::Result> { + tracing::info!("Downloading initial setup from {:?}", key_download_url); + + const DOWNLOAD_TIMEOUT: Duration = Duration::from_secs(120); + let client = reqwest::blocking::Client::builder() + .timeout(DOWNLOAD_TIMEOUT) + .build() + .unwrap(); + + const DOWNLOAD_RETRIES: usize = 5; + let mut retry_count = 0; + + while retry_count < DOWNLOAD_RETRIES { + let bytes = client + .get(key_download_url) + .send() + .and_then(|response| response.bytes().map(|bytes| bytes.to_vec())); + match bytes { + Ok(bytes) => return Ok(bytes), + Err(_) => retry_count += 1, + } + + tracing::warn!("Failed to download keys. Backing off for 5 second"); + std::thread::sleep(Duration::from_secs(5)); + } + + client + .get(key_download_url) + .send() + .and_then(|response| response.bytes().map(|bytes| bytes.to_vec())) +} + +pub fn download_initial_setup_keys_if_not_present( + initial_setup_key_path: &str, + key_download_url: &str, +) { + if Path::new(initial_setup_key_path).exists() { + tracing::info!( + "Initial setup already present at {:?}", + initial_setup_key_path + ); + return; + } + + let bytes = download_initial_setup(key_download_url).expect("Failed downloading initial setup"); + let initial_setup_key_dir = Path::new(initial_setup_key_path).parent().unwrap(); + create_dir_all(initial_setup_key_dir).unwrap_or_else(|_| { + panic!( + "Failed creating dirs recursively: {:?}", + initial_setup_key_dir + ) + }); + let mut file = std::fs::File::create(initial_setup_key_path) + .expect("Cannot create file for the initial setup"); + let mut content = Cursor::new(bytes); + std::io::copy(&mut content, &mut file).expect("Cannot write the downloaded key to the file"); +} diff --git a/prover/proof_fri_compressor/src/main.rs b/prover/proof_fri_compressor/src/main.rs index 90d937b6f309..33aaf3b9162b 100644 --- a/prover/proof_fri_compressor/src/main.rs +++ b/prover/proof_fri_compressor/src/main.rs @@ -11,9 +11,12 @@ use zksync_object_store::ObjectStoreFactory; use zksync_queued_job_processor::JobProcessor; use zksync_utils::wait_for_tasks::wait_for_tasks; -use crate::compressor::ProofCompressor; +use crate::{ + compressor::ProofCompressor, initial_setup_keys::download_initial_setup_keys_if_not_present, +}; mod compressor; +mod initial_setup_keys; mod metrics; #[derive(Debug, StructOpt)] @@ -76,7 +79,7 @@ async fn main() -> anyhow::Result<()> { }) .expect("Error setting Ctrl+C handler"); // Setting handler should always succeed. - zksync_prover_utils::ensure_initial_setup_keys_present( + download_initial_setup_keys_if_not_present( &config.universal_setup_path, &config.universal_setup_download_url, ); diff --git a/prover/prover_fri/Cargo.toml b/prover/prover_fri/Cargo.toml index bfef44fd65b3..dca42c966d27 100644 --- a/prover/prover_fri/Cargo.toml +++ b/prover/prover_fri/Cargo.toml @@ -15,7 +15,6 @@ zksync_env_config = { path = "../../core/lib/env_config" } prometheus_exporter = { path = "../../core/lib/prometheus_exporter" } vlog = { path = "../../core/lib/vlog" } zksync_object_store = { path = "../../core/lib/object_store" } -zksync_prover_utils = { path = "../../core/lib/prover_utils" } zksync_queued_job_processor = { path = "../../core/lib/queued_job_processor" } zksync_prover_fri_utils = { path = "../prover_fri_utils" } zksync_prover_fri_types = { path = "../prover_fri_types" } @@ -39,6 +38,8 @@ ctrlc = { version = "3.1", features = ["termination"] } serde = { version = "1.0", features = ["derive"] } async-trait = "0.1" local-ip-address = "0.5.0" +reqwest = { version = "0.11", features = ["blocking"] } +regex = "1.7.2" [features] default = [] diff --git a/prover/prover_fri/src/main.rs b/prover/prover_fri/src/main.rs index 980a499f5b48..06a71266a3da 100644 --- a/prover/prover_fri/src/main.rs +++ b/prover/prover_fri/src/main.rs @@ -17,8 +17,7 @@ use zksync_env_config::{ FromEnv, }; use zksync_object_store::{ObjectStore, ObjectStoreFactory}; -use zksync_prover_fri_utils::get_all_circuit_id_round_tuples_for; -use zksync_prover_utils::region_fetcher::get_zone; +use zksync_prover_fri_utils::{get_all_circuit_id_round_tuples_for, region_fetcher::get_zone}; use zksync_queued_job_processor::JobProcessor; use zksync_types::{ basic_fri_types::CircuitIdRoundTuple, diff --git a/prover/prover_fri_utils/Cargo.toml b/prover/prover_fri_utils/Cargo.toml index 148f68f5c0f8..e2023f92b2dc 100644 --- a/prover/prover_fri_utils/Cargo.toml +++ b/prover/prover_fri_utils/Cargo.toml @@ -13,6 +13,10 @@ zksync_config = { path = "../../core/lib/config" } zksync_types = { path = "../../core/lib/types" } zksync_prover_fri_types = { path = "../prover_fri_types" } zksync_dal = { path = "../../core/lib/dal" } +zksync_utils = { path = "../../core/lib/utils" } tracing = "0.1" serde = { version = "1.0", features = ["derive"] } +reqwest = { version = "0.11", features = ["blocking"] } +regex = "1.7.2" +anyhow = "1.0" diff --git a/prover/prover_fri_utils/src/lib.rs b/prover/prover_fri_utils/src/lib.rs index eee7293b591b..991683b7f9b8 100644 --- a/prover/prover_fri_utils/src/lib.rs +++ b/prover/prover_fri_utils/src/lib.rs @@ -19,6 +19,7 @@ use zksync_types::{ use crate::metrics::{CircuitLabels, PROVER_FRI_UTILS_METRICS}; pub mod metrics; +pub mod region_fetcher; pub mod socket_utils; pub async fn fetch_next_circuit( diff --git a/core/lib/prover_utils/src/region_fetcher.rs b/prover/prover_fri_utils/src/region_fetcher.rs similarity index 98% rename from core/lib/prover_utils/src/region_fetcher.rs rename to prover/prover_fri_utils/src/region_fetcher.rs index fceedd287245..cae211c26cbe 100644 --- a/core/lib/prover_utils/src/region_fetcher.rs +++ b/prover/prover_fri_utils/src/region_fetcher.rs @@ -1,4 +1,4 @@ -use anyhow::Context as _; +use anyhow::Context; use regex::Regex; use reqwest::{ header::{HeaderMap, HeaderValue}, diff --git a/prover/vk_setup_data_generator_server_fri/Cargo.toml b/prover/vk_setup_data_generator_server_fri/Cargo.toml index 59df5f37cf89..6186f121b25b 100644 --- a/prover/vk_setup_data_generator_server_fri/Cargo.toml +++ b/prover/vk_setup_data_generator_server_fri/Cargo.toml @@ -23,7 +23,6 @@ path = "src/commitment_generator.rs" [dependencies] vlog = { path = "../../core/lib/vlog" } zksync_types = { path = "../../core/lib/types" } -zksync_prover_utils = { path = "../../core/lib/prover_utils" } zksync_prover_fri_types = { path = "../prover_fri_types" } zkevm_test_harness = { git = "https://github.com/matter-labs/era-zkevm_test_harness.git", branch = "v1.4.0" } @@ -43,6 +42,7 @@ itertools = "0.10.5" bincode = "1" structopt = "0.3.26" once_cell = "1.8.0" +toml_edit = "0.14.4" [dev-dependencies] proptest = "1.2.0" diff --git a/prover/vk_setup_data_generator_server_fri/src/commitment_generator.rs b/prover/vk_setup_data_generator_server_fri/src/commitment_generator.rs index dffa1981d1c8..f2dd53b85283 100644 --- a/prover/vk_setup_data_generator_server_fri/src/commitment_generator.rs +++ b/prover/vk_setup_data_generator_server_fri/src/commitment_generator.rs @@ -1,8 +1,8 @@ -use anyhow::Context as _; -use zksync_prover_utils::vk_commitment_helper::{ - get_toml_formatted_value, read_contract_toml, write_contract_toml, +use anyhow::Context; +use zksync_vk_setup_data_server_fri::{ + commitment_utils::generate_commitments, + vk_commitment_helper::{get_toml_formatted_value, read_contract_toml, write_contract_toml}, }; -use zksync_vk_setup_data_server_fri::commitment_utils::generate_commitments; fn main() -> anyhow::Result<()> { tracing::info!("Starting commitment generation!"); diff --git a/prover/vk_setup_data_generator_server_fri/src/lib.rs b/prover/vk_setup_data_generator_server_fri/src/lib.rs index 19fab9af470b..37d0d827bfa2 100644 --- a/prover/vk_setup_data_generator_server_fri/src/lib.rs +++ b/prover/vk_setup_data_generator_server_fri/src/lib.rs @@ -59,6 +59,7 @@ use {shivini::cs::GpuSetup, std::alloc::Global}; pub mod commitment_utils; pub mod utils; +pub mod vk_commitment_helper; #[derive(Debug, Serialize, Deserialize)] #[serde( diff --git a/core/lib/prover_utils/src/vk_commitment_helper.rs b/prover/vk_setup_data_generator_server_fri/src/vk_commitment_helper.rs similarity index 100% rename from core/lib/prover_utils/src/vk_commitment_helper.rs rename to prover/vk_setup_data_generator_server_fri/src/vk_commitment_helper.rs diff --git a/prover/witness_generator/Cargo.toml b/prover/witness_generator/Cargo.toml index 9d8b10d73fa2..153bac7fa296 100644 --- a/prover/witness_generator/Cargo.toml +++ b/prover/witness_generator/Cargo.toml @@ -26,7 +26,6 @@ zksync_types = { path = "../../core/lib/types" } zksync_state = { path = "../../core/lib/state" } zksync_utils = { path = "../../core/lib/utils" } vk_setup_data_generator_server_fri = { path = "../vk_setup_data_generator_server_fri" } -zksync_prover_utils = { path = "../../core/lib/prover_utils" } zksync_prover_fri_types = { path = "../prover_fri_types" } zksync_prover_fri_utils = { path = "../prover_fri_utils" } diff --git a/prover/witness_generator/src/main.rs b/prover/witness_generator/src/main.rs index 3b3b8d460339..50b6f4caff87 100644 --- a/prover/witness_generator/src/main.rs +++ b/prover/witness_generator/src/main.rs @@ -3,6 +3,7 @@ use std::time::Instant; use anyhow::{anyhow, Context as _}; +use futures::{channel::mpsc, executor::block_on, SinkExt}; use prometheus_exporter::PrometheusExporterConfig; use structopt::StructOpt; use tokio::sync::watch; @@ -13,7 +14,6 @@ use zksync_config::{ use zksync_dal::ConnectionPool; use zksync_env_config::{object_store::ProverObjectStoreConfig, FromEnv}; use zksync_object_store::ObjectStoreFactory; -use zksync_prover_utils::get_stop_signal_receiver; use zksync_queued_job_processor::JobProcessor; use zksync_types::{proofs::AggregationRound, web3::futures::StreamExt}; use zksync_utils::wait_for_tasks::wait_for_tasks; @@ -229,7 +229,11 @@ async fn main() -> anyhow::Result<()> { SERVER_METRICS.init_latency[&(*round).into()].set(started_at.elapsed()); } - let mut stop_signal_receiver = get_stop_signal_receiver(); + let (mut stop_signal_sender, mut stop_signal_receiver) = mpsc::channel(256); + ctrlc::set_handler(move || { + block_on(stop_signal_sender.send(true)).expect("Ctrl+C signal send"); + }) + .expect("Error setting Ctrl+C handler"); let graceful_shutdown = None::>; let tasks_allowed_to_finish = true; tokio::select! { diff --git a/prover/witness_vector_generator/Cargo.toml b/prover/witness_vector_generator/Cargo.toml index 3f5a6ac55d9e..523514af207f 100644 --- a/prover/witness_vector_generator/Cargo.toml +++ b/prover/witness_vector_generator/Cargo.toml @@ -17,7 +17,6 @@ zksync_prover_fri_utils = { path = "../prover_fri_utils" } zksync_utils = { path = "../../core/lib/utils" } prometheus_exporter = { path = "../../core/lib/prometheus_exporter" } zksync_prover_fri_types = { path = "../prover_fri_types" } -zksync_prover_utils = { path = "../../core/lib/prover_utils" } zksync_queued_job_processor = { path = "../../core/lib/queued_job_processor" } vk_setup_data_generator_server_fri = { path = "../vk_setup_data_generator_server_fri" } vlog = { path = "../../core/lib/vlog" } diff --git a/prover/witness_vector_generator/src/main.rs b/prover/witness_vector_generator/src/main.rs index 552644782c0c..a880cf001829 100644 --- a/prover/witness_vector_generator/src/main.rs +++ b/prover/witness_vector_generator/src/main.rs @@ -11,8 +11,7 @@ use zksync_config::configs::{ use zksync_dal::ConnectionPool; use zksync_env_config::{object_store::ProverObjectStoreConfig, FromEnv}; use zksync_object_store::ObjectStoreFactory; -use zksync_prover_fri_utils::get_all_circuit_id_round_tuples_for; -use zksync_prover_utils::region_fetcher::get_zone; +use zksync_prover_fri_utils::{get_all_circuit_id_round_tuples_for, region_fetcher::get_zone}; use zksync_queued_job_processor::JobProcessor; use zksync_utils::wait_for_tasks::wait_for_tasks; use zksync_vk_setup_data_server_fri::commitment_utils::get_cached_commitments;