From 4ec3194f9d0528f3090320a7acce19549e395bfb Mon Sep 17 00:00:00 2001 From: Igor Aleksanov Date: Fri, 22 Dec 2023 09:18:06 +0400 Subject: [PATCH] refactor: Remove RocksDB backup leftovers (#721) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What ❔ Removes leftovers of RocksDB backup functionality. ## Why ❔ Support for backups has been removed long ago, and we currently don't have plans to restore it. ## Checklist - [x] PR title corresponds to the body of PR (we generate changelog entries from PRs). - [x] Tests for the changes have been added / updated. - [x] Documentation comments have been added / updated. - [x] Code has been formatted via `zk fmt` and `zk lint`. - [x] Spellcheck has been run via `cargo spellcheck --cfg=./spellcheck/era.cfg --code 1`. --- Cargo.lock | 12 --- Cargo.toml | 1 - core/bin/rocksdb_util/Cargo.toml | 22 ----- core/bin/rocksdb_util/src/main.rs | 85 ------------------- core/lib/config/src/configs/database.rs | 26 ------ core/lib/env_config/src/database.rs | 11 --- core/lib/zksync_core/src/lib.rs | 6 -- .../src/metadata_calculator/mod.rs | 3 - .../src/metadata_calculator/tests.rs | 2 - docker/server-v2/Dockerfile | 1 - 10 files changed, 169 deletions(-) delete mode 100644 core/bin/rocksdb_util/Cargo.toml delete mode 100644 core/bin/rocksdb_util/src/main.rs diff --git a/Cargo.lock b/Cargo.lock index 77ab90064454..c794cc5fe803 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5616,18 +5616,6 @@ dependencies = [ "librocksdb-sys", ] -[[package]] -name = "rocksdb_util" -version = "0.1.0" -dependencies = [ - "anyhow", - "clap 4.4.6", - "tempfile", - "zksync_config", - "zksync_env_config", - "zksync_storage", -] - [[package]] name = "route-recognizer" version = "0.3.1" diff --git a/Cargo.toml b/Cargo.toml index ac01673bfb99..34115e84bead 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,7 +5,6 @@ members = [ "core/bin/contract-verifier", "core/bin/external_node", "core/bin/merkle_tree_consistency_checker", - "core/bin/rocksdb_util", "core/bin/snapshots_creator", "core/bin/storage_logs_dedup_migration", "core/bin/system-constants-generator", diff --git a/core/bin/rocksdb_util/Cargo.toml b/core/bin/rocksdb_util/Cargo.toml deleted file mode 100644 index 75d1afe19797..000000000000 --- a/core/bin/rocksdb_util/Cargo.toml +++ /dev/null @@ -1,22 +0,0 @@ -[package] -name = "rocksdb_util" -version = "0.1.0" -edition = "2021" -authors = ["The Matter Labs Team "] -homepage = "https://zksync.io/" -repository = "https://github.com/matter-labs/zksync-era" -license = "MIT OR Apache-2.0" -keywords = ["blockchain", "zksync"] -categories = ["cryptography"] -publish = false # We don't want to publish our binaries. - -[dependencies] -zksync_config = { path = "../../lib/config" } -zksync_env_config = { path = "../../lib/env_config" } -zksync_storage = { path = "../../lib/storage" } - -anyhow = "1.0" -clap = { version = "4.2.4", features = ["derive"] } - -[dev-dependencies] -tempfile = "3.0.2" diff --git a/core/bin/rocksdb_util/src/main.rs b/core/bin/rocksdb_util/src/main.rs deleted file mode 100644 index 1fd60ca67c72..000000000000 --- a/core/bin/rocksdb_util/src/main.rs +++ /dev/null @@ -1,85 +0,0 @@ -use anyhow::Context as _; -use clap::{Parser, Subcommand}; -use zksync_config::DBConfig; -use zksync_env_config::FromEnv; -use zksync_storage::rocksdb::{ - backup::{BackupEngine, BackupEngineOptions, RestoreOptions}, - Env, Error, Options, DB, -}; - -#[derive(Debug, Parser)] -#[command(author = "Matter Labs", version, about = "RocksDB management utility", long_about = None)] -struct Cli { - #[command(subcommand)] - command: Command, -} - -#[derive(Debug, Subcommand)] -enum Command { - /// Creates new backup of running RocksDB instance. - #[command(name = "backup")] - Backup, - /// Restores RocksDB from backup. - #[command(name = "restore-from-backup")] - Restore, -} - -fn create_backup(config: &DBConfig) -> Result<(), Error> { - let mut engine = BackupEngine::open( - &BackupEngineOptions::new(&config.merkle_tree.backup_path)?, - &Env::new()?, - )?; - let db_dir = &config.merkle_tree.path; - let db = DB::open_for_read_only(&Options::default(), db_dir, false)?; - engine.create_new_backup(&db)?; - engine.purge_old_backups(config.backup_count) -} - -fn restore_from_latest_backup(config: &DBConfig) -> Result<(), Error> { - let mut engine = BackupEngine::open( - &BackupEngineOptions::new(&config.merkle_tree.backup_path)?, - &Env::new()?, - )?; - let db_dir = &config.merkle_tree.path; - engine.restore_from_latest_backup(db_dir, db_dir, &RestoreOptions::default()) -} - -fn main() -> anyhow::Result<()> { - let db_config = DBConfig::from_env().context("DBConfig::from_env()")?; - match Cli::parse().command { - Command::Backup => create_backup(&db_config).context("create_backup"), - Command::Restore => { - restore_from_latest_backup(&db_config).context("restore_from_latest_backup") - } - } -} - -#[cfg(test)] -mod tests { - use tempfile::TempDir; - - use super::*; - - #[test] - fn backup_restore_workflow() { - let backup_dir = TempDir::new().expect("failed to get temporary directory for RocksDB"); - let temp_dir = TempDir::new().expect("failed to get temporary directory for RocksDB"); - let mut db_config = DBConfig::from_env().unwrap(); - db_config.merkle_tree.path = temp_dir.path().to_str().unwrap().to_string(); - db_config.merkle_tree.backup_path = backup_dir.path().to_str().unwrap().to_string(); - let db_dir = &db_config.merkle_tree.path; - - let mut options = Options::default(); - options.create_if_missing(true); - let db = DB::open(&options, db_dir).unwrap(); - db.put(b"key", b"value").expect("failed to write to db"); - - create_backup(&db_config).expect("failed to create backup"); - // Drop original database - drop((db, temp_dir)); - - restore_from_latest_backup(&db_config).expect("failed to restore from backup"); - let db = DB::open(&Options::default(), db_dir).unwrap(); - assert_eq!(db.get(b"key").unwrap().unwrap(), b"value"); - } -} diff --git a/core/lib/config/src/configs/database.rs b/core/lib/config/src/configs/database.rs index dcff7d486a83..578cd6be46a6 100644 --- a/core/lib/config/src/configs/database.rs +++ b/core/lib/config/src/configs/database.rs @@ -23,9 +23,6 @@ pub struct MerkleTreeConfig { /// Path to the RocksDB data directory for Merkle tree. #[serde(default = "MerkleTreeConfig::default_path")] pub path: String, - /// Path to merkle tree backup directory. - #[serde(default = "MerkleTreeConfig::default_backup_path")] - pub backup_path: String, /// Operation mode for the Merkle tree. If not specified, the full mode will be used. #[serde(default)] pub mode: MerkleTreeMode, @@ -53,7 +50,6 @@ impl Default for MerkleTreeConfig { fn default() -> Self { Self { path: Self::default_path(), - backup_path: Self::default_backup_path(), mode: MerkleTreeMode::default(), multi_get_chunk_size: Self::default_multi_get_chunk_size(), block_cache_size_mb: Self::default_block_cache_size_mb(), @@ -69,10 +65,6 @@ impl MerkleTreeConfig { "./db/lightweight-new".to_owned() // named this way for legacy reasons } - fn default_backup_path() -> String { - "./db/backups".to_owned() - } - const fn default_multi_get_chunk_size() -> usize { 500 } @@ -120,30 +112,12 @@ pub struct DBConfig { // ^ Filled in separately in `Self::from_env()`. We cannot use `serde(flatten)` because it // doesn't work with 'envy`. pub merkle_tree: MerkleTreeConfig, - /// Number of backups to keep. - #[serde(default = "DBConfig::default_backup_count")] - pub backup_count: usize, - /// Time interval between performing backups. - #[serde(default = "DBConfig::default_backup_interval_ms")] - pub backup_interval_ms: u64, } impl DBConfig { fn default_state_keeper_db_path() -> String { "./db/state_keeper".to_owned() } - - const fn default_backup_count() -> usize { - 5 - } - - const fn default_backup_interval_ms() -> u64 { - 60_000 - } - - pub fn backup_interval(&self) -> Duration { - Duration::from_millis(self.backup_interval_ms) - } } /// Collection of different database URLs and general PostgreSQL options. diff --git a/core/lib/env_config/src/database.rs b/core/lib/env_config/src/database.rs index e350c487662a..e6269cd23768 100644 --- a/core/lib/env_config/src/database.rs +++ b/core/lib/env_config/src/database.rs @@ -59,29 +59,23 @@ mod tests { let mut lock = MUTEX.lock(); let config = r#" DATABASE_STATE_KEEPER_DB_PATH="/db/state_keeper" - DATABASE_MERKLE_TREE_BACKUP_PATH="/db/backups" DATABASE_MERKLE_TREE_PATH="/db/tree" DATABASE_MERKLE_TREE_MODE=lightweight DATABASE_MERKLE_TREE_MULTI_GET_CHUNK_SIZE=250 DATABASE_MERKLE_TREE_MEMTABLE_CAPACITY_MB=512 DATABASE_MERKLE_TREE_STALLED_WRITES_TIMEOUT_SEC=60 DATABASE_MERKLE_TREE_MAX_L1_BATCHES_PER_ITER=50 - DATABASE_BACKUP_COUNT=5 - DATABASE_BACKUP_INTERVAL_MS=60000 "#; lock.set_env(config); let db_config = DBConfig::from_env().unwrap(); assert_eq!(db_config.state_keeper_db_path, "/db/state_keeper"); assert_eq!(db_config.merkle_tree.path, "/db/tree"); - assert_eq!(db_config.merkle_tree.backup_path, "/db/backups"); assert_eq!(db_config.merkle_tree.mode, MerkleTreeMode::Lightweight); assert_eq!(db_config.merkle_tree.multi_get_chunk_size, 250); assert_eq!(db_config.merkle_tree.max_l1_batches_per_iter, 50); assert_eq!(db_config.merkle_tree.memtable_capacity_mb, 512); assert_eq!(db_config.merkle_tree.stalled_writes_timeout_sec, 60); - assert_eq!(db_config.backup_count, 5); - assert_eq!(db_config.backup_interval().as_secs(), 60); } #[test] @@ -97,22 +91,17 @@ mod tests { "DATABASE_MERKLE_TREE_MEMTABLE_CAPACITY_MB", "DATABASE_MERKLE_TREE_STALLED_WRITES_TIMEOUT_SEC", "DATABASE_MERKLE_TREE_MAX_L1_BATCHES_PER_ITER", - "DATABASE_BACKUP_COUNT", - "DATABASE_BACKUP_INTERVAL_MS", ]); let db_config = DBConfig::from_env().unwrap(); assert_eq!(db_config.state_keeper_db_path, "./db/state_keeper"); assert_eq!(db_config.merkle_tree.path, "./db/lightweight-new"); - assert_eq!(db_config.merkle_tree.backup_path, "./db/backups"); assert_eq!(db_config.merkle_tree.mode, MerkleTreeMode::Full); assert_eq!(db_config.merkle_tree.multi_get_chunk_size, 500); assert_eq!(db_config.merkle_tree.max_l1_batches_per_iter, 20); assert_eq!(db_config.merkle_tree.block_cache_size_mb, 128); assert_eq!(db_config.merkle_tree.memtable_capacity_mb, 256); assert_eq!(db_config.merkle_tree.stalled_writes_timeout_sec, 30); - assert_eq!(db_config.backup_count, 5); - assert_eq!(db_config.backup_interval().as_secs(), 60); // Check that new env variable for Merkle tree path is supported lock.set_env("DATABASE_MERKLE_TREE_PATH=/db/tree/main"); diff --git a/core/lib/zksync_core/src/lib.rs b/core/lib/zksync_core/src/lib.rs index de286db73d25..275a81e23229 100644 --- a/core/lib/zksync_core/src/lib.rs +++ b/core/lib/zksync_core/src/lib.rs @@ -218,7 +218,6 @@ pub enum Component { Tree, // TODO(BFT-273): Remove `TreeLightweight` component as obsolete TreeLightweight, - TreeBackup, /// Merkle tree API. TreeApi, EthWatcher, @@ -257,7 +256,6 @@ impl FromStr for Components { "tree_lightweight" | "tree_lightweight_new" => { Ok(Components(vec![Component::TreeLightweight])) } - "tree_backup" => Ok(Components(vec![Component::TreeBackup])), "tree_api" => Ok(Components(vec![Component::TreeApi])), "state_keeper" => Ok(Components(vec![Component::StateKeeper])), "housekeeper" => Ok(Components(vec![Component::Housekeeper])), @@ -756,10 +754,6 @@ async fn add_trees_to_task_futures( store_factory: &ObjectStoreFactory, stop_receiver: watch::Receiver, ) -> anyhow::Result<()> { - if components.contains(&Component::TreeBackup) { - anyhow::bail!("Tree backup mode is disabled"); - } - let db_config = configs.db_config.clone().context("db_config")?; let operation_config = configs .operations_manager_config diff --git a/core/lib/zksync_core/src/metadata_calculator/mod.rs b/core/lib/zksync_core/src/metadata_calculator/mod.rs index dd088b9da908..0cfea020203c 100644 --- a/core/lib/zksync_core/src/metadata_calculator/mod.rs +++ b/core/lib/zksync_core/src/metadata_calculator/mod.rs @@ -114,7 +114,6 @@ pub struct MetadataCalculator { impl MetadataCalculator { /// Creates a calculator with the specified `config`. pub async fn new(config: &MetadataCalculatorConfig<'_>) -> Self { - // TODO (SMA-1726): restore the tree from backup if appropriate assert!( config.max_l1_batches_per_iter > 0, "Maximum L1 batches per iteration is misconfigured to be 0; please update it to positive value" @@ -271,6 +270,4 @@ impl MetadataCalculator { tracing::trace!("L1 batch metadata: {metadata:?}"); metadata } - - // TODO (SMA-1726): Integrate tree backup mode } diff --git a/core/lib/zksync_core/src/metadata_calculator/tests.rs b/core/lib/zksync_core/src/metadata_calculator/tests.rs index 9aad3b3b93e6..4e480ce1d998 100644 --- a/core/lib/zksync_core/src/metadata_calculator/tests.rs +++ b/core/lib/zksync_core/src/metadata_calculator/tests.rs @@ -56,8 +56,6 @@ async fn genesis_creation() { assert_eq!(tree.next_l1_batch_number(), L1BatchNumber(1)); } -// TODO (SMA-1726): Restore tests for tree backup mode - #[tokio::test] async fn basic_workflow() { let pool = ConnectionPool::test_pool().await; diff --git a/docker/server-v2/Dockerfile b/docker/server-v2/Dockerfile index 6bd011d6d6cd..51048e11c167 100644 --- a/docker/server-v2/Dockerfile +++ b/docker/server-v2/Dockerfile @@ -31,7 +31,6 @@ EXPOSE 3030 COPY --from=builder /usr/src/zksync/target/release/zksync_server /usr/bin COPY --from=builder /usr/src/zksync/target/release/block_reverter /usr/bin COPY --from=builder /usr/src/zksync/target/release/merkle_tree_consistency_checker /usr/bin -COPY --from=builder /usr/src/zksync/target/release/rocksdb_util /usr/bin COPY contracts/system-contracts/bootloader/build/artifacts/ /contracts/system-contracts/bootloader/build/artifacts/ COPY contracts/system-contracts/contracts/artifacts/ /contracts/system-contracts/contracts/artifacts/ COPY contracts/system-contracts/contracts/precompiles/artifacts/ /contracts/system-contracts/contracts/precompiles/artifacts/