feat: Snapshot Creator (matter-labs#498)

## What ❔
Snapshot creator is small command line tool for creating a snapshot of
zkSync node for EN node to be able to initialize
to a certain L1 Batch.

Snapshots do not contain full transactions history, but rather a minimal
subset of information needed to bootstrap EN
node.

## Checklist

<!-- Check your PR fulfills the following items. -->
<!-- For draft PRs check the boxes as you complete them. -->

- [X] PR title corresponds to the body of PR (we generate changelog
entries from PRs).
- [X] Tests for the changes have been added / updated.
- [X] Documentation comments have been added / updated.
- [X] Code has been formatted via `zk fmt` and `zk lint`.

.github/workflows/ci-core-reusable.yml

@@ -280,7 +280,7 @@ jobs:
 
       # TODO(PLA-653): Restore bridge tests for EN.
       - name: Integration tests
-        run: ci_run zk test i server --testPathIgnorePatterns 'contract-verification|custom-erc20-bridge'
+        run: ci_run zk test i server --testPathIgnorePatterns 'contract-verification|custom-erc20-bridge|snapshots-creator'
 
       - name: Run Cross EN Checker
         run: ci_run zk run cross-en-checker

Cargo.toml

@@ -6,6 +6,7 @@ members = [
     "core/bin/external_node",
     "core/bin/merkle_tree_consistency_checker",
     "core/bin/rocksdb_util",
+    "core/bin/snapshots_creator",
     "core/bin/storage_logs_dedup_migration",
     "core/bin/system-constants-generator",
     "core/bin/verification_key_generator_and_server",

core/bin/external_node/src/config/mod.rs

@@ -338,7 +338,7 @@ impl OptionalENConfig {
     pub fn api_namespaces(&self) -> Vec<Namespace> {
         self.api_namespaces
             .clone()
-            .unwrap_or_else(|| Namespace::NON_DEBUG.to_vec())
+            .unwrap_or_else(|| Namespace::DEFAULT.to_vec())
     }
 
     pub fn max_response_body_size(&self) -> usize {

core/bin/snapshots_creator/Cargo.toml

@@ -0,0 +1,30 @@
+[package]
+name = "snapshots_creator"
+version = "0.1.0"
+edition = "2021"
+authors = ["The Matter Labs Team <[email protected]>"]
+homepage = "https://zksync.io/"
+repository = "https://github.com/matter-labs/zksync-era"
+license = "MIT OR Apache-2.0"
+keywords = ["blockchain", "zksync"]
+categories = ["cryptography"]
+publish = false # We don't want to publish our binaries.
+
+[dependencies]
+vise = { git = "https://github.com/matter-labs/vise.git", version = "0.1.0", rev = "dd05139b76ab0843443ab3ff730174942c825dae" }
+prometheus_exporter = { path = "../../lib/prometheus_exporter" }
+zksync_config = { path = "../../lib/config" }
+zksync_dal = { path = "../../lib/dal" }
+zksync_env_config = { path = "../../lib/env_config" }
+zksync_utils = { path = "../../lib/utils" }
+zksync_types = { path = "../../lib/types" }
+zksync_core = { path = "../../lib/zksync_core" }
+zksync_object_store = { path = "../../lib/object_store" }
+vlog = { path = "../../lib/vlog" }
+
+anyhow = "1.0"
+tokio = { version = "1", features = ["full"] }
+tracing = "0.1"
+futures = "0.3"
+serde = { version = "1.0.189", features = ["derive"] }
+serde_json = "1.0"

core/bin/snapshots_creator/README.md

@@ -0,0 +1,24 @@
+# Snapshots Creator
+
+Snapshot creator is small command line tool for creating a snapshot of zkSync node for EN node to be able to initialize
+to a certain L1 Batch.
+
+Snapshots do not contain full transactions history, but rather a minimal subset of information needed to bootstrap EN
+node.
+
+Usage (local development):\
+First run `zk env dev` \
+then the creator can be run using:  
+`zk run snapshots_creator`
+
+Snapshot contents can be stored based on blob_store config either in local filesystem or GS.
+
+## Snapshots format
+
+Each snapshot consists of three types of objects (see
+[snapshots.rs](https://github.com/matter-labs/zksync-era/blob/main/core/lib/types/src/snapshots.rs)) : header, storage
+logs chunks and factory deps:
+
+- Snapshot Header (currently returned by snapshots namespace of JSON-RPC API)
+- Snapshot Storage logs chunks (most likely to be stored in gzipped protobuf files, but this part is still WIP) :
+- Factory dependencies (most likely to be stored as protobufs in the very near future)

core/bin/snapshots_creator/src/chunking.rs

@@ -0,0 +1,22 @@
+use std::ops;
+
+use zksync_types::{H256, U256};
+use zksync_utils::u256_to_h256;
+
+pub fn get_chunk_hashed_keys_range(chunk_id: u64, chunks_count: u64) -> ops::RangeInclusive<H256> {
+    assert!(chunks_count > 0);
+    let mut stride = U256::MAX / chunks_count;
+    let stride_minus_one = if stride < U256::MAX {
+        stride += U256::one();
+        stride - 1
+    } else {
+        stride // `stride` is really 1 << 256 == U256::MAX + 1
+    };
+
+    let start = stride * chunk_id;
+    let (mut end, is_overflow) = stride_minus_one.overflowing_add(start);
+    if is_overflow {
+        end = U256::MAX;
+    }
+    u256_to_h256(start)..=u256_to_h256(end)
+}