From 67050eece174fa6d8e2b494d4ec40312772a25c8 Mon Sep 17 00:00:00 2001 From: Rim Rakhimov Date: Thu, 24 Oct 2024 12:17:05 +0400 Subject: [PATCH] feat(stylus-verifier): make use of docker api (#1074) * feat(stylus-verifier): make use of docker api * feat(stylus-verifier): add docker api url argument into settings * feat(stylus-verifier): add support unix-based docker api addresses * fix(stylus-verifier): upload project directory into github container explicitly * refactor(stylus-verifier): add trace logging on image build; rename start_container to run_container --- stylus-verifier/Cargo.lock | 194 +++++++++- stylus-verifier/Cargo.toml | 5 + stylus-verifier/Dockerfile | 3 +- .../stylus-verifier-logic/Cargo.toml | 5 + .../stylus-verifier-logic/src/docker.rs | 336 +++++++++++++----- .../src/stylus_sdk_rs.rs | 7 + .../stylus-verifier-server/Cargo.toml | 2 +- .../stylus-verifier-server/src/server.rs | 3 +- .../src/services/stylus_sdk_rs_verifier.rs | 9 +- .../stylus-verifier-server/src/settings.rs | 16 + 10 files changed, 471 insertions(+), 109 deletions(-) diff --git a/stylus-verifier/Cargo.lock b/stylus-verifier/Cargo.lock index 37b0419d2..b7e2189a9 100644 --- a/stylus-verifier/Cargo.lock +++ b/stylus-verifier/Cargo.lock @@ -264,6 +264,12 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +[[package]] +name = "adler2" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" + [[package]] name = "ahash" version = "0.7.8" @@ -628,7 +634,7 @@ dependencies = [ "cc", "cfg-if", "libc", - "miniz_oxide", + "miniz_oxide 0.7.1", "object", "rustc-demangle", ] @@ -766,6 +772,50 @@ dependencies = [ "uuid", ] +[[package]] +name = "bollard" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d41711ad46fda47cd701f6908e59d1bd6b9a2b7464c0d0aeab95c6d37096ff8a" +dependencies = [ + "base64 0.22.1", + "bollard-stubs", + "bytes", + "futures-core", + "futures-util", + "hex", + "http 1.1.0", + "http-body-util", + "hyper 1.4.1", + "hyper-named-pipe", + "hyper-util", + "hyperlocal", + "log", + "pin-project-lite", + "serde", + "serde_derive", + "serde_json", + "serde_repr", + "serde_urlencoded", + "thiserror", + "tokio", + "tokio-util", + "tower-service", + "url", + "winapi", +] + +[[package]] +name = "bollard-stubs" +version = "1.45.0-rc.26.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d7c5415e3a6bc6d3e99eff6268e488fd4ee25e7b28c10f08fa6760bd9de16e4" +dependencies = [ + "serde", + "serde_repr", + "serde_with 3.9.0", +] + [[package]] name = "brotli" version = "3.4.0" @@ -1222,6 +1272,18 @@ dependencies = [ "subtle", ] +[[package]] +name = "filetime" +version = "0.2.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35c0522e981e68cbfa8c3f978441a5f34b30b96e146b33cd3359176b50fe8586" +dependencies = [ + "cfg-if", + "libc", + "libredox", + "windows-sys 0.59.0", +] + [[package]] name = "fixed-hash" version = "0.8.0" @@ -1242,12 +1304,12 @@ checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" [[package]] name = "flate2" -version = "1.0.28" +version = "1.0.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46303f565772937ffe1d394a4fac6f411c6013172fadde9dcdb1e147a086940e" +checksum = "a1b589b4dc103969ad3cf85c950899926ec64300a1a46d76c03a6072957036f0" dependencies = [ "crc32fast", - "miniz_oxide", + "miniz_oxide 0.8.0", ] [[package]] @@ -1303,9 +1365,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff4dd66668b557604244583e3e1e1eada8c5c2e96a6d0d6653ede395b78bbacb" +checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" dependencies = [ "futures-core", "futures-sink", @@ -1313,9 +1375,9 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb1d22c66e66d9d72e1758f0bd7d4fd0bee04cad842ee34587d68c07e45d088c" +checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" [[package]] name = "futures-executor" @@ -1336,9 +1398,9 @@ checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" [[package]] name = "futures-macro" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53b153fd91e4b0147f4aced87be237c98248656bb01050b96bf3ee89220a8ddb" +checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", @@ -1347,21 +1409,21 @@ dependencies = [ [[package]] name = "futures-sink" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e36d3378ee38c2a36ad710c5d30c2911d752cb941c00c72dbabfb786a7970817" +checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5" [[package]] name = "futures-task" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "efd193069b0ddadc69c46389b740bbccdd97203899b48d09c5f7969591d6bae2" +checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004" [[package]] name = "futures-util" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a19526d624e703a3179b3d322efec918b6246ea0fa51d41124525f00f1cc8104" +checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" dependencies = [ "futures-channel", "futures-core", @@ -1632,6 +1694,7 @@ dependencies = [ "http 1.1.0", "http-body 1.0.1", "httparse", + "httpdate", "itoa", "pin-project-lite", "smallvec", @@ -1639,6 +1702,21 @@ dependencies = [ "want", ] +[[package]] +name = "hyper-named-pipe" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73b7d8abf35697b81a825e386fc151e0d503e8cb5fcb93cc8669c376dfd6f278" +dependencies = [ + "hex", + "hyper 1.4.1", + "hyper-util", + "pin-project-lite", + "tokio", + "tower-service", + "winapi", +] + [[package]] name = "hyper-rustls" version = "0.27.3" @@ -1717,6 +1795,21 @@ dependencies = [ "tracing", ] +[[package]] +name = "hyperlocal" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "986c5ce3b994526b3cd75578e62554abd09f0899d6206de48b3e96ab34ccc8c7" +dependencies = [ + "hex", + "http-body-util", + "hyper 1.4.1", + "hyper-util", + "pin-project-lite", + "tokio", + "tower-service", +] + [[package]] name = "iana-time-zone" version = "0.1.58" @@ -1924,6 +2017,17 @@ version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ec2a862134d2a7d32d7983ddcdd1c4923530833c9f2ea1a44fc5fa473989058" +[[package]] +name = "libredox" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" +dependencies = [ + "bitflags 2.4.1", + "libc", + "redox_syscall 0.5.6", +] + [[package]] name = "libssh2-sys" version = "0.3.0" @@ -2037,6 +2141,15 @@ dependencies = [ "adler", ] +[[package]] +name = "miniz_oxide" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2d80299ef12ff69b16a84bb182e3b9df68b5a91574d3d4fa6e41b65deec4df1" +dependencies = [ + "adler2", +] + [[package]] name = "mio" version = "0.8.9" @@ -2358,7 +2471,7 @@ checksum = "4c42a9226546d68acdd9c0a280d17ce19bfe27a46bf68784e4066115788d008e" dependencies = [ "cfg-if", "libc", - "redox_syscall", + "redox_syscall 0.4.1", "smallvec", "windows-targets 0.48.5", ] @@ -2762,6 +2875,15 @@ dependencies = [ "bitflags 1.3.2", ] +[[package]] +name = "redox_syscall" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "355ae415ccd3a04315d3f8246e86d67689ea74d88d915576e1589a351062a13b" +dependencies = [ + "bitflags 2.4.1", +] + [[package]] name = "regex" version = "1.10.2" @@ -3199,6 +3321,17 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_repr" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c64451ba24fc7a6a2d60fc75dd9c83c90903b19028d4eff35e88fc1e86564e9" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.77", +] + [[package]] name = "serde_spanned" version = "0.6.7" @@ -3427,18 +3560,23 @@ dependencies = [ "alloy-json-abi", "anyhow", "blockscout-display-bytes", + "bollard", "bytes", + "flate2", + "futures-util", "git2", "pretty_assertions", "semver 1.0.23", "serde_json", "stylus-verifier-proto", + "tar", "tempfile", "thiserror", "tokio", "toml 0.8.19", "tracing", "url", + "uuid", ] [[package]] @@ -3571,6 +3709,17 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" +[[package]] +name = "tar" +version = "0.4.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ff6c40d3aedb5e06b57c6f669ad17ab063dd1e63d977c6a88e7f4dfa4f04020" +dependencies = [ + "filetime", + "libc", + "xattr", +] + [[package]] name = "tempfile" version = "3.12.0" @@ -4475,6 +4624,17 @@ dependencies = [ "tap", ] +[[package]] +name = "xattr" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8da84f1a25939b27f6820d92aed108f83ff920fdf11a7b19366c27c4cda81d4f" +dependencies = [ + "libc", + "linux-raw-sys", + "rustix", +] + [[package]] name = "yaml-rust" version = "0.4.5" diff --git a/stylus-verifier/Cargo.toml b/stylus-verifier/Cargo.toml index 5f8d2ad23..ed4e3afd4 100644 --- a/stylus-verifier/Cargo.toml +++ b/stylus-verifier/Cargo.toml @@ -20,7 +20,10 @@ anyhow = { version = "1.0.87" } async-trait = { version = "0.1.82" } blockscout-display-bytes = { version = "1.1.0" } blockscout-service-launcher = { version = "0.13.1" } +bollard = { version = "0.17.1" } bytes = { version = "1.7.1" } +flate2 = { version = "1.0.34" } +futures-util = { version = "0.3.30" } git2 = { version = "0.19.0" } pretty_assertions = { version = "1.4.0" } prost = { version = "0.11" } @@ -30,6 +33,7 @@ semver = { version = "1.0.23" } serde = { version = "1" } serde_json = { version = "1.0.128" } serde_with = { version = "3.9" } +tar = { version = "0.4.42" } tempfile = { version = "3.12.0" } thiserror = { version = "1.0.63" } tokio = { version = "1.40.0" } @@ -38,3 +42,4 @@ tonic = { version = "0.8" } tonic-build = { version = "0.8" } tracing = { version = "0.1.40" } url = { version = "2.5.2" } +uuid = { version = "1.10.0" } diff --git a/stylus-verifier/Dockerfile b/stylus-verifier/Dockerfile index 974c408d5..97cde2891 100644 --- a/stylus-verifier/Dockerfile +++ b/stylus-verifier/Dockerfile @@ -19,8 +19,7 @@ RUN cargo build --release FROM ubuntu:20.04 AS run RUN apt-get update && \ - apt-get install -y libssl1.1 libssl-dev ca-certificates curl && \ - curl -sSL https://get.docker.com/ | sh + apt-get install -y libssl1.1 libssl-dev ca-certificates curl WORKDIR /app ENV APP_USER=app diff --git a/stylus-verifier/stylus-verifier-logic/Cargo.toml b/stylus-verifier/stylus-verifier-logic/Cargo.toml index 56965f8fb..bc8ead449 100644 --- a/stylus-verifier/stylus-verifier-logic/Cargo.toml +++ b/stylus-verifier/stylus-verifier-logic/Cargo.toml @@ -7,15 +7,20 @@ edition = "2021" alloy-json-abi = { workspace = true } anyhow = { workspace = true } blockscout-display-bytes = { workspace = true } # conversion related +bollard = { workspace = true } bytes = { workspace = true } +flate2 = { workspace = true } git2 = { workspace = true } semver = { workspace = true } +futures-util = { workspace = true } serde_json = { workspace = true } stylus-verifier-proto = { workspace = true } +tar = { workspace = true } tempfile = { workspace = true } thiserror = { workspace = true } tokio = { workspace = true, features = ["process"] } toml = { workspace = true } tracing = { workspace = true } url = { workspace = true } +uuid = { workspace = true } pretty_assertions = { workspace = true } diff --git a/stylus-verifier/stylus-verifier-logic/src/docker.rs b/stylus-verifier/stylus-verifier-logic/src/docker.rs index cfaa3f077..6291c6eb3 100644 --- a/stylus-verifier/stylus-verifier-logic/src/docker.rs +++ b/stylus-verifier/stylus-verifier-logic/src/docker.rs @@ -1,11 +1,46 @@ -// Adapted from https://github.com/OffchainLabs/cargo-stylus - use anyhow::Context; +use bollard::{ + container::{ + self, AttachContainerOptions, CreateContainerOptions, LogOutput, UploadToContainerOptions, + }, + image::{BuildImageOptions, BuilderVersion}, + models::HostConfig, + Docker, +}; +use futures_util::stream::StreamExt; use semver::Version; -use std::{io::Write, path::Path}; -use tokio::{io::AsyncWriteExt, process::Command}; +use std::{io::Write, path::Path, str}; +use url::Url; +use uuid::Uuid; + +/// Default timeout for all requests is 2 minutes. +const DEFAULT_TIMEOUT: u64 = 120; + +const WORKDIR: &str = "/source"; + +pub async fn connect(addr: &Url) -> Result { + let docker = match addr.scheme() { + "unix" => { + Docker::connect_with_local(addr.as_ref(), DEFAULT_TIMEOUT, bollard::API_DEFAULT_VERSION) + } + "http" | "tcp" => { + Docker::connect_with_http(addr.as_ref(), DEFAULT_TIMEOUT, bollard::API_DEFAULT_VERSION) + } + _ => anyhow::bail!( + "unsupported docker API scheme: {}. Expected one of 'unix', 'http', 'tcp'", + addr.scheme() + ), + } + .context("connection failed")?; + docker + .ping() + .await + .context("connected daemon ping failed")?; + Ok(docker) +} pub async fn run_reproducible( + docker: &Docker, cargo_stylus_version: &Version, toolchain: &Version, dir: &Path, @@ -20,8 +55,23 @@ pub async fn run_reproducible( for s in command_line.iter() { command.push(s); } - create_image(cargo_stylus_version, toolchain).await?; - run_in_docker_container(cargo_stylus_version, toolchain, dir, &command).await + let image_name = version_to_image_name(cargo_stylus_version, toolchain); + create_image(docker, &image_name, cargo_stylus_version, toolchain) + .await + .context("creating image")?; + + let container_id = create_container(docker, &image_name, &command) + .await + .context("creating container")?; + copy_directory_to_container(docker, &container_id, dir) + .await + .context("copying directory to container")?; + + let output = run_container(docker, &container_id) + .await + .context("running container")?; + + Ok(output) } fn version_to_image_name(cargo_stylus_version: &Version, toolchain: &Version) -> String { @@ -31,41 +81,26 @@ fn version_to_image_name(cargo_stylus_version: &Version, toolchain: &Version) -> ) } -fn validate_docker_output(output: &std::process::Output) -> anyhow::Result { - if !output.status.success() { - let stderr = - std::str::from_utf8(&output.stderr).context("failed to read Docker command stderr")?; - if stderr.contains("Cannot connect to the Docker daemon") { - tracing::error!("Docker is not found in the system"); - anyhow::bail!("Docker is not running"); - } - tracing::error!("Docker command failed: {stderr}"); - anyhow::bail!("Docker command failed: {stderr}"); +async fn image_exists(docker: &Docker, name: &str) -> Result { + match docker.inspect_image(name).await { + Ok(_) => Ok(true), + Err(bollard::errors::Error::DockerResponseServerError { + status_code: 404, .. + }) => Ok(false), + Err(err) => Err(err).context("failed to inspect docker image"), } - - let stdout = std::str::from_utf8(&output.stdout).context("failed to read Docker stdout")?; - Ok(stdout.to_string()) -} - -async fn image_exists(name: &str) -> Result { - let output = Command::new("docker") - .arg("images") - .arg(name) - .output() - .await - .context("failed to execute Docker command")?; - - let stdout = validate_docker_output(&output)?; - - Ok(stdout.chars().filter(|c| *c == '\n').count() > 1) } async fn create_image( + docker: &Docker, + image_name: &str, cargo_stylus_version: &Version, toolchain: &Version, ) -> Result<(), anyhow::Error> { - let name = version_to_image_name(cargo_stylus_version, toolchain); - if image_exists(&name).await.context("check if image exists")? { + if image_exists(docker, image_name) + .await + .context("check if image exists")? + { return Ok(()); } @@ -74,65 +109,194 @@ async fn create_image( cargo_stylus_version, toolchain ); - let mut child = Command::new("docker") - .arg("build") - .arg("-t") - .arg(name) - .arg(".") - .arg("-f-") - .stdin(std::process::Stdio::piped()) - .spawn() - .context("failed to execute Docker build command")?; - - let mut dockerfile = Vec::::new(); - write!( - dockerfile, - "\ - FROM --platform=linux/amd64 offchainlabs/cargo-stylus-base:{cargo_stylus_version} as base - RUN rustup toolchain install {toolchain}-x86_64-unknown-linux-gnu - RUN rustup default {toolchain}-x86_64-unknown-linux-gnu - RUN rustup target add wasm32-unknown-unknown - RUN rustup component add rust-src --toolchain {toolchain}-x86_64-unknown-linux-gnu - ", - ).expect("write into the vector should not fail"); - - child - .stdin - .as_mut() - .unwrap() - .write_all(&dockerfile) - .await - .context("failed to write dockerfile content into docker process stdin")?; - child.wait().await.context("wait failed")?; + + let dockerfile = format!( + "FROM offchainlabs/cargo-stylus-base:{cargo_stylus_version} as base +RUN rustup toolchain install {toolchain}-x86_64-unknown-linux-gnu +RUN rustup default {toolchain}-x86_64-unknown-linux-gnu +RUN rustup target add wasm32-unknown-unknown +RUN rustup component add rust-src --toolchain {toolchain}-x86_64-unknown-linux-gnu +" + ); + + let content = build_tar_with_dockerfile(&dockerfile)?; + let build_image_options = BuildImageOptions { + t: image_name.to_string(), + dockerfile: "Dockerfile".to_string(), + version: BuilderVersion::BuilderV1, + networkmode: "host".to_string(), + pull: true, + rm: true, + forcerm: true, + platform: "linux/amd64".to_string(), + ..Default::default() + }; + + let mut image_build_stream = + docker.build_image(build_image_options, None, Some(content.into())); + + let mut output = vec![]; + while let Some(result) = image_build_stream.next().await { + match result { + Ok(info) => { + if let Some(value) = info.stream { + tracing::trace!(image_name = image_name, value = value, "building an image"); + output.push(value) + } + } + Err(bollard::errors::Error::DockerStreamError { error }) => { + output.push(error); + let output = output.join(""); + tracing::error!( + image_name = image_name, + output = output, + "error while building an image" + ); + anyhow::bail!( + "error while building an image; image_name={image_name}, output={output}" + ); + } + Err(err) => { + let output = output.join(""); + tracing::error!( + image_name = image_name, + output = output, + error = err.to_string(), + "unknown error while building an image" + ); + anyhow::bail!("unknown error while building an image; image_name={image_name}, output={output}, error={err}"); + } + } + } Ok(()) } -async fn run_in_docker_container( - cargo_stylus_version: &Version, - toolchain: &Version, - dir: &Path, +async fn create_container( + docker: &Docker, + image_name: &str, command_line: &[&str], ) -> Result { - let name = version_to_image_name(cargo_stylus_version, toolchain); - if !image_exists(&name).await? { - anyhow::bail!("Docker image {name} doesn't exist"); - } + let container_suffix = Uuid::new_v4(); + let container_name = format!( + "{}-{container_suffix}", + image_name.replace(|c: char| !c.is_alphanumeric(), "_") + ); + + let options = CreateContainerOptions { + name: container_name.clone(), + ..Default::default() + }; + let config = container::Config { + image: Some(image_name), + working_dir: Some(WORKDIR), + host_config: Some(HostConfig { + network_mode: Some("host".to_string()), + auto_remove: Some(true), + ..Default::default() + }), + cmd: Some(command_line.into()), + ..Default::default() + }; - let output = Command::new("docker") - .arg("run") - .arg("--network") - .arg("host") - .arg("-w") - .arg("/source") - .arg("-v") - .arg(format!("{}:/source", dir.as_os_str().to_str().unwrap())) - .arg("--rm") - .arg(name) - .args(command_line) - .output() + let container = docker.create_container(Some(options), config).await?; + + Ok(container.id) +} + +async fn copy_directory_to_container( + docker: &Docker, + container_id: &str, + dir: &Path, +) -> Result<(), anyhow::Error> { + let tar = build_tar_from_directory(dir).context("building tar archive from directory")?; + + let options = UploadToContainerOptions { + path: WORKDIR, + no_overwrite_dir_non_dir: "", + }; + docker + .upload_to_container(container_id, Some(options), tar.into()) .await - .context("failed to execute Docker run command")?; + .context("uploading tar archive to container")?; + + Ok(()) +} + +async fn run_container(docker: &Docker, container_id: &str) -> Result { + docker.start_container::(container_id, None).await?; + let mut attach_results = docker + .attach_container::( + container_id, + Some(AttachContainerOptions { + stdout: Some(true), + stderr: Some(true), + stream: Some(true), + logs: Some(true), + ..Default::default() + }), + ) + .await?; + + let mut container_output = vec![]; + while let Some(result) = attach_results.output.next().await { + match result { + Ok(output) => match output { + LogOutput::StdErr { message } => container_output.push(message), + LogOutput::StdOut { message } => container_output.push(message), + _ => (), + }, + Err(err) => { + tracing::error!( + err = err.to_string(), + "reading output from attached container failed" + ); + Err(err).context("reading output from attached container")? + } + } + } + + Ok(container_output.into_iter().filter_map(|output| {match str::from_utf8(&output) { + Ok(s) => Some(s.to_string()), + Err(err) => { + tracing::warn!(line=?output, err=err.to_string(), "converting output line to utf8 string failed"); + None + } + }}).collect::>().join("")) +} + +fn build_tar_with_dockerfile(content: &str) -> Result, anyhow::Error> { + let mut header = tar::Header::new_gnu(); + header + .set_path("Dockerfile") + .context("set dockerfile path in the header")?; + header.set_size(content.len() as u64); + header.set_mode(0o755); + header.set_cksum(); + let mut tar = tar::Builder::new(Vec::new()); + tar.append(&header, content.as_bytes()) + .context("append dockerfile")?; + + let uncompressed = tar + .into_inner() + .context("convert tar into inner representation")?; + compress_archive(&uncompressed) +} + +fn build_tar_from_directory(dir: &Path) -> Result, anyhow::Error> { + let mut tar = tar::Builder::new(Vec::new()); + tar.append_dir_all("", dir) + .context("appending files from directory")?; + let uncompressed = tar + .into_inner() + .context("convert tar into inner representation")?; + compress_archive(&uncompressed) +} - validate_docker_output(&output) +fn compress_archive(uncompressed: &[u8]) -> Result, anyhow::Error> { + let mut encoder = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default()); + encoder + .write_all(uncompressed) + .context("write uncompressed data to encoder")?; + encoder.finish().context("finish encoding") } diff --git a/stylus-verifier/stylus-verifier-logic/src/stylus_sdk_rs.rs b/stylus-verifier/stylus-verifier-logic/src/stylus_sdk_rs.rs index c51bb26f2..96e7b92f2 100644 --- a/stylus-verifier/stylus-verifier-logic/src/stylus_sdk_rs.rs +++ b/stylus-verifier/stylus-verifier-logic/src/stylus_sdk_rs.rs @@ -69,7 +69,12 @@ pub enum Error { Internal(#[from] anyhow::Error), } +pub type Docker = bollard::Docker; + +pub use docker::connect as docker_connect; + pub async fn verify_github_repository( + docker: &Docker, request: VerifyGithubRepositoryRequest, ) -> Result { let repo_directory = @@ -84,6 +89,7 @@ pub async fn verify_github_repository( let toolchain = validate_toolchain_channel(&toolchain_channel)?; let verify_output = docker::run_reproducible( + docker, &request.cargo_stylus_version, &toolchain, &project_path, @@ -112,6 +118,7 @@ pub async fn verify_github_repository( } let export_abi_output = docker::run_reproducible( + docker, &request.cargo_stylus_version, &toolchain, &project_path, diff --git a/stylus-verifier/stylus-verifier-server/Cargo.toml b/stylus-verifier/stylus-verifier-server/Cargo.toml index 3d5f5257b..c2caf0aff 100644 --- a/stylus-verifier/stylus-verifier-server/Cargo.toml +++ b/stylus-verifier/stylus-verifier-server/Cargo.toml @@ -14,6 +14,7 @@ stylus-verifier-logic = { workspace = true } stylus-verifier-proto = { workspace = true } tokio = { workspace = true, features = ["macros", "rt-multi-thread", "fs"] } tonic = { workspace = true } +url = { workspace = true, features = ["serde"] } [dev-dependencies] blockscout-display-bytes = { workspace = true } @@ -23,5 +24,4 @@ pretty_assertions = { workspace = true } reqwest = { workspace = true, features = ["json"] } serde_json = { workspace = true } serde_with = { workspace = true, features = ["json"] } -url = { workspace = true, features = ["serde"] } diff --git a/stylus-verifier/stylus-verifier-server/src/server.rs b/stylus-verifier/stylus-verifier-server/src/server.rs index fc2c078a1..7cab8bffd 100644 --- a/stylus-verifier/stylus-verifier-server/src/server.rs +++ b/stylus-verifier/stylus-verifier-server/src/server.rs @@ -43,7 +43,8 @@ pub async fn run(settings: Settings) -> Result<(), anyhow::Error> { tracing::init_logs(SERVICE_NAME, &settings.tracing, &settings.jaeger)?; let health = Arc::new(HealthService::default()); - let stylus_sdk_rs_verifier = Arc::new(StylusSdkRsVerifierService::new()); + let stylus_sdk_rs_verifier = + Arc::new(StylusSdkRsVerifierService::new(settings.docker_api).await); // TODO: init services here diff --git a/stylus-verifier/stylus-verifier-server/src/services/stylus_sdk_rs_verifier.rs b/stylus-verifier/stylus-verifier-server/src/services/stylus_sdk_rs_verifier.rs index 43316cf8c..39e6ae094 100644 --- a/stylus-verifier/stylus-verifier-server/src/services/stylus_sdk_rs_verifier.rs +++ b/stylus-verifier/stylus-verifier-server/src/services/stylus_sdk_rs_verifier.rs @@ -1,3 +1,4 @@ +use crate::settings::DockerApiSettings; use async_trait::async_trait; use stylus_verifier_logic::stylus_sdk_rs; use stylus_verifier_proto::blockscout::stylus_verifier::v1::{ @@ -8,12 +9,16 @@ use stylus_verifier_proto::blockscout::stylus_verifier::v1::{ use tonic::{Request, Response, Status}; pub struct StylusSdkRsVerifierService { + docker_client: stylus_sdk_rs::Docker, supported_cargo_stylus_versions: Vec, } impl StylusSdkRsVerifierService { - pub fn new() -> Self { + pub async fn new(docker_api_settings: DockerApiSettings) -> Self { Self { + docker_client: stylus_sdk_rs::docker_connect(&docker_api_settings.addr) + .await + .expect("failed to connect to docker daemon"), // TODO: to be automatically retrieved from the dockerhub registry supported_cargo_stylus_versions: vec![ semver::Version::new(0, 5, 0), @@ -48,7 +53,7 @@ impl StylusSdkRsVerifier for StylusSdkRsVerifierService { ))); } - let result = stylus_sdk_rs::verify_github_repository(request).await; + let result = stylus_sdk_rs::verify_github_repository(&self.docker_client, request).await; process_verify_result(result) } diff --git a/stylus-verifier/stylus-verifier-server/src/settings.rs b/stylus-verifier/stylus-verifier-server/src/settings.rs index 29d602faf..2bab59431 100644 --- a/stylus-verifier/stylus-verifier-server/src/settings.rs +++ b/stylus-verifier/stylus-verifier-server/src/settings.rs @@ -3,6 +3,7 @@ use blockscout_service_launcher::{ tracing::{JaegerSettings, TracingSettings}, }; use serde::Deserialize; +use url::Url; #[derive(Debug, Default, Deserialize, Clone, PartialEq, Eq)] #[serde(default, deny_unknown_fields)] @@ -11,6 +12,21 @@ pub struct Settings { pub metrics: MetricsSettings, pub tracing: TracingSettings, pub jaeger: JaegerSettings, + pub docker_api: DockerApiSettings, +} + +#[derive(Debug, Deserialize, Clone, PartialEq, Eq)] +#[serde(default, deny_unknown_fields)] +pub struct DockerApiSettings { + pub addr: Url, +} + +impl Default for DockerApiSettings { + fn default() -> Self { + Self { + addr: Url::parse("unix:///var/run/docker.sock").expect("default docker api url"), + } + } } impl ConfigSettings for Settings {