Skip to content

Commit

Permalink
chore: build rust binaries with nix flake (matter-labs#1175)
Browse files Browse the repository at this point in the history
## What ❔

<!-- What are the changes this PR brings about? -->
<!-- Example: This PR adds a PR template to the repo. -->
<!-- (For bigger PRs adding more context is appreciated) -->
Build the rust binaries with the nix flake.

To do that, we have to vendor zksync-era with a custom patched
`cargo`... **sigh** See
matter-labs#1086

To build the rust components with this flake, first run:
```
$ nix build .#cargoDeps
```
set `cargoHash` in `flake.nix` to the result of the build then
```
$ nix build .#zksync_server
```
or
```
$ nix build .#zksync_server.contract_verifier
$ nix build .#zksync_server.external_node
$ nix build .#zksync_server.server
$ nix build .#zksync_server.snapshots_creator
$ nix build .#zksync_server.block_reverter
```

To enter the development shell, run:
```
$ nix develop --impure
```

To vendor the dependencies manually, run:
```
$ nix shell .#cargo-vendor -c cargo vendor --no-merge-sources
```

## Why ❔

- [x] Enable reproducible builds for the binaries

## Checklist

<!-- Check your PR fulfills the following items. -->
<!-- For draft PRs check the boxes as you complete them. -->

- [ ] PR title corresponds to the body of PR (we generate changelog
entries from PRs).
- [ ] Tests for the changes have been added / updated.
- [ ] Documentation comments have been added / updated.
- [ ] Code has been formatted via `zk fmt` and `zk lint`.
- [ ] Spellcheck has been run via `zk spellcheck`.
- [ ] Linkcheck has been run via `zk linkcheck`.

Signed-off-by: Harald Hoyer <[email protected]>
  • Loading branch information
haraldh authored Mar 13, 2024
1 parent c3228e7 commit a3c2fae
Show file tree
Hide file tree
Showing 2 changed files with 321 additions and 45 deletions.
115 changes: 109 additions & 6 deletions flake.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

251 changes: 212 additions & 39 deletions flake.nix
Original file line number Diff line number Diff line change
@@ -1,43 +1,216 @@
###################################################################################################
#
# To build the rust components with this flake, run:
# $ nix build .#cargoDeps
# set `cargoHash` below to the result of the build
# then
# $ nix build .#zksync_server
# or
# $ nix build .#zksync_server.contract_verifier
# $ nix build .#zksync_server.external_node
# $ nix build .#zksync_server.server
# $ nix build .#zksync_server.snapshots_creator
# $ nix build .#zksync_server.block_reverter
#
# To enter the development shell, run:
# $ nix develop --impure
#
# To vendor the dependencies manually, run:
# $ nix shell .#cargo-vendor -c cargo vendor --no-merge-sources
#
###################################################################################################
{
description = "zkSync development shell";
description = "zkSync-era";
inputs = {
stable.url = "github:NixOS/nixpkgs/nixos-23.11";
};
outputs = { self, stable }: {
formatter.x86_64-linux = stable.legacyPackages.x86_64-linux.nixpkgs-fmt;
devShells.x86_64-linux.default =
with import stable { system = "x86_64-linux"; };
pkgs.mkShell.override { stdenv = pkgs.stdenvAdapters.useMoldLinker pkgs.gccStdenv; } {
name = "zkSync";
src = ./.;
buildInputs = [
docker-compose
nodejs
yarn
axel
libclang
openssl
pkg-config
postgresql
python3
solc
sqlx-cli
rustup
];

# for RocksDB and other Rust bindgen libraries
LIBCLANG_PATH = lib.makeLibraryPath [ libclang.lib ];
BINDGEN_EXTRA_CLANG_ARGS = ''-I"${libclang.lib}/lib/clang/${builtins.elemAt (builtins.splitVersion libclang.version) 0}/include"'';

shellHook = ''
export ZKSYNC_HOME=$PWD
export PATH=$ZKSYNC_HOME/bin:$PATH
'';

# hardhat solc requires ld-linux
# Nixos has to fake it with nix-ld
NIX_LD_LIBRARY_PATH = lib.makeLibraryPath [];
NIX_LD = builtins.readFile "${stdenv.cc}/nix-support/dynamic-linker";
};
nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.11";
flake-utils.url = "github:numtide/flake-utils";
rust-overlay.url = "github:oxalica/rust-overlay";
};
outputs = { self, nixpkgs, flake-utils, rust-overlay }:
flake-utils.lib.eachDefaultSystem (system:
let
###########################################################################################
# This changes every time `Cargo.lock` changes. Set to `null` to force re-vendoring
cargoHash = null;
# cargoHash = "sha256-UzIR9KhfOXFIHpj1pCjzfBmvuyvqrtbtAvVunS7djPQ=";
###########################################################################################
officialRelease = false;

versionSuffix =
if officialRelease
then ""
else "pre${builtins.substring 0 8 (self.lastModifiedDate or self.lastModified or "19700101")}_${self.shortRev or "dirty"}";

pkgs = import nixpkgs { inherit system; overlays = [ rust-overlay.overlays.default ]; };

# patched version of cargo to support `cargo vendor` for vendoring dependencies
# see https://github.com/matter-labs/zksync-era/issues/1086
# used as `cargo vendor --no-merge-sources`
cargo-vendor = pkgs.rustPlatform.buildRustPackage rec {
pname = "cargo-vendor";
version = "0.78.0";
src = pkgs.fetchFromGitHub {
owner = "haraldh";
repo = "cargo";
rev = "3ee1557d2bd95ca9d0224c5dbf1d1e2d67186455";
hash = "sha256-A8xrOG+NmF8dQ7tA9I2vJSNHlYxsH44ZRXdptLblCXk=";
};
doCheck = false;
cargoHash = "sha256-LtuNtdoX+FF/bG5LQc+L2HkFmgCtw5xM/m0/0ShlX2s=";
nativeBuildInputs = [
pkgs.pkg-config
pkgs.rustPlatform.bindgenHook
];
buildInputs = [
pkgs.openssl
];
};

# custom import-cargo-lock to import Cargo.lock file and vendor dependencies
# see https://github.com/matter-labs/zksync-era/issues/1086
import-cargo-lock = { lib, cacert, runCommand }: { src, cargoHash ? null } @ args:
runCommand "import-cargo-lock"
{
inherit src;
nativeBuildInputs = [ cargo-vendor cacert ];
preferLocalBuild = true;
outputHashMode = "recursive";
outputHashAlgo = "sha256";
outputHash = if cargoHash != null then cargoHash else lib.fakeSha256;
}
''
mkdir -p $out/.cargo
mkdir -p $out/cargo-vendor-dir
HOME=$(pwd)
pushd ${src}
HOME=$HOME cargo vendor --no-merge-sources $out/cargo-vendor-dir > $out/.cargo/config
sed -i -e "s#$out#import-cargo-lock#g" $out/.cargo/config
cp $(pwd)/Cargo.lock $out/Cargo.lock
popd
''
;
cargoDeps = pkgs.buildPackages.callPackage import-cargo-lock { } { inherit src; inherit cargoHash; };

rustVersion = pkgs.rust-bin.fromRustupToolchainFile ./rust-toolchain;

stdenv = pkgs.stdenvAdapters.useMoldLinker pkgs.clangStdenv;

rustPlatform = (pkgs.makeRustPlatform {
cargo = rustVersion;
rustc = rustVersion;
inherit stdenv;
});
zksync_server_cargoToml = (builtins.fromTOML (builtins.readFile ./core/bin/zksync_server/Cargo.toml));

hardeningEnable = [ "fortify3" "pie" "relro" ];

src = with pkgs.lib.fileset; toSource {
root = ./.;
fileset = unions [
./Cargo.lock
./Cargo.toml
./core
./sdk
./.github/release-please/manifest.json
];
};

zksync_server = with pkgs; stdenv.mkDerivation {
pname = "zksync";
version = zksync_server_cargoToml.package.version + versionSuffix;

updateAutotoolsGnuConfigScriptsPhase = ":";

nativeBuildInputs = [
pkg-config
rustPlatform.bindgenHook
rustPlatform.cargoSetupHook
rustPlatform.cargoBuildHook
rustPlatform.cargoInstallHook
];

buildInputs = [
libclang
openssl
];

inherit src;
cargoBuildFlags = "--all";
cargoBuildType = "release";
inherit cargoDeps;

inherit hardeningEnable;

outputs = [
"out"
"contract_verifier"
"external_node"
"server"
"snapshots_creator"
"block_reverter"
];

postInstall = ''
mkdir -p $out/nix-support
for i in $outputs; do
[[ $i == "out" ]] && continue
mkdir -p "''${!i}/bin"
echo "''${!i}" >> $out/nix-support/propagated-user-env-packages
if [[ -e "$out/bin/zksync_$i" ]]; then
mv "$out/bin/zksync_$i" "''${!i}/bin"
else
mv "$out/bin/$i" "''${!i}/bin"
fi
done
mkdir -p $external_node/nix-support
echo "block_reverter" >> $external_node/nix-support/propagated-user-env-packages
mv $out/bin/merkle_tree_consistency_checker $server/bin
mkdir -p $server/nix-support
echo "block_reverter" >> $server/nix-support/propagated-user-env-packages
'';
};
in
{
formatter = pkgs.nixpkgs-fmt;

packages = {
inherit zksync_server;
default = zksync_server;
inherit cargo-vendor;
inherit cargoDeps;
};

devShells = with pkgs;{
default = pkgs.mkShell.override { inherit stdenv; } {
inputsFrom = [ zksync_server ];

packages = [
docker-compose
nodejs
yarn
axel
postgresql
python3
solc
sqlx-cli
];

inherit hardeningEnable;

shellHook = ''
export ZKSYNC_HOME=$PWD
export PATH=$ZKSYNC_HOME/bin:$PATH
'';

# hardhat solc requires ld-linux
# Nixos has to fake it with nix-ld
NIX_LD_LIBRARY_PATH = lib.makeLibraryPath [ ];
NIX_LD = builtins.readFile "${clangStdenv.cc}/nix-support/dynamic-linker";
};
};
});
}

0 comments on commit a3c2fae

Please sign in to comment.