From 0141d4cbb935e30c20ce904f30e48eb1ca23a01f Mon Sep 17 00:00:00 2001 From: ChengyuZhu6 Date: Sun, 17 Sep 2023 15:11:50 +0800 Subject: [PATCH 1/3] integration: update nydus tests To support the v0.12.0 nydus-snapshotter, we need to update the config files and the commandline to start nydus-snapshotter. Depends-on: github.com/kata-containers/kata-containers#7961 Fixes #5770 Signed-off-by: ChengyuZhu6 --- integration/nydus/nydus_tests.sh | 22 ++-- integration/nydus/nydusd-config.json | 12 +- integration/nydus/snapshotter-config.toml | 128 ++++++++++++++++++++++ 3 files changed, 146 insertions(+), 16 deletions(-) create mode 100644 integration/nydus/snapshotter-config.toml diff --git a/integration/nydus/nydus_tests.sh b/integration/nydus/nydus_tests.sh index fe49580da..ff09acf93 100755 --- a/integration/nydus/nydus_tests.sh +++ b/integration/nydus/nydus_tests.sh @@ -67,18 +67,12 @@ function setup_nydus() { # Config nydus snapshotter sudo -E cp "$dir_path/nydusd-config.json" /etc/ + sudo -E cp "$dir_path/snapshotter-config.toml" /etc/ # start nydus-snapshotter nohup /usr/local/bin/containerd-nydus-grpc \ - --config-path /etc/nydusd-config.json \ - --shared-daemon \ - --log-level debug \ - --root /var/lib/containerd/io.containerd.snapshotter.v1.nydus \ - --cache-dir /var/lib/nydus/cache \ - --nydusd-path /usr/local/bin/nydusd \ - --nydusimg-path /usr/local/bin/nydus-image \ - --disable-cache-manager true \ - --enable-nydus-overlayfs true \ + --config /etc/snapshotter-config.toml \ + --nydusd-config /etc/nydusd-config.json \ --log-to-stdout >/dev/null 2>&1 & } @@ -142,12 +136,22 @@ function config_containerd() { EOF } +function check_nydus_snapshotter_exist() { + bin="containerd-nydus-grpc" + if pgrep -f "$bin" >/dev/null; then + echo "nydus-snapshotter is running" + else + die "nydus-snapshotter is not running" + fi +} + function setup() { setup_nydus config_kata config_containerd restart_containerd_service check_processes + check_nydus_snapshotter_exist extract_kata_env } diff --git a/integration/nydus/nydusd-config.json b/integration/nydus/nydusd-config.json index 654b39353..4b75426d7 100644 --- a/integration/nydus/nydusd-config.json +++ b/integration/nydus/nydusd-config.json @@ -3,17 +3,13 @@ "backend": { "type": "registry", "config": { - "scheme": "https", "timeout": 5, "connect_timeout": 5, "retry_limit": 2 } }, "cache": { - "type": "blobcache", - "config": { - "work_dir": "/var/lib/nydus/cache" - } + "type": "blobcache" } }, "mode": "direct", @@ -22,6 +18,8 @@ "enable_xattr": true, "fs_prefetch": { "enable": true, - "threads_count": 2 + "threads_count": 8, + "merging_size": 1048576, + "prefetch_all": true } -} +} \ No newline at end of file diff --git a/integration/nydus/snapshotter-config.toml b/integration/nydus/snapshotter-config.toml new file mode 100644 index 000000000..7b668236c --- /dev/null +++ b/integration/nydus/snapshotter-config.toml @@ -0,0 +1,128 @@ +version = 1 +# Snapshotter's own home directory where it stores and creates necessary resources +root = "/var/lib/containerd-nydus" +# The snapshotter's GRPC server socket, containerd will connect to plugin on this socket +address = "/run/containerd-nydus/containerd-nydus-grpc.sock" +daemon_mode = "dedicated" +# Whether snapshotter should try to clean up resources when it is closed +cleanup_on_close = false + +[system] +# Snapshotter's debug and trace HTTP server interface +enable = true +# Unix domain socket path where system controller is listening on +address = "/run/containerd-nydus/system.sock" + +[system.debug] +# Snapshotter can profile the CPU utilization of each nydusd daemon when it is being started. +# This option specifies the profile duration when nydusd is downloading and uncomproessing data. +daemon_cpu_profile_duration_secs = 5 +# Enable by assigning an address, empty indicates pprof server is disabled +pprof_address = "" + +[daemon] +# Specify a configuration file for nydusd +nydusd_config = "/etc/nydusd-config.json" +nydusd_path = "/usr/local/bin/nydusd" +nydusimage_path = "/usr/local/bin/nydus-image" +# fusedev or fscache +fs_driver = "fusedev" +# How to process when daemon dies: "none", "restart" or "failover" +recover_policy = "restart" +# Nydusd worker thread number to handle FUSE or fscache requests, [0-1024]. +# Setting to 0 will use the default configuration of nydusd. +threads_number = 4 +# Log rotation size for nydusd, in unit MB(megabytes) +log_rotation_size = 100 + + +[cgroup] +# Whether to use separate cgroup for nydusd. +enable = true +# The memory limit for nydusd cgroup, which contains all nydusd processes. +# Percentage is supported as well, please ensure it is end with "%". +# The default unit is bytes. Acceptable values include "209715200", "200MiB", "200Mi" and "10%". +memory_limit = "" + +[log] +# Print logs to stdout rather than logging files +log_to_stdout = false +# Snapshotter's log level +level = "info" +log_rotation_compress = true +log_rotation_local_time = true +# Max number of days to retain logs +log_rotation_max_age = 7 +log_rotation_max_backups = 5 +# In unit MB(megabytes) +log_rotation_max_size = 100 + +[metrics] +# Enable by assigning an address, empty indicates metrics server is disabled +address = ":9110" + +[remote] +convert_vpc_registry = false + +[remote.mirrors_config] +# Snapshotter will overwrite daemon's mirrors configuration +# if the values loaded from this driectory are not null before starting a daemon. +# Set to "" or an empty directory to disable it. +#dir = "/etc/nydus/certs.d" + +[remote.auth] +# Fetch the private registry auth by listening to K8s API server +enable_kubeconfig_keychain = false +# synchronize `kubernetes.io/dockerconfigjson` secret from kubernetes API server with specified kubeconfig (default `$KUBECONFIG` or `~/.kube/config`) +kubeconfig_path = "" +# Fetch the private registry auth as CRI image service proxy +enable_cri_keychain = false +# the target image service when using image proxy +#image_service_address = "/run/containerd/containerd.sock" + +[snapshot] +# Let containerd use nydus-overlayfs mount helper +enable_nydus_overlayfs = true +# Insert Kata Virtual Volume option to `Mount.Options` +enable_kata_volume = false +# Whether to remove resources when a snapshot is removed +sync_remove = false + +[cache_manager] +disable = false +gc_period = "24h" +# Directory to host cached files +cache_dir = "" + +[image] +public_key_file = "" +validate_signature = false + +# The configuraions for features that are not production ready +[experimental] +# Whether to enable stargz support +enable_stargz = false +# Whether to enable referrers support +# The option enables trying to fetch the Nydus image associated with the OCI image and run it. +# Also see https://github.com/opencontainers/distribution-spec/blob/main/spec.md#listing-referrers +enable_referrer_detect = false +[experimental.tarfs] +# Whether to enable nydus tarfs mode. Tarfs is supported by: +# - The EROFS filesystem driver since Linux 6.4 +# - Nydus Image Service release v2.3 +enable_tarfs = false +# Mount rafs on host by loopdev and EROFS +mount_tarfs_on_host = false +# Only enable nydus tarfs mode for images with `tarfs hint` label when true +tarfs_hint = false +# Maximum of concurrence to converting OCIv1 images to tarfs, 0 means default +max_concurrent_proc = 0 +# Mode to export tarfs images: +# - "none" or "": do not export tarfs +# - "layer_verity_only": only generate disk verity information for a layer blob +# - "image_verity_only": only generate disk verity information for all blobs of an image +# - "layer_block": generate a raw block disk image with tarfs for a layer +# - "image_block": generate a raw block disk image with tarfs for an image +# - "layer_block_with_verity": generate a raw block disk image with tarfs for a layer with dm-verity info +# - "image_block_with_verity": generate a raw block disk image with tarfs for an image with dm-verity info +export_mode = "" \ No newline at end of file From 62afa00a666a9539a20feb7cf82881285076ca0a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Mon, 18 Sep 2023 13:37:24 +0200 Subject: [PATCH 2/3] nydus: Skip tests on dragonball for now MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is needed to workaround the following issue: https://github.com/kata-containers/tests/issues/5771 dragomball maintainers will help us to get this back in place later on. Signed-off-by: Fabiano Fidêncio --- integration/nydus/nydus_tests.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/integration/nydus/nydus_tests.sh b/integration/nydus/nydus_tests.sh index ff09acf93..69faa8428 100755 --- a/integration/nydus/nydus_tests.sh +++ b/integration/nydus/nydus_tests.sh @@ -30,8 +30,8 @@ containerd_config_backup="/tmp/containerd.config.toml" # test image for container IMAGE="${IMAGE:-ghcr.io/dragonflyoss/image-service/alpine:nydus-latest}" -if [ "$KATA_HYPERVISOR" != "qemu" ] && [ "$KATA_HYPERVISOR" != "cloud-hypervisor" ] && [ "$KATA_HYPERVISOR" != "dragonball" ]; then - echo "Skip nydus test for $KATA_HYPERVISOR, it only works for QEMU/CLH/DB now." +if [ "$KATA_HYPERVISOR" != "qemu" ] && [ "$KATA_HYPERVISOR" != "cloud-hypervisor" ]; then + echo "Skip nydus test for $KATA_HYPERVISOR, it only works for QEMU/CLH. now." exit 0 fi From 6bf08c833e3663cdff81e75b446325326bec500c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Mon, 18 Sep 2023 14:20:51 +0200 Subject: [PATCH 3/3] ci: Ensure we have yq installed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit There was a recent change on the Kata Containers side that ended up breaking our CI here as yq is not automagically installed anymore. Signed-off-by: Fabiano Fidêncio --- .ci/jenkins_job_build.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.ci/jenkins_job_build.sh b/.ci/jenkins_job_build.sh index f023c872a..e0ddac5b5 100755 --- a/.ci/jenkins_job_build.sh +++ b/.ci/jenkins_job_build.sh @@ -227,6 +227,8 @@ run_unit_test() { clone_katacontainers_repo pushd "${GOPATH}/src/${katacontainers_repo}" + echo "Installing yq" + sudo -E INSTALL_IN_GOPATH=false ./ci/install_yq.sh echo "Installing libseccomp library from sources" libseccomp_install_dir=$(mktemp -d -t libseccomp.XXXXXXXXXX) gperf_install_dir=$(mktemp -d -t gperf.XXXXXXXXXX)