From f908e5fd063ba8ceada8003fa51661a6ae3d9c5c Mon Sep 17 00:00:00 2001 From: pavelbezpravel Date: Wed, 5 Jun 2024 15:08:44 +0300 Subject: [PATCH 1/3] docker: lint dockerfile reduce image size reduce layers amount format combine apt-get update with apt-get install in the same RUN statement use arguments JSON notation for CMD: https://github.com/hadolint/hadolint/wiki/DL3025\#rationale --- docker/dev/Dockerfile | 44 ++++++++++++++++++++++++++++--------------- 1 file changed, 29 insertions(+), 15 deletions(-) diff --git a/docker/dev/Dockerfile b/docker/dev/Dockerfile index f9a3c939849..a1329284ff1 100644 --- a/docker/dev/Dockerfile +++ b/docker/dev/Dockerfile @@ -1,23 +1,37 @@ +# syntax=docker/dockerfile:1 + FROM ubuntu:mantic -RUN apt-get -y update -RUN apt-get -y install gnupg curl -RUN echo "deb http://apt.llvm.org/mantic/ llvm-toolchain-mantic-17 main" \ - >> /etc/apt/sources.list.d/llvm.list -RUN echo "deb http://apt.llvm.org/mantic/ llvm-toolchain-mantic-18 main" \ - >> /etc/apt/sources.list.d/llvm.list -RUN curl -sSL https://apt.llvm.org/llvm-snapshot.gpg.key -o /etc/apt/trusted.gpg.d/apt.llvm.org.asc -RUN apt -y update \ - && apt -y install build-essential \ - && apt -y install gcc-12 g++-12 gcc-13 g++-13 pandoc \ + +COPY install-dependencies.sh /tmp/ + +RUN apt-get update && apt-get install -y \ + curl \ + gnupg \ + && echo "deb http://apt.llvm.org/mantic/ llvm-toolchain-mantic-17 main" \ + >> /etc/apt/sources.list.d/llvm.list \ + && echo "deb http://apt.llvm.org/mantic/ llvm-toolchain-mantic-18 main" \ + >> /etc/apt/sources.list.d/llvm.list \ + && curl -sSL https://apt.llvm.org/llvm-snapshot.gpg.key -o /etc/apt/trusted.gpg.d/apt.llvm.org.asc \ + && apt-get update && apt-get install -y \ + build-essential \ + clang-17 \ + clang-18 \ + clang-tools-18 \ + gcc-12 \ + g++-12 \ + gcc-13 \ + g++-13 \ + pandoc \ && update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 12 \ && update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-12 12 \ && update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-13 13 \ && update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-13 13 \ - && apt -y install clang-17 clang-18 clang-tools-18 \ && update-alternatives --install /usr/bin/clang clang /usr/bin/clang-17 17 \ && update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-17 17 \ && update-alternatives --install /usr/bin/clang clang /usr/bin/clang-18 18 \ - && update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-18 18 -COPY install-dependencies.sh /tmp/ -RUN bash /tmp/install-dependencies.sh -CMD /bin/bash + && update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-18 18 \ + && bash /tmp/install-dependencies.sh \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +CMD ["/bin/bash"] From 2b4ddd26662f42c2f2d2910742ca7739bff4aaa3 Mon Sep 17 00:00:00 2001 From: pavelbezpravel Date: Wed, 5 Jun 2024 15:54:09 +0300 Subject: [PATCH 2/3] docker: bind the file instead of copying during the build stage it reduces the layer's size. also we do not need this file in running container --- docker/dev/Dockerfile | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/docker/dev/Dockerfile b/docker/dev/Dockerfile index a1329284ff1..12d2ebc3813 100644 --- a/docker/dev/Dockerfile +++ b/docker/dev/Dockerfile @@ -2,9 +2,8 @@ FROM ubuntu:mantic -COPY install-dependencies.sh /tmp/ - -RUN apt-get update && apt-get install -y \ +RUN --mount=type=bind,source=./install-dependencies.sh,target=./install-dependencies.sh \ + apt-get update && apt-get install -y \ curl \ gnupg \ && echo "deb http://apt.llvm.org/mantic/ llvm-toolchain-mantic-17 main" \ @@ -30,7 +29,7 @@ RUN apt-get update && apt-get install -y \ && update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-17 17 \ && update-alternatives --install /usr/bin/clang clang /usr/bin/clang-18 18 \ && update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-18 18 \ - && bash /tmp/install-dependencies.sh \ + && bash ./install-dependencies.sh \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* From 14df7e6d8b3b6f62c710630a5f164343efd0a8e3 Mon Sep 17 00:00:00 2001 From: pavelbezpravel Date: Wed, 5 Jun 2024 16:14:46 +0300 Subject: [PATCH 3/3] scripts: sort packages alphanumerically it makes maintenance easier and helps to avoid duplication of packages and make the list much easier to update --- install-dependencies.sh | 171 +++++++++++++++++++++------------------- 1 file changed, 88 insertions(+), 83 deletions(-) diff --git a/install-dependencies.sh b/install-dependencies.sh index 184d8a5ec78..542bcda8445 100755 --- a/install-dependencies.sh +++ b/install-dependencies.sh @@ -28,39 +28,39 @@ else fi debian_packages=( - ninja-build - ragel - libhwloc-dev - libnuma-dev - libpciaccess-dev - libcrypto++-dev + cmake + diffutils + doxygen + g++ + gcc libboost-all-dev - libxml2-dev - xfslibs-dev + libc-ares-dev + libcrypto++-dev + libfmt-dev libgnutls28-dev + libhwloc-dev liblz4-dev + libnuma-dev + libpciaccess-dev + libprotobuf-dev libsctp-dev + libtool liburing-dev - gcc + libxml2-dev + libyaml-cpp-dev make meson + ninja-build + openssl + pkg-config + protobuf-compiler python3 python3-pyelftools - systemtap-sdt-dev - libtool - cmake - libyaml-cpp-dev - libc-ares-dev + ragel stow - g++ - libfmt-dev - diffutils + systemtap-sdt-dev valgrind - doxygen - openssl - pkg-config - libprotobuf-dev - protobuf-compiler + xfslibs-dev ) # seastar doesn't directly depend on these packages. They are @@ -68,85 +68,90 @@ debian_packages=( # has no way of saying "static seastar, but dynamic transitive # dependencies". They provide the various .so -> .so.ver symbolic # links. -transitive=(libtool-ltdl-devel trousers-devel libidn2-devel libunistring-devel) +transitive=( + libidn2-devel + libtool-ltdl-devel + libunistring-devel + trousers-devel +) redhat_packages=( + boost-devel + c-ares-devel + cmake + diffutils + doxygen + fmt-devel + gcc + gnutls-devel hwloc-devel - numactl-devel libpciaccess-devel + libtool + liburing-devel libxml2-devel - xfsprogs-devel - gnutls-devel lksctp-tools-devel lz4-devel - liburing-devel - gcc make meson + numactl-devel + openssl + protobuf-compiler + protobuf-devel python3 python3-pyelftools - systemtap-sdt-devel - libtool - cmake - yaml-cpp-devel - c-ares-devel stow - diffutils - doxygen - openssl - fmt-devel - boost-devel + systemtap-sdt-devel valgrind-devel - protobuf-devel - protobuf-compiler + xfsprogs-devel + yaml-cpp-devel "${transitive[@]}" ) fedora_packages=( "${redhat_packages[@]}" - gcc-c++ - ninja-build - ragel boost-devel fmt-devel - libubsan + gcc-c++ libasan libatomic + libubsan + ninja-build + ragel valgrind-devel ) centos7_packages=( "${redhat_packages[@]}" - ninja-build - ragel cmake3 - rh-mongodb36-boost-devel devtoolset-11-gcc-c++ - devtoolset-11-libubsan devtoolset-11-libasan devtoolset-11-libatomic + devtoolset-11-libubsan + ninja-build + ragel + rh-mongodb36-boost-devel ) centos8_packages=( "${redhat_packages[@]}" - ninja-build - ragel gcc-toolset-11-gcc gcc-toolset-11-gcc-c++ - gcc-toolset-11-libubsan-devel gcc-toolset-11-libasan-devel gcc-toolset-11-libatomic-devel + gcc-toolset-11-libubsan-devel + ninja-build + ragel ) centos9_packages=( "${redhat_packages[@]}" - ninja-build - ragel gcc-toolset-13-gcc gcc-toolset-13-gcc-c++ - gcc-toolset-13-libubsan-devel gcc-toolset-13-libasan-devel gcc-toolset-13-libatomic-devel + gcc-toolset-13-libubsan-devel + ninja-build + ragel ) # 1) glibc 2.30-3 has sys/sdt.h (systemtap include) @@ -157,43 +162,49 @@ centos9_packages=( # 3) aur installations require having sudo and being # a sudoer. makepkg does not work otherwise. arch_packages=( - gcc - ninja - ragel boost boost-libs + c-ares + cmake + crypto++ + filesystem + fmt + gcc + glibc + gnutls hwloc - numactl libpciaccess - crypto++ + libtool + liburing libxml2 - xfsprogs - gnutls lksctp-tools lz4 make meson - python-pyelftools - protobuf - libtool - cmake - yaml-cpp - stow - c-ares + ninja + numactl + openssl pkgconf - fmt + protobuf python3 - glibc - filesystem + python-pyelftools + ragel + stow valgrind - openssl - liburing + xfsprogs + yaml-cpp ) opensuse_packages=( c-ares-devel cmake hwloc-devel + libboost_atomic1_66_0 + libboost_atomic1_66_0-devel + libboost_chrono1_66_0 + libboost_chrono1_66_0-devel + libboost_date_time1_66_0 + libboost_date_time1_66_0-devel libboost_filesystem1_66_0 libboost_filesystem1_66_0-devel libboost_program_options1_66_0 @@ -204,26 +215,20 @@ opensuse_packages=( libboost_test1_66_0-devel libboost_thread1_66_0 libboost_thread1_66_0-devel - libboost_atomic1_66_0 - libboost_atomic1_66_0-devel - libboost_date_time1_66_0 - libboost_date_time1_66_0-devel - libboost_chrono1_66_0 - libboost_chrono1_66_0-devel libgnutls-devel libgnutlsxx28 liblz4-devel libnuma-devel + libtool lksctp-tools-devel meson ninja + openssl + protobuf-devel ragel + stow xfsprogs-devel yaml-cpp-devel - protobuf-devel - libtool - stow - openssl ) case "$ID" in