From 619c12cc640823047bb3f582ec76fac4b9509120 Mon Sep 17 00:00:00 2001 From: sellskin Date: Mon, 25 Mar 2024 12:53:53 +0800 Subject: [PATCH 01/16] remove code that will not be executed Signed-off-by: sellskin --- management/middleware_test.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/management/middleware_test.go b/management/middleware_test.go index 5af7e8f31c2..0023536e7e5 100644 --- a/management/middleware_test.go +++ b/management/middleware_test.go @@ -52,13 +52,11 @@ func testRequest(t *testing.T, ts *httptest.Server, method, path string, body io req, err := http.NewRequest(method, ts.URL+path, body) if err != nil { t.Fatal(err) - return nil, nil } resp, err := ts.Client().Do(req) if err != nil { t.Fatal(err) - return nil, nil } var claims managementErrorResponse err = json.NewDecoder(resp.Body).Decode(&claims) From 75752b681b5a1b5337ddd01837b4bef74cc70436 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Oliveirinha?= Date: Tue, 12 Dec 2023 14:05:15 +0000 Subject: [PATCH 02/16] TUN-8057: cloudflared uses new PQ curve ID --- supervisor/pqtunnels.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/supervisor/pqtunnels.go b/supervisor/pqtunnels.go index f8dce98d4da..70a3fd690a7 100644 --- a/supervisor/pqtunnels.go +++ b/supervisor/pqtunnels.go @@ -12,7 +12,7 @@ import ( // to https://pqtunnels.cloudflareresearch.com. const ( - PQKex = tls.CurveID(0xfe31) // X25519Kyber768Draft00 + PQKex = tls.CurveID(0x6399) // X25519Kyber768Draft00 PQKexName = "X25519Kyber768Draft00" ) From c95959e845f0825abe4f681f70ea4d0eb59a696c Mon Sep 17 00:00:00 2001 From: lneto Date: Fri, 5 Jul 2024 19:39:22 +0100 Subject: [PATCH 03/16] TUN-8520: add macos arm64 build - refactor build script for macos to include arm64 build - refactor Makefile to upload all the artifacts instead of issuing one by one - update cfsetup due to 2. - place build files in specific folders - cleanup build directory before/after creating build artifacts --- .teamcity/mac/build.sh | 95 +++++++++++++++++++++++------------------- Makefile | 3 +- 2 files changed, 53 insertions(+), 45 deletions(-) diff --git a/.teamcity/mac/build.sh b/.teamcity/mac/build.sh index d2c80be436e..d1eaa5689c3 100755 --- a/.teamcity/mac/build.sh +++ b/.teamcity/mac/build.sh @@ -12,8 +12,6 @@ export GO111MODULE=on # build 'cloudflared-darwin-amd64.tgz' mkdir -p artifacts -FILENAME="$(pwd)/artifacts/cloudflared-darwin-amd64.tgz" -PKGNAME="$(pwd)/artifacts/cloudflared-amd64.pkg" TARGET_DIRECTORY=".build" BINARY_NAME="cloudflared" VERSION=$(git describe --tags --always --dirty="-dev") @@ -28,7 +26,6 @@ export PATH="$PATH:/usr/local/bin" mkdir -p ../src/github.com/cloudflare/ cp -r . ../src/github.com/cloudflare/cloudflared cd ../src/github.com/cloudflare/cloudflared -GOCACHE="$PWD/../../../../" GOPATH="$PWD/../../../../" CGO_ENABLED=1 make cloudflared # Add code signing private key to the key chain if [[ ! -z "$CFD_CODE_SIGN_KEY" ]]; then @@ -138,47 +135,59 @@ else fi fi -# sign the cloudflared binary -if [[ ! -z "$CODE_SIGN_NAME" ]]; then - codesign -s "${CODE_SIGN_NAME}" -f -v --timestamp --options runtime ${BINARY_NAME} +# cleanup the build directory because the previous execution might have failed without cleaning up. +rm -rf "${TARGET_DIRECTORY}" +archs=("amd64" "arm64") +export TARGET_OS=darwin +for arch in ${archs[@]}; do - # notarize the binary - # TODO: TUN-5789 -fi + FILENAME="$(pwd)/artifacts/cloudflared-darwin-$arch.tgz" + PKGNAME="$(pwd)/artifacts/cloudflared-$arch.pkg" + TARGET_ARCH=$arch GOCACHE="$PWD/../../../../" GOPATH="$PWD/../../../../" CGO_ENABLED=1 make cloudflared + + # sign the cloudflared binary + if [[ ! -z "$CODE_SIGN_NAME" ]]; then + codesign -s "${CODE_SIGN_NAME}" -f -v --timestamp --options runtime ${BINARY_NAME} + + # notarize the binary + # TODO: TUN-5789 + fi -# creating build directory -rm -rf $TARGET_DIRECTORY -mkdir "${TARGET_DIRECTORY}" -mkdir "${TARGET_DIRECTORY}/contents" -cp -r ".mac_resources/scripts" "${TARGET_DIRECTORY}/scripts" - -# copy cloudflared into the build directory -cp ${BINARY_NAME} "${TARGET_DIRECTORY}/contents/${PRODUCT}" - -# compress cloudflared into a tar and gzipped file -tar czf "$FILENAME" "${BINARY_NAME}" - -# build the installer package -if [[ ! -z "$PKG_SIGN_NAME" ]]; then - pkgbuild --identifier com.cloudflare.${PRODUCT} \ - --version ${VERSION} \ - --scripts ${TARGET_DIRECTORY}/scripts \ - --root ${TARGET_DIRECTORY}/contents \ - --install-location /usr/local/bin \ - --sign "${PKG_SIGN_NAME}" \ - ${PKGNAME} - - # notarize the package - # TODO: TUN-5789 -else - pkgbuild --identifier com.cloudflare.${PRODUCT} \ - --version ${VERSION} \ - --scripts ${TARGET_DIRECTORY}/scripts \ - --root ${TARGET_DIRECTORY}/contents \ - --install-location /usr/local/bin \ - ${PKGNAME} -fi + ARCH_TARGET_DIRECTORY="${TARGET_DIRECTORY}/${arch}-build" + # creating build directory + rm -rf $ARCH_TARGET_DIRECTORY + mkdir "${ARCH_TARGET_DIRECTORY}" + mkdir "${ARCH_TARGET_DIRECTORY}/contents" + cp -r ".mac_resources/scripts" "${ARCH_TARGET_DIRECTORY}/scripts" + # copy cloudflared into the build directory + cp ${BINARY_NAME} "${ARCH_TARGET_DIRECTORY}/contents/${PRODUCT}" + + # compress cloudflared into a tar and gzipped file + tar czf "$FILENAME" "${BINARY_NAME}" + + # build the installer package + if [[ ! -z "$PKG_SIGN_NAME" ]]; then + pkgbuild --identifier com.cloudflare.${PRODUCT} \ + --version ${VERSION} \ + --scripts ${ARCH_TARGET_DIRECTORY}/scripts \ + --root ${ARCH_TARGET_DIRECTORY}/contents \ + --install-location /usr/local/bin \ + --sign "${PKG_SIGN_NAME}" \ + ${PKGNAME} + + # notarize the package + # TODO: TUN-5789 + else + pkgbuild --identifier com.cloudflare.${PRODUCT} \ + --version ${VERSION} \ + --scripts ${ARCH_TARGET_DIRECTORY}/scripts \ + --root ${ARCH_TARGET_DIRECTORY}/contents \ + --install-location /usr/local/bin \ + ${PKGNAME} + fi +done -# cleaning up the build directory -rm -rf $TARGET_DIRECTORY +# cleanup build the build directory because this script is not ran within containers, +# which might lead to future issues in subsequent runs. +rm -rf "${TARGET_DIRECTORY}" diff --git a/Makefile b/Makefile index 1961f03d9c0..ce7d8f40981 100644 --- a/Makefile +++ b/Makefile @@ -241,8 +241,7 @@ github-message: .PHONY: github-mac-upload github-mac-upload: - python3 github_release.py --path artifacts/cloudflared-darwin-amd64.tgz --release-version $(VERSION) --name cloudflared-darwin-amd64.tgz - python3 github_release.py --path artifacts/cloudflared-amd64.pkg --release-version $(VERSION) --name cloudflared-amd64.pkg + python3 github_release.py --path artifacts --release-version $(VERSION) .PHONY: github-windows-upload github-windows-upload: From 47733ba25e9dc84b01ad2b98f97baa550e566615 Mon Sep 17 00:00:00 2001 From: lneto Date: Fri, 5 Jul 2024 19:39:22 +0100 Subject: [PATCH 04/16] TUN-8523: refactor makefile and cfsetup - remove unused targets in Makefile - order deps in cfsetup.yaml - only build cloudflared not all linux targets - rename stages to be more explicit - adjust build deps of build-linux-release - adjust build deps of build-linux-fips-release - rename github_release_pkgs_pre_cache to build_release_pre_cache - only build release release artifacts within build-linux-release - only build release release artifacts within build-linux-fips-release - remove github-release-macos - remove github-release-windows - adjust builddeps of test and test-fips - create builddeps anchor for component-test and use it in component-test-fips - remove wixl from build-linux-* - rename release-pkgs-linux to r2-linux-release - add github-release: artifacts uplooad and set release message - clean build directory before build - add step to package windows binaries - refactor windows script One of TeamCity changes is moving the artifacts to the built artifacts, hence, there is no need to cp files from artifacts to built_artifacts - create anchor for release builds - create anchor for tests stages - remove reprepro and createrepo as they are only called by release_pkgs.py --- .teamcity/mac/build.sh | 2 +- .teamcity/package-windows.sh | 5 +- Makefile | 31 ++------ cfsetup.yaml | 132 +++++++++++++++-------------------- 4 files changed, 65 insertions(+), 105 deletions(-) diff --git a/.teamcity/mac/build.sh b/.teamcity/mac/build.sh index d1eaa5689c3..865b296e740 100755 --- a/.teamcity/mac/build.sh +++ b/.teamcity/mac/build.sh @@ -188,6 +188,6 @@ for arch in ${archs[@]}; do fi done -# cleanup build the build directory because this script is not ran within containers, +# cleanup build directory because this script is not ran within containers, # which might lead to future issues in subsequent runs. rm -rf "${TARGET_DIRECTORY}" diff --git a/.teamcity/package-windows.sh b/.teamcity/package-windows.sh index 9d92a21f0a3..0139a51b57c 100755 --- a/.teamcity/package-windows.sh +++ b/.teamcity/package-windows.sh @@ -9,9 +9,8 @@ windowsArchs=("amd64" "386") for arch in ${windowsArchs[@]}; do export TARGET_ARCH=$arch # Copy exe into final directory - cp ./artifacts/cloudflared-windows-$arch.exe $ARTIFACT_DIR/cloudflared-windows-$arch.exe - cp ./artifacts/cloudflared-windows-$arch.exe ./cloudflared.exe + cp $ARTIFACT_DIR/cloudflared-windows-$arch.exe ./cloudflared.exe make cloudflared-msi # Copy msi into final directory mv cloudflared-$VERSION-$arch.msi $ARTIFACT_DIR/cloudflared-windows-$arch.msi -done \ No newline at end of file +done diff --git a/Makefile b/Makefile index ce7d8f40981..1d044ac957a 100644 --- a/Makefile +++ b/Makefile @@ -218,37 +218,14 @@ cloudflared-pkg: cloudflared cloudflared.1 cloudflared-msi: wixl --define Version=$(VERSION) --define Path=$(EXECUTABLE_PATH) --output cloudflared-$(VERSION)-$(TARGET_ARCH).msi cloudflared.wxs -.PHONY: cloudflared-darwin-amd64.tgz -cloudflared-darwin-amd64.tgz: cloudflared - tar czf cloudflared-darwin-amd64.tgz cloudflared - rm cloudflared - .PHONY: github-release -github-release: cloudflared - python3 github_release.py --path $(EXECUTABLE_PATH) --release-version $(VERSION) - -.PHONY: github-release-built-pkgs -github-release-built-pkgs: +github-release: python3 github_release.py --path $(PWD)/built_artifacts --release-version $(VERSION) - -.PHONY: release-pkgs-linux -release-pkgs-linux: - python3 ./release_pkgs.py - -.PHONY: github-message -github-message: python3 github_message.py --release-version $(VERSION) -.PHONY: github-mac-upload -github-mac-upload: - python3 github_release.py --path artifacts --release-version $(VERSION) - -.PHONY: github-windows-upload -github-windows-upload: - python3 github_release.py --path built_artifacts/cloudflared-windows-amd64.exe --release-version $(VERSION) --name cloudflared-windows-amd64.exe - python3 github_release.py --path built_artifacts/cloudflared-windows-amd64.msi --release-version $(VERSION) --name cloudflared-windows-amd64.msi - python3 github_release.py --path built_artifacts/cloudflared-windows-386.exe --release-version $(VERSION) --name cloudflared-windows-386.exe - python3 github_release.py --path built_artifacts/cloudflared-windows-386.msi --release-version $(VERSION) --name cloudflared-windows-386.msi +.PHONY: r2-linux-release +r2-linux-release: + python3 ./release_pkgs.py .PHONY: capnp capnp: diff --git a/cfsetup.yaml b/cfsetup.yaml index 12afa0e94c1..953a80a1118 100644 --- a/cfsetup.yaml +++ b/cfsetup.yaml @@ -3,34 +3,27 @@ pinned_go: &pinned_go go-boring=1.22.2-1 build_dir: &build_dir /cfsetup_build default-flavor: bullseye buster: &buster - build: + build-linux: build_dir: *build_dir builddeps: &build_deps - *pinned_go - build-essential - - gotest-to-teamcity - fakeroot - rubygem-fpm - rpm - libffi-dev - - reprepro - - createrepo pre-cache: &build_pre_cache - export GOCACHE=/cfsetup_build/.cache/go-build - go install golang.org/x/tools/cmd/goimports@latest post-cache: - # TODO: TUN-8126 this is temporary to make sure packages can be built before release - - ./build-packages.sh # Build binary for component test - GOOS=linux GOARCH=amd64 make cloudflared - build-fips: + build-linux-fips: build_dir: *build_dir builddeps: *build_deps pre-cache: *build_pre_cache post-cache: - export FIPS=true - # TODO: TUN-8126 this is temporary to make sure packages can be built before release - - ./build-packages-fips.sh # Build binary for component test - GOOS=linux GOARCH=amd64 make cloudflared cover: @@ -39,28 +32,21 @@ buster: &buster pre-cache: *build_pre_cache post-cache: - make cover - # except FIPS (handled in github-fips-release-pkgs) and macos (handled in github-release-macos-amd64) - github-release-pkgs: + # except FIPS and macos + build-linux-release: build_dir: *build_dir - builddeps: + builddeps: &build_deps_release - *pinned_go - build-essential - fakeroot - rubygem-fpm - rpm - - wget - # libmsi and libgcab are libraries the wixl binary depends on. - - libmsi-dev - - libgcab-dev - - python3-dev - libffi-dev - - python3-setuptools + - python3-dev - python3-pip - - reprepro - - createrepo - pre-cache: &github_release_pkgs_pre_cache - - wget https://github.com/sudarshan-reddy/msitools/releases/download/v0.101b/wixl -P /usr/local/bin - - chmod a+x /usr/local/bin/wixl + - python3-setuptools + - wget + pre-cache: &build_release_pre_cache - pip3 install pynacl==1.4.0 - pip3 install pygithub==1.55 - pip3 install boto3==1.22.9 @@ -68,32 +54,14 @@ buster: &buster post-cache: # build all packages (except macos and FIPS) and move them to /cfsetup/built_artifacts - ./build-packages.sh - # release the packages built and moved to /cfsetup/built_artifacts - - make github-release-built-pkgs - # publish packages to linux repos - - make release-pkgs-linux # handle FIPS separately so that we built with gofips compiler - github-fips-release-pkgs: + build-linux-fips-release: build_dir: *build_dir - builddeps: - - *pinned_go - - build-essential - - fakeroot - - rubygem-fpm - - rpm - - wget - # libmsi and libgcab are libraries the wixl binary depends on. - - libmsi-dev - - libgcab-dev - - python3-dev - - libffi-dev - - python3-setuptools - - python3-pip - pre-cache: *github_release_pkgs_pre_cache + builddeps: *build_deps_release + pre-cache: *build_release_pre_cache post-cache: # same logic as above, but for FIPS packages only - ./build-packages-fips.sh - - make github-release-built-pkgs generate-versions-file: build_dir: *build_dir builddeps: @@ -152,21 +120,7 @@ buster: &buster - export GOOS=linux - export GOARCH=arm64 - make cloudflared-deb - github-release-macos-amd64: - build_dir: *build_dir - builddeps: &build_pygithub - - *pinned_go - - build-essential - - python3-dev - - libffi-dev - - python3-setuptools - - python3-pip - pre-cache: &install_pygithub - - pip3 install pynacl==1.4.0 - - pip3 install pygithub==1.55 - post-cache: - - make github-mac-upload - github-release-windows: + package-windows: build_dir: *build_dir builddeps: - *pinned_go @@ -186,10 +140,16 @@ buster: &buster - pip3 install pygithub==1.55 post-cache: - .teamcity/package-windows.sh - - make github-windows-upload test: build_dir: *build_dir - builddeps: *build_deps + builddeps: &build_deps_tests + - *pinned_go + - build-essential + - fakeroot + - rubygem-fpm + - rpm + - libffi-dev + - gotest-to-teamcity pre-cache: *build_pre_cache post-cache: - export GOOS=linux @@ -199,7 +159,7 @@ buster: &buster - make test | gotest-to-teamcity test-fips: build_dir: *build_dir - builddeps: *build_deps + builddeps: *build_deps_tests pre-cache: *build_pre_cache post-cache: - export GOOS=linux @@ -210,7 +170,7 @@ buster: &buster - make test | gotest-to-teamcity component-test: build_dir: *build_dir - builddeps: + builddeps: &build_deps_component_test - *pinned_go - python3.7 - python3-pip @@ -230,24 +190,48 @@ buster: &buster - python3 component-tests/setup.py --type cleanup component-test-fips: build_dir: *build_dir - builddeps: - - *pinned_go - - python3.7 - - python3-pip - - python3-setuptools - # procps installs the ps command which is needed in test_sysv_service because the init script - # uses ps pid to determine if the agent is running - - procps + builddeps: *build_deps_component_test pre-cache-copy-paths: - component-tests/requirements.txt pre-cache: *component_test_pre_cache post-cache: *component_test_post_cache - github-message-release: + github-release: build_dir: *build_dir - builddeps: *build_pygithub - pre-cache: *install_pygithub + builddeps: + - *pinned_go + - build-essential + - python3-dev + - libffi-dev + - python3-setuptools + - python3-pip + pre-cache: + - pip3 install pynacl==1.4.0 + - pip3 install pygithub==1.55 post-cache: + - make github-release - make github-message + r2-linux-release: + build_dir: *build_dir + builddeps: + - *pinned_go + - build-essential + - fakeroot + - rubygem-fpm + - rpm + - wget + - python3-dev + - libffi-dev + - python3-setuptools + - python3-pip + - reprepro + - createrepo + pre-cache: + - pip3 install pynacl==1.4.0 + - pip3 install pygithub==1.55 + - pip3 install boto3==1.22.9 + - pip3 install python-gnupg==0.4.9 + post-cache: + - make r2-linux-release bullseye: *buster bookworm: *buster From 13b2e423eda8d8c21737ba6442c486f47ba2c920 Mon Sep 17 00:00:00 2001 From: lneto Date: Mon, 15 Jul 2024 14:24:16 +0100 Subject: [PATCH 05/16] Release 2024.7.0 --- RELEASE_NOTES | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/RELEASE_NOTES b/RELEASE_NOTES index 9b1c972744e..2ce3edc0236 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -1,3 +1,11 @@ +2024.7.0 +- 2024-07-05 TUN-8520: add macos arm64 build +- 2024-07-05 TUN-8523: refactor makefile and cfsetup +- 2024-07-02 TUN-8504: Use pre-installed python version instead of downloading it on Windows builds +- 2024-06-26 TUN-8489: Add default noop logger for capnprpc +- 2024-06-25 TUN-8487: Add user-agent for quick-tunnel requests +- 2023-12-12 TUN-8057: cloudflared uses new PQ curve ID + 2024.6.1 - 2024-06-12 TUN-8461: Don't log Failed to send session payload if the error is EOF - 2024-06-07 TUN-8456: Update quic-go to 0.45 and collect mtu and congestion control metrics From 26ae1ca3c8ade8184847c0968613f5e57ce9421f Mon Sep 17 00:00:00 2001 From: lneto Date: Tue, 16 Jul 2024 12:10:31 +0100 Subject: [PATCH 06/16] TUN-8543: use -p flag to create intermediate directories --- .teamcity/mac/build.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.teamcity/mac/build.sh b/.teamcity/mac/build.sh index 865b296e740..588ef42f294 100755 --- a/.teamcity/mac/build.sh +++ b/.teamcity/mac/build.sh @@ -156,8 +156,8 @@ for arch in ${archs[@]}; do ARCH_TARGET_DIRECTORY="${TARGET_DIRECTORY}/${arch}-build" # creating build directory rm -rf $ARCH_TARGET_DIRECTORY - mkdir "${ARCH_TARGET_DIRECTORY}" - mkdir "${ARCH_TARGET_DIRECTORY}/contents" + mkdir -p "${ARCH_TARGET_DIRECTORY}" + mkdir -p "${ARCH_TARGET_DIRECTORY}/contents" cp -r ".mac_resources/scripts" "${ARCH_TARGET_DIRECTORY}/scripts" # copy cloudflared into the build directory From db239e7319243c9cd80d82a570bd4cb661730c72 Mon Sep 17 00:00:00 2001 From: lneto Date: Tue, 16 Jul 2024 16:24:52 +0100 Subject: [PATCH 07/16] Release 2024.7.1 --- RELEASE_NOTES | 3 +++ 1 file changed, 3 insertions(+) diff --git a/RELEASE_NOTES b/RELEASE_NOTES index 2ce3edc0236..36972cebbb9 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -1,3 +1,6 @@ +2024.7.1 +- 2024-07-16 TUN-8543: use -p flag to create intermediate directories + 2024.7.0 - 2024-07-05 TUN-8520: add macos arm64 build - 2024-07-05 TUN-8523: refactor makefile and cfsetup From c2183bd814c13740f29ea09576176789a4103949 Mon Sep 17 00:00:00 2001 From: lneto Date: Wed, 17 Jul 2024 15:39:51 +0100 Subject: [PATCH 08/16] TUN-8546: rework MacOS build script The rework consists in building and packaging the cloudflared binary based on the OS & ARCH of the system. read TARGET_ARCH from export and exit if TARGET_ARCH is not set --- .teamcity/mac/build.sh | 94 +++++++++++++++++++++--------------------- 1 file changed, 48 insertions(+), 46 deletions(-) diff --git a/.teamcity/mac/build.sh b/.teamcity/mac/build.sh index 588ef42f294..d303144be68 100755 --- a/.teamcity/mac/build.sh +++ b/.teamcity/mac/build.sh @@ -7,6 +7,12 @@ if [[ "$(uname)" != "Darwin" ]] ; then exit 1 fi +if [[ "amd64" != "${TARGET_ARCH}" && "arm64" != "${TARGET_ARCH}" ]] +then + echo "TARGET_ARCH must be amd64 or arm64" + exit 1 +fi + go version export GO111MODULE=on @@ -137,56 +143,52 @@ fi # cleanup the build directory because the previous execution might have failed without cleaning up. rm -rf "${TARGET_DIRECTORY}" -archs=("amd64" "arm64") -export TARGET_OS=darwin -for arch in ${archs[@]}; do - - FILENAME="$(pwd)/artifacts/cloudflared-darwin-$arch.tgz" - PKGNAME="$(pwd)/artifacts/cloudflared-$arch.pkg" - TARGET_ARCH=$arch GOCACHE="$PWD/../../../../" GOPATH="$PWD/../../../../" CGO_ENABLED=1 make cloudflared - - # sign the cloudflared binary - if [[ ! -z "$CODE_SIGN_NAME" ]]; then - codesign -s "${CODE_SIGN_NAME}" -f -v --timestamp --options runtime ${BINARY_NAME} - - # notarize the binary - # TODO: TUN-5789 - fi +export TARGET_OS="darwin" +FILENAME="$(pwd)/artifacts/cloudflared-darwin-$TARGET_ARCH.tgz" +PKGNAME="$(pwd)/artifacts/cloudflared-$TARGET_ARCH.pkg" +GOCACHE="$PWD/../../../../" GOPATH="$PWD/../../../../" CGO_ENABLED=1 make cloudflared - ARCH_TARGET_DIRECTORY="${TARGET_DIRECTORY}/${arch}-build" - # creating build directory - rm -rf $ARCH_TARGET_DIRECTORY - mkdir -p "${ARCH_TARGET_DIRECTORY}" - mkdir -p "${ARCH_TARGET_DIRECTORY}/contents" - cp -r ".mac_resources/scripts" "${ARCH_TARGET_DIRECTORY}/scripts" +# sign the cloudflared binary +if [[ ! -z "$CODE_SIGN_NAME" ]]; then + codesign -s "${CODE_SIGN_NAME}" -f -v --timestamp --options runtime ${BINARY_NAME} - # copy cloudflared into the build directory - cp ${BINARY_NAME} "${ARCH_TARGET_DIRECTORY}/contents/${PRODUCT}" - - # compress cloudflared into a tar and gzipped file - tar czf "$FILENAME" "${BINARY_NAME}" + # notarize the binary + # TODO: TUN-5789 +fi - # build the installer package - if [[ ! -z "$PKG_SIGN_NAME" ]]; then +ARCH_TARGET_DIRECTORY="${TARGET_DIRECTORY}/${arch}-build" +# creating build directory +rm -rf $ARCH_TARGET_DIRECTORY +mkdir -p "${ARCH_TARGET_DIRECTORY}" +mkdir -p "${ARCH_TARGET_DIRECTORY}/contents" +cp -r ".mac_resources/scripts" "${ARCH_TARGET_DIRECTORY}/scripts" + +# copy cloudflared into the build directory +cp ${BINARY_NAME} "${ARCH_TARGET_DIRECTORY}/contents/${PRODUCT}" + +# compress cloudflared into a tar and gzipped file +tar czf "$FILENAME" "${BINARY_NAME}" + +# build the installer package +if [[ ! -z "$PKG_SIGN_NAME" ]]; then + pkgbuild --identifier com.cloudflare.${PRODUCT} \ + --version ${VERSION} \ + --scripts ${ARCH_TARGET_DIRECTORY}/scripts \ + --root ${ARCH_TARGET_DIRECTORY}/contents \ + --install-location /usr/local/bin \ + --sign "${PKG_SIGN_NAME}" \ + ${PKGNAME} + + # notarize the package + # TODO: TUN-5789 +else pkgbuild --identifier com.cloudflare.${PRODUCT} \ - --version ${VERSION} \ - --scripts ${ARCH_TARGET_DIRECTORY}/scripts \ - --root ${ARCH_TARGET_DIRECTORY}/contents \ - --install-location /usr/local/bin \ - --sign "${PKG_SIGN_NAME}" \ - ${PKGNAME} - - # notarize the package - # TODO: TUN-5789 - else - pkgbuild --identifier com.cloudflare.${PRODUCT} \ - --version ${VERSION} \ - --scripts ${ARCH_TARGET_DIRECTORY}/scripts \ - --root ${ARCH_TARGET_DIRECTORY}/contents \ - --install-location /usr/local/bin \ - ${PKGNAME} - fi -done + --version ${VERSION} \ + --scripts ${ARCH_TARGET_DIRECTORY}/scripts \ + --root ${ARCH_TARGET_DIRECTORY}/contents \ + --install-location /usr/local/bin \ + ${PKGNAME} +fi # cleanup build directory because this script is not ran within containers, # which might lead to future issues in subsequent runs. From 3bb3d71093c932297d2fed87780b37edd0dc07e3 Mon Sep 17 00:00:00 2001 From: lneto Date: Wed, 31 Jul 2024 11:18:57 +0100 Subject: [PATCH 09/16] Release 2024.7.2 --- RELEASE_NOTES | 3 +++ 1 file changed, 3 insertions(+) diff --git a/RELEASE_NOTES b/RELEASE_NOTES index 36972cebbb9..6bce201dbaf 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -1,3 +1,6 @@ +2024.7.2 +- 2024-07-17 TUN-8546: rework MacOS build script + 2024.7.1 - 2024-07-16 TUN-8543: use -p flag to create intermediate directories From c7cd4e02b867db54cbc3605f409299383f71b7b1 Mon Sep 17 00:00:00 2001 From: lneto Date: Wed, 31 Jul 2024 15:51:02 +0100 Subject: [PATCH 10/16] TUN-8546: Fix final artifacts paths - The build artifacts must be placed in the checkout directory so that they can be picked up from cfsetup --- .teamcity/mac/build.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.teamcity/mac/build.sh b/.teamcity/mac/build.sh index d303144be68..413f113346f 100755 --- a/.teamcity/mac/build.sh +++ b/.teamcity/mac/build.sh @@ -29,6 +29,8 @@ INSTALLER_CERT="installer.cer" BUNDLE_ID="com.cloudflare.cloudflared" SEC_DUP_MSG="security: SecKeychainItemImport: The specified item already exists in the keychain." export PATH="$PATH:/usr/local/bin" +FILENAME="$(pwd)/artifacts/cloudflared-darwin-$TARGET_ARCH.tgz" +PKGNAME="$(pwd)/artifacts/cloudflared-$TARGET_ARCH.pkg" mkdir -p ../src/github.com/cloudflare/ cp -r . ../src/github.com/cloudflare/cloudflared cd ../src/github.com/cloudflare/cloudflared @@ -144,8 +146,6 @@ fi # cleanup the build directory because the previous execution might have failed without cleaning up. rm -rf "${TARGET_DIRECTORY}" export TARGET_OS="darwin" -FILENAME="$(pwd)/artifacts/cloudflared-darwin-$TARGET_ARCH.tgz" -PKGNAME="$(pwd)/artifacts/cloudflared-$TARGET_ARCH.pkg" GOCACHE="$PWD/../../../../" GOPATH="$PWD/../../../../" CGO_ENABLED=1 make cloudflared # sign the cloudflared binary @@ -156,7 +156,7 @@ if [[ ! -z "$CODE_SIGN_NAME" ]]; then # TODO: TUN-5789 fi -ARCH_TARGET_DIRECTORY="${TARGET_DIRECTORY}/${arch}-build" +ARCH_TARGET_DIRECTORY="${TARGET_DIRECTORY}/${TARGET_ARCH}-build" # creating build directory rm -rf $ARCH_TARGET_DIRECTORY mkdir -p "${ARCH_TARGET_DIRECTORY}" From c7f0f90bede391ff7897e28a862e710bde4432ec Mon Sep 17 00:00:00 2001 From: lneto Date: Wed, 31 Jul 2024 16:29:18 +0100 Subject: [PATCH 11/16] Release 2024.7.3 --- RELEASE_NOTES | 3 +++ 1 file changed, 3 insertions(+) diff --git a/RELEASE_NOTES b/RELEASE_NOTES index 6bce201dbaf..200a04bdbb7 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -1,3 +1,6 @@ +2024.7.3 +- 2024-07-31 TUN-8546: Fix final artifacts paths + 2024.7.2 - 2024-07-17 TUN-8546: rework MacOS build script From ae7f7fa7e814cae8f4f6b000a2b6f3e27db99c35 Mon Sep 17 00:00:00 2001 From: lneto Date: Thu, 1 Aug 2024 09:36:59 +0100 Subject: [PATCH 12/16] TUN-8546: remove call to non existant make target --- cfsetup.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/cfsetup.yaml b/cfsetup.yaml index 953a80a1118..613cb0f85e5 100644 --- a/cfsetup.yaml +++ b/cfsetup.yaml @@ -209,7 +209,6 @@ buster: &buster - pip3 install pygithub==1.55 post-cache: - make github-release - - make github-message r2-linux-release: build_dir: *build_dir builddeps: From b03ea055b0bb42d21941fe9c7ef8b367de86db47 Mon Sep 17 00:00:00 2001 From: lneto Date: Thu, 1 Aug 2024 16:26:45 +0100 Subject: [PATCH 13/16] TUN-8581: create dry run for github release --- Makefile | 4 ++++ cfsetup.yaml | 16 +++++++++++++++- github_release.py | 12 ++++++++++-- 3 files changed, 29 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 1d044ac957a..46fee2a979b 100644 --- a/Makefile +++ b/Makefile @@ -218,6 +218,10 @@ cloudflared-pkg: cloudflared cloudflared.1 cloudflared-msi: wixl --define Version=$(VERSION) --define Path=$(EXECUTABLE_PATH) --output cloudflared-$(VERSION)-$(TARGET_ARCH).msi cloudflared.wxs +.PHONY: github-release-dryrun +github-release-dryrun: + python3 github_release.py --path $(PWD)/built_artifacts --release-version $(VERSION) --dry-run + .PHONY: github-release github-release: python3 github_release.py --path $(PWD)/built_artifacts --release-version $(VERSION) diff --git a/cfsetup.yaml b/cfsetup.yaml index 613cb0f85e5..62e8de5dc05 100644 --- a/cfsetup.yaml +++ b/cfsetup.yaml @@ -195,9 +195,23 @@ buster: &buster - component-tests/requirements.txt pre-cache: *component_test_pre_cache post-cache: *component_test_post_cache + github-release-dryrun: + build_dir: *build_dir + builddeps: + - *pinned_go + - build-essential + - python3-dev + - libffi-dev + - python3-setuptools + - python3-pip + pre-cache: + - pip3 install pynacl==1.4.0 + - pip3 install pygithub==1.55 + post-cache: + - make github-release-dryrun github-release: build_dir: *build_dir - builddeps: + builddeps: - *pinned_go - build-essential - python3-dev diff --git a/github_release.py b/github_release.py index e28a89cce5d..db6120863bf 100755 --- a/github_release.py +++ b/github_release.py @@ -17,7 +17,7 @@ from github import Github, GithubException, UnknownObjectException FORMAT = "%(levelname)s - %(asctime)s: %(message)s" -logging.basicConfig(format=FORMAT) +logging.basicConfig(format=FORMAT, level=logging.INFO) CLOUDFLARED_REPO = os.environ.get("GITHUB_REPO", "cloudflare/cloudflared") GITHUB_CONFLICT_CODE = "already_exists" @@ -219,7 +219,15 @@ def main(): release = get_or_create_release(repo, args.release_version, args.dry_run) if args.dry_run: - logging.info("Skipping asset upload because of dry-run") + if os.path.isdir(args.path): + onlyfiles = [f for f in listdir(args.path) if isfile(join(args.path, f))] + for filename in onlyfiles: + binary_path = os.path.join(args.path, filename) + logging.info("binary: " + binary_path) + elif os.path.isfile(args.path): + logging.info("binary: " + binary_path) + else: + logging.error("dryrun failed") return if os.path.isdir(args.path): From bd9e020df93206ef32d35229d91349526f2e10e8 Mon Sep 17 00:00:00 2001 From: lneto Date: Mon, 5 Aug 2024 10:44:33 +0100 Subject: [PATCH 14/16] TUN-8583: change final directory of artifacts --- .teamcity/package-windows.sh | 10 ++++++---- build-packages-fips.sh | 4 ++-- build-packages.sh | 2 +- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/.teamcity/package-windows.sh b/.teamcity/package-windows.sh index 0139a51b57c..a4b91d93d0a 100755 --- a/.teamcity/package-windows.sh +++ b/.teamcity/package-windows.sh @@ -3,14 +3,16 @@ echo $VERSION export TARGET_OS=windows # This controls the directory the built artifacts go into -export ARTIFACT_DIR=built_artifacts/ -mkdir -p $ARTIFACT_DIR +export BUILT_ARTIFACT_DIR=built_artifacts/ +export FINAL_ARTIFACT_DIR=artifacts/ +mkdir -p $BUILT_ARTIFACT_DIR +mkdir -p $FINAL_ARTIFACT_DIR windowsArchs=("amd64" "386") for arch in ${windowsArchs[@]}; do export TARGET_ARCH=$arch # Copy exe into final directory - cp $ARTIFACT_DIR/cloudflared-windows-$arch.exe ./cloudflared.exe + cp $BUILT_ARTIFACT_DIR/cloudflared-windows-$arch.exe ./cloudflared.exe make cloudflared-msi # Copy msi into final directory - mv cloudflared-$VERSION-$arch.msi $ARTIFACT_DIR/cloudflared-windows-$arch.msi + mv cloudflared-$VERSION-$arch.msi $FINAL_ARTIFACT_DIR/cloudflared-windows-$arch.msi done diff --git a/build-packages-fips.sh b/build-packages-fips.sh index a7401eab636..0ec3b3c9e84 100755 --- a/build-packages-fips.sh +++ b/build-packages-fips.sh @@ -3,7 +3,7 @@ VERSION=$(git describe --tags --always --match "[0-9][0-9][0-9][0-9].*.*") echo $VERSION # This controls the directory the built artifacts go into -export ARTIFACT_DIR=built_artifacts/ +export ARTIFACT_DIR=artifacts/ mkdir -p $ARTIFACT_DIR arch=("amd64") @@ -23,4 +23,4 @@ make cloudflared-rpm mv cloudflared-fips-$RPMVERSION-1.$RPMARCH.rpm $ARTIFACT_DIR/cloudflared-fips-linux-$RPMARCH.rpm # finally move the linux binary as well. -mv ./cloudflared $ARTIFACT_DIR/cloudflared-fips-linux-$arch \ No newline at end of file +mv ./cloudflared $ARTIFACT_DIR/cloudflared-fips-linux-$arch diff --git a/build-packages.sh b/build-packages.sh index 9570dab0cca..df5dc7bb244 100755 --- a/build-packages.sh +++ b/build-packages.sh @@ -7,7 +7,7 @@ export GOEXPERIMENT=noboringcrypto export CGO_ENABLED=0 # This controls the directory the built artifacts go into -export ARTIFACT_DIR=built_artifacts/ +export ARTIFACT_DIR=artifacts/ mkdir -p $ARTIFACT_DIR linuxArchs=("386" "amd64" "arm" "armhf" "arm64") From 86f33005b98ec2a21534f82bdf76a9e2bf667dfd Mon Sep 17 00:00:00 2001 From: lneto Date: Mon, 5 Aug 2024 14:27:56 +0100 Subject: [PATCH 15/16] TUN-8585: Avoid creating GH client when dry-run is true - copy exe files from windows build --- .teamcity/package-windows.sh | 1 + github_release.py | 27 ++++++++++++++------------- 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/.teamcity/package-windows.sh b/.teamcity/package-windows.sh index a4b91d93d0a..6715af928ce 100755 --- a/.teamcity/package-windows.sh +++ b/.teamcity/package-windows.sh @@ -15,4 +15,5 @@ for arch in ${windowsArchs[@]}; do make cloudflared-msi # Copy msi into final directory mv cloudflared-$VERSION-$arch.msi $FINAL_ARTIFACT_DIR/cloudflared-windows-$arch.msi + cp $BUILT_ARTIFACT_DIR/cloudflared-windows-$arch.exe $FINAL_ARTIFACT_DIR/cloudflared-windows-$arch.exe done diff --git a/github_release.py b/github_release.py index db6120863bf..8773fc43f43 100755 --- a/github_release.py +++ b/github_release.py @@ -214,9 +214,6 @@ def main(): """ Attempts to upload Asset to Github Release. Creates Release if it doesn't exist """ try: args = parse_args() - client = Github(args.api_key) - repo = client.get_repo(CLOUDFLARED_REPO) - release = get_or_create_release(repo, args.release_version, args.dry_run) if args.dry_run: if os.path.isdir(args.path): @@ -229,17 +226,21 @@ def main(): else: logging.error("dryrun failed") return - - if os.path.isdir(args.path): - onlyfiles = [f for f in listdir(args.path) if isfile(join(args.path, f))] - for filename in onlyfiles: - binary_path = os.path.join(args.path, filename) - upload_asset(release, binary_path, filename, args.release_version, args.kv_account_id, args.namespace_id, - args.kv_api_token) - move_asset(binary_path, filename) else: - upload_asset(release, args.path, args.name, args.release_version, args.kv_account_id, args.namespace_id, - args.kv_api_token) + client = Github(args.api_key) + repo = client.get_repo(CLOUDFLARED_REPO) + release = get_or_create_release(repo, args.release_version, args.dry_run) + + if os.path.isdir(args.path): + onlyfiles = [f for f in listdir(args.path) if isfile(join(args.path, f))] + for filename in onlyfiles: + binary_path = os.path.join(args.path, filename) + upload_asset(release, binary_path, filename, args.release_version, args.kv_account_id, args.namespace_id, + args.kv_api_token) + move_asset(binary_path, filename) + else: + upload_asset(release, args.path, args.name, args.release_version, args.kv_account_id, args.namespace_id, + args.kv_api_token) except Exception as e: logging.exception(e) From 9f0002db4053de1e9792df526c4fc5d16aaf5150 Mon Sep 17 00:00:00 2001 From: lneto Date: Mon, 5 Aug 2024 18:25:12 +0100 Subject: [PATCH 16/16] Release 2024.8.2 --- RELEASE_NOTES | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/RELEASE_NOTES b/RELEASE_NOTES index 200a04bdbb7..a40a02516b5 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -1,3 +1,7 @@ +2024.8.2 +- 2024-08-05 TUN-8583: change final directory of artifacts +- 2024-08-05 TUN-8585: Avoid creating GH client when dry-run is true + 2024.7.3 - 2024-07-31 TUN-8546: Fix final artifacts paths