From ec619b3c7ff0ce9a5576f1dee73e4563d332ad70 Mon Sep 17 00:00:00 2001 From: Zhou Cheng Date: Mon, 13 May 2024 15:21:56 +0800 Subject: [PATCH] CI: refactor fstests (#4841) --- .github/workflows/integrationtests.yml | 165 +++++++++++++++++-------- .github/workflows/mutate-test.yml | 2 +- .github/workflows/pjdfstest.yml | 18 +-- .github/workflows/xattr.yml | 145 ++++++++++++++++++++++ .gitignore | 3 - .golangci.yml | 2 - fstests/Makefile | 43 ------- fstests/node1/redis.conf | 5 - fstests/node2/redis.conf | 5 - fstests/node3/redis.conf | 5 - 10 files changed, 265 insertions(+), 128 deletions(-) create mode 100644 .github/workflows/xattr.yml delete mode 100644 fstests/Makefile delete mode 100644 fstests/node1/redis.conf delete mode 100644 fstests/node2/redis.conf delete mode 100644 fstests/node3/redis.conf diff --git a/.github/workflows/integrationtests.yml b/.github/workflows/integrationtests.yml index 60a9151675c3..01f94cd59e99 100644 --- a/.github/workflows/integrationtests.yml +++ b/.github/workflows/integrationtests.yml @@ -5,18 +5,20 @@ on: branches: - 'main' - 'release-*' - paths-ignore: - - 'docs/**' - - '**.md' + paths: + - '**.c' + - '**.go' + - 'Makefile' + - 'integrationtests.yml' pull_request: - #The branches below must be a subset of the branches above branches: - 'main' - 'release-*' - paths-ignore: - - 'docs/**' - - '**.md' - - '.github/**' + paths: + - '**.c' + - '**.go' + - 'Makefile' + - 'integrationtests.yml' workflow_dispatch: inputs: debug: @@ -26,37 +28,74 @@ on: default: false jobs: + build-matrix: + runs-on: ubuntu-20.04 + steps: + - id: set-matrix + run: | + echo "github.event_name is ${{github.event_name}}" + echo "GITHUB_REF_NAME is ${GITHUB_REF_NAME}" + if [[ "${{github.event_name}}" == "schedule" || "${{github.event_name}}" == "workflow_dispatch" ]]; then + echo 'meta_matrix=["sqlite3", "redis", "mysql", "tikv", "tidb", "postgres", "badger", "mariadb", "fdb"]' >> $GITHUB_OUTPUT + elif [[ "${{github.event_name}}" == "pull_request" || "${{github.event_name}}" == "push" ]]; then + echo 'meta_matrix=["redis", "mysql"]' >> $GITHUB_OUTPUT + else + echo "event_name is not supported" && exit 1 + fi + outputs: + meta_matrix: ${{ steps.set-matrix.outputs.meta_matrix }} + integrationtests: - timeout-minutes: 60 + timeout-minutes: 120 runs-on: ubuntu-20.04 + needs: build-matrix + strategy: + fail-fast: false + matrix: + meta: ${{ fromJson(needs.build-matrix.outputs.meta_matrix) }} steps: - name: Checkout uses: actions/checkout@v3 with: fetch-depth: 1 + - name: Set Variable + id: vars + run: | + if [ "${{matrix.meta}}" == "fdb" ]; then + echo "target=juicefs.fdb" >> $GITHUB_OUTPUT + else + echo "target=juicefs" >> $GITHUB_OUTPUT + fi + - name: Build - timeout-minutes: 10 uses: ./.github/actions/build + with: + target: ${{steps.vars.outputs.target}} - - name: Copy - run: | - cp juicefs /tmp/mount.juicefs - - - name: Run Redis - run: | - sudo docker run -d --name redis -v redis-data:/data \ - -p 6379:6379 redis redis-server --appendonly yes + - name: Prepare meta db + run: | + chmod +x .github/scripts/start_meta_engine.sh + source .github/scripts/start_meta_engine.sh + start_meta_engine ${{matrix.meta}} + meta_url=$(get_meta_url ${{matrix.meta}}) + create_database $meta_url - name: Juicefs Format run: | - sudo ./juicefs format redis://127.0.0.1:6379/1 pics + source .github/scripts/start_meta_engine.sh + meta_url=$(get_meta_url ${{matrix.meta}}) + sudo ./juicefs format $meta_url --trash-days 0 pics - name: Juicefs Mount run: | - sudo ./juicefs mount -d redis://127.0.0.1:6379/1 /jfs/ --enable-xattr --no-usage-report & + source .github/scripts/start_meta_engine.sh + meta_url=$(get_meta_url ${{matrix.meta}}) + sudo ./juicefs mount -d $meta_url /jfs --no-usage-report --enable-xattr + stat /jfs/.accesslog - name: Fslock Test + timeout-minutes: 5 run: | cd /jfs/ git clone https://github.com/danjacques/gofslock.git @@ -64,52 +103,74 @@ jobs: go test -v ./fslock/... stat /jfs/ - - - name: Pyxattr Test + - name: flock test + timeout-minutes: 5 run: | - git clone https://github.com/iustin/pyxattr.git - cd pyxattr - pip3 install pytest - pip3 install pyxattr - stat /jfs/ - TEST_DIR=/jfs/ python3 -m pytest tests - + git clone https://github.com/gofrs/flock.git + mkdir /jfs/tmp + cd flock && go mod init github.com/gofrs/flock.git && go mod tidy && TMPDIR=/jfs/tmp go test . - - name: Fstests + - name: make secfs.test run: | - sudo .github/scripts/apt_install.sh libacl1-dev attr - sudo DURATION=60 make -C fstests fsx - sudo make -C fstests xattrs - sudo make -C fstests flock - - - - name: Fsracer - #https://github.com/gfx/example-github-actions-with-tty - shell: 'script -q -e -c "bash {0}"' + sudo .github/scripts/apt_install.sh libacl1-dev + git clone https://github.com/billziss-gh/secfs.test.git + make -C secfs.test + + - name: Fsx Test + timeout-minutes: 15 run: | - sudo DURATION=60 make -C fstests fsracer - - - name: MountJuiceFSTest + if [[ "${{github.event_name}}" == "schedule" || "${{github.event_name}}" == "workflow_dispatch" ]] ; then + duration=900 + else + duration=300 + fi + sudo touch /jfs/fsx.out + sudo rm -f /tmp/fsx.out + sudo ln -s /jfs/fsx.out /tmp/fsx.out + sudo secfs.test/tools/bin/fsx -d $duration -p 10000 -F 10000000 /tmp/fsx.out + # sudo secfs.test/tools/bin/fsx -d 180 -p 10000 -F 100000 /jfs/fsx.out + + - name: Fsracer Test + timeout-minutes: 15 + shell: 'script -q -e -c "bash {0}"' run: | - sudo /tmp/mount.juicefs redis://127.0.0.1:6379/1 /tmp/mount-jfs - [ `stat --format "%i" /tmp/mount-jfs` -eq 1 ] - - + if [[ "${{github.event_name}}" == "schedule" || "${{github.event_name}}" == "workflow_dispatch" ]] ; then + duration=900 + else + duration=300 + fi + sudo secfs.test/tools/bin/fsracer $duration /jfs + - name: log if: always() run: | tail -300 /var/log/juicefs.log grep ":" /var/log/juicefs.log && exit 1 || true + - name: Setup upterm session + if: failure() && (github.event.inputs.debug == 'true' || github.run_attempt != 1) + timeout-minutes: 60 + uses: lhotari/action-upterm@v1 + + success-all-test: + runs-on: ubuntu-latest + needs: [integrationtests] + if: always() + steps: + - uses: technote-space/workflow-conclusion-action@v3 + - uses: actions/checkout@v3 + + - name: Check Failure + if: env.WORKFLOW_CONCLUSION == 'failure' + run: exit 1 + - name: Send Slack Notification - if: failure() + if: failure() && github.event_name != 'workflow_dispatch' uses: juicedata/slack-notify-action@main with: channel-id: "${{ secrets.SLACK_CHANNEL_ID_FOR_PR_CHECK_NOTIFY }}" slack_bot_token: "${{ secrets.SLACK_BOT_TOKEN }}" - - - name: Setup upterm session - if: failure() && (github.event.inputs.debug == 'true' || github.run_attempt != 1) - timeout-minutes: 60 - uses: lhotari/action-upterm@v1 + - name: Success + if: success() + run: echo "All Done" \ No newline at end of file diff --git a/.github/workflows/mutate-test.yml b/.github/workflows/mutate-test.yml index e4ec3856e1b4..bdf1ab34c821 100644 --- a/.github/workflows/mutate-test.yml +++ b/.github/workflows/mutate-test.yml @@ -148,7 +148,7 @@ jobs: docker run -d -p 9000:9000 -p 9001:9001 -e "MINIO_ROOT_USER=testUser" -e "MINIO_ROOT_PASSWORD=testUserPassword" quay.io/minio/minio:RELEASE.2022-01-25T19-56-04Z server /data --console-address ":9001" go install github.com/minio/mc@RELEASE.2022-01-07T06-01-38Z && mc config host add local http://127.0.0.1:9000 testUser testUserPassword && mc mb local/testbucket make - sudo make -C fstests setup + # sudo make -C fstests setup - name: run mutate test # timeout-minutes: 120 diff --git a/.github/workflows/pjdfstest.yml b/.github/workflows/pjdfstest.yml index 627c8889a24a..295de7d7d1f5 100644 --- a/.github/workflows/pjdfstest.yml +++ b/.github/workflows/pjdfstest.yml @@ -8,6 +8,7 @@ on: paths-ignore: - 'docs/**' - '**.md' + - '.github/**' pull_request: #The branches below must be a subset of the branches above branches: @@ -34,15 +35,12 @@ jobs: - id: set-matrix run: | echo "github.event_name is ${{github.event_name}}" - echo "GITHUB_REF_NAME is ${GITHUB_REF_NAME}" - if [ "${{github.event_name}}" == "schedule" ]; then + if [[ "${{github.event_name}}" == "schedule" || "${{github.event_name}}" == "workflow_dispatch" ]]; then echo 'meta_matrix=["sqlite3", "redis", "mysql", "tikv", "tidb", "postgres", "badger", "mariadb", "fdb"]' >> $GITHUB_OUTPUT - elif [ "${{github.event_name}}" == "pull_request" ]; then - echo 'meta_matrix=["redis"]' >> $GITHUB_OUTPUT - elif [ "${{github.event_name}}" == "workflow_dispatch" ]; then + elif [[ "${{github.event_name}}" == "pull_request" || "${{github.event_name}}" == "push" ]]; then echo 'meta_matrix=["redis"]' >> $GITHUB_OUTPUT else - echo 'meta_matrix=["redis"]' >> $GITHUB_OUTPUT + echo 'event name is not supported' && exit 1 fi outputs: meta_matrix: ${{ steps.set-matrix.outputs.meta_matrix }} @@ -100,12 +98,8 @@ jobs: meta_url=$(get_meta_url ${{matrix.meta}}) # sudo mkdir /var/jfs # sudo chmod 777 /var/jfs - sudo ./juicefs mount -d $meta_url /tmp/jfs --no-usage-report & - sleep 5 - if [ ! -f /tmp/jfs/.accesslog ]; then - echo ": mount failed" - exit 1 - fi + sudo ./juicefs mount -d $meta_url /tmp/jfs --no-usage-report + stat /tmp/jfs/.accesslog - name: Pjdfstest run: | diff --git a/.github/workflows/xattr.yml b/.github/workflows/xattr.yml new file mode 100644 index 000000000000..5a3d592189ae --- /dev/null +++ b/.github/workflows/xattr.yml @@ -0,0 +1,145 @@ +name: "xattr" + +on: + push: + branches: + - 'main' + - 'release-**' + paths: + - '**.go' + - '**.c' + - '**/xattr.yml' + pull_request: + branches: + - 'main' + - 'release-**' + paths: + - '**.go' + - '**.c' + - '**/xattr.yml' + schedule: + - cron: '0 19 * * *' + workflow_dispatch: + inputs: + debug: + type: boolean + description: "Run the build with tmate debugging enabled" + required: false + default: false + +jobs: + build-matrix: + runs-on: ubuntu-20.04 + steps: + - id: set-matrix + run: | + echo "github.event_name is ${{github.event_name}}" + if [[ "${{github.event_name}}" == "schedule" || "${{github.event_name}}" == "workflow_dispatch" ]]; then + echo 'meta_matrix=["sqlite3", "redis", "mysql", "postgres", "badger", "mariadb", "fdb"]' >> $GITHUB_OUTPUT + elif [[ "${{github.event_name}}" == "pull_request" || "${{github.event_name}}" == "push" ]]; then + echo 'meta_matrix=["redis"]' >> $GITHUB_OUTPUT + else + echo 'event name is not supported' && exit 1 + fi + outputs: + meta_matrix: ${{ steps.set-matrix.outputs.meta_matrix }} + + xattr: + needs: build-matrix + strategy: + fail-fast: false + matrix: + meta: ${{ fromJson(needs.build-matrix.outputs.meta_matrix) }} + runs-on: ubuntu-20.04 + + steps: + - uses: shogo82148/actions-setup-perl@v1 + with: + perl-version: '5.34' + + - name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 1 + + - name: Set Variable + id: vars + run: | + if [ "${{matrix.meta}}" == "fdb" ]; then + echo "target=juicefs.fdb" >> $GITHUB_OUTPUT + else + echo "target=juicefs" >> $GITHUB_OUTPUT + fi + + - name: Build + uses: ./.github/actions/build + with: + target: ${{steps.vars.outputs.target}} + + - name: Prepare meta db + run: | + chmod +x .github/scripts/start_meta_engine.sh + source .github/scripts/start_meta_engine.sh + start_meta_engine ${{matrix.meta}} + meta_url=$(get_meta_url ${{matrix.meta}}) + create_database $meta_url + + - name: Juicefs Format + run: | + source .github/scripts/start_meta_engine.sh + meta_url=$(get_meta_url ${{matrix.meta}}) + sudo ./juicefs format $meta_url --trash-days 0 pics + + - name: Juicefs Mount + run: | + source .github/scripts/start_meta_engine.sh + meta_url=$(get_meta_url ${{matrix.meta}}) + # sudo mkdir /var/jfs + # sudo chmod 777 /var/jfs + sudo ./juicefs mount -d $meta_url /tmp/jfs --no-usage-report --enable-xattr + stat /tmp/jfs/.accesslog + + - name: Test + run: | + git clone https://github.com/iustin/pyxattr.git + cd pyxattr + pip3 install pytest + pip3 install pyxattr + stat /tmp/jfs/ + TEST_DIR=/tmp/jfs/ python3 -m pytest tests + + - name: log + if: always() + run: | + if [ -f /var/log/juicefs.log ]; then + tail -300 /var/log/juicefs.log + grep ":" /var/log/juicefs.log && exit 1 || true + fi + + - name: Setup upterm session + if: failure() && (github.event.inputs.debug == 'true' || github.run_attempt != 1) + timeout-minutes: 60 + uses: lhotari/action-upterm@v1 + + success-all-test: + runs-on: ubuntu-latest + needs: [xattr] + if: always() + steps: + - uses: technote-space/workflow-conclusion-action@v3 + - uses: actions/checkout@v3 + + - name: Check Failure + if: env.WORKFLOW_CONCLUSION == 'failure' + run: exit 1 + + - name: Send Slack Notification + if: failure() && github.event_name != 'workflow_dispatch' + uses: juicedata/slack-notify-action@main + with: + channel-id: "${{ secrets.SLACK_CHANNEL_ID_FOR_PR_CHECK_NOTIFY }}" + slack_bot_token: "${{ secrets.SLACK_BOT_TOKEN }}" + + - name: Success + if: success() + run: echo "All Done" \ No newline at end of file diff --git a/.gitignore b/.gitignore index 6c00982f2cd4..34b0a6cb0cfe 100644 --- a/.gitignore +++ b/.gitignore @@ -5,9 +5,6 @@ ltmain.sh *.rej .deps .dirstamp -fstests/secfs.test -fstests/flock -!fstests/Makefile jfs *.rdb .release-env diff --git a/.golangci.yml b/.golangci.yml index b0572f1a3694..285aca6b3a51 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -1,5 +1,3 @@ run: timeout: 5m tests: false - skip-dirs: - - fstests diff --git a/fstests/Makefile b/fstests/Makefile deleted file mode 100644 index c19712e6ab99..000000000000 --- a/fstests/Makefile +++ /dev/null @@ -1,43 +0,0 @@ -DURATION ?= 10 - -all: fsracer fsx xattrs - -xattrs: - touch /jfs/test_xattrs - setfattr -n user.k -v value /jfs/test_xattrs - getfattr -n user.k /jfs/test_xattrs | grep -q user.k= - -fsracer: healthcheck secfs.test/tools/bin/fsracer - secfs.test/tools/bin/fsracer $(DURATION) /jfs >fsracer.log - make healthcheck - -fsx: healthcheck secfs.test/tools/bin/fsx - secfs.test/tools/bin/fsx -d $(DURATION) -p 10000 -F 100000 /jfs/fsx.out - make healthcheck - -setup: - redis-server & - cd node1 && redis-server redis.conf & - cd node2 && redis-server redis.conf & - cd node3 && redis-server redis.conf & - sleep 1 - echo yes | redis-cli --cluster create 127.0.0.1:7001 127.0.0.1:7002 127.0.0.1:7003 - mkdir -p /jfs - ../juicefs format localhost unittest - ../juicefs mount -d --no-usage-report --enable-xattr localhost /jfs - -healthcheck: - pgrep juicefs - -secfs.test/tools/bin/fsx: secfs.test - -secfs.test/tools/bin/fsracer: secfs.test - -secfs.test: - git clone https://github.com/billziss-gh/secfs.test.git - make -C secfs.test >secfs.test-build.log 2>&1 - -flock: - git clone https://github.com/gofrs/flock.git - mkdir /jfs/tmp - cd flock && go mod init github.com/gofrs/flock.git && go mod tidy && TMPDIR=/jfs/tmp go test . diff --git a/fstests/node1/redis.conf b/fstests/node1/redis.conf deleted file mode 100644 index 689dc1153910..000000000000 --- a/fstests/node1/redis.conf +++ /dev/null @@ -1,5 +0,0 @@ -port 7001 -cluster-enabled yes -cluster-config-file nodes.conf -cluster-node-timeout 5000 -appendonly yes diff --git a/fstests/node2/redis.conf b/fstests/node2/redis.conf deleted file mode 100644 index ac6106579e26..000000000000 --- a/fstests/node2/redis.conf +++ /dev/null @@ -1,5 +0,0 @@ -port 7002 -cluster-enabled yes -cluster-config-file nodes.conf -cluster-node-timeout 5000 -appendonly yes diff --git a/fstests/node3/redis.conf b/fstests/node3/redis.conf deleted file mode 100644 index 54933907ad2a..000000000000 --- a/fstests/node3/redis.conf +++ /dev/null @@ -1,5 +0,0 @@ -port 7003 -cluster-enabled yes -cluster-config-file nodes.conf -cluster-node-timeout 5000 -appendonly yes