CI: refactor fstests (#4841)

.github/workflows/integrationtests.yml

@@ -5,18 +5,20 @@ on:
     branches:
       - 'main'
       - 'release-*'
-    paths-ignore:
-      - 'docs/**'
-      - '**.md'
+    paths:
+      - '**.c'
+      - '**.go'
+      - 'Makefile'
+      - 'integrationtests.yml'
   pull_request:
-    #The branches below must be a subset of the branches above
     branches:
       - 'main'
       - 'release-*'
-    paths-ignore:
-      - 'docs/**'
-      - '**.md'
-      - '.github/**'
+    paths:
+      - '**.c'
+      - '**.go'
+      - 'Makefile'
+      - 'integrationtests.yml'
   workflow_dispatch:
     inputs:
       debug:
@@ -26,90 +28,149 @@ on:
         default: false
 
 jobs:
+  build-matrix:
+    runs-on: ubuntu-20.04
+    steps:
+      - id: set-matrix
+        run: |
+          echo "github.event_name is ${{github.event_name}}"
+          echo "GITHUB_REF_NAME is ${GITHUB_REF_NAME}"
+          if [[ "${{github.event_name}}" == "schedule" || "${{github.event_name}}" == "workflow_dispatch"  ]]; then
+            echo 'meta_matrix=["sqlite3", "redis", "mysql", "tikv", "tidb", "postgres", "badger", "mariadb", "fdb"]' >> $GITHUB_OUTPUT
+          elif [[ "${{github.event_name}}" == "pull_request" ||  "${{github.event_name}}" == "push" ]]; then
+            echo 'meta_matrix=["redis", "mysql"]' >> $GITHUB_OUTPUT
+          else
+            echo "event_name is not supported" && exit 1
+          fi
+    outputs:
+      meta_matrix: ${{ steps.set-matrix.outputs.meta_matrix }}
+
   integrationtests:
-    timeout-minutes: 60
+    timeout-minutes: 120
     runs-on: ubuntu-20.04
+    needs: build-matrix
+    strategy:
+      fail-fast: false
+      matrix:
+        meta: ${{ fromJson(needs.build-matrix.outputs.meta_matrix) }}
     steps:
       - name: Checkout
         uses: actions/checkout@v3
         with:
           fetch-depth: 1
 
+      - name: Set Variable
+        id: vars
+        run: |
+          if [ "${{matrix.meta}}" == "fdb" ]; then
+            echo "target=juicefs.fdb" >> $GITHUB_OUTPUT
+          else
+            echo "target=juicefs" >> $GITHUB_OUTPUT
+          fi
+
       - name: Build
-        timeout-minutes: 10
         uses: ./.github/actions/build
+        with: 
+          target: ${{steps.vars.outputs.target}}
 
-      - name: Copy
-        run: |
-          cp juicefs /tmp/mount.juicefs
-
-      - name: Run Redis
-        run: |
-          sudo docker run -d --name redis -v redis-data:/data  \
-          -p 6379:6379  redis redis-server --appendonly yes
+      - name: Prepare meta db
+        run: | 
+          chmod +x .github/scripts/start_meta_engine.sh
+          source .github/scripts/start_meta_engine.sh
+          start_meta_engine ${{matrix.meta}}
+          meta_url=$(get_meta_url ${{matrix.meta}})
+          create_database $meta_url
 
       - name: Juicefs Format
         run: |
-          sudo ./juicefs format  redis://127.0.0.1:6379/1 pics
+          source .github/scripts/start_meta_engine.sh
+          meta_url=$(get_meta_url ${{matrix.meta}})
+          sudo ./juicefs format $meta_url --trash-days 0 pics
 
       - name: Juicefs Mount
         run: |
-          sudo ./juicefs mount -d redis://127.0.0.1:6379/1 /jfs/ --enable-xattr --no-usage-report &
+          source .github/scripts/start_meta_engine.sh
+          meta_url=$(get_meta_url ${{matrix.meta}})
+          sudo ./juicefs mount -d $meta_url /jfs --no-usage-report --enable-xattr
+          stat /jfs/.accesslog
 
       - name: Fslock Test
+        timeout-minutes: 5
         run: |
           cd /jfs/
           git clone https://github.com/danjacques/gofslock.git
           cd gofslock && git checkout cc7f001fe0e7df1710adc8f0cd9e9d6d21fdb3a9
           go test -v ./fslock/...
           stat /jfs/
 
-
-      - name: Pyxattr Test
+      - name: flock test
+        timeout-minutes: 5
         run: |
-          git clone https://github.com/iustin/pyxattr.git
-          cd pyxattr
-          pip3 install pytest
-          pip3 install pyxattr
-          stat /jfs/
-          TEST_DIR=/jfs/ python3 -m pytest tests
-
+          git clone https://github.com/gofrs/flock.git
+          mkdir /jfs/tmp
+          cd flock && go mod init github.com/gofrs/flock.git && go mod tidy && TMPDIR=/jfs/tmp go test .
 
-      - name: Fstests
+      - name: make secfs.test
         run: |
-          sudo .github/scripts/apt_install.sh  libacl1-dev attr
-          sudo DURATION=60 make -C fstests fsx
-          sudo make -C fstests xattrs
-          sudo make -C fstests flock
-
-
-      - name: Fsracer
-        #https://github.com/gfx/example-github-actions-with-tty
-        shell: 'script -q -e -c "bash {0}"'
+          sudo .github/scripts/apt_install.sh  libacl1-dev
+          git clone https://github.com/billziss-gh/secfs.test.git
+          make -C secfs.test
+  
+      - name: Fsx Test
+        timeout-minutes: 15
         run: |
-          sudo DURATION=60 make -C fstests fsracer
-
-      - name: MountJuiceFSTest
+          if [[ "${{github.event_name}}" == "schedule" || "${{github.event_name}}" == "workflow_dispatch"  ]] ; then
+            duration=900
+          else
+            duration=300
+          fi
+          sudo touch /jfs/fsx.out
+          sudo rm -f /tmp/fsx.out
+          sudo ln -s /jfs/fsx.out /tmp/fsx.out
+          sudo secfs.test/tools/bin/fsx -d $duration -p 10000 -F 10000000 /tmp/fsx.out
+          # sudo secfs.test/tools/bin/fsx -d 180 -p 10000 -F 100000 /jfs/fsx.out
+
+      - name: Fsracer Test
+        timeout-minutes: 15
+        shell: 'script -q -e -c "bash {0}"'
         run: |
-          sudo /tmp/mount.juicefs redis://127.0.0.1:6379/1 /tmp/mount-jfs
-          [ `stat --format "%i" /tmp/mount-jfs` -eq 1 ]
-
-
+          if [[ "${{github.event_name}}" == "schedule" || "${{github.event_name}}" == "workflow_dispatch"  ]] ; then
+            duration=900
+          else
+            duration=300
+          fi
+          sudo secfs.test/tools/bin/fsracer $duration /jfs
+  
       - name: log
         if: always()
         run: |
           tail -300 /var/log/juicefs.log
           grep "<FATAL>:" /var/log/juicefs.log && exit 1 || true
 
+      - name: Setup upterm session
+        if: failure() && (github.event.inputs.debug == 'true' || github.run_attempt != 1)
+        timeout-minutes: 60
+        uses: lhotari/action-upterm@v1
+
+  success-all-test:
+    runs-on: ubuntu-latest
+    needs: [integrationtests]
+    if: always()
+    steps:
+      - uses: technote-space/workflow-conclusion-action@v3
+      - uses: actions/checkout@v3
+
+      - name: Check Failure
+        if: env.WORKFLOW_CONCLUSION == 'failure'
+        run: exit 1
+
       - name: Send Slack Notification
-        if: failure()
+        if: failure() && github.event_name != 'workflow_dispatch'
         uses: juicedata/slack-notify-action@main
         with:
           channel-id: "${{ secrets.SLACK_CHANNEL_ID_FOR_PR_CHECK_NOTIFY }}"
           slack_bot_token: "${{ secrets.SLACK_BOT_TOKEN }}"
 
-
-      - name: Setup upterm session
-        if: failure() && (github.event.inputs.debug == 'true' || github.run_attempt != 1)
-        timeout-minutes: 60
-        uses: lhotari/action-upterm@v1
+      - name: Success
+        if: success() 
+        run: echo "All Done"

.github/workflows/mutate-test.yml

@@ -148,7 +148,7 @@ jobs:
           docker run -d -p 9000:9000 -p 9001:9001 -e "MINIO_ROOT_USER=testUser" -e "MINIO_ROOT_PASSWORD=testUserPassword" quay.io/minio/minio:RELEASE.2022-01-25T19-56-04Z server /data --console-address ":9001"
           go install github.com/minio/[email protected] && mc config host add local http://127.0.0.1:9000 testUser testUserPassword && mc mb local/testbucket
           make
-          sudo make -C fstests setup
+          # sudo make -C fstests setup
 
       - name: run mutate test
         # timeout-minutes: 120

.github/workflows/pjdfstest.yml

@@ -8,6 +8,7 @@ on:
     paths-ignore:
       - 'docs/**'
       - '**.md'
+      - '.github/**'
   pull_request:
     #The branches below must be a subset of the branches above
     branches:
@@ -34,15 +35,12 @@ jobs:
       - id: set-matrix
         run: |
           echo "github.event_name is ${{github.event_name}}"
-          echo "GITHUB_REF_NAME is ${GITHUB_REF_NAME}"
-          if [ "${{github.event_name}}" == "schedule"  ]; then
+          if [[ "${{github.event_name}}" == "schedule" || "${{github.event_name}}" == "workflow_dispatch" ]]; then
             echo 'meta_matrix=["sqlite3", "redis", "mysql", "tikv", "tidb", "postgres", "badger", "mariadb", "fdb"]' >> $GITHUB_OUTPUT
-          elif [ "${{github.event_name}}" == "pull_request"  ]; then
-            echo 'meta_matrix=["redis"]' >> $GITHUB_OUTPUT
-          elif [ "${{github.event_name}}" == "workflow_dispatch"  ]; then
+          elif [[ "${{github.event_name}}" == "pull_request" || "${{github.event_name}}" == "push"  ]]; then
             echo 'meta_matrix=["redis"]' >> $GITHUB_OUTPUT
           else
-            echo 'meta_matrix=["redis"]' >> $GITHUB_OUTPUT
+            echo 'event name is not supported' && exit 1
           fi
     outputs:
       meta_matrix: ${{ steps.set-matrix.outputs.meta_matrix }}
@@ -100,12 +98,8 @@ jobs:
           meta_url=$(get_meta_url ${{matrix.meta}})
           # sudo mkdir /var/jfs
           # sudo chmod 777 /var/jfs
-          sudo ./juicefs mount -d $meta_url /tmp/jfs --no-usage-report &
-          sleep 5
-          if [ ! -f /tmp/jfs/.accesslog ]; then
-            echo "<FATAL>: mount failed"
-            exit 1
-          fi
+          sudo ./juicefs mount -d $meta_url /tmp/jfs --no-usage-report 
+          stat /tmp/jfs/.accesslog
 
       - name: Pjdfstest
         run: |