CI: Test dump load

.github/scripts/command/load_dump.sh

@@ -1,12 +1,17 @@
-#!/bin/bash -e
+#!/bin/bash -ex
 source .github/scripts/common/common.sh
 
 [[ -z "$META" ]] && META=sqlite3
 source .github/scripts/start_meta_engine.sh
 start_meta_engine $META
 META_URL=$(get_meta_url $META)
+META_URL2=$(get_meta_url2 $META)
 [[ -z "$SEED" ]] && SEED=$(date +%s)
+HEARTBEAT_INTERVAL=2
+DIR_QUOTA_FLUSH_INTERVAL=4
 # [[ -z "$SEED" ]] && SEED=1711594639
+[[ -z "$BINARY" ]] && BINARY=false
+[[ -z "$FAST" ]] && FAST=false
 
 trap "echo random seed is $SEED" EXIT
 
@@ -23,103 +28,142 @@ sleep 3s
 mc alias set myminio http://localhost:9000 minioadmin minioadmin
 python3 -c "import xattr" || sudo pip install xattr
 
-test_dump_load_with_iflag(){
-    do_dump_load_with_iflag
+test_dump_load_sustained_file(){
+    prepare_test
+    ./juicefs format $META_URL myjfs
+    ./juicefs mount -d $META_URL /jfs
+    # create a file, open it and keep it open, then remove it
+    echo "hello" > /jfs/hello.txt
+    exec 3<>/jfs/hello.txt
+    rm /jfs/hello.txt
+    # lsof -p $$ 
+    ./juicefs dump $META_URL dump.json $(get_dump_option)
+    exec 3>&-
+    if [[ "$BINARY" == "true" ]]; then
+        sustained=$(./juicefs load dump.json --binary --stat | grep sustained | awk -F"|" '{print $2}')
+    else
+        sustained=$(jq '.Sustained | length' dump.json)
+    fi
+    echo "sustained file count: $sustained"
+    # [[ "$sustained" -eq "$file_count" ]] || (echo "sustained file count($sustained) should be $file_count" && exit 1)
+    umount_jfs /jfs $META_URL
+    python3 .github/scripts/flush_meta.py $META_URL
+    ./juicefs load $META_URL dump.json $(get_load_option)
+    ./juicefs mount -d $META_URL /jfs 
+    ls /jfs
 }
 
-test_dump_load_with_iflag_binary(){
-    do_dump_load_with_iflag --binary
+test_dump_load_with_copy_file_range(){
+    prepare_test
+    ./juicefs format $META_URL myjfs
+    ./juicefs mount -d $META_URL /jfs
+    dd if=/dev/zero of=/tmp/test bs=1M count=4096
+    cp /tmp/test /jfs/test
+    node .github/scripts/copyFile.js /jfs/test /jfs/test1
+    ./juicefs dump $META_URL dump.json $(get_dump_option)
+    umount_jfs /jfs $META_URL
+    python3 .github/scripts/flush_meta.py $META_URL
+    ./juicefs load $META_URL dump.json $(get_load_option)
+    ./juicefs mount -d $META_URL /jfs
+    compare_md5sum /tmp/test /jfs/test1
 }
 
-do_dump_load_with_iflag(){
-    option=$@
+test_dump_load_with_clone(){
+    prepare_test
+    ./juicefs format $META_URL myjfs
+    ./juicefs mount -d $META_URL /jfs
+    dd if=/dev/urandom of=/tmp/test bs=1M count=1024
+    cp /tmp/test /jfs/test
+    ./juicefs clone /jfs/test /jfs/test2
+    ./juicefs dump $META_URL dump.json $(get_dump_option)
+    umount_jfs /jfs $META_URL
+    python3 .github/scripts/flush_meta.py $META_URL
+    ./juicefs load $META_URL dump.json $(get_load_option)
+    ./juicefs mount -d $META_URL /jfs
+    compare_md5sum /tmp/test /jfs/test2
+}
+
+test_dump_load_with_quota(){
+    prepare_test
+    ./juicefs format $META_URL myjfs 
+    ./juicefs mount -d $META_URL /jfs --heartbeat $HEARTBEAT_INTERVAL
+    mkdir -p /jfs/d
+    ./juicefs quota set $META_URL --path /d --inodes 1000 --capacity 1
+    ./juicefs dump --log-level error $META_URL $(get_dump_option) > dump.json
+    umount_jfs /jfs $META_URL
+    python3 .github/scripts/flush_meta.py $META_URL
+    ./juicefs load $META_URL dump.json $(get_load_option)
+    ./juicefs mount $META_URL /jfs -d --heartbeat $HEARTBEAT_INTERVAL
+    ./juicefs quota get $META_URL --path /d
+    dd if=/dev/zero of=/jfs/d/test1 bs=1G count=1
+    sleep $DIR_QUOTA_FLUSH_INTERVAL
+    echo a | tee -a /jfs/d/test1 2>error.log && echo "write should fail on out of space" && exit 1 || true
+    grep "Disk quota exceeded" error.log || (echo "grep failed" && exit 1)
+}
+
+test_dump_load_with_iflag(){
     prepare_test
     ./juicefs format $META_URL myjfs
     ./juicefs mount -d $META_URL /jfs --enable-ioctl
     echo "hello" > /jfs/hello.txt
     chattr +i /jfs/hello.txt
-    ./juicefs dump $META_URL dump.json $option
+    ./juicefs dump $META_URL dump.json $(get_dump_option)
     umount_jfs /jfs $META_URL
     python3 .github/scripts/flush_meta.py $META_URL
-    ./juicefs load $META_URL dump.json $option
+    ./juicefs load $META_URL dump.json $(get_load_option)
     ./juicefs mount -d $META_URL /jfs --enable-ioctl
     echo "hello" > /jfs/hello.txt && echo "write should fail" && exit 1 || true
     chattr -i /jfs/hello.txt
     echo "world" > /jfs/hello.txt
     cat /jfs/hello.txt | grep world
 }
 
-test_dump_load_with_keep_secret_key(){
-    do_dump_load_with_keep_secret_key
-}
-
-test_dump_load_with_keep_secret_key_in_binary(){
-    do_dump_load_with_keep_secret_key --binary
-}
-
-do_dump_load_with_keep_secret_key()
+test_dump_load_with_keep_secret_key()
 {
     option=$@
     prepare_test
     ./juicefs format $META_URL myjfs --storage minio --bucket http://localhost:9000/test --access-key minioadmin --secret-key minioadmin
-    ./juicefs dump --keep-secret-key $META_URL dump.json $option
+    ./juicefs dump --keep-secret-key $META_URL dump.json $(get_dump_option)
     python3 .github/scripts/flush_meta.py $META_URL
-    ./juicefs load $META_URL dump.json $option
+    ./juicefs load $META_URL dump.json $(get_load_option)
     ./juicefs mount -d $META_URL /jfs
     echo "hello" > /jfs/hello.txt
     cat /jfs/hello.txt | grep hello
 
     umount_jfs /jfs $META_URL
-    ./juicefs dump $META_URL dump.json $option
+    ./juicefs dump $META_URL dump.json $(get_dump_option)
     python3 .github/scripts/flush_meta.py $META_URL
-    ./juicefs load $META_URL dump.json $option
+    ./juicefs load $META_URL dump.json $(get_load_option)
     ./juicefs mount -d $META_URL /jfs && echo "mount should fail" && exit 1 || true
     ./juicefs config --secret-key minioadmin $META_URL
     ./juicefs mount -d $META_URL /jfs
     echo "hello" > /jfs/hello.txt
     cat /jfs/hello.txt | grep hello
 }
 
-
-test_dump_load_with_trash_enable(){
-    do_dump_load_with_fsrand 1
-}
-test_dump_load_with_trash_disable(){
-    do_dump_load_with_fsrand 0
+test_dump_load_with_trash(){
+    echo "TOOD"
 }
 
-do_dump_load_with_fsrand(){
-    trash_days=$1
+test_dump_load_with_fsrand()
+{
     prepare_test
-    ./juicefs format $META_URL myjfs --trash-days $trash_days --enable-acl
+    ./juicefs format $META_URL myjfs --trash-days 0 --enable-acl
     ./juicefs mount -d $META_URL /jfs --enable-xattr
-    SEED=$SEED LOG_LEVEL=WARNING MAX_EXAMPLE=30 STEP_COUNT=20 PROFILE=generate ROOT_DIR1=/jfs/fsrand ROOT_DIR2=/tmp/fsrand python3 .github/scripts/hypo/fs.py || true
-    do_dump_load_and_compare 
-    do_dump_load_and_compare --binary
-    do_dump_load_and_compare --fast
-    do_dump_load_and_compare --skip-trash
-    do_dump_load_and_compare --fast --skip-trash
-}
-
-do_dump_load_and_compare()
-{
-    option=$@
-    echo option is $option
-    ./juicefs dump $META_URL dump.json $option
-    rm -rf test2.db 
-    if [[ "$option" == *"--binary"* ]]; then
-        ./juicefs load sqlite3://test2.db dump.json $option
-    else
-        ./juicefs load sqlite3://test2.db dump.json
-    fi
-    ./juicefs dump sqlite3://test2.db dump2.json $option
-    # if [[ "$option" != *"--binary"* ]]; then
+    SEED=$SEED LOG_LEVEL=WARNING MAX_EXAMPLE=30 STEP_COUNT=20 PROFILE=generate ROOT_DIR1=/jfs/fsrand ROOT_DIR2=/tmp/fsrand python3 .github/scripts/hypo/fs.py || true    
+    ./juicefs dump $META_URL dump.json $(get_dump_option)
+    create_database $META_URL2
+    ./juicefs load $META_URL2 dump.json $(get_load_option)
+    ./juicefs dump $META_URL2 dump2.json $(get_dump_option)
+    # if [[ "$BINARY" == "false" ]]; then
     #     compare_dump_json
     # fi
-    ./juicefs mount -d sqlite3://test2.db /jfs2
+    ./juicefs mount -d $META_URL2 /jfs2
     diff -ur /jfs/fsrand /jfs2/fsrand --no-dereference
     compare_stat_acl_xattr /jfs/fsrand /jfs2/fsrand
-    umount /jfs2
+    umount_jfs /jfs2 $META_URL2
+    ./juicefs status $META_URL2 && UUID=$(./juicefs status $META_URL2 | grep UUID | cut -d '"' -f 4)
+    ./juicefs destroy --yes $META_URL2 $UUID
 }
 
 compare_dump_json(){
@@ -178,6 +222,27 @@ test_load_encrypted_meta_backup()
     rm test2.db -rf
 }
 
+
+get_dump_option(){
+    if [[ "$BINARY" == "true" ]]; then 
+        option="--binary"
+    elif [[ "$FAST" == "true" ]]; then
+        option="--fast"
+    else
+        option=""
+    fi
+    echo $option
+}
+
+get_load_option(){
+    if [[ "$BINARY" == "true" ]]; then 
+        option="--binary"
+    else
+        option=""
+    fi
+    echo $option
+}
+
 prepare_test(){
     umount_jfs /jfs $META_URL
     umount_jfs /jfs2 sqlite3://test2.db

.github/scripts/copyFile.js

@@ -0,0 +1,19 @@
+const fs = require('fs');
+const path = require('path');
+const crypto = require('crypto');
+
+if (process.argv.length !== 4) {
+  console.error('Usage: node copyFile.js <sourceFile> <destinationFile>');
+  process.exit(1);
+}
+
+const sourceFile = path.resolve(process.argv[2]);
+const destinationFile = path.resolve(process.argv[3]);
+
+fs.copyFile(sourceFile, destinationFile, async (err) => {
+  if (err) {
+    console.error('Error copying file:', err);
+    process.exit(1);
+  }
+  console.log('File copied successfully.');
+});

.github/scripts/hypo/fs.py

@@ -44,7 +44,7 @@ class JuicefsMachine(RuleBasedStateMachine):
     if os.getenv('EXCLUDE_RULES'):
         EXCLUDE_RULES = os.getenv('EXCLUDE_RULES').split(',')
     else:
-        EXCLUDE_RULES = ['clone_cp_file', 'clone_cp_dir', 'loop_symlink', 'symlink', 'readlines', 'readline']
+        EXCLUDE_RULES = ['readlines', 'readline']
         # EXCLUDE_RULES = ['rebalance_dir', 'rebalance_file', 'clone_cp_file', 'clone_cp_dir', 'loop_symlink', 'hardlink', 'rename_dir', 'chown']
     ROOT_DIR1=os.environ.get('ROOT_DIR1', '/tmp/fsrand')
     ROOT_DIR2=os.environ.get('ROOT_DIR2', '/tmp/jfs/fsrand')

.github/workflows/load.yml

@@ -35,7 +35,7 @@ jobs:
           if [ "${{github.event_name}}" == "schedule"  ] || [ "${{github.event_name}}" == "workflow_dispatch"  ]; then
             echo 'meta_matrix=["sqlite3", "redis", "mysql", "tikv", "tidb", "postgres", "mariadb", "fdb"]' >> $GITHUB_OUTPUT
           else
-            echo 'meta_matrix=["redis", "tikv", "mysql"]' >> $GITHUB_OUTPUT
+            echo 'meta_matrix=["redis"]' >> $GITHUB_OUTPUT
           fi
     outputs:
       meta_matrix: ${{ steps.set-matrix.outputs.meta_matrix }}
@@ -80,12 +80,36 @@ jobs:
         with: 
           target: ${{steps.vars.outputs.target}}
 
-      - name: Test Load & Dump 
+      - name: Install nodejs
+        run: |
+          curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.1/install.sh | bash
+          export NVM_DIR="$HOME/.nvm"
+          [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" # This loads nvm
+          [ -s "$NVM_DIR/bash_completion" ] && \. "$NVM_DIR/bash_completion" # This loads nvm bash_completion
+          nvm install 22
+          nvm use 22
+          node -v 
+          nvm current
+
+      - name: Test Load & Dump with Binary
+        timeout-minutes: 30
+        run: |
+          sudo BINARY=true GOCOVERDIR=$(pwd)/cover META=${{matrix.meta}} .github/scripts/command/load_dump.sh  test_dump_load_sustained_file    
+
+      - name: Test Load & Dump with Json Fast Mode
+        if: false
         timeout-minutes: 30
         run: |
-          sudo GOCOVERDIR=$(pwd)/cover META=${{matrix.meta}} .github/scripts/command/load_dump.sh          
+          sudo FAST=true GOCOVERDIR=$(pwd)/cover META=${{matrix.meta}} .github/scripts/command/load_dump.sh  
+
+      - name: Test Load & Dump with Json
+        if: false
+        timeout-minutes: 30
+        run: |
+          sudo GOCOVERDIR=$(pwd)/cover META=${{matrix.meta}} .github/scripts/command/load_dump.sh     
 
       - name: Benchmark dump load in binary format
+        if : false
         timeout-minutes: 60
         env:
           AWS_ACCESS_KEY_ID: ${{secrets.CI_COVERAGE_AWS_AK}}
@@ -97,6 +121,7 @@ jobs:
           sudo -E GOCOVERDIR=$(pwd)/cover .github/scripts/command/load_dump_bench.sh test_dump_load_in_binary
 
       - name: Benchmark dump load fast
+        if : false
         timeout-minutes: 60
         env:
           AWS_ACCESS_KEY_ID: ${{secrets.CI_COVERAGE_AWS_AK}}
@@ -108,6 +133,7 @@ jobs:
           sudo -E GOCOVERDIR=$(pwd)/cover .github/scripts/command/load_dump_bench.sh test_dump_load_fast
 
       - name: Benchmark dump load
+        if : false
         timeout-minutes: 60
         env:
           AWS_ACCESS_KEY_ID: ${{secrets.CI_COVERAGE_AWS_AK}}
@@ -123,13 +149,13 @@ jobs:
         run: | 
           tail -500 /var/log/juicefs.log
           grep "<FATAL>:" /var/log/juicefs.log && exit 1 || true
-          
+
       - name: upload coverage report
         timeout-minutes: 5
         uses: ./.github/actions/upload-coverage
         with:
           UPLOAD_TOKEN: ${{ secrets.CI_COVERAGE_FILE_UPLOAD_AUTH_TOKEN }}
-          
+
       - name: Setup upterm session
         if: failure() && (github.event.inputs.debug == 'true' || github.run_attempt != 1)
         # if: failure()
@@ -148,6 +174,15 @@ jobs:
         if: env.WORKFLOW_CONCLUSION == 'failure'
         run: exit 1
 
+      - name: mount coverage dir
+        timeout-minutes: 5
+        uses: ./.github/actions/mount-coverage-dir
+        with:
+          mount_point: cover
+          access_key: ${{ secrets.CI_COVERAGE_AWS_AK }}
+          secret_key: ${{ secrets.CI_COVERAGE_AWS_SK }}
+          token: ${{ secrets.CI_COVERAGE_AWS_TOKEN }}
+
       - name: Send Slack Notification
         if: failure() && github.event_name != 'workflow_dispatch' 
         uses: juicedata/slack-notify-action@main